Re-organize directory structure (#105)

* Python SDK directory refactor
* Rename Python SDK crate to `rerun_py`, located in repository root
* Rename `example_dummy.py` to `example_car.py`
* Added `rerun_py/justfile` for automation of the Python build/develop workflows

.github/workflows/rust.yml

@@ -119,10 +119,10 @@ jobs:
         with:
           python-version: "3.8"
           cache: "pip"
-          cache-dependency-path: "crates/re_sdk_python/requirements-build.txt"
-      - run: pip install -r crates/re_sdk_python/requirements-build.txt
+          cache-dependency-path: "rerun_py/requirements-build.txt"
+      - run: pip install -r rerun_py/requirements-build.txt
       - name: Build wheels
-        run: maturin build -m crates/re_sdk_python/Cargo.toml -o dist
+        run: maturin build -m rerun_py/Cargo.toml -o dist
 
   cargo-deny:
     name: cargo deny
@@ -157,7 +157,7 @@ jobs:
         with:
           maturin-version: "0.13.1"
           command: build
-          args: -m crates/re_sdk_python/Cargo.toml -o dist
+          args: -m rerun_py/Cargo.toml -o dist
 
   maturin-macos:
     name: Maturin MacOS
@@ -176,4 +176,4 @@ jobs:
         with:
           maturin-version: "0.13.1"
           command: build
-          args: -m crates/re_sdk_python/Cargo.toml  -o dist --universal2
+          args: -m rerun_py/Cargo.toml  -o dist --universal2

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = ["crates/*", "examples/*"]
+members = ["crates/*", "examples/*", "rerun_py"]
 exclude = ["examples/nyud"]
 
 
@@ -15,10 +15,10 @@ debug = true
 
 [patch.crates-io]
 # 2022-09-21 (updated arboard)
-eframe      = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
-egui        = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
+eframe = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
+egui = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
 egui_extras = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
-egui_glow   = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
+egui_glow = { git = "https://github.com/emilk/egui", rev = "12dc61ed0f1a23b45e336799005d62e0a6f47e4f" }
 
 # Because gltf hasn't published a new version: https://github.com/gltf-rs/gltf/issues/357
 gltf = { git = "https://github.com/rerun-io/gltf", rev = "3c14ded73755d1ce9e47010edb06db63cb7e2cca" }

README.md

@@ -33,17 +33,18 @@ You should now be able to run `rerun --help` in any terminal.
 
 ```
 python3 -m pip install --upgrade pip
-pip3 install "crates/re_sdk_python[tests,examples]"
+pip3 install "rerun_py[tests,examples]"
 ```
 Note: If you are unable to upgrade pip to version `>=21.3`, you need to pass `--use-feature=in-tree-build` to the `pip3 install` command.
 
+
 ### Run the example
 ```sh
-python3 crates/re_sdk_python/example.py
+example_car
 ```
 
 ### Using the Rerun Python SDK
-See [`crates/re_sdk_python/USAGE.md`](crates/re_sdk_python/USAGE.md).
+See [`rerun_py/USAGE.md`](rerun_py/USAGE.md).
 
 
 # Development

rerun_py/.flake8

@@ -0,0 +1,19 @@
+[flake8]
+max-line-length = 88
+ban-relative-imports = true
+docstring-convention = all
+extend-ignore =
+    # Satisfy black: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
+    E203,
+    # pydocstyle: http://www.pydocstyle.org/en/stable/error_codes.html
+    # numpy convention with a few additional lints
+    D107, D203, D212, D401, D402, D415, D416,
+    # TODO: Remove errors below to further improve docstring linting
+    D1,
+    # flake8-simplify
+    SIM102, SIM117,
+
+extend-exclude =
+    # Automatically generated test artifacts
+    venv/,
+    target/,

rerun_py/.gitignore

@@ -0,0 +1,5 @@
+!Cargo.lock
+.DS_Store
+target/
+venv/
+wheels/

crates/re_sdk_python/Cargo.toml → rerun_py/Cargo.toml

@@ -1,18 +1,16 @@
 [package]
 edition = "2021"
 license = "MIT OR Apache-2.0"
-name = "re_sdk_python" # name of the rust crate
+name = "re_sdk_python"        # name of the rust crate
 publish = false
 rust-version = "1.63"
 version = "0.1.0"
 
-[package.metadata.maturin]
-python-source = "python"
 
 
 [lib]
 crate-type = ["cdylib"]
-name = "rerun_sdk" # name of the Python library
+name = "rerun_sdk"      # name of the Python library
 
 
 [features]
@@ -22,10 +20,10 @@ extension-module = ["pyo3/extension-module"]
 
 
 [dependencies]
-re_log = { path = "../re_log" }
-re_log_types = { path = "../re_log_types" }
-re_sdk_comms = { path = "../re_sdk_comms", features = ["client"] }
-re_viewer = { path = "../re_viewer", optional = true }
+re_log = { path = "../crates/re_log" }
+re_log_types = { path = "../crates/re_log_types" }
+re_sdk_comms = { path = "../crates/re_sdk_comms", features = ["client"] }
+re_viewer = { path = "../crates/re_viewer", optional = true }
 
 bytemuck = { version = "1.11", features = ["extern_crate_alloc"] }
 image = { version = "0.24", default-features = false, features = ["jpeg", "jpeg_rayon"] }

crates/re_sdk_python/README.md → rerun_py/README.md

@@ -4,42 +4,67 @@ Goal: an ergonomic Python library for logging rich data, over TCP, to a rerun se
 
 ℹ️ Note: The rust crate is called `re_sdk_python`, while the Python library is called `rerun_sdk`.
 
-## Setup
-
-ℹ️ Note: these instructions all assume you're running them from the root of the Rerun repository and have set up an environment with Python 3.7 or later.
-
-To set up a new virtualenv:
+## Build and install
+To build and install the `rerun_sdk` into your *current* Python environment run:
 
 ```sh
-python3 -m venv env
-source env/bin/activate
 python3 -m pip install --upgrade pip
+pip3 install "rerun_py"
 ```
 
-## Build, test, and run
-### Build and install
-To build and install the `rerun_sdk` into your current Python environment run:
-
-```sh
-python3 -m pip install --upgrade pip
-pip3 install "crates/re_sdk_python[tests,examples]"
-```
 ℹ️ Notes:
 - If you are unable to upgrade pip to version `>=21.3`, you need to pass `--use-feature=in-tree-build` to the `pip3 install` command.
 -  `[tests,examples]` here is used to specify that we also want to install the dependencies needed for running tests and examples.
 
-### Test
+## Usage
+See [`USAGE.md`](USAGE.md).
+
+## Running the example code
 ```sh
-mypy crates/re_sdk_python  # Static type checking
-pytest crates/re_sdk_python  # Unit tests
+python rerun_sdk/examples/example_car.py
 ```
 
-### Running the example code
+By default, the example runs Rerun in buffered mode, in the same process as the example code. This means all logged data is buffered until `rerun_sdk.show()` is called in the end, which shows the viewer and blocks until the viewer is closed.
+
+## Development
+
+ℹ️ Note:
+- These instructions assume you're running from the `rerun_py` folder and have Python 3.7 or later available.
+- We make use of the [`just`](https://github.com/casey/just) command runner tool for repository automation. See [here](https://github.com/casey/just#installation) for installation instructions.
+
+## Setup
+
+To set up a new virtualenv for development:
+
 ```sh
-python3 crates/re_sdk_python/example.py
+just develop-venv
+# For bash/zsh users:
+source venv/bin/activate
+# Or if you're using fish:
+source venv/bin/activate.fish
 ```
 
-By default, the example runs Rerun in buffered mode, in the same process as the example code. This means all logged data is buffered until `rerun_sdk.show()` is called in the end, which shows the viewer and blocks until the viewer is closed.
+## Build, test, and run
+
+For ease of development you can build and install in "editable" mode. This means you can edit the `rerun_sdk` Python code without having to re-build and install to see changes.
+
+```sh
+# Build the SDK and install in develop mode into the virtualenv
+# Re-run this if the Rust code has changed!
+just build
+```
+
+### Test
+```sh
+# Run the unit tests
+just test
+
+# Run the linting checks
+just lint
+
+# Run an example
+example_car
+```
 
 ### Logging and viewing in different processes
 
@@ -52,25 +77,16 @@ cargo run -p rerun --release
 
 In a second terminal, run the example with the `--connect` option:
 ```sh
-python3 crates/re_sdk_python/example.py --connect
-```
-
-## Usage
-See [`USAGE.md`](USAGE.md).
-
-## Developing
-For ease of development you can build and install in "editable" mode by passing the `-e` (or `--editable` ) flag to `pip install`. This means you can edit the `rerun_sdk` Python code without having to re-build and install to see changes.
-```sh
-pip install -e "crates/re_sdk_python[tests,examples]"
+example_car --connect
 ```
 
 ## Building an installable Python Wheel
 The Python bindings to the core Rust library are built using https://github.com/PyO3/pyo3.
 
 To build an installable Python wheel run:
 ```
-pip install -r crates/re_sdk_python/requirements-build.txt
-maturin build -m ./crates/re_sdk_python/Cargo.toml --release
+pip install -r rerun_py/requirements-build.txt
+maturin build -m rerun_py/Cargo.toml --release
 ```
 
 By default the wheels will be built to `target/wheels` (use the `-o` flag to set a different output directory).

crates/re_sdk_python/USAGE.md → rerun_py/USAGE.md

@@ -18,7 +18,7 @@ rerun.connect() # Connect to the separate `rerun` process.
 rerun.log_image("rgb_image", image)
 ```
 
-See more in [`example.py`](./example.py).
+See more in [`example_car.py`](rerun_sdk/examples/example_car.py).
 
 ## Paths
 The first argument to each log function is an _object path_. Each time you log to a specific object path you will update the object, i.e. log a new instance of it along the timeline. Each logging to a path bust be of the same type (you cannot log an image to the same path as a point cloud).

rerun_py/justfile

@@ -0,0 +1,34 @@
+# Set up virtual environment for development
+develop-env:
+    #!/usr/bin/env bash
+    echo "Setting up Python virtual environment in venv"
+    # set -euxo pipefail
+    python -m venv venv
+    venv/bin/pip install --upgrade pip
+    venv/bin/pip install -r requirements-build.txt
+    venv/bin/pip install -r requirements-lint.txt
+    echo "Do 'source venv/bin/activate' to use the virtual environment!"
+
+# Build and install the package into the venv
+build mode="release":
+    #!/usr/bin/env bash
+    unset CONDA_PREFIX && \
+        source venv/bin/activate && \
+        maturin develop  --extras="examples,tests" {{ if mode == "debug" { "" } else { "--release" } }}
+
+# Run autoformatting
+format:
+    isort .
+    black .
+    blackdoc .
+    pyupgrade --py37-plus `find rerun_sdk/ tests/ -name "*.py" -type f`
+    cargo fmt --all
+
+# Run linting
+lint:
+    mypy
+    flake8
+
+# Run fast unittests
+test:
+    python -m pytest tests/unit/

crates/re_sdk_python/pyproject.toml → rerun_py/pyproject.toml

@@ -5,30 +5,28 @@ build-backend = "maturin"
 requires = ["maturin==0.13,<0.14"]
 
 [project]
-authors = [
-    {name = "Rerun.io", email="opensource@rerun.io"}
-]
+name = "rerun_sdk"
+authors = [{ name = "Rerun.io", email = "opensource@rerun.io" }]
 classifiers = [
   "Programming Language :: Rust",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-
 description = "The Rerun Logging SDK"
 keywords = ["computer-vision", "logging", "rerun"]
-license = {text = "MIT OR Apache-2.0"}
-name = "rerun_sdk"
+license = { text = "MIT OR Apache-2.0" }
 requires-python = ">=3.7"
 
 dependencies = ["numpy"]
 
 [project.optional-dependencies]
-tests = [
-    "mypy==0.971",
-    "pytest==7.1.2",
-]
+tests = ["pytest==7.1.2"]
 examples = ["opencv-python~=4.6.0"]
 
+[project.scripts]
+example_car = "rerun_sdk.examples.example_car:main"
+example_multiprocessing = "rerun_sdk.examples.example_multiprocessing:main"
+
 [project.urls]
 homepage = "https://www.rerun.io/"
 documentation = "https://github.com/rerun-io/rerun"
@@ -37,3 +35,13 @@ repository = "https://github.com/rerun-io/rerun"
 [tool.maturin]
 all-features = true
 locked = true
+
+[tool.isort]
+profile = "black"
+
+[tool.mypy]
+files = ["rerun_sdk", "tests"]
+namespace_packages = true
+show_error_codes = true
+strict = true
+enable_error_code = ["redundant-expr", "truthy-bool", "ignore-without-code"]

rerun_py/requirements-lint.txt

@@ -0,0 +1,12 @@
+black==22.8.0
+blackdoc==0.3.6
+flake8-bugbear==22.8.23
+flake8-comprehensions==3.10.0
+flake8-docstrings==1.6.0
+flake8-simplify==0.19.3
+flake8-tidy-imports==4.8.0
+flake8==5.0.4
+isort==5.10.1
+mypy==0.971
+pyupgrade==2.37.3
+types-requests==2.28.10

crates/re_sdk_python/python/rerun_sdk/__init__.py → rerun_py/rerun_sdk/__init__.py

@@ -6,8 +6,8 @@
 from pathlib import Path
 from dataclasses import dataclass
 
-from . import rerun_sdk as rerun_rs  # type: ignore
-from .color_conversion import linear_to_gamma_u8_pixel
+from rerun_sdk import rerun_sdk as rerun_rs # type: ignore
+from rerun_sdk.color_conversion import linear_to_gamma_u8_pixel
 
 
 def rerun_shutdown():

rerun_py/rerun_sdk/examples/data/camera.glb

@@ -0,0 +1 @@
+../../../../crates/re_viewer/data/camera.glb