diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 695c4a750..10b02ba38 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -5,6 +5,6 @@ tag = True
tag_name = v{new_version}
message = Bump version to {new_version} [skip ci]
-[bumpversion:file:neuracore/__init__.py]
-search = __version__ = "{current_version}"
-replace = __version__ = "{new_version}"
+[bumpversion:file:pyproject.toml]
+
+[bumpversion:file:packaging/neuracore-data-daemon/pyproject.toml]
diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml
index 4f0e118ba..b9492ac6e 100644
--- a/.github/workflows/build-wheels.yaml
+++ b/.github/workflows/build-wheels.yaml
@@ -1,8 +1,12 @@
-name: Build Wheels
+name: Build Daemon Wheels
-# Builds the Python wheel that bundles the Rust data-daemon binary and the
-# producer cdylib. See docs/rust_data_daemon_development.md#packaging-the-wheel
-# for the build pipeline rationale (cargo → build_wheel_artefacts.sh → python -m build).
+# Builds the Linux-only `neuracore-data-daemon` wheels (the Rust `_data_bridge`
+# extension + the bundled `data-daemon` binary). maturin owns the build
+# (packaging/neuracore-data-daemon/pyproject.toml); the daemon binary is a
+# separate crate, so rust/scripts/build_wheel_artefacts.sh compiles it into the
+# package tree first. The pure-Python `neuracore` wheel is NOT built here — it
+# needs no compilation (built with `python -m build`).
+# See docs/rust_data_daemon_development.md#packaging-the-wheel.
on:
pull_request:
@@ -10,18 +14,14 @@ on:
- main
paths:
- 'rust/**'
- - 'neuracore/data_daemon/**'
- - 'setup.py'
- - 'MANIFEST.in'
+ - 'packaging/neuracore-data-daemon/**'
- '.github/workflows/build-wheels.yaml'
push:
branches:
- main
paths:
- 'rust/**'
- - 'neuracore/data_daemon/**'
- - 'setup.py'
- - 'MANIFEST.in'
+ - 'packaging/neuracore-data-daemon/**'
- '.github/workflows/build-wheels.yaml'
workflow_call:
@@ -31,16 +31,14 @@ concurrency:
jobs:
build:
- name: wheel (${{ matrix.os }}, py${{ matrix.python-version }})
+ name: daemon wheel (${{ matrix.target }}, py${{ matrix.python-version }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
- # Linux-first per docs/data-daemon-rewrite.md §Open items.
- # aarch64 ships when there's demand — the helper script is platform-
- # agnostic but cross-compilation isn't wired up here yet.
- os: [ubuntu-22.04]
- python-version: ["3.10", "3.11"]
+ include:
+ - { os: ubuntu-22.04, target: x86_64-unknown-linux-gnu, python-version: "3.10" }
+ - { os: ubuntu-22.04, target: x86_64-unknown-linux-gnu, python-version: "3.11" }
steps:
- name: Checkout code
@@ -48,6 +46,8 @@ jobs:
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
+ with:
+ targets: ${{ matrix.target }}
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
@@ -55,37 +55,79 @@ jobs:
workspaces: rust
- name: Set up Python ${{ matrix.python-version }}
- id: setup-python
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- - name: Install build tooling
- run: python -m pip install --upgrade pip build
-
- - name: Build Rust artefacts
- # PYO3_PYTHON pins pyo3-build-config to the matrix interpreter so the
- # produced cdylib's ABI matches the wheel's python tag. Ubuntu images
- # ship multiple python3 binaries; relying on PATH order would silently
- # pick the wrong one and produce a wheel that imports under the wrong
- # interpreter.
- env:
- PYO3_PYTHON: ${{ steps.setup-python.outputs.python-path }}
- run: ./rust/scripts/build_wheel_artefacts.sh
-
- - name: Build wheel
- run: python -m build --wheel
+ - name: Build daemon wheel
+ uses: PyO3/maturin-action@v1
+ with:
+ working-directory: packaging/neuracore-data-daemon
+ target: ${{ matrix.target }}
+ # manylinux_2_28 (AlmaLinux 8), NOT auto/manylinux2014: iceoryx2's
+ # bindgen calls clang_getTranslationUnitTargetInfo, which needs
+ # libclang >= 5.0. manylinux2014's `yum install clang` is clang 3.4 and
+ # panics with "unsupported version"; 2_28's `dnf install clang` is 15+.
+ # Cost: wheel glibc floor rises to 2.28 (RHEL8/Ubuntu 18.10+/Debian 10+).
+ manylinux: 2_28
+ # Cache compiled objects (extension + the in-container daemon binary,
+ # which inherits this wrapper) across runs. Swatinem/rust-cache only
+ # covers the host filesystem; the manylinux container builds in its own
+ # filesystem and can't see it, so without sccache every crate recompiles
+ # from scratch. sccache uses the GHA cache backend, which the container can.
+ sccache: 'true'
+ args: >-
+ --release
+ --out dist
+ --interpreter python${{ matrix.python-version }}
+ # Build the daemon binary inside the manylinux container so its glibc
+ # matches the wheel, and install clang/libclang for iceoryx2's bindgen.
+ # $GITHUB_WORKSPACE anchors the script path regardless of the build
+ # working-directory.
+ before-script-linux: |
+ (command -v yum && yum install -y clang) || (command -v dnf && dnf install -y clang) || true
+ # maturin-action exports RUSTC_WRAPPER=sccache plus the GHA-cache env,
+ # but installs the sccache binary only just before ITS own build —
+ # after this script — so the inherited wrapper points at a missing
+ # exe here. The daemon binary compiles the whole shared dep graph
+ # FIRST and dominates wall-clock, so leaving it un-wrapped meant every
+ # crate recompiled from scratch each run (Swatinem/rust-cache can't
+ # reach inside this container). Install a matching sccache ourselves
+ # and keep the wrapper set: the daemon build now reads/writes the same
+ # cross-run GHA cache, and maturin's extension build inherits the warm
+ # cache for the deps they share. Falls back to local disk cache if the
+ # GHA backend env is absent, so this never fails the build.
+ if ! command -v sccache >/dev/null 2>&1; then
+ ver="$(curl -fsSL https://api.github.com/repos/mozilla/sccache/releases/latest | grep -oP '"tag_name": *"\K[^"]+')"
+ curl -fsSL "https://github.com/mozilla/sccache/releases/download/${ver}/sccache-${ver}-x86_64-unknown-linux-musl.tar.gz" | tar -xz -C /tmp
+ install -m0755 "/tmp/sccache-${ver}-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache
+ fi
+ bash "$GITHUB_WORKSPACE/rust/scripts/build_wheel_artefacts.sh" --target ${{ matrix.target }}
- - name: Smoke-test the built wheel
+ - name: Smoke-test the daemon wheel
run: |
- python -m pip install dist/neuracore-*.whl
- python -c "import neuracore.data_daemon._native_producer as p; print('producer ok:', p)"
- python -c "from importlib.resources import files; b = files('neuracore.data_daemon') / 'bin' / 'data-daemon'; assert b.is_file(), b; print('binary ok:', b)"
+ python -m pip install --upgrade pip
+ python -m pip install --find-links packaging/neuracore-data-daemon/dist neuracore-data-daemon
+ # Run the import checks from OUTSIDE the checkout. At the repo root the
+ # in-tree `neuracore/` source package sits on sys.path[0] and shadows
+ # the installed namespace wheel, so importing it would execute the full
+ # SDK __init__ and pull in deps (pydantic, ...) this job never installs.
+ # cd-ing away tests the actually-installed daemon wheel.
+ cd "$RUNNER_TEMP"
+ python -c "import neuracore.data_daemon._data_bridge as p; print('extension ok:', p)"
+ python - <<'PY'
+ import os
+ from importlib.resources import files
+ b = files("neuracore.data_daemon") / "bin" / "data-daemon"
+ assert b.is_file(), b
+ assert os.access(b, os.X_OK), f"daemon binary is not executable: {b}"
+ print("binary ok:", b)
+ PY
- name: Upload wheel artefact
uses: actions/upload-artifact@v4
with:
- name: neuracore-wheel-${{ matrix.os }}-py${{ matrix.python-version }}
- path: dist/*.whl
+ name: neuracore-data-daemon-wheel-${{ matrix.target }}-py${{ matrix.python-version }}
+ path: packaging/neuracore-data-daemon/dist/*.whl
if-no-files-found: error
retention-days: 7
diff --git a/.github/workflows/integration-platform-production.yaml b/.github/workflows/integration-platform-production.yaml
index 125a82dc4..8cc58ab5d 100644
--- a/.github/workflows/integration-platform-production.yaml
+++ b/.github/workflows/integration-platform-production.yaml
@@ -31,7 +31,6 @@ jobs:
max-parallel: 1
matrix:
python-version: ['3.11']
- ffmpeg: [true, false]
steps:
- name: Free Disk Space (Ubuntu)
@@ -97,7 +96,6 @@ jobs:
cache: 'pip'
- name: Install FFmpeg
- if: matrix.ffmpeg == 'true'
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
diff --git a/.github/workflows/integration-platform-staging.yaml b/.github/workflows/integration-platform-staging.yaml
index 0799d0ee1..5e3feca64 100644
--- a/.github/workflows/integration-platform-staging.yaml
+++ b/.github/workflows/integration-platform-staging.yaml
@@ -31,7 +31,6 @@ jobs:
max-parallel: 1
matrix:
python-version: ['3.11']
- ffmpeg: [true, false]
steps:
- name: Free Disk Space (Ubuntu)
@@ -76,14 +75,61 @@ jobs:
cache: 'pip'
- name: Install FFmpeg
- if: matrix.ffmpeg == 'true'
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
+ # The bundled data daemon (Rust `_data_bridge` extension + the `data-daemon`
+ # binary) ships in the separate Linux-only `neuracore-data-daemon` wheel,
+ # which the base `neuracore` dependency pins from PyPI. PyPI only ever has
+ # the *last released* daemon, so build the wheel from the harness ref under
+ # test instead and install it locally — that satisfies the pin and exercises
+ # this ref's daemon code. Mirrors .github/workflows/build-wheels.yaml.
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ targets: x86_64-unknown-linux-gnu
+
+ - name: Cache cargo build
+ uses: Swatinem/rust-cache@v2
+ with:
+ workspaces: rust
+
+ - name: Build bundled data daemon wheel
+ uses: PyO3/maturin-action@v1
+ with:
+ working-directory: packaging/neuracore-data-daemon
+ target: x86_64-unknown-linux-gnu
+ # manylinux_2_28 (AlmaLinux 8), NOT auto: iceoryx2's bindgen needs
+ # libclang >= 5.0, but manylinux2014's clang is 3.4. See build-wheels.yaml.
+ manylinux: 2_28
+ # Cache compiled objects (extension + the in-container daemon binary)
+ # across runs via the GHA cache backend — the manylinux container can't
+ # see Swatinem/rust-cache's host-side cache. See build-wheels.yaml.
+ sccache: 'true'
+ args: >-
+ --release
+ --out dist
+ --interpreter python${{ matrix.python-version }}
+ # Build the daemon binary inside the manylinux container so its glibc
+ # matches the wheel, and install clang/libclang for iceoryx2's bindgen.
+ # $GITHUB_WORKSPACE anchors the script path regardless of the build
+ # working-directory.
+ before-script-linux: |
+ (command -v yum && yum install -y clang) || (command -v dnf && dnf install -y clang) || true
+ # maturin-action exports RUSTC_WRAPPER=sccache, but sccache isn't on PATH
+ # yet during before-script, so the daemon binary's cargo build can't exec
+ # it. Unset it here — the extension build still gets sccache.
+ unset RUSTC_WRAPPER
+ bash "$GITHUB_WORKSPACE/rust/scripts/build_wheel_artefacts.sh" --target x86_64-unknown-linux-gnu
+
- name: Install Python dependencies
run: |
python -m pip install --no-cache-dir --upgrade pip
+ # Install the locally-built daemon wheel first so the
+ # `neuracore-data-daemon==…` pin is already satisfied and the next
+ # install does not pull the released wheel from PyPI.
+ pip install --no-cache-dir packaging/neuracore-data-daemon/dist/*.whl
pip install --no-cache-dir ".[dev,ml,examples]"
- name: Resolve test harness
@@ -100,20 +146,12 @@ jobs:
print(f"Installed neuracore version: {neuracore.__version__}")
EOF
- - name: Run Consume Stream Integration Test
- env:
- NEURACORE_API_URL: https://staging.api.neuracore.com/api
- NEURACORE_API_KEY: ${{ secrets.STAGING_SERVICE_API_KEY }}
- NEURACORE_ORG_ID: ${{ vars.STAGING_SERVICE_ORG_ID }}
- continue-on-error: true
- run: |
- if ! pytest -o log_cli=true --log-cli-level=INFO tests/integration/platform/test_consume_stream.py; then
- echo "::warning::test_consume_stream.py failed, but this is non-blocking."
- fi
-
- name: Run Data Daemon Integration Tests
env:
NEURACORE_API_URL: https://staging.api.neuracore.com/api
NEURACORE_API_KEY: ${{ secrets.STAGING_SERVICE_API_KEY }}
NEURACORE_ORG_ID: ${{ vars.STAGING_SERVICE_ORG_ID }}
- run: pytest -o log_cli=true --log-cli-level=INFO tests/integration/platform/data_daemon/
+ # Exercise the bundled Rust data-daemon binary (installed via the daemon
+ # wheel above) rather than the in-process Python runner.
+ NCD_RUST_DAEMON: "1"
+ run: pytest -o log_cli=true --log-cli-level=INFO --import-mode=importlib tests/integration/platform/data_daemon/
diff --git a/.github/workflows/pr-merge-test.yaml b/.github/workflows/pr-merge-test.yaml
index 3db486dee..8738c2848 100644
--- a/.github/workflows/pr-merge-test.yaml
+++ b/.github/workflows/pr-merge-test.yaml
@@ -10,7 +10,7 @@ on:
- 'tests/unit/ml/**'
- 'tests/unit/conftest.py'
- 'neuracore/__init__.py'
- - 'setup.py'
+ - 'pyproject.toml'
- '.github/workflows/pr-merge-test.yaml'
concurrency:
@@ -34,7 +34,7 @@ jobs:
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- cache-dependency-glob: "setup.py"
+ cache-dependency-glob: "pyproject.toml"
python-version: ${{ matrix.python-version }}
- name: Install Python dependencies
diff --git a/.github/workflows/pr-pre-commit-and-test.yaml b/.github/workflows/pr-pre-commit-and-test.yaml
index 23bf3a7ae..87e2b959c 100644
--- a/.github/workflows/pr-pre-commit-and-test.yaml
+++ b/.github/workflows/pr-pre-commit-and-test.yaml
@@ -37,7 +37,7 @@ jobs:
- 'tests/unit/**'
- '!tests/unit/ml/**'
deps:
- - 'setup.py'
+ - 'pyproject.toml'
ci:
- '.github/workflows/pr-pre-commit-and-test.yaml'
- '.pre-commit-config.yaml'
@@ -80,12 +80,15 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
+ # neuracore is pure-Python — no Rust toolchain needed. The Rust data daemon
+ # ships in the separate Linux-only `neuracore-data-daemon` package, which
+ # unit tests don't install (the daemon path is exercised separately).
- name: Set up uv
if: env.SHOULD_RUN == 'true' && matrix.package-manager == 'uv'
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- cache-dependency-glob: "setup.py"
+ cache-dependency-glob: "pyproject.toml"
python-version: ${{ matrix.python-version }}
- name: Install Python dependencies
@@ -181,7 +184,7 @@ jobs:
uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- cache-dependency-glob: "setup.py"
+ cache-dependency-glob: "pyproject.toml"
python-version: ${{ matrix.python-version }}
- name: Install Python dependencies
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index fcc9e0979..60dd0b1bc 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -369,7 +369,7 @@ jobs:
run: |
NEW_VERSION="${{ needs.generate-changelog.outputs.new_version }}"
- git add .bumpversion.cfg neuracore/__init__.py changelogs/pending-changelog.md
+ git add .bumpversion.cfg pyproject.toml packaging/neuracore-data-daemon/pyproject.toml changelogs/pending-changelog.md
git commit -m "Bump version to ${NEW_VERSION} [skip ci]"
TARGET_BRANCH="${{ inputs.emergency_deploy && github.ref_name || 'main' }}"
@@ -377,14 +377,13 @@ jobs:
echo "Pushed version bump to ${TARGET_BRANCH}"
- - name: Build Python package
- run: python -m build
-
- - name: Publish to PyPI
+ - name: Build and publish neuracore (pure Python)
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
- run: twine upload --skip-existing dist/*
+ run: |
+ python -m build
+ twine upload --skip-existing dist/*
- name: Create git tag
run: |
@@ -444,8 +443,8 @@ jobs:
summary += `**Version:** \`${version}\`\n\n`;
summary += `## Actions Completed\n\n`;
summary += `- Version bumped and pushed to \`main\`\n`;
- summary += `- Package published to PyPI\n`;
summary += `- Git tag \`v${version}\` created\n`;
+ summary += `- Wheels + sdist publishing to PyPI (see publish jobs)\n`;
summary += `- GitHub Release created\n\n`;
summary += `### Links\n\n`;
summary += `- [PyPI Package](https://pypi.org/project/neuracore/${version}/)\n`;
@@ -454,6 +453,77 @@ jobs:
summary += `\`\`\`bash\npip install neuracore==${version}\n\`\`\`\n`;
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
+ # Build and publish the Linux-only `neuracore-data-daemon` wheels (the Rust
+ # extension + bundled binary). The pure-Python `neuracore` wheel + sdist are
+ # already built and published by the `release` job above. Runs after the
+ # version bump + tag are pushed and checks out that tag so each wheel carries
+ # the bumped version; twine --skip-existing makes re-running a failed matrix
+ # leg idempotent. Mirrors .github/workflows/build-wheels.yaml.
+ publish-daemon-wheels:
+ needs: release
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - { os: ubuntu-22.04, target: x86_64-unknown-linux-gnu, python-version: "3.10" }
+ - { os: ubuntu-22.04, target: x86_64-unknown-linux-gnu, python-version: "3.11" }
+ steps:
+ - name: Checkout the tagged release
+ uses: actions/checkout@v6
+ with:
+ ref: v${{ needs.release.outputs.new_version }}
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ targets: ${{ matrix.target }}
+
+ - name: Cache cargo build
+ uses: Swatinem/rust-cache@v2
+ with:
+ workspaces: rust
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v6
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Build daemon wheel
+ uses: PyO3/maturin-action@v1
+ with:
+ working-directory: packaging/neuracore-data-daemon
+ target: ${{ matrix.target }}
+ # manylinux_2_28 (AlmaLinux 8), NOT auto: iceoryx2's bindgen needs
+ # libclang >= 5.0, but manylinux2014's clang is 3.4. See build-wheels.yaml.
+ manylinux: 2_28
+ # Cache compiled objects (extension + in-container daemon binary) across
+ # runs via the GHA cache backend — the manylinux container can't see
+ # Swatinem/rust-cache's host-side cache. See build-wheels.yaml.
+ sccache: 'true'
+ args: >-
+ --release
+ --out dist
+ --interpreter python${{ matrix.python-version }}
+ # Build the daemon binary inside the manylinux container (glibc match)
+ # and install clang/libclang for iceoryx2's bindgen. $GITHUB_WORKSPACE
+ # anchors the script path regardless of the build working-directory.
+ before-script-linux: |
+ (command -v yum && yum install -y clang) || (command -v dnf && dnf install -y clang) || true
+ # maturin-action exports RUSTC_WRAPPER=sccache, but sccache isn't on
+ # PATH yet during before-script, so the daemon binary's cargo build
+ # can't exec it. Unset it here — the extension build still gets sccache.
+ unset RUSTC_WRAPPER
+ bash "$GITHUB_WORKSPACE/rust/scripts/build_wheel_artefacts.sh" --target ${{ matrix.target }}
+
+ - name: Publish daemon wheel to PyPI
+ env:
+ TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+ run: |
+ python -m pip install --upgrade twine
+ twine upload --skip-existing packaging/neuracore-data-daemon/dist/*.whl
+
dry-run-summary:
needs: [analyze-changes, generate-changelog]
if: inputs.dry_run == true
diff --git a/.gitignore b/.gitignore
index 69f03c827..229c51c5d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -126,5 +126,5 @@ examples/test_streaming/
# `rust/scripts/build_wheel_artefacts.sh`. Both are per-machine and per-build,
# so we never commit them; the wheel-build CI job recreates them on every run.
# See docs/rust_data_daemon_development.md#packaging-the-wheel.
-neuracore/data_daemon/bin/
-neuracore/data_daemon/_native_producer*.so
+packaging/neuracore-data-daemon/neuracore/data_daemon/bin/
+packaging/neuracore-data-daemon/neuracore/data_daemon/_data_bridge*.so
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index 430ced192..000000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,16 +0,0 @@
-# Sources required to rebuild the bundled Rust artefacts from an sdist.
-# The artefacts themselves (neuracore/data_daemon/bin/data-daemon and
-# neuracore/data_daemon/_native_producer.so) are gitignored and only land
-# in the *wheel* — the sdist ships the Rust sources so a downstream
-# packager can run rust/scripts/build_wheel_artefacts.sh themselves.
-#
-# See docs/rust_data_daemon_development.md#packaging-the-wheel.
-
-include README.md
-include LICENSE
-
-recursive-include rust *.rs *.toml *.lock *.sql *.md *.sh
-
-prune rust/target
-global-exclude __pycache__
-global-exclude *.py[cod]
diff --git a/README.md b/README.md
index 7c6807ad0..44435c115 100644
--- a/README.md
+++ b/README.md
@@ -152,7 +152,7 @@ predictions = policy.predict(timeout=5)
- [Environment Variables](./docs/environment_variable.md)
- [Contribution Guide](./docs/contribution_guide.md)
- [Data Daemon](./docs/data_daemon.md)
-- [Rust Data Daemon — Developer Guide](./docs/rust_data_daemon_development.md) — building the [rust/](./rust/) workspace that ships inside the wheel as the data-daemon binary + `neuracore.data_daemon._native_producer` cdylib.
+- [Rust Data Daemon — Developer Guide](./docs/rust_data_daemon_development.md) — building the [rust/](./rust/) workspace that ships inside the wheel as the data-daemon binary + `neuracore.data_daemon._data_bridge` cdylib.
# 💬 Community
diff --git a/docs/data_daemon.md b/docs/data_daemon.md
index b87a43867..b457e7e87 100644
--- a/docs/data_daemon.md
+++ b/docs/data_daemon.md
@@ -128,7 +128,8 @@ daemon implementations and the launcher picks one based on the `NCD_RUST_DAEMON`
flag (see [rust_data_daemon_development.md](rust_data_daemon_development.md)):
- **Rust daemon** — when `NCD_RUST_DAEMON` is truthy,
- the launcher `exec`s the bundled native binary shipped in the wheel at
+ the launcher `exec`s the native binary shipped by the optional, Linux-only
+ `neuracore-data-daemon` package (`pip install neuracore[daemon]`) at
`neuracore/data_daemon/bin/data-daemon`. This is the implementation described
throughout this guide.
- **Legacy Python daemon (default)** — when `NCD_RUST_DAEMON` is unset or not
diff --git a/docs/rust_data_daemon_development.md b/docs/rust_data_daemon_development.md
index be75f7890..56e104617 100644
--- a/docs/rust_data_daemon_development.md
+++ b/docs/rust_data_daemon_development.md
@@ -12,7 +12,7 @@ The [rust/](../rust/) directory is a Cargo workspace with three members declared
|---|---|---|
| `data-daemon` | [rust/data_daemon/](../rust/data_daemon/) | The daemon binary — CLI, lifecycle, SQLite state, IPC listener, per-trace pipeline, encoding. |
| `data_daemon_shared` | [rust/data_daemon_shared/](../rust/data_daemon_shared/) | Shared library — IPC envelope types and service-name constants, plus the daemon configuration model and filesystem-path resolution the two processes must compute identically. Linked by both the daemon and the producer crate. |
-| `data_daemon_producer` | [rust/data_daemon_producer/](../rust/data_daemon_producer/) | PyO3 `cdylib` — producer-side IPC client exposed to Python as `neuracore.data_daemon._native_producer`. |
+| `data_daemon_bridge` | [rust/data_daemon_bridge/](../rust/data_daemon_bridge/) | PyO3 `cdylib` — producer-side IPC client exposed to Python as `neuracore.data_daemon._data_bridge`. |
Shared workspace dependencies and the Rust edition (`2021`) are pinned in [rust/Cargo.toml](../rust/Cargo.toml); individual crates inherit them via `.workspace = true`.
@@ -30,7 +30,7 @@ flowchart LR
subgraph SDK["Python SDK"]
LOG["log_joints / log_json / log_frame
start / stop / cancel_recording"]
end
- subgraph PROD["data_daemon_producer (PyO3 cdylib)"]
+ subgraph PROD["data_daemon_bridge (PyO3 cdylib)"]
LOG -->|GIL released| PUBT["publisher thread"]
LOG -->|RGB frames| WRT["writer thread → NUT spool on disk"]
end
@@ -105,7 +105,7 @@ CI uses `stable` (via `dtolnay/rust-toolchain@stable`), so any recent stable too
sudo apt-get update && sudo apt-get install -y ffmpeg
```
-- **maturin** (only when working on the `data_daemon_producer` PyO3 crate):
+- **maturin** (only when working on the `data_daemon_bridge` PyO3 crate):
```bash
pip install maturin
@@ -127,7 +127,7 @@ cargo build --workspace
cargo build --release -p data-daemon
# Producer cdylib only
-cargo build -p data_daemon_producer
+cargo build -p data_daemon_bridge
```
The release binary lands at [rust/target/release/data-daemon](../rust/target/release/data-daemon).
@@ -211,76 +211,109 @@ RUST_LOG=data_daemon=trace,iceoryx2=warn cargo run -p data-daemon -- launch
## Working on the PyO3 producer
-The `data_daemon_producer` crate compiles to a `cdylib` that Python imports as `neuracore.data_daemon._native_producer`. During development, use `maturin develop` from the producer crate directory to build and install it into your active virtualenv in one step:
+The `data_daemon_bridge` crate compiles to a `cdylib` that Python imports as `neuracore.data_daemon._data_bridge` (shipped by the separate `neuracore-data-daemon` package). During development, use `maturin develop` from that package directory — its `pyproject.toml` carries the `module-name` and `manifest-path` that point maturin at the crate:
```bash
-cd rust/data_daemon_producer
+cd packaging/neuracore-data-daemon
maturin develop
-python -c "import neuracore.data_daemon._native_producer as p; print(p)"
+python -c "import neuracore.data_daemon._data_bridge as p; print(p)"
```
-To route the Python SDK through the native producer instead of the legacy zmq one, set the rollout flag:
+To route the Python SDK through the data bridge instead of the legacy zmq one, set the rollout flag:
```bash
export NCD_RUST_DAEMON=1
python your_script.py
```
-Selection logic lives in [neuracore/data_daemon/rust_selection.py](../neuracore/data_daemon/rust_selection.py); both the daemon binary handoff and the SDK's `DataStream` construction read it. A small shim bridges the native producer to the Python `ProducerChannel` contract.
+Selection logic lives in [neuracore/data_daemon/rust_selection.py](../neuracore/data_daemon/rust_selection.py); both the daemon binary handoff and the SDK's `DataStream` construction read it. A small shim bridges the data bridge to the Python `ProducerChannel` contract.
---
-## Packaging the wheel
+## Packaging: two distributions
-The Python wheel ships two Rust artefacts inside the `neuracore.data_daemon` package:
+The Rust daemon is shipped as a **separate, Linux-only PyPI distribution** so that
+`neuracore` itself stays pure-Python and installs on every platform.
-| Artefact | Wheel location | Source crate | Imported / executed as |
+| Distribution | Build | Installs on | Ships |
|---|---|---|---|
-| Daemon binary | `neuracore/data_daemon/bin/data-daemon` | `data-daemon` (bin) | Re-exec'd by [neuracore/data_daemon/__main__.py](../neuracore/data_daemon/__main__.py) when `NCD_RUST_DAEMON` is truthy |
-| Producer cdylib | `neuracore/data_daemon/_native_producer*.so` | `data_daemon_producer` (cdylib) | `import neuracore.data_daemon._native_producer` from the SDK producer shim |
+| `neuracore` | pure-Python (setuptools, [pyproject.toml](../pyproject.toml)) | everywhere (`py3-none-any`) | all the Python SDK + the `daemon` extra |
+| `neuracore-data-daemon` | maturin ([packaging/neuracore-data-daemon/pyproject.toml](../packaging/neuracore-data-daemon/pyproject.toml)) | Linux x86_64 only | the `_data_bridge` PyO3 extension + the `data-daemon` binary |
-Both paths are inside the Python package tree, so vanilla setuptools `package_data` is enough to package them once they're built — there is no `pyproject.toml`/maturin build-backend migration. The trade-off is that each wheel build runs cargo twice (once per crate) before `python -m build` packages the result.
+`neuracore-data-daemon` is a base dependency of `neuracore` carrying the PEP 508
+marker `; sys_platform == 'linux'`, so `pip install neuracore` pulls the daemon
+**automatically on Linux** and is a no-op on macOS/Windows (where the SDK falls
+back to the Python daemon). The `daemon` extra is kept as a backward-compatible
+explicit alias — `pip install neuracore[daemon]` resolves to the same thing.
-### One-shot local build
+The two artefacts and how `neuracore` finds them:
-Use the helper script to compile both crates in release mode and copy the artefacts into the package tree at the locations the runtime expects:
+| Artefact | Location (in `neuracore-data-daemon`) | Source crate | Reached from `neuracore` via |
+|---|---|---|---|
+| Daemon binary | `neuracore/data_daemon/bin/data-daemon` | `data-daemon` (bin) | `rust_selection.rust_daemon_binary_path()` → `files("neuracore.data_daemon")/"bin"/"data-daemon"` |
+| Producer extension | `neuracore/data_daemon/_data_bridge*.so` | `data_daemon_bridge` (cdylib) | `recording_context._load_native()` → `import neuracore.data_daemon._data_bridge` |
-```bash
-./rust/scripts/build_wheel_artefacts.sh
-```
+Both lookups degrade gracefully when `neuracore-data-daemon` is absent (binary path → `None`, extension import → a helpful `RuntimeError`).
-What it does:
+The daemon wheel installs its files **into** the `neuracore/data_daemon/` import path the `neuracore` wheel owns, but ships **no** `__init__.py` for those dirs — so the two distributions share the directory with no file collision: `neuracore` owns the `.py` + `__init__.py`, `neuracore-data-daemon` owns the `.so` + binary. (Installed alone, the daemon wheel's `neuracore/data_daemon/` imports as a PEP 420 namespace package; installed alongside `neuracore`, it's a normal package — both ways `import neuracore.data_daemon._data_bridge` resolves.)
-1. `cargo build --release -p data-daemon` and copies the binary to [neuracore/data_daemon/bin/data-daemon](../neuracore/data_daemon/bin/data-daemon).
-2. `cargo build --release -p data_daemon_producer` and copies the cdylib to [neuracore/data_daemon/_native_producer.so](../neuracore/data_daemon/_native_producer.so) (renames `libdata_daemon_producer.so` → `_native_producer.so` so PyO3's `PyInit__native_producer` is discoverable).
+### Local build
-Both targets are gitignored (`neuracore/data_daemon/bin/` and `*.so`); the script is idempotent so re-running it after a `cargo` edit refreshes the in-tree copies. `pip install -e .` after the script picks the new artefacts up automatically via `package_data`.
+The daemon binary is a *separate* crate that maturin doesn't build, so the helper
+script compiles it into the daemon package tree first; then maturin builds the
+extension:
-For day-to-day iteration on the producer crate only, prefer `maturin develop` from [rust/data_daemon_producer/](../rust/data_daemon_producer/) — it skips the binary build, only refreshes the cdylib, and is faster.
+```bash
+./rust/scripts/build_wheel_artefacts.sh # cargo build -p data-daemon -> packaging/.../neuracore/data_daemon/bin/data-daemon
+cd packaging/neuracore-data-daemon
+maturin develop # builds + editable-installs _data_bridge into the active env
+```
-### Building a wheel
+`maturin develop` builds and installs the **extension only** — it doesn't run the
+binary build or evaluate `include`, so run the helper script first for an
+end-to-end daemon. For the SDK itself, `pip install -e .` (or `.[dev]`) at the repo
+root installs `neuracore` pure-Python with no Rust.
+
+### Building wheels
```bash
+# neuracore (pure Python, py3-none-any)
+python -m build
+
+# neuracore-data-daemon (Linux only)
./rust/scripts/build_wheel_artefacts.sh
-python -m build --wheel
+cd packaging/neuracore-data-daemon
+maturin build --release --out dist --interpreter python3.11
```
-The wheel is platform-tagged (Linux x86_64 today) because [setup.py](../setup.py) sets `Distribution.has_ext_modules` so setuptools tags the wheel for the host platform — without that hook setuptools would tag it `py3-none-any` and pip would happily install a Linux .so onto macOS. `package_data` ships both artefacts; `MANIFEST.in` ships the script and the `rust/` sources for the sdist.
+The daemon wheel is platform-specific: pyo3 uses `extension-module` without `abi3`,
+so each is one Python minor × one platform and `--interpreter pythonX.Y` must be
+passed. `include` bundles the daemon binary; no sdist is shipped for the daemon
+package (its Rust sources live outside the package dir).
-### CI
+> **Daemon is Linux only.** The daemon stack uses `iceoryx2` shared-memory IPC and
+> Linux-only syscalls (`sync_file_range`, `gettid`). `neuracore-data-daemon` builds
+> and ships `linux-x86_64` wheels only; `neuracore` is unaffected on other platforms.
-The wheel job runs in [.github/workflows/build-wheels.yaml](../.github/workflows/build-wheels.yaml):
-
-1. Installs the Rust toolchain + ffmpeg (for unit tests).
-2. Runs the helper script above.
-3. Runs `python -m build --wheel` to produce the wheel.
-4. Uploads the wheel as an artefact for the release job to consume.
+### CI
-The matrix is Linux x86_64 only for v1; aarch64 ships when there's demand (the script is platform-agnostic — only the cross-compilation toolchain would need to grow). Each wheel is one Python version × one platform, matching the cdylib's ABI.
+[.github/workflows/build-wheels.yaml](../.github/workflows/build-wheels.yaml) builds
+the `neuracore-data-daemon` wheels — a `linux-x86_64` × py3.10/3.11 `PyO3/maturin-action`
+matrix that builds the daemon binary inside the `manylinux_2_28` container (glibc
+match, plus a `clang` install for iceoryx2's bindgen — `manylinux_2_28` is required
+over the default `manylinux2014` because iceoryx2's bindgen needs libclang >= 5.0,
+which 2014's clang 3.4 can't provide), runs `maturin build`, and smoke-tests
+`import neuracore.data_daemon._data_bridge` + the executable binary. The pure-Python
+`neuracore` wheel needs no special CI — it's built with `python -m build`.
### Release path
-The [release workflow](../.github/workflows/release.yaml) wires the wheel job into its publish step: it depends on `build-wheels.yaml`, downloads the matrix of wheels, and `twine upload`s them alongside the sdist. The sdist remains useful as a portable fallback (users build the Rust artefacts themselves at install time) but is not the recommended install path — the bundled-binary wheel is.
+The [release workflow](../.github/workflows/release.yaml) bumps the version (in both
+pyprojects + the `daemon` extra pin, kept in lockstep by [.bumpversion.cfg](../.bumpversion.cfg)),
+pushes the tag, then: the `release` job builds + `twine`-publishes the pure-Python
+`neuracore` wheel + sdist (`python -m build`), and a `publish-daemon-wheels`
+linux-x86_64 matrix checks out the tag, builds the `neuracore-data-daemon` wheels,
+and publishes them. `twine --skip-existing` makes re-running any leg idempotent.
---
diff --git a/neuracore-dictionary.txt b/neuracore-dictionary.txt
index 6b3aad594..bf0d9098b 100644
--- a/neuracore-dictionary.txt
+++ b/neuracore-dictionary.txt
@@ -8,8 +8,11 @@ addfinalizer
addinivalue
agentview
agilex
+aiofiles
+aiohttp
aiolimiter
aiortc
+aiosqlite
allclose
allocvec
altclip
@@ -38,6 +41,7 @@ bodyless
Brawner
broadcastable
buildtool
+bumpversion
byteswap
calcsize
caplog
@@ -121,6 +125,7 @@ DONTNEED
dtype
EADDRINUSE
eigenpy
+einops
elementwise
elems
embs
@@ -147,11 +152,13 @@ facecolor
facenum
fadvise
falsey
+fastapi
fcntl
feedforward
fflags
ffprobe
figsize
+filelock
filtergraph
finetune
finetuning
@@ -257,7 +264,7 @@ loglik
logsigmoid
logvar
makereport
-maxpool
+matplotlib
maxs
mcap
MCAP
@@ -311,9 +318,7 @@ nprocs
numpy
octomap
offsamples
-oieb
-Oieb
-OIEB
+omegaconf
oneshot
openarm
openarm_description
@@ -353,16 +358,20 @@ pretraining
prio
proprio
proprios
-PSNR
+psutil
pthread
pyav
pycache
pydantic
+pyee
pyfunction
pygments
+pyinstrument
pymodule
pyquaternion
pytest
+pyyaml
+pyzmq
Qbcaa
qpos
qposadr
@@ -427,6 +436,7 @@ softmax
solimp
solref
splitn
+sqlalchemy
sqlx
squaredcos
startcode
@@ -458,6 +468,7 @@ tobytes
tolist
torchdynamo
torchserve
+torchvision
torq
tqdm
traj
@@ -489,6 +500,7 @@ urdfdom
usefixtures
userspace
utaustin
+uvicorn
varint
Vaswani
vdecode
@@ -510,6 +522,7 @@ WRONLY
wxyz
XARM
xdata
+xdist
xlabel
xmat
xmls
diff --git a/neuracore/__init__.py b/neuracore/__init__.py
index 0910431c3..02c9f382b 100644
--- a/neuracore/__init__.py
+++ b/neuracore/__init__.py
@@ -1,5 +1,8 @@
"""Init."""
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
from .api.core import * # noqa: F403
from .api.datasets import * # noqa: F403
from .api.endpoints import * # noqa: F403
@@ -7,4 +10,11 @@
from .api.training import * # noqa: F403
from .core.exceptions import * # noqa: F403
-__version__ = "13.3.0"
+try:
+ # The version lives in pyproject.toml [project].version (the single source
+ # of truth). Read it back from the installed metadata so it never drifts.
+ __version__ = _pkg_version("neuracore")
+except PackageNotFoundError:
+ # Running from a source tree that was never installed (e.g. a bare checkout
+ # used as a sys.path entry). Keep a sentinel rather than crashing on import.
+ __version__ = "0.0.0+unknown"
diff --git a/neuracore/api/core.py b/neuracore/api/core.py
index f66eb5d5e..08d8c5549 100644
--- a/neuracore/api/core.py
+++ b/neuracore/api/core.py
@@ -20,7 +20,7 @@
)
from neuracore.core.streaming.recording_state_manager import get_recording_state_manager
from neuracore.core.utils import backend_utils
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from ..core.auth import get_auth
from ..core.data.dataset import Dataset
@@ -334,7 +334,7 @@ def stop_recording(
recording_id = robot.get_current_recording_id()
if not recording_id:
raise ValueError("Recording_id is None, no current recording")
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
cloud_recording_id = robot.get_cloud_recording_id() if wait else None
robot.stop_recording(
recording_id, wait_for_producer_drain=wait, timestamp=timestamp
diff --git a/neuracore/api/logging.py b/neuracore/api/logging.py
index 55d58fc00..5c77fed86 100644
--- a/neuracore/api/logging.py
+++ b/neuracore/api/logging.py
@@ -54,7 +54,7 @@
BatchedJointDataItemPayload,
BatchedJointDataPayload,
)
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
logger = logging.getLogger(__name__)
@@ -246,7 +246,7 @@ def _record_json_to_daemon(
data: Data object to serialize and persist.
timestamp: Capture timestamp in seconds.
"""
- if not (rust_daemon_enabled() and robot.get_current_recording_id() is not None):
+ if not (is_rust_daemon_enabled() and robot.get_current_recording_id() is not None):
return
payload = json.dumps(data.model_dump(mode="json")).encode("utf-8")
robot._get_daemon_recording_context().log_json(
@@ -461,7 +461,7 @@ def _log_group_of_joint_data(
_smoke_validate_joint_values(joint_data)
robot = _get_robot(robot_name, instance)
- rust_mode = rust_daemon_enabled()
+ rust_mode = is_rust_daemon_enabled()
binding_cache = robot._joint_stream_bindings
bindings_for_type = binding_cache.get(data_type)
@@ -659,7 +659,7 @@ def _log_camera_data(
# or having to make two copies for streaming and bucket storage.
stream.log(camera_data_without_frame, frame=image)
- if rust_daemon_enabled() and robot.get_current_recording_id() is not None:
+ if is_rust_daemon_enabled() and robot.get_current_recording_id() is not None:
contiguous = image if image.flags.c_contiguous else np.ascontiguousarray(image)
robot._get_daemon_recording_context().log_frame(
camera_type.value,
diff --git a/neuracore/core/robot.py b/neuracore/core/robot.py
index c59f4b983..b45160f4d 100644
--- a/neuracore/core/robot.py
+++ b/neuracore/core/robot.py
@@ -31,7 +31,7 @@
from neuracore.data_daemon.communications_management.shared_transport import (
recording_context,
)
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from .auth import Auth, get_auth
from .const import API_URL, MAX_DATA_STREAMS
@@ -300,7 +300,7 @@ def start_recording(
if not self.id:
raise RobotError("Robot not initialized. Call init() first.")
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
local_handle = str(uuid.uuid4())
self._get_daemon_recording_context().start_recording(
robot_id=self.id,
@@ -389,7 +389,7 @@ def stop_recording(
raise RobotError("Robot not initialized. Call init() first.")
end_time = time.time()
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
active_handle = get_recording_state_manager().get_current_recording_id(
self.id, self.instance
)
@@ -546,7 +546,7 @@ def get_cloud_recording_id(
"""
if not self.id:
raise RobotError("Robot not initialized. Call init() first.")
- if not rust_daemon_enabled():
+ if not is_rust_daemon_enabled():
return self.get_current_recording_id()
return self._get_daemon_recording_context().get_recording_id(
timestamp_ns=timestamp_ns, timeout_s=timeout_s
@@ -828,7 +828,7 @@ def cancel_recording(
self._stop_all_streams()
daemon_context = self._get_daemon_recording_context()
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
daemon_context.cancel_recording(timestamp=timestamp)
active_handle = get_recording_state_manager().get_current_recording_id(
self.id, self.instance
diff --git a/neuracore/core/streaming/data_stream.py b/neuracore/core/streaming/data_stream.py
index 99cb155c2..ed2941bd7 100644
--- a/neuracore/core/streaming/data_stream.py
+++ b/neuracore/core/streaming/data_stream.py
@@ -17,7 +17,7 @@
from neuracore_types import CameraData, DataType, JointData, NCData
from neuracore.data_daemon.communications_management.producer import ProducerChannel
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
logger = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ def __init__(self, data_type: DataType, stream_name: str) -> None:
self._data_type = data_type
self._stream_name = stream_name
self._producer_channel: ProducerChannel | None = None
- self._use_native_producer = rust_daemon_enabled()
+ self._use_data_bridge = is_rust_daemon_enabled()
@property
def data_type(self) -> DataType:
@@ -109,7 +109,7 @@ def _handle_ensure_producer_channel(self, context: DataRecordingContext) -> None
context: Recording context containing identifiers for
the recording session, robot, and dataset.
"""
- if self._use_native_producer:
+ if self._use_data_bridge:
return
if self._producer_channel is None:
channel_id = f"{self._data_type.value}:\
@@ -133,7 +133,7 @@ def prepare_recording_stopped(self) -> tuple[ProducerChannel | None, int]:
"""
producer_channel = self.get_producer_channel()
if producer_channel is None:
- if not rust_daemon_enabled():
+ if not is_rust_daemon_enabled():
raise MissingProducerChannelError(
"stream has no active producer channel"
)
@@ -155,7 +155,7 @@ def stop_recording(
self._producer_channel = None
if producer_channel is None:
- if not rust_daemon_enabled():
+ if not is_rust_daemon_enabled():
# Legacy daemon: a stream with no producer channel is stale —
# raise so the caller prunes it.
raise MissingProducerChannelError(
@@ -270,7 +270,7 @@ def log(self, data: NCData, *, send_to_daemon: bool = True) -> None:
self._latest_data = data
if not self.is_recording() or not send_to_daemon:
return
- if self._use_native_producer:
+ if self._use_data_bridge:
# Rust daemon: the logging layer delivers the sample to the daemon
# via RecordingContext; the stream only keeps `_latest_data`.
return
@@ -367,7 +367,7 @@ def log(self, metadata: CameraData, frame: np.ndarray) -> None:
self._latest_data = metadata
if not self.is_recording():
return
- if self._use_native_producer:
+ if self._use_data_bridge:
# Rust daemon: the frame is delivered to the daemon by the logging
# layer (RecordingContext.log_frame); the stream only keeps
# `_latest_data` for live-data consumers.
diff --git a/neuracore/core/streaming/recording_state_manager.py b/neuracore/core/streaming/recording_state_manager.py
index 73eba41fc..bb88f7cfd 100644
--- a/neuracore/core/streaming/recording_state_manager.py
+++ b/neuracore/core/streaming/recording_state_manager.py
@@ -36,12 +36,12 @@
recording_context as _recording_context,
)
from neuracore.data_daemon.lifecycle.daemon_os_control import ensure_daemon_running
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
logger = logging.getLogger(__name__)
-def _notify_native_producer_of_expiry(robot_id: str, instance: int) -> None:
+def _notify_data_bridge_of_expiry(robot_id: str, instance: int) -> None:
"""Tell the Rust producer a source's recording has been locally auto-expired.
Calls the native ``stop_recording`` for the source so the producer flushes
@@ -58,7 +58,7 @@ def _notify_native_producer_of_expiry(robot_id: str, instance: int) -> None:
)
except Exception:
logger.exception(
- "Failed to notify native producer of recording expiry for %s:%s",
+ "Failed to notify data bridge of recording expiry for %s:%s",
robot_id,
instance,
)
@@ -223,8 +223,8 @@ def expire_if_still_active() -> None:
)
self._expired_recording_ids.add(recording_id)
self.recording_stopped(robot_id, instance, recording_id)
- if rust_daemon_enabled():
- _notify_native_producer_of_expiry(robot_id, instance)
+ if is_rust_daemon_enabled():
+ _notify_data_bridge_of_expiry(robot_id, instance)
loop = get_running_loop()
@@ -272,6 +272,10 @@ def recording_stopped(
if current_recording != recording_id:
return
self.recording_robot_instances.pop(instance_key, None)
+ # Note: the data bridge stop is driven by the recording context
+ # (normal/remote stop) or the expiry timer — NOT here — so the daemon
+ # receives exactly one StopRecording carrying the correct data-clock
+ # boundary.
if recording_id is not None:
self._cancel_recording_timers(recording_id)
diff --git a/neuracore/data_daemon/__main__.py b/neuracore/data_daemon/__main__.py
index 8bd4f8568..54d443468 100644
--- a/neuracore/data_daemon/__main__.py
+++ b/neuracore/data_daemon/__main__.py
@@ -12,14 +12,14 @@
import sys
from neuracore.data_daemon.rust_selection import (
+ is_rust_daemon_enabled,
rust_daemon_binary_path,
- rust_daemon_enabled,
)
def main() -> None:
"""Dispatch to the Rust data daemon when enabled, else the Python CLI."""
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
binary = rust_daemon_binary_path()
if binary is None:
print(
@@ -29,6 +29,14 @@ def main() -> None:
)
else:
try:
+ # maturin's wheel `include` can drop the bundled binary without
+ # the executable bit, which would make execv fail and silently
+ # fall back to the Python daemon. Restore it best-effort first.
+ if not os.access(binary, os.X_OK):
+ try:
+ os.chmod(binary, 0o755)
+ except OSError:
+ pass
os.execv(str(binary), [str(binary), *sys.argv[1:]])
except OSError as error:
# The binary is present but couldn't be executed (e.g. not
diff --git a/neuracore/data_daemon/communications_management/shared_transport/recording_context.py b/neuracore/data_daemon/communications_management/shared_transport/recording_context.py
index d5a394c2e..d8a11114c 100644
--- a/neuracore/data_daemon/communications_management/shared_transport/recording_context.py
+++ b/neuracore/data_daemon/communications_management/shared_transport/recording_context.py
@@ -7,31 +7,37 @@
from types import ModuleType
from neuracore.data_daemon.models import CommandType
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from .communications_manager import CommunicationsManager, MessageEnvelope
logger = logging.getLogger(__name__)
-_NATIVE_MODULE: ModuleType | None = None
+_DATA_BRIDGE_MODULE: ModuleType | None = None
-_NATIVE_IMPORT_HINT = (
- "neuracore.data_daemon._native_producer is not available. Build the Rust "
- "data_daemon_producer crate with maturin and ensure the resulting "
- "extension is on sys.path, or unset NCD_RUST_DAEMON to fall back to the "
- "legacy Python producer."
+_DATA_BRIDGE_IMPORT_HINT = (
+ "neuracore.data_daemon._data_bridge is not available. The Rust extension "
+ "ships in the separate, Linux-only 'neuracore-data-daemon' distribution "
+ "(installed automatically with `neuracore` on Linux); reinstall with "
+ "`pip install neuracore` on Linux, or unset NCD_RUST_DAEMON to fall back to "
+ "the legacy Python data daemon."
)
def _load_native() -> ModuleType:
- """Lazily import and cache the PyO3 producer module for the process."""
- global _NATIVE_MODULE
- if _NATIVE_MODULE is None:
+ """Lazily import and cache the PyO3 data daemon bridge module for the process.
+
+ The compiled extension is contributed into ``neuracore.data_daemon`` by the
+ optional, Linux-only ``neuracore-data-daemon`` distribution, so this raises a
+ helpful error when that distribution is absent.
+ """
+ global _DATA_BRIDGE_MODULE
+ if _DATA_BRIDGE_MODULE is None:
try:
- _NATIVE_MODULE = import_module("neuracore.data_daemon._native_producer")
+ _DATA_BRIDGE_MODULE = import_module("neuracore.data_daemon._data_bridge")
except ImportError as error:
- raise RuntimeError(_NATIVE_IMPORT_HINT) from error
- return _NATIVE_MODULE
+ raise RuntimeError(_DATA_BRIDGE_IMPORT_HINT) from error
+ return _DATA_BRIDGE_MODULE
class RecordingContext:
@@ -39,7 +45,7 @@ class RecordingContext:
Under the Rust daemon this is a *thin shipper* bridge: ``start_recording``
/ ``log_joints`` / ``log_frame`` / ``log_json`` / ``stop_recording`` /
- ``cancel_recording`` forward straight through to ``_native_producer`` over
+ ``cancel_recording`` forward straight through to ``_data_bridge`` over
iceoryx2, tagged only with the **source** ``(robot_id, robot_instance)``.
The daemon owns all recording identity — there is no recording id on the
wire. Routing is by a producer-stamped *publish* timestamp (wall clock),
@@ -58,11 +64,11 @@ def __init__(
"""Initialize the recording context.
Under the Rust daemon the ZMQ producer socket is unused — every
- envelope flows through ``_native_producer`` over iceoryx2 — so we skip
+ envelope flows through ``_data_bridge`` over iceoryx2 — so we skip
creating it.
"""
self.recording_id = recording_id
- self._rust_mode = rust_daemon_enabled()
+ self._rust_mode = is_rust_daemon_enabled()
self._robot_id: str | None = None
self._robot_instance: int = 0
self._recording_marker_ns: int = 0
diff --git a/neuracore/data_daemon/lifecycle/daemon_os_control.py b/neuracore/data_daemon/lifecycle/daemon_os_control.py
index 2dee2015f..c568a4eb1 100644
--- a/neuracore/data_daemon/lifecycle/daemon_os_control.py
+++ b/neuracore/data_daemon/lifecycle/daemon_os_control.py
@@ -21,8 +21,8 @@
from neuracore.data_daemon.helpers import get_daemon_db_path, get_daemon_pid_path
from neuracore.data_daemon.lifecycle.auth_preflight import ensure_daemon_auth_ready
from neuracore.data_daemon.rust_selection import (
+ is_rust_daemon_enabled,
rust_daemon_binary_path,
- rust_daemon_enabled,
)
# cspell:ignore WNOHANG waitpid
@@ -118,7 +118,7 @@ def _build_daemon_runner_command() -> list[str]:
foreground so it inherits the same process semantics the Python runner
relies on (signal handling, parent-side ``Popen.wait``).
"""
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
binary = rust_daemon_binary_path()
if binary is not None:
return [str(binary), "launch"]
@@ -146,7 +146,7 @@ def _build_daemon_launch_env(
environment = os.environ.copy()
environment["NEURACORE_DAEMON_PID_PATH"] = str(pid_path)
environment["NEURACORE_DAEMON_DB_PATH"] = str(db_path)
- if not rust_daemon_enabled():
+ if not is_rust_daemon_enabled():
environment["NEURACORE_DAEMON_MANAGE_PID"] = "0"
if env_overrides:
environment.update(env_overrides)
@@ -327,7 +327,7 @@ def launch_daemon_subprocess(
)
poll_interval_s = 0.05
daemon_startup_timeout_s = time.monotonic() + timeout_s
- rust_mode = rust_daemon_enabled() and rust_daemon_binary_path() is not None
+ rust_mode = is_rust_daemon_enabled() and rust_daemon_binary_path() is not None
def _ready() -> bool:
if rust_mode:
diff --git a/neuracore/data_daemon/rust_selection.py b/neuracore/data_daemon/rust_selection.py
index 349015be0..9186f1a14 100644
--- a/neuracore/data_daemon/rust_selection.py
+++ b/neuracore/data_daemon/rust_selection.py
@@ -13,19 +13,25 @@
from __future__ import annotations
import os
-from importlib.resources import files
from pathlib import Path
_TRUTHY_VALUES = frozenset({"1", "true", "yes", "y"})
-def rust_daemon_enabled() -> bool:
+def is_rust_daemon_enabled() -> bool:
"""Return True when ``NCD_RUST_DAEMON`` selects the Rust data daemon."""
return os.environ.get("NCD_RUST_DAEMON", "").strip().lower() in _TRUTHY_VALUES
def rust_daemon_binary_path() -> Path | None:
- """Return the path to the bundled Rust data-daemon binary, if present."""
+ """Return the path to the Rust data-daemon binary, if available.
+
+ The binary is contributed into ``neuracore/data_daemon/bin/`` by the
+ optional, Linux-only ``neuracore-data-daemon`` distribution. Returns
+ ``None`` when that distribution is not installed (the file is simply absent).
+ """
+ from importlib.resources import files
+
candidate = files("neuracore.data_daemon") / "bin" / "data-daemon"
path = Path(str(candidate))
return path if path.is_file() else None
diff --git a/packaging/neuracore-data-daemon/pyproject.toml b/packaging/neuracore-data-daemon/pyproject.toml
new file mode 100644
index 000000000..ca9170ef1
--- /dev/null
+++ b/packaging/neuracore-data-daemon/pyproject.toml
@@ -0,0 +1,46 @@
+[build-system]
+requires = ["maturin>=1.7,<2.0"]
+build-backend = "maturin"
+
+# Linux-only companion distribution for `neuracore`. Ships the Rust data daemon:
+# the `_data_bridge` PyO3 extension (built by maturin from the workspace at
+# ../../rust) and the standalone `data-daemon` binary (built separately by
+# ../../rust/scripts/build_wheel_artefacts.sh and bundled via `include`).
+# `neuracore` pulls this in automatically on Linux (base dependency).
+#
+# It contributes the extension + binary INTO `neuracore/data_daemon/` (the same
+# import path the `neuracore` wheel owns) but ships NO __init__.py for those
+# dirs, so the two wheels share the directory without a file collision: the
+# `neuracore` wheel owns the .py + __init__.py, this wheel owns the .so + binary.
+
+[project]
+name = "neuracore-data-daemon"
+# Bumped in lockstep with neuracore (see /.bumpversion.cfg). neuracore's `daemon`
+# extra pins `neuracore-data-daemon==` so the two always match.
+version = "13.3.0"
+description = "Linux-only Rust data daemon and bridge for neuracore."
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Stephen James", email = "support@neuracore.com" }]
+classifiers = [
+ "Development Status :: 3 - Alpha",
+ "Operating System :: POSIX :: Linux",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+]
+
+[tool.maturin]
+# The package tree (`neuracore/data_daemon/`, deliberately WITHOUT __init__.py)
+# lives next to this file; maturin treats it as a namespace contribution.
+python-source = "."
+# Drops the compiled extension at neuracore/data_daemon/_data_bridge —
+# the same import path the `neuracore` wheel owns. Last path component must match
+# the `#[pymodule] fn _data_bridge` in ../../rust/data_daemon_bridge/src/lib.rs.
+module-name = "neuracore.data_daemon._data_bridge"
+# Build the data bridge cdylib from the workspace crate.
+manifest-path = "../../rust/data_daemon_bridge/Cargo.toml"
+# Bundle the standalone data-daemon binary (built into the package tree by
+# build_wheel_artefacts.sh before `maturin build`). `include` only copies; a
+# missing path is a no-op. Linux wheels only — no sdist (the Rust sources live
+# outside this project dir), so we don't ship an sdist for this package.
+include = [{ path = "neuracore/data_daemon/bin/data-daemon", format = "wheel" }]
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..77a342951
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,154 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "neuracore"
+# `neuracore` is a PURE-PYTHON distribution (py3-none-any) — it installs on every
+# platform and never compiles anything. The Rust data daemon ships separately in
+# the optional, Linux-only `neuracore-data-daemon` package (see the `daemon`
+# extra below and packaging/neuracore-data-daemon/). neuracore/__init__.py
+# re-derives __version__ from the installed metadata, so this is the single
+# source of truth.
+version = "13.3.0"
+description = "Neuracore Client Library"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Stephen James", email = "support@neuracore.com" }]
+keywords = ["robotics", "machine-learning", "ai", "client-library"]
+classifiers = [
+ "Development Status :: 3 - Alpha",
+ "Intended Audience :: Developers",
+ "Operating System :: OS Independent",
+ "License :: OSI Approved :: MIT License",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+ "numpy>=2.0.0",
+ "requests>=2.31.0",
+ "pillow>=10.0.0",
+ "pyyaml>=6.0.1",
+ "tqdm>=4.66.0",
+ "requests-oauthlib==2.0.0",
+ "pydantic>=2.10",
+ "av==15.1.0",
+ "aiortc==1.14.0",
+ "aiohttp-sse-client==0.2.1",
+ "numpy-stl",
+ "wget",
+ "uvicorn[standard]==0.42.0",
+ "fastapi==0.135.2",
+ "psutil",
+ "typer>=0.20.0",
+ "neuracore_types>=10.0.0,<11.0.0",
+ "ordered_set",
+ "pyzmq==27.1.0",
+ "sqlalchemy>=2.0.0",
+ "aiosqlite>=0.19.0",
+ "aiohttp>=3.9.0",
+ "aiofiles>=23.0.0",
+ "aiolimiter",
+ "pyee==13.0.0",
+ "greenlet",
+ "filelock>=3.0.0",
+ "omegaconf",
+ "msgpack>=1.0.8",
+ # The Rust data daemon, shipped as a separate Linux-only distribution. The
+ # platform marker keeps it off macOS/Windows (where it can't build), so
+ # `pip install neuracore` stays installable everywhere — Linux additionally
+ # gets the daemon automatically. Bumped in lockstep by .bumpversion.cfg.
+ "neuracore-data-daemon==13.3.0; sys_platform == 'linux'",
+]
+
+[project.urls]
+Homepage = "https://github.com/neuracoreai/neuracore"
+
+[project.optional-dependencies]
+examples = ["matplotlib>=3.3.0", "mujoco==2.3.7", "pyquaternion>=0.9.5"]
+mjcf = ["mujoco>3"]
+ml = [
+ # torch==2.10.0 is minimum version defaulting to CUDA 12.8.
+ # Do NOT upgrade to 2.11.0+: it defaults to CUDA 13 which drops
+ # V100 support, which Neuracore still uses.
+ # hub pinned to 0.35.3 to match lerobot==0.4.4's
+ # <0.36.0 constraint, allowing [ml] and [import] to coexist.
+ # We have dropped support for P100 and P4 due to CUDA 12.8
+ # dropping support for them.
+ "torch==2.10.0",
+ "torchvision==0.25.0",
+ "transformers==4.53.2",
+ "huggingface-hub==0.35.3",
+ "diffusers==0.35.1",
+ "safetensors==0.6.2",
+ "einops",
+ "hydra-core>=1.3.0",
+ "tensorboard>=2",
+ "names-generator>=0.2.0",
+]
+dev = [
+ "pytest>=6.2.5",
+ "pytest-cov>=2.12.1",
+ "diff-cover",
+ "pytest-asyncio>=0.15.1",
+ "pytest-xdist",
+ "twine>=3.4.2",
+ "requests-mock>=1.9.3",
+ "pre-commit",
+ "types-aiofiles",
+ "pyinstrument",
+ "plotly",
+]
+import = [
+ # lerobot is used only for dataset importing
+ # (LeRobotDataset/LeRobotDatasetMetadata).
+ # Do NOT install lerobot[transformers-dep] — that extra requires
+ # transformers>=4.57.1 which conflicts with our transformers==4.53.2 pin.
+ # lerobot>=0.4.x drops v2 dataset format support.
+ # torch is not pinned here as lerobot==0.4.4 constrains it to
+ # >=2.2.1,<2.11.0, compatible with [ml]'s torch==2.10.0.
+ # hub pinned to match [ml], allowing both extras to coexist.
+ # The importer only uses torch for CPU tensor ops
+ # (data loading + .numpy()), so CUDA version does not matter.
+ "lerobot==0.4.4",
+ "huggingface-hub==0.35.3",
+ "tensorflow-datasets>=4.9.9",
+ "tensorflow-cpu>=2.20.0",
+ "pin==3.9.0",
+ "pin-pink==4.2.0",
+ "coal==3.0.2",
+ "eigenpy==3.12.0",
+ "cmeel-boost==1.89.0",
+ "cmeel-urdfdom==4.0.1",
+ "cmeel-tinyxml2==10.0.0",
+ "cmeel-assimp==6.0.5",
+ "cmeel-octomap==1.10.0",
+ "mcap>=1.3.1,<2",
+ "mcap-protobuf-support>=0.5.4,<0.6",
+ "mcap-ros1-support>=0.7.4,<0.8",
+ "mcap-ros2-support>=0.5.7,<0.6",
+]
+# The Rust data daemon is in the base `dependencies` above, so it installs
+# automatically on Linux. This extra is kept as a backward-compatible, explicit
+# opt-in alias — `pip install neuracore[daemon]` resolves to the same pinned,
+# Linux-only requirement (a no-op on macOS/Windows).
+daemon = ["neuracore-data-daemon==13.3.0; sys_platform == 'linux'"]
+
+[project.scripts]
+neuracore = "neuracore.core.cli.app:main"
+
+[tool.setuptools.packages.find]
+include = ["neuracore*"]
+exclude = ["tests*", "examples*"]
+
+[tool.setuptools]
+# Don't rely on the VCS file-finder (it's skipped on a direct `build --wheel`);
+# the package-data globs below ship the data files deterministically.
+include-package-data = false
+
+[tool.setuptools.package-data]
+# Ship the hydra/algorithm/importer config trees (yaml) and the algorithm
+# requirements/readme files (txt/md) nested under the neuracore packages.
+"*" = ["**/*.yaml", "**/*.yml", "**/*.txt", "**/*.md"]
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 00229f264..b5531322d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -448,7 +448,7 @@ dependencies = [
]
[[package]]
-name = "data_daemon_producer"
+name = "data_daemon_bridge"
version = "0.1.0"
dependencies = [
"data_daemon_shared",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 0e20c8537..307aa8d81 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -1,5 +1,5 @@
[workspace]
-members = ["data_daemon", "data_daemon_shared", "data_daemon_producer"]
+members = ["data_daemon", "data_daemon_shared", "data_daemon_bridge"]
resolver = "2"
[workspace.package]
@@ -63,6 +63,6 @@ uuid = { version = "1", features = ["v4"] }
# a prod [dependencies] table by inheriting `.workspace = true`.
[profile.release]
-# Strip symbols from the shipped daemon binary / producer cdylib (~3 MB off the
+# Strip symbols from the shipped daemon binary / data bridge cdylib (~3 MB off the
# 16 MB debug-symbol binary); the wheel ships only the stripped artefacts.
strip = true
diff --git a/rust/data_daemon/src/cli/launch.rs b/rust/data_daemon/src/cli/launch.rs
index e6956d527..bdffbf077 100644
--- a/rust/data_daemon/src/cli/launch.rs
+++ b/rust/data_daemon/src/cli/launch.rs
@@ -229,11 +229,11 @@ fn run_daemon(
let db_path = runtime_env.db_path.clone();
let recordings_root = runtime_env.recordings_root.clone();
- // The recordings root is shared with the producer, which lives in a
+ // The recordings root is shared with the data bridge, which lives in a
// *separate* process and resolves it from `NEURACORE_DAEMON_RECORDINGS_ROOT`
// (or the db-dir sibling) — it never reads the daemon profile. So a
// profile `path_to_store_record` that disagrees with the effective root
- // cannot be silently honoured here without stranding the producer's spooled
+ // cannot be silently honoured here without stranding the data bridge's spooled
// video under a path the daemon never scans. Surface the mismatch loudly
// instead and point the operator at the knob that actually coordinates both
// processes.
@@ -247,7 +247,7 @@ fn run_daemon(
configured,
effective = %recordings_root.display(),
"profile `path_to_store_record` is ignored; the recordings root is set by \
- NEURACORE_DAEMON_RECORDINGS_ROOT (read by both daemon and producer). \
+ NEURACORE_DAEMON_RECORDINGS_ROOT (read by both daemon and bridge). \
Set that env var to relocate recordings."
);
}
diff --git a/rust/data_daemon/src/cli/mod.rs b/rust/data_daemon/src/cli/mod.rs
index 1af77e154..490312387 100644
--- a/rust/data_daemon/src/cli/mod.rs
+++ b/rust/data_daemon/src/cli/mod.rs
@@ -91,7 +91,7 @@ enum ProfileCommand {
/// Bandwidth limit in bytes per second.
#[arg(long = "bandwidth-limit", visible_alias = "bandwidth_limit", value_parser = parse_bytes)]
bandwidth_limit: Option,
- /// Producer video spool-backlog cap in bytes (0 disables the bound).
+ /// Video spool-backlog cap in bytes (0 disables the bound).
#[arg(long = "spool-limit", visible_alias = "spool_limit", value_parser = parse_bytes)]
spool_limit: Option,
/// Path where records should be stored.
diff --git a/rust/data_daemon_producer/Cargo.toml b/rust/data_daemon_bridge/Cargo.toml
similarity index 95%
rename from rust/data_daemon_producer/Cargo.toml
rename to rust/data_daemon_bridge/Cargo.toml
index afafcac13..e714da128 100644
--- a/rust/data_daemon_producer/Cargo.toml
+++ b/rust/data_daemon_bridge/Cargo.toml
@@ -1,12 +1,12 @@
[package]
-name = "data_daemon_producer"
+name = "data_daemon_bridge"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Producer-side IPC client for the Neuracore data daemon, exposed to Python via PyO3."
[lib]
-# `cdylib` produces the `.so` Python imports as `neuracore.data_daemon._native_producer`.
+# `cdylib` produces the `.so` Python imports as `neuracore.data_daemon._data_bridge`.
# `rlib` lets Rust integration tests still link against the library.
crate-type = ["cdylib", "rlib"]
path = "src/lib.rs"
diff --git a/rust/data_daemon_producer/src/lib.rs b/rust/data_daemon_bridge/src/lib.rs
similarity index 99%
rename from rust/data_daemon_producer/src/lib.rs
rename to rust/data_daemon_bridge/src/lib.rs
index 2c4188859..cdf166db8 100644
--- a/rust/data_daemon_producer/src/lib.rs
+++ b/rust/data_daemon_bridge/src/lib.rs
@@ -12,7 +12,7 @@
//! PyO3 producer client for the Neuracore data daemon — a *thin shipper*.
//!
-//! This crate ships as `neuracore.data_daemon._native_producer` inside the
+//! This crate ships as `neuracore.data_daemon._data_bridge` inside the
//! Python wheel. It knows nothing about recordings: it publishes
//! source/sensor/timestamp-tagged data and three fire-and-forget lifecycle
//! events, and the daemon decides which recording (if any) each datum belongs
@@ -433,9 +433,9 @@ fn get_recording_id(
})
}
-/// Python module entrypoint registered as `neuracore.data_daemon._native_producer`.
+/// Python module entrypoint registered as `neuracore.data_daemon._data_bridge`.
#[pymodule]
-fn _native_producer(module: &Bound<'_, PyModule>) -> PyResult<()> {
+fn _data_bridge(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(start_recording, module)?)?;
module.add_function(wrap_pyfunction!(log_joints, module)?)?;
module.add_function(wrap_pyfunction!(log_frame, module)?)?;
diff --git a/rust/data_daemon_producer/src/nut_writer.rs b/rust/data_daemon_bridge/src/nut_writer.rs
similarity index 100%
rename from rust/data_daemon_producer/src/nut_writer.rs
rename to rust/data_daemon_bridge/src/nut_writer.rs
diff --git a/rust/data_daemon_producer/src/paths.rs b/rust/data_daemon_bridge/src/paths.rs
similarity index 100%
rename from rust/data_daemon_producer/src/paths.rs
rename to rust/data_daemon_bridge/src/paths.rs
diff --git a/rust/data_daemon_producer/src/publisher.rs b/rust/data_daemon_bridge/src/publisher.rs
similarity index 100%
rename from rust/data_daemon_producer/src/publisher.rs
rename to rust/data_daemon_bridge/src/publisher.rs
diff --git a/rust/data_daemon_producer/src/query.rs b/rust/data_daemon_bridge/src/query.rs
similarity index 100%
rename from rust/data_daemon_producer/src/query.rs
rename to rust/data_daemon_bridge/src/query.rs
diff --git a/rust/data_daemon_producer/src/writer.rs b/rust/data_daemon_bridge/src/writer.rs
similarity index 100%
rename from rust/data_daemon_producer/src/writer.rs
rename to rust/data_daemon_bridge/src/writer.rs
diff --git a/rust/data_daemon_shared/src/lib.rs b/rust/data_daemon_shared/src/lib.rs
index 61bf211c7..922242f71 100644
--- a/rust/data_daemon_shared/src/lib.rs
+++ b/rust/data_daemon_shared/src/lib.rs
@@ -1,7 +1,7 @@
//! Shared definitions for the Neuracore data daemon.
//!
//! Both the daemon binary and the PyO3 producer crate
-//! (`data_daemon_producer`) depend on this crate so they agree on everything
+//! (`data_daemon_bridge`) depend on this crate so they agree on everything
//! that crosses the process boundary:
//!
//! - the iceoryx2 service-name conventions ([`service_name`]),
@@ -125,7 +125,7 @@ pub mod service_name {
/// Maximum number of concurrent publishers per service.
///
/// iceoryx2's default cap of 2 is unworkable for the SDK's threading
- /// model: the native producer parks its iceoryx2 publisher in a
+ /// model: the data bridge parks its iceoryx2 publisher in a
/// `thread_local!` (publishers are `!Sync`), so each Python OS thread
/// that calls into the producer builds its own. The integration matrix
/// fans up to ~32 worker threads (`parallel_contexts=8` × three joint
@@ -152,7 +152,7 @@ pub mod service_name {
/// Maximum number of concurrent iceoryx2 nodes attached to any service.
///
/// One node is built per **thread** (the `thread_local!` PRODUCER slot in
- /// the native producer). The integration matrix fans to 8 parallel worker
+ /// the data bridge). The integration matrix fans to 8 parallel worker
/// subprocesses each running 5+ threads (main + RGB + joint roles), giving
/// 40+ nodes plus the daemon. 512 gives enough headroom that the cap is
/// never approached in any test configuration.
@@ -188,7 +188,7 @@ pub mod service_name {
pub const QUERIES_MAX_PAYLOAD_BYTES: usize = 4 * 1024;
/// Maximum number of concurrent query clients. Mirrors
- /// [`MAX_PUBLISHERS_PER_SERVICE`]: the native producer parks one client port
+ /// [`MAX_PUBLISHERS_PER_SERVICE`]: the data bridge parks one client port
/// per OS thread (iceoryx2 ports are `!Sync`), so the cap must cover the
/// integration matrix's full thread fan-out.
pub const MAX_QUERY_CLIENTS_PER_SERVICE: usize = 128;
diff --git a/rust/scripts/build_wheel_artefacts.sh b/rust/scripts/build_wheel_artefacts.sh
index 3a4e8c126..338873889 100755
--- a/rust/scripts/build_wheel_artefacts.sh
+++ b/rust/scripts/build_wheel_artefacts.sh
@@ -1,76 +1,69 @@
#!/usr/bin/env bash
-# Build the Rust artefacts shipped in the neuracore wheel and place them
-# inside the Python package tree where setup.py's package_data expects them.
+# Build the standalone data-daemon binary and place it inside the
+# neuracore-data-daemon package tree where `maturin build`'s [tool.maturin]
+# include expects it:
#
-# Two artefacts are produced:
-# 1. The data-daemon binary -> neuracore/data_daemon/bin/data-daemon
-# Re-exec'd by neuracore/data_daemon/__main__.py when NCD_RUST_DAEMON
-# is truthy.
-# 2. The data_daemon_producer cdylib -> neuracore/data_daemon/_native_producer.so
-# Renamed from libdata_daemon_producer.so so PyO3's PyInit__native_producer
-# is discoverable by the Python import machinery.
+# packaging/neuracore-data-daemon/neuracore/data_daemon/bin/data-daemon
#
-# See docs/rust_data_daemon_development.md#packaging-the-wheel for the rationale.
+# Re-exec'd by neuracore/data_daemon/__main__.py when NCD_RUST_DAEMON is truthy.
+#
+# The producer cdylib (_data_bridge) is NOT built here — maturin builds it from
+# rust/data_daemon_bridge and names it correctly per platform. This script
+# only handles the daemon binary, which lives in a *different* crate
+# (data-daemon) that maturin does not build. Run it before `maturin build` of
+# the neuracore-data-daemon package.
+#
+# Usage:
+# ./rust/scripts/build_wheel_artefacts.sh # native host target
+# ./rust/scripts/build_wheel_artefacts.sh --target
+# e.g. --target x86_64-unknown-linux-gnu inside the manylinux container.
+#
+# See docs/rust_data_daemon_development.md#packaging-the-wheel for the pipeline.
set -euo pipefail
+target=""
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --target)
+ target="${2:?--target requires a triple}"
+ shift 2
+ ;;
+ --target=*)
+ target="${1#--target=}"
+ shift
+ ;;
+ *)
+ echo "error: unknown argument '$1'" >&2
+ echo "usage: $0 [--target ]" >&2
+ exit 1
+ ;;
+ esac
+done
+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
workspace_root="$(cd "$script_dir/.." && pwd)"
repo_root="$(cd "$workspace_root/.." && pwd)"
-package_dir="$repo_root/neuracore/data_daemon"
+package_dir="$repo_root/packaging/neuracore-data-daemon/neuracore/data_daemon"
bin_dst="$package_dir/bin/data-daemon"
-cdylib_dst="$package_dir/_native_producer.so"
-# PyO3's build-config probes (in order) PYO3_PYTHON, VIRTUAL_ENV/bin/python,
-# CONDA_PREFIX/bin/python, then /usr/bin/python. On minimal Debian/Ubuntu
-# images only python3 is on PATH and some dev environments set VIRTUAL_ENV to
-# a host (e.g. /usr) where neither python nor python3 lives — both cases
-# leave pyo3 with no interpreter and the build fails.
-#
-# When PYO3_PYTHON isn't set explicitly, walk the same probe chain ourselves
-# and fall back to whatever `python3` resolves to on PATH if none of the
-# usual candidates exist. Caller-supplied PYO3_PYTHON always wins.
-if [[ -z "${PYO3_PYTHON:-}" ]]; then
- pyo3_candidates=()
- [[ -n "${VIRTUAL_ENV:-}" ]] && pyo3_candidates+=("$VIRTUAL_ENV/bin/python")
- [[ -n "${CONDA_PREFIX:-}" ]] && pyo3_candidates+=("$CONDA_PREFIX/bin/python")
- pyo3_candidates+=("/usr/bin/python")
- for candidate in "${pyo3_candidates[@]}"; do
- if [[ -x "$candidate" ]]; then
- export PYO3_PYTHON="$candidate"
- break
- fi
- done
- if [[ -z "${PYO3_PYTHON:-}" ]]; then
- if command -v python3 >/dev/null 2>&1; then
- export PYO3_PYTHON
- PYO3_PYTHON="$(command -v python3)"
- echo "==> using PYO3_PYTHON=$PYO3_PYTHON (no python found in VIRTUAL_ENV/CONDA_PREFIX/system)"
- else
- echo "error: no python interpreter found; set PYO3_PYTHON or install python3" >&2
- exit 1
- fi
- fi
+cargo_args=(build --release --manifest-path "$workspace_root/Cargo.toml" -p data-daemon)
+if [[ -n "$target" ]]; then
+ cargo_args+=(--target "$target")
+ bin_src="$workspace_root/target/$target/release/data-daemon"
+else
+ bin_src="$workspace_root/target/release/data-daemon"
fi
-echo "==> cargo build --release -p data-daemon"
-cargo build --release --manifest-path "$workspace_root/Cargo.toml" -p data-daemon
+echo "==> cargo ${cargo_args[*]}"
+cargo "${cargo_args[@]}"
-echo "==> cargo build --release -p data_daemon_producer"
-cargo build --release --manifest-path "$workspace_root/Cargo.toml" -p data_daemon_producer
+if [[ ! -f "$bin_src" ]]; then
+ echo "error: data-daemon binary not found at $bin_src" >&2
+ exit 1
+fi
mkdir -p "$(dirname "$bin_dst")"
-install -m 0755 "$workspace_root/target/release/data-daemon" "$bin_dst"
+install -m 0755 "$bin_src" "$bin_dst"
echo " wrote $bin_dst"
-
-# cdylib filename varies by platform: libfoo.so on Linux, libfoo.dylib on macOS,
-# foo.dll on Windows. Linux-only support per data-daemon-rewrite.md §Open items.
-cdylib_src="$workspace_root/target/release/libdata_daemon_producer.so"
-if [[ ! -f "$cdylib_src" ]]; then
- echo "error: cdylib not found at $cdylib_src" >&2
- echo " (data-daemon-rewrite.md is Linux-first; macOS/Windows are not supported)" >&2
- exit 1
-fi
-install -m 0755 "$cdylib_src" "$cdylib_dst"
-echo " wrote $cdylib_dst"
diff --git a/rust/scripts/run_integration_tests.sh b/rust/scripts/run_integration_tests.sh
index 9813b581f..889a33fbd 100755
--- a/rust/scripts/run_integration_tests.sh
+++ b/rust/scripts/run_integration_tests.sh
@@ -138,6 +138,58 @@ cleanup_state() {
shopt -u nullglob
}
+# ---------------------------------------------------------------------------
+# Build + materialize the daemon artefacts into the source tree
+# ---------------------------------------------------------------------------
+
+# pytest is invoked from $repo_root below, and the test dirs form an unbroken
+# __init__.py package chain up to a repo root that has none, so pytest's default
+# (prepend) import mode puts $repo_root on sys.path and `import neuracore`
+# resolves to the in-tree ./neuracore/ — shadowing any site-packages install.
+# The Rust daemon path therefore imports `neuracore.data_daemon._data_bridge`
+# (and resolves the `data-daemon` binary) out of THIS tree, so the compiled
+# extension and the binary must physically live under ./neuracore/data_daemon/.
+# `maturin develop` installs the extension into site-packages (unreachable here)
+# and build_wheel_artefacts.sh drops the binary into the packaging tree, so
+# neither lands where the imported package can see it. Build both and copy them
+# into place — the same artefacts the CI staging job extracts from the wheel.
+materialize_artefacts() {
+ log "building data bridge extension (cargo build -p data_daemon_bridge --release)"
+ cargo build --release \
+ --manifest-path "$workspace_root/Cargo.toml" \
+ -p data_daemon_bridge 2>&1 | tee -a "$log_file"
+
+ local cdylib_src="$workspace_root/target/release/libdata_daemon_bridge.so"
+ if [[ ! -f "$cdylib_src" ]]; then
+ log "error: bridge cdylib not found at $cdylib_src"
+ exit 1
+ fi
+
+ # Name the extension with the running interpreter's ABI suffix (matching
+ # maturin) and drop any stale build first, so a Python-minor switch can't
+ # leave behind an ABI-incompatible _data_bridge that import picks up instead.
+ local ext_suffix
+ ext_suffix="$(python3 -c 'import importlib.machinery as m; print(m.EXTENSION_SUFFIXES[0])')"
+ local dd_dir="$repo_root/neuracore/data_daemon"
+ rm -f "$dd_dir"/_data_bridge*.so
+ install -m 0644 "$cdylib_src" "$dd_dir/_data_bridge${ext_suffix}"
+ log " wrote $dd_dir/_data_bridge${ext_suffix}"
+
+ # Build the standalone binary into the packaging tree, then copy it next to
+ # the extension so rust_daemon_binary_path() (importlib.resources over this
+ # tree) can find it.
+ log "building data-daemon binary (build_wheel_artefacts.sh)"
+ bash "$script_dir/build_wheel_artefacts.sh" 2>&1 | tee -a "$log_file"
+ local bin_src="$repo_root/packaging/neuracore-data-daemon/neuracore/data_daemon/bin/data-daemon"
+ if [[ ! -f "$bin_src" ]]; then
+ log "error: data-daemon binary not found at $bin_src"
+ exit 1
+ fi
+ mkdir -p "$dd_dir/bin"
+ install -m 0755 "$bin_src" "$dd_dir/bin/data-daemon"
+ log " wrote $dd_dir/bin/data-daemon"
+}
+
# ---------------------------------------------------------------------------
# Environment
# ---------------------------------------------------------------------------
@@ -261,6 +313,7 @@ main() {
log "==== integration test run starting ===="
stop_daemon
cleanup_state
+ materialize_artefacts
set +e
run_tests
diff --git a/setup.py b/setup.py
deleted file mode 100644
index f1c681bff..000000000
--- a/setup.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import os
-
-from setuptools import find_packages, setup
-from setuptools.dist import Distribution
-
-version = None
-with open("neuracore/__init__.py", encoding="utf-8") as f:
- for line in f:
- if line.startswith("__version__"):
- version = line.strip().split()[-1][1:-1]
- break
-assert version is not None, "Could not find version string"
-
-with open("README.md", encoding="utf-8") as fh:
- long_description = fh.read()
-
-
-# The prebuilt Rust artefacts are placed under ``neuracore/data_daemon/`` by
-# rust/scripts/build_wheel_artefacts.sh before ``python -m build``. They are
-# gitignored and absent from a plain checkout and from the standard release
-# build (which does not run that script), so their presence is what decides
-# whether this is a binary wheel.
-_DATA_DAEMON_DIR = os.path.join(os.path.dirname(__file__), "neuracore", "data_daemon")
-_RUST_ARTEFACTS = (
- os.path.join(_DATA_DAEMON_DIR, "bin", "data-daemon"),
- os.path.join(_DATA_DAEMON_DIR, "_native_producer.so"),
-)
-_HAS_RUST_ARTEFACTS = all(os.path.exists(path) for path in _RUST_ARTEFACTS)
-
-
-class BinaryDistribution(Distribution):
- """Force a platform-specific wheel tag when the Rust artefacts are bundled.
-
- The Rust data-daemon binary and producer cdylib shipped under
- ``neuracore/data_daemon/`` are platform-specific, so a wheel that contains
- them must not be tagged ``py3-none-any`` (pip would install a Linux ``.so``
- onto macOS). When the artefacts are absent — e.g. the standard
- ``python -m build`` release, which does not run
- ``build_wheel_artefacts.sh`` — this returns ``False`` so the usual pure
- ``py3-none-any`` wheel is produced and the release pipeline is unchanged.
- See ``docs/rust_data_daemon_development.md#packaging-the-wheel``.
- """
-
- def has_ext_modules(self) -> bool:
- return _HAS_RUST_ARTEFACTS
-
-
-setup(
- name="neuracore",
- distclass=BinaryDistribution,
- include_package_data=True,
- package_data={
- "neuracore.data_daemon": [
- # Pre-built Rust artefacts. Generated by
- # rust/scripts/build_wheel_artefacts.sh before `python -m build`.
- # Both paths are gitignored; the wheel build is responsible for
- # placing them. Empty trees just produce an any-platform wheel
- # without these payloads (the Python daemon still works).
- "bin/data-daemon",
- "_native_producer.so",
- ],
- },
- version=version,
- author="Stephen James",
- author_email="support@neuracore.com",
- description="Neuracore Client Library",
- long_description=long_description,
- long_description_content_type="text/markdown",
- url="https://github.com/neuracoreai/neuracore",
- packages=find_packages(exclude=["tests*", "examples*"]),
- classifiers=[
- "Development Status :: 3 - Alpha",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: MIT License",
- "Operating System :: OS Independent",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- ],
- python_requires=">=3.10",
- install_requires=[
- "numpy>=2.0.0",
- "requests>=2.31.0",
- "pillow>=10.0.0",
- "pyyaml>=6.0.1",
- "tqdm>=4.66.0",
- "requests-oauthlib==2.0.0",
- "pydantic>=2.10",
- "av==15.1.0",
- "aiortc==1.14.0",
- "aiohttp-sse-client==0.2.1",
- "numpy-stl",
- "wget",
- "uvicorn[standard]==0.42.0",
- "fastapi==0.135.2",
- "psutil",
- "typer>=0.20.0",
- "neuracore_types>=10.0.0,<11.0.0",
- "ordered_set",
- "pyzmq==27.1.0",
- "sqlalchemy>=2.0.0",
- "aiosqlite>=0.19.0",
- "aiohttp>=3.9.0",
- "aiofiles>=23.0.0",
- "aiolimiter",
- "pyee==13.0.0",
- "greenlet",
- "filelock>=3.0.0",
- "omegaconf",
- "msgpack>=1.0.8",
- ],
- extras_require={
- "examples": [
- "matplotlib>=3.3.0",
- "mujoco==2.3.7",
- "pyquaternion>=0.9.5",
- ],
- "mjcf": [
- "mujoco>3",
- ],
- "ml": [
- # torch==2.8.0 is minimum version defaulting to CUDA 12.8.
- # Do NOT upgrade to 2.11.0+: it defaults to CUDA 13 which drops
- # V100 support, which Neuracore still uses.
- # hub pinned to 0.35.3 to match lerobot==0.4.4's
- # <0.36.0 constraint, allowing [ml] and [import] to coexist.
- # We have dropped support for P100 and P4 due to CUDA 12.8
- # dropping support for them.
- "torch==2.10.0",
- "torchvision==0.25.0",
- "transformers==4.53.2",
- "huggingface-hub==0.35.3",
- "diffusers==0.35.1",
- "safetensors==0.6.2",
- "einops",
- "hydra-core>=1.3.0",
- "tensorboard>=2",
- "names-generator>=0.2.0",
- ],
- "dev": [
- "pytest>=6.2.5",
- "pytest-cov>=2.12.1",
- "diff-cover",
- "pytest-asyncio>=0.15.1",
- "pytest-xdist",
- "twine>=3.4.2",
- "requests-mock>=1.9.3",
- "pre-commit",
- "types-aiofiles",
- "pyinstrument",
- "plotly",
- ],
- "import": [
- # lerobot is used only for dataset importing
- # (LeRobotDataset/LeRobotDatasetMetadata).
- # Do NOT install lerobot[transformers-dep] — that extra requires
- # transformers>=4.57.1 which conflicts with our transformers==4.53.2 pin.
- # lerobot>=0.4.x drops v2 dataset format support.
- # torch is not pinned here as lerobot==0.4.4 constrains it to
- # >=2.2.1,<2.11.0, compatible with [ml]'s torch==2.10.0.
- # hub pinned to match [ml], allowing both extras to coexist.
- # The importer only uses torch for CPU tensor ops
- # (data loading + .numpy()), so CUDA version does not matter.
- "lerobot==0.4.4",
- "huggingface-hub==0.35.3",
- "tensorflow-datasets>=4.9.9",
- "tensorflow-cpu>=2.20.0",
- "pin==3.9.0",
- "pin-pink==4.2.0",
- "coal==3.0.2",
- "eigenpy==3.12.0",
- "cmeel-boost==1.89.0",
- "cmeel-urdfdom==4.0.1",
- "cmeel-tinyxml2==10.0.0",
- "cmeel-assimp==6.0.5",
- "cmeel-octomap==1.10.0",
- "mcap>=1.3.1,<2",
- "mcap-protobuf-support>=0.5.4,<0.6",
- "mcap-ros1-support>=0.7.4,<0.8",
- "mcap-ros2-support>=0.5.7,<0.6",
- ],
- },
- entry_points={
- "console_scripts": [
- "neuracore = neuracore.core.cli.app:main",
- ]
- },
- keywords="robotics machine-learning ai client-library",
-)
diff --git a/tests/integration/platform/data_daemon/behavioural_correctness/test_offline_to_online.py b/tests/integration/platform/data_daemon/behavioural_correctness/test_offline_to_online.py
index a0ca795c6..f6a3ab10d 100644
--- a/tests/integration/platform/data_daemon/behavioural_correctness/test_offline_to_online.py
+++ b/tests/integration/platform/data_daemon/behavioural_correctness/test_offline_to_online.py
@@ -2,7 +2,7 @@
import pytest
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.shared.assertions import (
assert_exactly_one_daemon_pid,
verify_cloud_results,
@@ -89,7 +89,7 @@ def test_offline_pending_data_recovers_when_online(
with online_daemon_running():
for result in results:
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
for recording_index in result.recording_indexes:
wait_for_upload_complete_in_db(recording_index)
else:
diff --git a/tests/integration/platform/data_daemon/data_integrity/test_network.py b/tests/integration/platform/data_daemon/data_integrity/test_network.py
index 53bd74211..638336cfa 100644
--- a/tests/integration/platform/data_daemon/data_integrity/test_network.py
+++ b/tests/integration/platform/data_daemon/data_integrity/test_network.py
@@ -4,7 +4,7 @@
import pytest
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.daemon_test_cases import (
PRE_NETWORK_INTEGRITY_CASES,
)
@@ -59,7 +59,7 @@ def _assert_online_verification_invariants(
``recording_id`` under the legacy daemon.
"""
for result in results:
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
for recording_index in result.recording_indexes:
wait_for_upload_complete_in_db(
recording_index, timeout_s=timeout_seconds
diff --git a/tests/integration/platform/data_daemon/shared/assertions.py b/tests/integration/platform/data_daemon/shared/assertions.py
index 1427a72cb..a96c1847a 100644
--- a/tests/integration/platform/data_daemon/shared/assertions.py
+++ b/tests/integration/platform/data_daemon/shared/assertions.py
@@ -48,7 +48,7 @@
from neuracore.core.data.recording import Recording
from neuracore.data_daemon.const import SOCKET_PATH
from neuracore.data_daemon.helpers import get_daemon_pid_path
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.shared.db_helpers import (
wait_for_dataset_ready,
wait_for_recordings_finalized,
@@ -91,7 +91,7 @@
def assert_context_mode(case: DataDaemonTestCase, results: list[ContextResult]) -> None:
"""Assert that context timing matches the expected mode."""
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
active_results = [result for result in results if result.recording_indexes]
else:
active_results = [result for result in results if result.recording_ids]
diff --git a/tests/integration/platform/data_daemon/shared/db_helpers.py b/tests/integration/platform/data_daemon/shared/db_helpers.py
index 4bf1be824..303adf3b0 100644
--- a/tests/integration/platform/data_daemon/shared/db_helpers.py
+++ b/tests/integration/platform/data_daemon/shared/db_helpers.py
@@ -40,7 +40,7 @@
import neuracore as nc
from neuracore.data_daemon.helpers import get_daemon_db_path
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.shared.db_constants import (
COLUMN_EXPECTED_TRACE_COUNT,
COLUMN_EXPECTED_TRACE_COUNT_REPORTED,
@@ -100,7 +100,7 @@ def _recording_correlation_column() -> str:
daemon: recordings are keyed by the cloud ``recording_id`` (TEXT PK), which
is also the traces foreign key.
"""
- return COLUMN_RECORDING_INDEX if rust_daemon_enabled() else COLUMN_RECORDING_ID
+ return COLUMN_RECORDING_INDEX if is_rust_daemon_enabled() else COLUMN_RECORDING_ID
class DaemonDbStore:
@@ -368,7 +368,7 @@ def assert_offline_recordings_pending(results: list[ContextResult]) -> None:
Raises:
AssertionError: If a recording row is missing or already has a cloud id.
"""
- if not rust_daemon_enabled():
+ if not is_rust_daemon_enabled():
return
for result in results:
rows_by_index = {
@@ -413,7 +413,7 @@ def resolve_cloud_recording_ids(
Raises:
AssertionError: If any recording's cloud id is not populated in time.
"""
- if not rust_daemon_enabled():
+ if not is_rust_daemon_enabled():
return results
resolved: list[ContextResult] = []
@@ -837,7 +837,7 @@ def wait_for_offline_db_ready(
RECORDINGS_TABLE,
TRACES_TABLE,
}
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
target_recording_keys: set[int] | set[str] = normalize_recording_indexes(
expected_recording_keys
)
@@ -923,7 +923,7 @@ def wait_for_all_traces_written(
min_poll_interval_s = 0.05
max_poll_interval_s = 1.0
- use_rust = rust_daemon_enabled()
+ use_rust = is_rust_daemon_enabled()
correlation_column = COLUMN_RECORDING_INDEX if use_rust else COLUMN_RECORDING_ID
def _raw_keys() -> list:
diff --git a/tests/integration/platform/data_daemon/shared/disk_helpers.py b/tests/integration/platform/data_daemon/shared/disk_helpers.py
index 43096fa19..c49874660 100644
--- a/tests/integration/platform/data_daemon/shared/disk_helpers.py
+++ b/tests/integration/platform/data_daemon/shared/disk_helpers.py
@@ -10,7 +10,7 @@
from typing import TYPE_CHECKING
from neuracore.data_daemon.helpers import get_daemon_recordings_root_path
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.shared.test_case.constants import (
TIMESTAMP_MODE_REAL,
TIMESTAMP_MODE_STOCHASTIC,
@@ -141,7 +141,7 @@ def _result_recording_keys(result: ContextResult) -> list[tuple[str, int | str]]
string. Keeping these two values together lets the assertion body stay
identical across modes.
"""
- if rust_daemon_enabled():
+ if is_rust_daemon_enabled():
return [
(str(recording_index), recording_index)
for recording_index in result.recording_indexes
diff --git a/tests/integration/platform/data_daemon/shared/storage_assertions.py b/tests/integration/platform/data_daemon/shared/storage_assertions.py
index 0a14707c4..cf08e4bc9 100644
--- a/tests/integration/platform/data_daemon/shared/storage_assertions.py
+++ b/tests/integration/platform/data_daemon/shared/storage_assertions.py
@@ -14,7 +14,7 @@
get_daemon_db_path,
get_daemon_recordings_root_path,
)
-from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.shared.test_case.constants import (
OFFLINE_DB_PATH,
OFFLINE_RECORDINGS_ROOT,
@@ -32,7 +32,7 @@ def harness_db_path() -> Path:
target the wrong folder. When the Rust daemon is active and the env var is
unset, resolve the real shared test-state path the daemon actually used.
"""
- if rust_daemon_enabled() and not os.getenv("NEURACORE_DAEMON_DB_PATH"):
+ if is_rust_daemon_enabled() and not os.getenv("NEURACORE_DAEMON_DB_PATH"):
return OFFLINE_DB_PATH
return get_daemon_db_path()
@@ -42,7 +42,7 @@ def harness_recordings_root() -> Path:
See :func:`harness_db_path` for why the Rust daemon needs special handling.
"""
- if rust_daemon_enabled() and not os.getenv("NEURACORE_DAEMON_RECORDINGS_ROOT"):
+ if is_rust_daemon_enabled() and not os.getenv("NEURACORE_DAEMON_RECORDINGS_ROOT"):
return OFFLINE_RECORDINGS_ROOT
return get_daemon_recordings_root_path()
diff --git a/tests/integration/platform/data_daemon/shared/test_case/build_test_case_context.py b/tests/integration/platform/data_daemon/shared/test_case/build_test_case_context.py
index 00d912c16..f7d9fc7a9 100644
--- a/tests/integration/platform/data_daemon/shared/test_case/build_test_case_context.py
+++ b/tests/integration/platform/data_daemon/shared/test_case/build_test_case_context.py
@@ -751,12 +751,12 @@ def _subprocess_context_worker(spec: ContextSpec) -> ContextResult:
def context_worker(spec: ContextSpec) -> ContextResult:
"""Execute recordings for a single parallel context."""
- from neuracore.data_daemon.rust_selection import rust_daemon_enabled
+ from neuracore.data_daemon.rust_selection import is_rust_daemon_enabled
from tests.integration.platform.data_daemon.shared.db_helpers import (
wait_for_recording_index_for_source,
)
- use_rust = rust_daemon_enabled()
+ use_rust = is_rust_daemon_enabled()
case = spec.case
use_real_timestamps = case.timestamp_mode == TIMESTAMP_MODE_REAL
joint_name_list = joint_names_for_count(case.joint_count)
diff --git a/tests/unit/core/test_data_stream.py b/tests/unit/core/test_data_stream.py
index 63a5e58d6..a777c18a7 100644
--- a/tests/unit/core/test_data_stream.py
+++ b/tests/unit/core/test_data_stream.py
@@ -233,7 +233,7 @@ def test_rgb_stream_owns_no_channel_under_rust_daemon(monkeypatch) -> None:
_FakeProducerChannel,
)
monkeypatch.setattr(
- "neuracore.core.streaming.data_stream.rust_daemon_enabled",
+ "neuracore.core.streaming.data_stream.is_rust_daemon_enabled",
lambda: True,
)
diff --git a/tests/unit/data_daemon/upload_management/test_multifile_upload.py b/tests/unit/data_daemon/upload_management/test_multifile_upload.py
index 03769352e..42797ec25 100644
--- a/tests/unit/data_daemon/upload_management/test_multifile_upload.py
+++ b/tests/unit/data_daemon/upload_management/test_multifile_upload.py
@@ -1139,11 +1139,14 @@ def put_side_effect(*args, **kwargs):
else make_response(200, headers={"x-goog-hash": f"md5={md5_b64}"})
)
- with patch(
- "neuracore.data_daemon.upload_management.resumable_file_uploader.get_auth"
- ) as mock_rfu_auth, patch(
- "neuracore.data_daemon.upload_management.resumable_file_uploader.get_current_org",
- return_value="test-org",
+ with (
+ patch(
+ "neuracore.data_daemon.upload_management.resumable_file_uploader.get_auth"
+ ) as mock_rfu_auth,
+ patch(
+ "neuracore.data_daemon.upload_management.resumable_file_uploader.get_current_org",
+ return_value="test-org",
+ ),
):
rfu_auth = MagicMock()
diff --git a/tests/unit/importer/test_lerobot_importer_behavior.py b/tests/unit/importer/test_lerobot_importer_behavior.py
index 3a5a895ed..ce1a3c400 100644
--- a/tests/unit/importer/test_lerobot_importer_behavior.py
+++ b/tests/unit/importer/test_lerobot_importer_behavior.py
@@ -117,9 +117,12 @@ def test_lerobot_import_item_step_mode_skips_failing_steps():
)
importer._record_step = MagicMock(side_effect=[ValueError("bad step"), None])
- with patch("neuracore.importer.lerobot_importer.nc.start_recording"), patch(
- "neuracore.importer.lerobot_importer.nc.stop_recording"
- ) as stop_recording:
+ with (
+ patch("neuracore.importer.lerobot_importer.nc.start_recording"),
+ patch(
+ "neuracore.importer.lerobot_importer.nc.stop_recording"
+ ) as stop_recording,
+ ):
importer.import_item(ImportItem(index=0))
assert importer._record_step.call_count == 2
@@ -160,8 +163,9 @@ def test_lerobot_import_item_non_step_mode_re_raises():
importer._iter_episode_steps = MagicMock(return_value=(iter([{"v": 1}]), 1))
importer._record_step = MagicMock(side_effect=RuntimeError("explode"))
- with patch("neuracore.importer.lerobot_importer.nc.start_recording"), patch(
- "neuracore.importer.lerobot_importer.nc.stop_recording"
+ with (
+ patch("neuracore.importer.lerobot_importer.nc.start_recording"),
+ patch("neuracore.importer.lerobot_importer.nc.stop_recording"),
):
with pytest.raises(RuntimeError, match="explode"):
importer.import_item(ImportItem(index=0))
diff --git a/tests/unit/ml/test_server_policy_endpoint.py b/tests/unit/ml/test_server_policy_endpoint.py
index d54c13275..de3f52741 100644
--- a/tests/unit/ml/test_server_policy_endpoint.py
+++ b/tests/unit/ml/test_server_policy_endpoint.py
@@ -416,13 +416,16 @@ def test_set_checkpoint_raises_endpoint_error_for_connection_error(
nc.login(TEST_API_KEY)
endpoint = _connect_test_remote_endpoint(mock_auth_requests, mocked_org_id)
mock_session.post.side_effect = requests.exceptions.ConnectionError()
- with patch(
- "neuracore.core.endpoint.thread_local_session", return_value=mock_session
- ), pytest.raises(
- EndpointError,
- match=(
- "Failed to connect to endpoint, please check your internet "
- "connection and try again."
+ with (
+ patch(
+ "neuracore.core.endpoint.thread_local_session", return_value=mock_session
+ ),
+ pytest.raises(
+ EndpointError,
+ match=(
+ "Failed to connect to endpoint, please check your internet "
+ "connection and try again."
+ ),
),
):
endpoint.set_checkpoint(epoch=1)
@@ -439,9 +442,12 @@ def test_set_checkpoint_raises_endpoint_error_for_request_exception(
nc.login(TEST_API_KEY)
endpoint = _connect_test_remote_endpoint(mock_auth_requests, mocked_org_id)
mock_session.post.side_effect = requests.exceptions.Timeout("timeout")
- with patch(
- "neuracore.core.endpoint.thread_local_session", return_value=mock_session
- ), pytest.raises(EndpointError, match="Failed to set checkpoint: timeout"):
+ with (
+ patch(
+ "neuracore.core.endpoint.thread_local_session", return_value=mock_session
+ ),
+ pytest.raises(EndpointError, match="Failed to set checkpoint: timeout"),
+ ):
endpoint.set_checkpoint(epoch=1)