From a301a254a81af4d41acf9ae6a31f374f43b8109d Mon Sep 17 00:00:00 2001 From: Marton Mayer Date: Fri, 4 Sep 2026 10:29:27 +0200 Subject: [PATCH 1/4] feat(ci): add CI and release automation --- .github/actionlint.yaml | 6 + .github/release-matrix.json | 8 + .github/workflows/ci.yml | 241 +++ .github/workflows/release-audio.yml | 139 ++ .github/workflows/release-docker.yml | 396 +++++ .github/workflows/release-helm.yml | 116 ++ .github/workflows/release-native.yml | 115 ++ .github/workflows/release-npm.yml | 62 + .github/workflows/release-python.yml | 62 + .github/workflows/release.yml | 373 ++++ .gitignore | 2 + .npmrc | 1 - .release-please-manifest.json | 3 + AGENTS.md | 60 + CONTRIBUTING.md | 84 + README.md | 6 + RELEASE.md | 221 +++ deny.toml | 9 +- deploy/helm/sie-cluster/README.md | 15 + examples/document-ocr/README.md | 6 +- examples/document-ocr/compose.gpu.yml | 4 +- examples/document-ocr/compose.yml | 4 +- .../sie_ts_chroma/tests/embedding.test.ts | 90 +- integrations/sie_ts_lancedb/src/index.ts | 11 +- .../sie_ts_lancedb/tests/embedding.test.ts | 146 +- .../sie_ts_langchain/tests/embeddings.test.ts | 94 +- .../sie_ts_langchain/tests/extractors.test.ts | 70 +- .../sie_ts_langchain/tests/rerankers.test.ts | 56 +- .../sie_ts_llamaindex/src/extractors.ts | 6 +- .../sie_ts_llamaindex/tests/embedding.test.ts | 106 +- .../tests/extractors.test.ts | 46 +- .../sie_ts_llamaindex/tests/rerankers.test.ts | 56 +- mise.toml | 15 +- package.json | 5 +- .../client/test_transport_error_retry.py | 58 +- packages/sie_sdk/tests/test_cache.py | 24 +- .../tests/adapters/test_docling_smoke.py | 2 +- packages/sie_server_sidecar/Dockerfile | 4 +- packages/sie_ts_sdk/package.json | 10 +- packages/sie_ts_sdk/pnpm-lock.yaml | 1515 ----------------- packages/sie_ts_sdk/src/encoding.ts | 7 +- pnpm-lock.yaml | 12 +- release-please-config.json | 41 + telemetry/README.md | 6 +- telemetry/contract.yaml | 4 +- tests/parity/README.md | 49 + tests/parity/run_batch_empty.json | 15 + tests/parity/run_batch_encode_lora.json | 78 + tests/parity/run_batch_encode_no_lora.json | 64 + tests/parity/run_batch_extract_lora.json | 52 + tests/parity/run_batch_mixed_op.json | 56 + tests/parity/run_batch_score_basic.json | 41 + tests/parity/run_batch_score_lora_warns.json | 38 + tests/parity/run_batch_unknown_op.json | 36 + tests/parity/run_parity.sh | 50 + tools/ci/build_audio_prep_release_asset.py | 50 + tools/ci/build_sidecar_release_asset.py | 109 ++ tools/ci/check_public_tree.py | 78 + tools/ci/check_release_contract.py | 763 +++++++++ tools/ci/cpu_stack_smoke.py | 213 +++ tools/ci/distributions.py | 299 ++++ tools/ci/fresh_bootstrap.bash | 11 + tools/ci/live_sdk.py | 126 ++ tools/ci/live_typescript.mjs | 23 + tools/ci/publish_helm_archive.py | 54 + tools/ci/release_artifact.py | 125 ++ tools/ci/release_guard.py | 165 ++ tools/ci/release_recovery.py | 182 ++ tools/ci/required_ci.py | 42 + tools/ci/restore_release_artifact.py | 91 + tools/ci/rust_tests.py | 75 + tools/ci/tests/test_cpu_checks.py | 85 + tools/ci/tests/test_distributions.py | 222 +++ tools/ci/tests/test_docker_task.py | 393 +++++ tools/ci/tests/test_helm_task.py | 47 + tools/ci/tests/test_public_tree.py | 80 + tools/ci/tests/test_release_artifact.py | 170 ++ tools/ci/tests/test_release_contract.py | 271 +++ tools/ci/tests/test_release_guard.py | 321 ++++ tools/ci/tests/test_required_ci.py | 130 ++ tools/ci/upload_audio_prep_release_asset.bash | 5 + tools/ci/upload_native_release_asset.bash | 64 + tools/mise_tasks/common/device.py | 52 + tools/mise_tasks/docker-push-loaded.bash | 16 + tools/mise_tasks/docker.bash | 9 + tools/mise_tasks/docker_task.py | 620 +++++++ tools/mise_tasks/full-sync.bash | 9 +- tools/mise_tasks/helm.py | 60 +- tools/mise_tasks/test-integrations.bash | 9 +- 89 files changed, 7849 insertions(+), 1916 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/release-matrix.json create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release-audio.yml create mode 100644 .github/workflows/release-docker.yml create mode 100644 .github/workflows/release-helm.yml create mode 100644 .github/workflows/release-native.yml create mode 100644 .github/workflows/release-npm.yml create mode 100644 .github/workflows/release-python.yml create mode 100644 .github/workflows/release.yml delete mode 100644 .npmrc create mode 100644 .release-please-manifest.json create mode 100644 AGENTS.md create mode 100644 CONTRIBUTING.md create mode 100644 RELEASE.md delete mode 100644 packages/sie_ts_sdk/pnpm-lock.yaml create mode 100644 release-please-config.json create mode 100644 tests/parity/README.md create mode 100644 tests/parity/run_batch_empty.json create mode 100644 tests/parity/run_batch_encode_lora.json create mode 100644 tests/parity/run_batch_encode_no_lora.json create mode 100644 tests/parity/run_batch_extract_lora.json create mode 100644 tests/parity/run_batch_mixed_op.json create mode 100644 tests/parity/run_batch_score_basic.json create mode 100644 tests/parity/run_batch_score_lora_warns.json create mode 100644 tests/parity/run_batch_unknown_op.json create mode 100755 tests/parity/run_parity.sh create mode 100755 tools/ci/build_audio_prep_release_asset.py create mode 100755 tools/ci/build_sidecar_release_asset.py create mode 100755 tools/ci/check_public_tree.py create mode 100755 tools/ci/check_release_contract.py create mode 100644 tools/ci/cpu_stack_smoke.py create mode 100755 tools/ci/distributions.py create mode 100644 tools/ci/fresh_bootstrap.bash create mode 100644 tools/ci/live_sdk.py create mode 100644 tools/ci/live_typescript.mjs create mode 100755 tools/ci/publish_helm_archive.py create mode 100755 tools/ci/release_artifact.py create mode 100755 tools/ci/release_guard.py create mode 100755 tools/ci/release_recovery.py create mode 100644 tools/ci/required_ci.py create mode 100755 tools/ci/restore_release_artifact.py create mode 100644 tools/ci/rust_tests.py create mode 100644 tools/ci/tests/test_cpu_checks.py create mode 100644 tools/ci/tests/test_distributions.py create mode 100644 tools/ci/tests/test_docker_task.py create mode 100644 tools/ci/tests/test_helm_task.py create mode 100644 tools/ci/tests/test_public_tree.py create mode 100644 tools/ci/tests/test_release_artifact.py create mode 100644 tools/ci/tests/test_release_contract.py create mode 100644 tools/ci/tests/test_release_guard.py create mode 100644 tools/ci/tests/test_required_ci.py create mode 100755 tools/ci/upload_audio_prep_release_asset.bash create mode 100755 tools/ci/upload_native_release_asset.bash create mode 100644 tools/mise_tasks/common/device.py create mode 100755 tools/mise_tasks/docker-push-loaded.bash create mode 100755 tools/mise_tasks/docker.bash create mode 100755 tools/mise_tasks/docker_task.py diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..f106530df --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +self-hosted-runner: + labels: + - blacksmith-2vcpu-ubuntu-2404 + - blacksmith-4vcpu-ubuntu-2404 + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-16vcpu-ubuntu-2404 diff --git a/.github/release-matrix.json b/.github/release-matrix.json new file mode 100644 index 000000000..1d6aaf567 --- /dev/null +++ b/.github/release-matrix.json @@ -0,0 +1,8 @@ +{ + "bundles": ["default", "ctranslate2", "sglang", "transformers5"], + "platforms": ["cuda12", "cpu"], + "include": [ + { "platform": "cuda13", "bundle": "sglang-cu130" }, + { "platform": "cuda13", "bundle": "tensorrt-llm" } + ] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..09d3f9eda --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,241 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + policy: + name: CI / Policy + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: >- + mise exec -- uv run --frozen --project . pytest -q + tools/ci/tests/test_required_ci.py tools/ci/tests/test_public_tree.py + - run: mise exec -- python tools/ci/check_public_tree.py + - run: >- + mise exec -- actionlint + -ignore '^unexpected key "queue" for "concurrency" section\. expected one of "cancel-in-progress", "group"$' + + bootstrap: + name: CI / Fresh bootstrap + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + env: + UV_NO_CACHE: "1" + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + install: false + - run: bash tools/ci/fresh_bootstrap.bash + + python: + name: CI / Python + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise exec -- uv lock --check --project . + - run: mise run sync + - run: mise run lint + - run: mise run typecheck + - name: Offline public unit tests + env: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + run: mise run test + - run: mise run test-integrations -- --python-only + - name: Public tooling syntax and tests + run: | + mise exec -- uv run --frozen --project . --no-sync ruff format --check tools/ci + mise exec -- uv run --frozen --project . --no-sync ruff check --select E,F,I,UP,B tools/ci + mise exec -- uv run --frozen --project . --no-sync ruff check --select E9,F63,F7,F82 tools/mise_tasks + mise exec -- uv run --frozen --project . --no-sync pytest -q tools/ci/tests --ignore tools/ci/tests/test_required_ci.py --ignore tools/ci/tests/test_public_tree.py + + typescript: + name: CI / TypeScript + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise exec -- pnpm install --frozen-lockfile + - run: mise run ts -- build + - run: mise run ts -- typecheck + - run: mise run ts -- lint + - run: mise run ts -- test + + rust: + name: CI / Rust + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise run sync + - run: mise run rust-fmt -- --check + - run: mise run rust-check + - run: mise run rust-clippy + - run: mise run gateway-deny + - name: Audit standalone Rust worker dependencies + run: >- + mise exec -- cargo-deny --locked + --manifest-path packages/sie_server_rust/Cargo.toml --all-features + --config deny.toml check + - run: mise exec -- cargo fmt --manifest-path packages/sie_server_rust/Cargo.toml -- --check + - run: mise exec -- cargo check --manifest-path packages/sie_server_rust/Cargo.toml --locked --all-targets + - run: mise exec -- cargo clippy --manifest-path packages/sie_server_rust/Cargo.toml --locked --all-targets -- -D warnings + - run: mise exec -- uv run --frozen --project . --no-sync python tools/ci/rust_tests.py + - name: Preserve diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rust-logs + path: .cache/ci-logs/ + retention-days: 30 + if-no-files-found: ignore + + contracts: + name: CI / Contracts + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 25 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise run sync + - run: mise exec -- python tools/check_ipc_types_parity.py + - run: mise exec -- python tools/check_response_chunk_protocol.py + - run: tests/parity/run_parity.sh + + helm: + name: CI / Helm + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 25 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise run helm -- dependencies + - run: mise run helm -- lint --set payloadStore.enabled=false + - run: mise run helm -- template --set payloadStore.enabled=false >/dev/null + - run: | + destination="$(mktemp -d)" + mise exec -- helm package deploy/helm/sie-cluster --destination "$destination" + git diff --exit-code -- deploy/helm/sie-cluster/Chart.lock + + live-sdk: + name: CI / Live SDK CPU + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise run sync + - run: mise exec -- pnpm install --frozen-lockfile + - run: mise run ts -- build + - run: mise exec -- uv run --frozen --project . --no-sync python tools/ci/live_sdk.py + - run: >- + mise exec -- uv run --frozen --project . --no-sync pytest -q + packages/sie_server/tests/fake_stack/test_sdk_surface.py + -m integration + - name: Preserve diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: live-sdk-logs + path: .cache/ci-logs/ + retention-days: 30 + if-no-files-found: ignore + + cpu-stack: + name: CI / CPU containers + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 180 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + cache: false + - run: mise run sync + - run: mise exec -- uv run --frozen --project . --no-sync python -m tools.ci.cpu_stack_smoke + - name: Preserve diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: cpu-stack-logs + path: .cache/ci-logs/ + retention-days: 30 + if-no-files-found: ignore + + python-distributions: + name: CI / Python distributions + uses: ./.github/workflows/release-python.yml + with: + source_ref: ${{ github.sha }} + build_only: true + + npm-distributions: + name: CI / npm distributions + uses: ./.github/workflows/release-npm.yml + with: + source_ref: ${{ github.sha }} + build_only: true + + required: + name: CI / Required + if: ${{ always() }} + needs: [policy, bootstrap, python, typescript, rust, contracts, helm, live-sdk, cpu-stack, python-distributions, npm-distributions] + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - env: + NEEDS: ${{ toJSON(needs) }} + run: python3 tools/ci/required_ci.py diff --git a/.github/workflows/release-audio.yml b/.github/workflows/release-audio.yml new file mode 100644 index 000000000..015a36ff7 --- /dev/null +++ b/.github/workflows/release-audio.yml @@ -0,0 +1,139 @@ +name: Release native audio asset + +on: + workflow_call: + inputs: + version: + required: true + type: string + tag_name: + required: true + type: string + sha: + required: true + type: string + publish: + required: true + type: boolean + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: blacksmith-8vcpu-ubuntu-2404 + container: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb + timeout-minutes: 30 + outputs: + filename: ${{ steps.contract.outputs.filename }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + version: 2026.7.11 + install: false + cache: false + - name: Install pinned native build tools + run: | + mise --no-config install python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0 + test "$(mise --no-config exec python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0 -- rustc --version | cut -d' ' -f2)" = 1.97.0 + test "$(mise --no-config exec python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0 -- cargo --version | cut -d' ' -f2)" = 1.97.0 + test "$(mise --no-config exec python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0 -- zig version)" = 0.13.0 + - id: contract + name: Validate exact release identity and asset filename + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + run: | + set -euo pipefail + test "$RELEASE_TAG" = "v$RELEASE_VERSION" + test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + filename="$(mise --no-config exec python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0 -- python -c 'import runpy; print(runpy.run_path("packages/sie_audio_prep/build_wheel.py")["AUDIO_WHEEL_FILENAME"])')" + test "$filename" = "sie_audio_prep-$RELEASE_VERSION-cp312-abi3-manylinux_2_28_x86_64.whl" + echo "filename=$filename" >> "$GITHUB_OUTPUT" + - name: Build and validate exact Linux wheel + run: >- + mise --no-config exec python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0 -- + python tools/ci/build_audio_prep_release_asset.py --out dist + - name: Record exact tested wheel and original run provenance + run: >- + mise --no-config exec python@3.12.12 -- python -m tools.ci.release_artifact stamp + --directory dist --kind audio --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: audio-prep-${{ inputs.version }} + path: dist/ + if-no-files-found: error + retention-days: 30 + + publish: + needs: build + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 15 + environment: github-release + permissions: + contents: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + fetch-depth: 1 + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + GH_TOKEN: ${{ github.token }} + run: >- + python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" + --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: audio-prep-${{ inputs.version }} + path: artifact + - name: Attach and verify exact GitHub Release asset + env: + AUDIO_WHEEL_FILENAME: ${{ needs.build.outputs.filename }} + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag_name }} + run: | + set -euo pipefail + wheel="artifact/$AUDIO_WHEEL_FILENAME" + test -f "$wheel" + python3 -m tools.ci.release_artifact check --directory artifact --kind audio --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + python3 -c 'import runpy,sys; runpy.run_path("packages/sie_audio_prep/build_wheel.py")["_validate_wheel"](__import__("pathlib").Path(sys.argv[1]))' "$wheel" + tools/ci/upload_audio_prep_release_asset.bash "$wheel" + + complete: + if: always() && inputs.publish + needs: [build, publish] + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Require every release build and publisher to succeed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 -c 'import json,os; results=json.loads(os.environ["RESULTS"]); failed={name: job["result"] for name,job in results.items() if job["result"] != "success"}; print(failed); assert not failed' diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml new file mode 100644 index 000000000..bdf0738d5 --- /dev/null +++ b/.github/workflows/release-docker.yml @@ -0,0 +1,396 @@ +name: Release Docker + +on: + workflow_call: + inputs: + version: + required: true + type: string + tag_name: + required: true + type: string + sha: + required: true + type: string + publish: + required: true + type: boolean + +permissions: + contents: read + +jobs: + matrix: + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 10 + outputs: + server: ${{ steps.resolve.outputs.server }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - id: resolve + name: Validate source closure and coordinated versions + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + run: | + set -euo pipefail + test "$RELEASE_TAG" = "v$RELEASE_VERSION" + test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + matrix="$(mise run docker -- matrix --version "$RELEASE_VERSION")" + echo "server=$(jq -c . <<<"$matrix")" >> "$GITHUB_OUTPUT" + + build-server: + needs: matrix + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 120 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.server) }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - id: restore + name: Reuse an already retained original-run image on retry + if: github.run_attempt > 1 + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 -m tools.ci.restore_release_artifact + --directory artifact --name 'docker-server-${{ matrix.platform }}-${{ matrix.bundle }}-${{ inputs.version }}' + --kind docker --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + - name: Build once, smoke the loaded image, and save those bytes + if: steps.restore.outputs.restored != 'true' + run: >- + mise run docker -- build-server + --registry ghcr.io/superlinked --version '${{ inputs.version }}' + --platform '${{ matrix.platform }}' --bundle '${{ matrix.bundle }}' + --source-revision '${{ inputs.sha }}' + --run-id '${{ github.run_id }}' --archive-dir artifact --evidence-dir evidence + - if: steps.restore.outputs.restored != 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docker-server-${{ matrix.platform }}-${{ matrix.bundle }}-${{ inputs.version }} + path: artifact/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + - id: restore-evidence + name: Reuse original image verification evidence on retry + if: github.run_attempt > 1 + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 -m tools.ci.restore_release_artifact + --directory retained-evidence --evidence-of artifact + --name 'docker-evidence-server-${{ matrix.platform }}-${{ matrix.bundle }}-${{ inputs.version }}' + --kind docker --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + - name: Derive verification evidence from the retained archive + if: steps.restore-evidence.outputs.restored != 'true' + run: | + mkdir -p evidence + cp artifact/provenance.json evidence/provenance.json + - if: steps.restore-evidence.outputs.restored != 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docker-evidence-server-${{ matrix.platform }}-${{ matrix.bundle }}-${{ inputs.version }} + path: evidence/ + if-no-files-found: error + retention-days: 30 + + push-server: + needs: [matrix, build-server, build-service] + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + environment: ghcr + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.server) }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + GH_TOKEN: ${{ github.token }} + run: >- + python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" + --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: docker-server-${{ matrix.platform }}-${{ matrix.bundle }}-${{ inputs.version }} + path: artifact + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Publish only the retained tested image; reject different existing bytes + run: >- + mise run docker-push-loaded -- + --image 'ghcr.io/superlinked/sie-server:v${{ inputs.version }}-${{ matrix.platform }}-${{ matrix.bundle }}' + --archive-dir artifact --version '${{ inputs.version }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + + build-service: + needs: matrix + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 120 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: + service: [sie-gateway, sie-config, sie-mcp, sie-server-sidecar, sie-server-rust] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - id: restore + name: Reuse an already retained original-run image on retry + if: github.run_attempt > 1 + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 -m tools.ci.restore_release_artifact + --directory artifact --name 'docker-service-${{ matrix.service }}-${{ inputs.version }}' + --kind docker --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + - name: Build once, smoke the loaded image, and save those bytes + if: steps.restore.outputs.restored != 'true' + run: >- + mise run docker -- build-service + --registry ghcr.io/superlinked --version '${{ inputs.version }}' + --service '${{ matrix.service }}' + --source-revision '${{ inputs.sha }}' + --run-id '${{ github.run_id }}' --archive-dir artifact --evidence-dir evidence + - if: steps.restore.outputs.restored != 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docker-service-${{ matrix.service }}-${{ inputs.version }} + path: artifact/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + - id: restore-evidence + name: Reuse original image verification evidence on retry + if: github.run_attempt > 1 + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 -m tools.ci.restore_release_artifact + --directory retained-evidence --evidence-of artifact + --name 'docker-evidence-service-${{ matrix.service }}-${{ inputs.version }}' + --kind docker --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + - name: Derive verification evidence from the retained archive + if: steps.restore-evidence.outputs.restored != 'true' + run: | + mkdir -p evidence + cp artifact/provenance.json evidence/provenance.json + - if: steps.restore-evidence.outputs.restored != 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: docker-evidence-service-${{ matrix.service }}-${{ inputs.version }} + path: evidence/ + if-no-files-found: error + retention-days: 30 + + push-service: + needs: [matrix, build-server, build-service] + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + environment: ghcr + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + service: [sie-gateway, sie-config, sie-mcp, sie-server-sidecar, sie-server-rust] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + GH_TOKEN: ${{ github.token }} + run: >- + python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" + --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: docker-service-${{ matrix.service }}-${{ inputs.version }} + path: artifact + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Publish only the retained tested image; reject different existing bytes + run: >- + mise run docker-push-loaded -- + --image 'ghcr.io/superlinked/${{ matrix.service }}:v${{ inputs.version }}${{ matrix.service == 'sie-server-rust' && '-cuda12-sm89' || '' }}' + --archive-dir artifact --version '${{ inputs.version }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + + verify: + needs: [matrix, push-server, push-service] + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 25 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: docker-evidence-*-${{ inputs.version }} + path: evidence + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Verify every versioned image against source-bound build evidence + run: >- + mise run docker -- verify --registry ghcr.io/superlinked + --version '${{ inputs.version }}' --source-revision '${{ inputs.sha }}' + --run-id '${{ github.run_id }}' --evidence-dir evidence + + complete: + if: always() && inputs.publish + needs: [matrix, build-server, build-service, push-server, push-service, verify, alias] + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Require every release build and publisher to succeed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 -c 'import json,os; results=json.loads(os.environ["RESULTS"]); failed={name: job["result"] for name,job in results.items() if job["result"] != "success"}; print(failed); assert not failed' + + alias: + needs: [matrix, verify] + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 25 + environment: ghcr + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + GH_TOKEN: ${{ github.token }} + run: >- + python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" + --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: docker-evidence-*-${{ inputs.version }} + path: evidence + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Reverify the complete versioned set before moving any alias + env: + GH_TOKEN: ${{ github.token }} + run: >- + mise run docker -- alias --registry ghcr.io/superlinked + --version '${{ inputs.version }}' --source-revision '${{ inputs.sha }}' + --run-id '${{ github.run_id }}' --evidence-dir evidence diff --git a/.github/workflows/release-helm.yml b/.github/workflows/release-helm.yml new file mode 100644 index 000000000..ddc99180a --- /dev/null +++ b/.github/workflows/release-helm.yml @@ -0,0 +1,116 @@ +name: Release Helm + +on: + workflow_call: + inputs: + version: + required: true + type: string + tag_name: + required: true + type: string + sha: + required: true + type: string + publish: + required: true + type: boolean + +permissions: + contents: read + +jobs: + build: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 25 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - name: Validate, render, and retain the exact chart + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + run: | + set -euo pipefail + test "$RELEASE_TAG" = "v$RELEASE_VERSION" + test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + test "$(mise exec -- uv run --frozen --project . python -c 'import sys,yaml; print(yaml.safe_load(open(sys.argv[1]))["version"])' deploy/helm/sie-cluster/Chart.yaml)" = "$RELEASE_VERSION" + test "$(mise exec -- uv run --frozen --project . python -c 'import sys,yaml; print(yaml.safe_load(open(sys.argv[1]))["appVersion"])' deploy/helm/sie-cluster/Chart.yaml)" = "$RELEASE_TAG" + mise run helm -- dependencies + mise run helm -- lint --set payloadStore.enabled=false + mise run helm -- template --set payloadStore.enabled=false >/dev/null + mkdir -p artifact + mise exec -- helm package deploy/helm/sie-cluster --destination artifact + python3 -m tools.ci.release_artifact stamp --directory artifact --kind helm --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: helm-sie-cluster-${{ inputs.version }} + path: artifact/ + if-no-files-found: error + retention-days: 30 + + publish: + needs: build + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 20 + environment: helm + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + GH_TOKEN: ${{ github.token }} + run: >- + python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" + --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: helm-sie-cluster-${{ inputs.version }} + path: artifact + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Publish and verify the same chart bytes without replacing a version + run: >- + mise exec -- python -m tools.ci.publish_helm_archive + --directory artifact --version '${{ inputs.version }}' + --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + + complete: + if: always() && inputs.publish + needs: [build, publish] + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Require every release build and publisher to succeed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 -c 'import json,os; results=json.loads(os.environ["RESULTS"]); failed={name: job["result"] for name,job in results.items() if job["result"] != "success"}; print(failed); assert not failed' diff --git a/.github/workflows/release-native.yml b/.github/workflows/release-native.yml new file mode 100644 index 000000000..1664e0d28 --- /dev/null +++ b/.github/workflows/release-native.yml @@ -0,0 +1,115 @@ +name: Release native sidecar asset + +on: + workflow_call: + inputs: + version: + required: true + type: string + tag_name: + required: true + type: string + sha: + required: true + type: string + publish: + required: true + type: boolean + +permissions: + contents: read + +jobs: + build: + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: docker-service-sie-server-sidecar-${{ inputs.version }} + path: image-artifact + - name: Extract and validate the binary from the same tested image + env: + RELEASE_SHA: ${{ inputs.sha }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + test "$RELEASE_TAG" = "v$RELEASE_VERSION" + mise exec -- uv run --frozen --project . python -m tools.ci.build_sidecar_release_asset \ + --directory image-artifact --out artifact --version "$RELEASE_VERSION" \ + --source-revision "$RELEASE_SHA" --run-id '${{ github.run_id }}' + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-sidecar-${{ inputs.version }} + path: artifact/ + if-no-files-found: error + retention-days: 30 + + publish: + needs: build + if: >- + inputs.publish == true && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', inputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == inputs.sha + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 15 + environment: github-release + permissions: + contents: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_SHA: ${{ inputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + GH_TOKEN: ${{ github.token }} + run: >- + python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" + --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: native-sidecar-${{ inputs.version }} + path: artifact + - name: Revalidate and attach only missing or byte-identical sidecar assets + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag_name }} + RELEASE_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + python3 -m tools.ci.release_artifact check --directory artifact --kind native-sidecar --version '${{ inputs.version }}' --tag-name '${{ inputs.tag_name }}' --source-revision '${{ inputs.sha }}' --run-id '${{ github.run_id }}' + binary="sie-server-sidecar-v$RELEASE_VERSION-linux-amd64" + for filename in "$binary" "$binary.sha256" "$binary.json"; do + RELEASE_ASSET_FILENAME="$filename" bash tools/ci/upload_native_release_asset.bash "artifact/$filename" + done + + complete: + if: always() && inputs.publish + needs: [build, publish] + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Require every release build and publisher to succeed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 -c 'import json,os; results=json.loads(os.environ["RESULTS"]); failed={name: job["result"] for name,job in results.items() if job["result"] != "success"}; print(failed); assert not failed' diff --git a/.github/workflows/release-npm.yml b/.github/workflows/release-npm.yml new file mode 100644 index 000000000..f0bb59f0c --- /dev/null +++ b/.github/workflows/release-npm.yml @@ -0,0 +1,62 @@ +name: Build npm distributions + +on: + workflow_call: + inputs: + source_ref: + required: true + type: string + version: + type: string + default: '' + tag_name: + type: string + default: '' + build_only: + type: boolean + default: true + +permissions: + contents: read + +jobs: + build: + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 60 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.source_ref }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + install_args: python uv node pnpm + - name: Validate source and optional release identity + env: + GH_TOKEN: ${{ github.token }} + SOURCE_REF: ${{ inputs.source_ref }} + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + BUILD_ONLY: ${{ inputs.build_only }} + run: | + set -euo pipefail + if [ "$BUILD_ONLY" = false ]; then test -n "$RELEASE_VERSION"; fi + mise exec -- python tools/ci/release_guard.py build --source-ref "$SOURCE_REF" --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - name: Build complete archives and test clean packed consumers + env: + RELEASE_VERSION: ${{ inputs.version }} + run: mise exec -- python tools/ci/distributions.py build npm --directory artifacts --version "$RELEASE_VERSION" + - name: Bind tested release bytes to the original run + if: inputs.build_only == false + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + SOURCE_REF: ${{ inputs.source_ref }} + run: | + mise exec -- python -m tools.ci.release_artifact stamp --directory artifacts --kind npm --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" --source-revision "$SOURCE_REF" --run-id "$GITHUB_RUN_ID" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: npm-distributions + path: artifacts/* + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml new file mode 100644 index 000000000..c44c41ceb --- /dev/null +++ b/.github/workflows/release-python.yml @@ -0,0 +1,62 @@ +name: Build Python distributions + +on: + workflow_call: + inputs: + source_ref: + required: true + type: string + version: + type: string + default: '' + tag_name: + type: string + default: '' + build_only: + type: boolean + default: true + +permissions: + contents: read + +jobs: + build: + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 60 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ inputs.source_ref }} + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + install_args: python uv + - name: Validate source and optional release identity + env: + GH_TOKEN: ${{ github.token }} + SOURCE_REF: ${{ inputs.source_ref }} + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + BUILD_ONLY: ${{ inputs.build_only }} + run: | + set -euo pipefail + if [ "$BUILD_ONLY" = false ]; then test -n "$RELEASE_VERSION"; fi + mise exec -- python tools/ci/release_guard.py build --source-ref "$SOURCE_REF" --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - name: Build complete archives and test clean packed consumers + env: + RELEASE_VERSION: ${{ inputs.version }} + run: mise exec -- python tools/ci/distributions.py build python --directory artifacts --version "$RELEASE_VERSION" + - name: Bind tested release bytes to the original run + if: inputs.build_only == false + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.tag_name }} + SOURCE_REF: ${{ inputs.source_ref }} + run: | + mise exec -- python -m tools.ci.release_artifact stamp --directory artifacts --kind python --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" --source-revision "$SOURCE_REF" --run-id "$GITHUB_RUN_ID" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-distributions + path: artifacts/* + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..db8e730d2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,373 @@ +name: Release + +on: + push: + branches: [main] + release: + types: [published] + workflow_dispatch: + inputs: + version: + description: Stable release version newer than 0.7.3 + required: true + type: string + original_run: + description: Original failed release workflow run ID + required: true + type: string + family: + description: Failed artifact family to retry on the original run + type: choice + default: all + options: [all, python, npm, docker, helm, audio, native] + +permissions: + contents: read + +concurrency: + group: public-release-main + queue: max + cancel-in-progress: false + +jobs: + release-please: + if: >- + vars.PUBLIC_RELEASE_AUTOMATION_ENABLED == 'true' && + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + github.ref_protected == true && + github.repository == 'superlinked/sie' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: release-automation + permissions: + contents: read + outputs: + release_pr_head: ${{ steps.release-pr-head.outputs.sha }} + steps: + - name: Require activated public release GitHub App credentials + env: + APP_ID: ${{ vars.PUBLIC_RELEASE_APP_ID }} + APP_PRIVATE_KEY: ${{ secrets.PUBLIC_RELEASE_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + test -n "$APP_ID" + [[ "$APP_ID" =~ ^[0-9]+$ ]] + test -n "$APP_PRIVATE_KEY" + [[ "$APP_PRIVATE_KEY" == *"BEGIN"*"PRIVATE KEY"* ]] + - id: app-token + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 + with: + app-id: ${{ vars.PUBLIC_RELEASE_APP_ID }} + private-key: ${{ secrets.PUBLIC_RELEASE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-contents: write + permission-pull-requests: write + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + - id: seed + name: Require the genuine stable seed tag and ancestral release + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: python3 tools/ci/release_guard.py seed + - id: release + uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5 + with: + token: ${{ steps.app-token.outputs.token }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + skip-github-release: ${{ steps.seed.outputs.at_seed }} + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + if: steps.release.outputs.prs != '' && steps.release.outputs.prs != '[]' + - name: Refresh coupled locks on the release pull request + if: steps.release.outputs.prs != '' && steps.release.outputs.prs != '[]' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RELEASE_PRS: ${{ steps.release.outputs.prs }} + run: | + set -euo pipefail + branch="$(jq -r '.[0].headBranchName // empty' <<<"$RELEASE_PRS")" + test -n "$branch" + git fetch origin "$branch" + git checkout -B "$branch" FETCH_HEAD + mise exec -- uv lock --project . + mise exec -- pnpm install --lockfile-only + mise exec -- cargo metadata --format-version 1 > /dev/null + if git diff --quiet -- uv.lock pnpm-lock.yaml Cargo.lock; then + exit 0 + fi + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add uv.lock pnpm-lock.yaml Cargo.lock + git commit -m 'chore: refresh release locks' + git push origin "HEAD:refs/heads/$branch" + - id: release-pr-head + name: Verify the release PR final head after the App-authored push + if: steps.release.outputs.prs != '' && steps.release.outputs.prs != '[]' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RELEASE_PRS: ${{ steps.release.outputs.prs }} + run: | + set -euo pipefail + pr_number="$(jq -r '.[0].number // empty' <<<"$RELEASE_PRS")" + expected_sha="$(git rev-parse HEAD)" + test -n "$pr_number" + [[ "$expected_sha" =~ ^[0-9a-f]{40}$ ]] + remote_sha="" + for _ in 1 2 3 4 5 6; do + remote_sha="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$pr_number" --jq .head.sha)" + if [ "$remote_sha" = "$expected_sha" ]; then + break + fi + sleep 2 + done + test "$remote_sha" = "$expected_sha" + echo "sha=$expected_sha" >> "$GITHUB_OUTPUT" + + prepare: + if: >- + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.repository == 'superlinked/sie' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + sha: ${{ steps.identity.outputs.sha }} + version: ${{ steps.identity.outputs.version }} + tag_name: ${{ steps.identity.outputs.tag_name }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - id: identity + name: Verify published tag and protected-main ancestry + env: + GH_TOKEN: ${{ github.token }} + run: python3 tools/ci/release_guard.py prepare + + python: + needs: prepare + if: github.event_name == 'release' && needs.prepare.result == 'success' + uses: ./.github/workflows/release-python.yml + with: + source_ref: ${{ needs.prepare.outputs.sha }} + version: ${{ needs.prepare.outputs.version }} + tag_name: ${{ needs.prepare.outputs.tag_name }} + build_only: false + + npm: + needs: prepare + if: github.event_name == 'release' && needs.prepare.result == 'success' + uses: ./.github/workflows/release-npm.yml + with: + source_ref: ${{ needs.prepare.outputs.sha }} + version: ${{ needs.prepare.outputs.version }} + tag_name: ${{ needs.prepare.outputs.tag_name }} + build_only: false + + python-publish: + needs: [prepare, python] + if: >- + needs.prepare.result == 'success' && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', needs.prepare.outputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == needs.prepare.outputs.sha + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: pypi + permissions: + contents: read + id-token: write + env: + RELEASE_VERSION: ${{ needs.prepare.outputs.version }} + RELEASE_TAG: ${{ needs.prepare.outputs.tag_name }} + RELEASE_SHA: ${{ needs.prepare.outputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ needs.prepare.outputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + GH_TOKEN: ${{ github.token }} + run: python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-distributions + path: artifacts + - name: Verify the complete original tested archive set + run: | + python3 -m tools.ci.release_artifact check --directory artifacts --kind python --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" --source-revision "$RELEASE_SHA" --run-id "$GITHUB_RUN_ID" + python3 tools/ci/distributions.py verify python --directory artifacts --version "$RELEASE_VERSION" + - id: pending + name: Reject conflicting published bytes and select absent archives + run: python3 tools/ci/distributions.py prepare-pypi python --directory artifacts --destination pending --version "$RELEASE_VERSION" + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + if: steps.pending.outputs.pending == 'true' + with: + packages-dir: pending + + npm-publish: + needs: [prepare, npm] + if: >- + needs.prepare.result == 'success' && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' && + github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false && + github.ref_protected == true && + github.ref == format('refs/tags/{0}', needs.prepare.outputs.tag_name) && + github.repository == 'superlinked/sie' && + github.sha == needs.prepare.outputs.sha + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: npm + permissions: + contents: read + id-token: write + env: + RELEASE_VERSION: ${{ needs.prepare.outputs.version }} + RELEASE_TAG: ${{ needs.prepare.outputs.tag_name }} + RELEASE_SHA: ${{ needs.prepare.outputs.sha }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ needs.prepare.outputs.sha }} + persist-credentials: false + - name: Validate trusted publication context and tag binding + env: + GH_TOKEN: ${{ github.token }} + run: python3 tools/ci/release_guard.py publish --source-ref "$RELEASE_SHA" --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: npm-distributions + path: artifacts + - name: Verify the complete original tested archive set + run: | + python3 -m tools.ci.release_artifact check --directory artifacts --kind npm --version "$RELEASE_VERSION" --tag-name "$RELEASE_TAG" --source-revision "$RELEASE_SHA" --run-id "$GITHUB_RUN_ID" + python3 tools/ci/distributions.py verify npm --directory artifacts --version "$RELEASE_VERSION" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24.12.0' + - name: Install the pinned OIDC-capable npm client + run: npm install --global npm@11.6.2 --ignore-scripts + - name: Publish absent versions or verify identical existing bytes + run: python3 tools/ci/distributions.py publish-npm npm --directory artifacts --version "$RELEASE_VERSION" + + docker: + needs: prepare + if: github.event_name == 'release' && needs.prepare.result == 'success' + permissions: + contents: read + packages: write + actions: read + uses: ./.github/workflows/release-docker.yml + with: + version: ${{ needs.prepare.outputs.version }} + tag_name: ${{ needs.prepare.outputs.tag_name }} + sha: ${{ needs.prepare.outputs.sha }} + publish: true + + helm: + needs: [prepare, docker] + if: github.event_name == 'release' && needs.prepare.result == 'success' + permissions: + contents: read + packages: write + uses: ./.github/workflows/release-helm.yml + with: + version: ${{ needs.prepare.outputs.version }} + tag_name: ${{ needs.prepare.outputs.tag_name }} + sha: ${{ needs.prepare.outputs.sha }} + publish: true + + audio: + needs: prepare + if: github.event_name == 'release' && needs.prepare.result == 'success' + permissions: + contents: write + uses: ./.github/workflows/release-audio.yml + with: + version: ${{ needs.prepare.outputs.version }} + tag_name: ${{ needs.prepare.outputs.tag_name }} + sha: ${{ needs.prepare.outputs.sha }} + publish: true + + native: + needs: [prepare, docker] + if: github.event_name == 'release' && needs.prepare.result == 'success' + permissions: + contents: write + uses: ./.github/workflows/release-native.yml + with: + version: ${{ needs.prepare.outputs.version }} + tag_name: ${{ needs.prepare.outputs.tag_name }} + sha: ${{ needs.prepare.outputs.sha }} + publish: true + + complete: + needs: [prepare, python-publish, npm-publish, docker, helm, audio, native] + if: >- + always() && github.event_name == 'release' && + github.event.action == 'published' && + github.event.release.draft == false && + github.event.release.prerelease == false + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require every release family to finish successfully + env: + RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json, os + results = json.loads(os.environ["RESULTS"]) + bad = {name: job["result"] for name, job in results.items() if job["result"] != "success"} + if bad: + raise SystemExit(f"Incomplete release: {bad}") + PY + + recover: + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/main' && + github.ref_protected == true && + github.repository == 'superlinked/sie' && + vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: release-automation + permissions: + contents: read + actions: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Validate retained original-run evidence and request failed-job retry + env: + GH_TOKEN: ${{ github.token }} + PUBLIC_RELEASE_PUBLISHING_ENABLED: ${{ vars.PUBLIC_RELEASE_PUBLISHING_ENABLED }} + RELEASE_VERSION: ${{ inputs.version }} + ORIGINAL_RUN: ${{ inputs.original_run }} + FAMILY: ${{ inputs.family }} + run: python3 -m tools.ci.release_recovery --version "$RELEASE_VERSION" --original-run "$ORIGINAL_RUN" --family "$FAMILY" diff --git a/.gitignore b/.gitignore index d5d72e4f7..f54066c62 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ # Byte-compiled / optimized / DLL files +.codex/artifacts/worktree-task/ + __pycache__/ *.py[codz] *$py.class diff --git a/.npmrc b/.npmrc deleted file mode 100644 index f3b1a9fcb..000000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN} diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 000000000..657149d5d --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.7.3" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..d0cd46b47 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,60 @@ +# AGENTS.md — Public SIE contributor guide + +## Scope + +This repository is the source of truth for the open-source SIE server, SDKs, +gateway, sidecar, configuration service, MCP service, Helm chart, and public +release automation. Keep changes self-contained: a clean clone must build and +test without access to another repository, private registry, or secret. + +## Commands + +- Use ordinary shell tools for read-only inspection. +- Run project tasks with `mise run ` and pass task arguments after `--`. +- Run version-managed executables that are not tasks with `mise exec --`. +- Bootstrap a clean checkout with `./tools/init.sh`. +- Run `mise tasks` before assuming a task exists. + +Every pull request runs the public lint, typecheck, unit, integration, and +package/container checks; these are not selected by changed paths. Benchmark +and quality-evaluation jobs are not part of this CI. + +The main tasks are `mise run lint`, `mise run typecheck`, `mise run test`, +`mise run test-integrations`, `mise run ts -- build`, `mise run ts -- lint`, +`mise run rust-check`, `mise run rust-test`, and `mise run helm -- lint`. +Build the TypeScript workspace before checking dependent packages in isolation. +The standalone Candle worker uses +`mise exec -- cargo --manifest-path packages/sie_server_rust/Cargo.toml --locked`. + +## Development boundaries + +- Keep the root Python and pnpm locks authoritative; standalone Rust crates keep + their checked-in locks. +- Use `sie_sdk.SIEClient` for SIE API examples. Use current `gateway` + terminology; legacy wire names remain only where compatibility requires it. +- Keep `__init__.py` files empty and imports at module scope except for optional + dependencies. +- Do not commit secrets, credentials, generated Helm dependencies, staged chart + model/bundle files, build output, or package archives. +- Trust-boundary changes to ingress, authentication, identity handling, billing, + or wire protocols require an adversarial security review. +- Use Conventional Commits. + +## Releases + +Release automation is intentionally fail-closed. Package, image, and chart +builds may run without publication authority; external publication additionally +requires the protected workflow inputs and repository publishing latch described +in `RELEASE.md`. Never bypass version, tag, source-revision, or full-set +verification. + +Release-PR authoring is independently default-off and requires the repository +variable `PUBLIC_RELEASE_AUTOMATION_ENABLED` to be exactly `true`. Configure the +release App and protected `release-automation` environment before enabling it; +do not confuse this authoring gate with artifact publication enablement. + +The release-please baseline is 0.7.3 at exact public commit +`60996d9c30168e0f8e85b680295f147fdee87f61`. Keep that full SHA as the +checked-in bootstrap boundary; do not create a substitute baseline tag or move +the boundary to a later commit. Publication retries reuse the original release +run and artifacts, not a newer-main rebuild. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..9899e1ee4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,84 @@ +# Contributing to SIE + +Thank you for contributing. SIE accepts focused changes that keep the public +repository independently buildable and preserve compatibility across the +Python, TypeScript, Rust, wire-contract, and Helm surfaces. + +## Set up a checkout + +Install [mise](https://mise.jdx.dev/getting-started.html), clone the repository, +and run: + +```bash +./tools/init.sh +``` + +The bootstrap installs the pinned toolchain, synchronizes the root Python lock, +and installs every TypeScript workspace from the root pnpm lock. It does not +need package-registry credentials. + +## Validate a change + +Start with the checks that own the files you changed, then run the wider gates +before requesting review: + +```bash +mise run lint +mise run typecheck +mise run test + +mise run ts -- lint +mise run ts -- typecheck +mise run ts -- build +mise run ts -- test + +mise run rust-fmt -- --check +mise run rust-check +mise run rust-clippy +mise run rust-test + +mise exec -- python tools/check_ipc_types_parity.py +mise exec -- python tools/check_response_chunk_protocol.py + +mise run helm -- dependencies +mise run helm -- lint --set payloadStore.enabled=false +mise run helm -- template --set payloadStore.enabled=false +``` + +The standalone Candle worker is outside the root Rust workspace: + +```bash +mise exec -- cargo fmt --manifest-path packages/sie_server_rust/Cargo.toml -- --check +mise exec -- cargo check --manifest-path packages/sie_server_rust/Cargo.toml --locked --all-targets +mise exec -- cargo clippy --manifest-path packages/sie_server_rust/Cargo.toml --locked --all-targets -- -D warnings +mise exec -- cargo test --manifest-path packages/sie_server_rust/Cargo.toml --locked +``` + +Native release-wheel builds additionally use the repository's pinned Zig +toolchain and run on Linux x86_64. The public release workflow is the authority +for the exact manylinux asset; it is not part of the PyPI package matrix. + +The public Docker task resolves the checked-in release matrix and requires the +full source commit for every build. A CPU-only Candle image can be built on a +normal Docker host without publication credentials: + +```bash +mise run docker -- matrix +mise run docker -- build-service \ + --registry local \ + --version 0.7.2 \ + --service sie-server-rust-cpu \ + --source-revision <40-character-git-sha> +``` + +Helm dependency archives and the model/bundle files temporarily staged under +the chart are generated and ignored. Do not add them to a commit. + +## Pull requests + +Keep diffs minimal, explain compatibility or security implications, and include +the exact validation evidence. Use a Conventional Commit title. Tests must be +deterministic and must not depend on credentials, external writes, or mutable +package versions. + +See [RELEASE.md](RELEASE.md) for the public version and publication contract. diff --git a/README.md b/README.md index 58ec3d2eb..f560182c5 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,9 @@ SIE is an open-source inference engine that runs the models behind every agent t ## Development +Read [CONTRIBUTING.md](CONTRIBUTING.md) for the complete public development +workflow and [AGENTS.md](AGENTS.md) for repository automation boundaries. + Install [mise](https://mise.jdx.dev/getting-started.html), then bootstrap the versioned Python, Rust, Node.js, and Helm toolchains from the repository root: @@ -57,6 +60,9 @@ mise run rust-check mise run rust-test mise run gateway-test mise run server-sidecar-test +mise run helm -- dependencies +mise run helm -- lint --set payloadStore.enabled=false +mise run helm -- template --set payloadStore.enabled=false ``` The Python workspace uses the committed root lock. Package membership is diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..d1fd076a7 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,221 @@ +# Releases + +SIE uses one release-please train with `vX.Y.Z` tags. The last released version +at setup is **0.7.3**. Changes after that release determine the next version; +the setup does not create or republish 0.7.3. + +## Versioning + +`.release-please-manifest.json` records the current released version, and +`release-please-config.json` fixes the public history boundary at the actual +`v0.7.3` commit `60996d9c30168e0f8e85b680295f147fdee87f61`. The workflow +also verifies that the public `v0.7.3` stable release and tag exist and that the +tag belongs to the release branch. Release-please discovers the matching +release natively; the checked-in bootstrap SHA remains the fail-closed fallback +for that same historical boundary. + +The current pre-1.0 policy is retained: ordinary features and fixes advance the +patch version; breaking changes advance the minor version. Public conventional +commits generate `CHANGELOG.md`. No old changelog sections are rewritten. + +The release PR updates the coordinated Python/npm package versions, gateway, +sidecar and audio release fields, TypeScript runtime version, and Helm metadata. +It also refreshes the coupled public locks. Config and MCP join this train for +their first PyPI publication. Independently versioned implementation crates are +not silently renumbered: a Rust worker image follows the release image tag even +where its crate has an independent version. + +Release PRs receive the same mandatory CI checks as other PRs. Release-please +and its lock refresh use a repository-scoped GitHub App so their PR updates +trigger normal CI. No PR build is a publication job. + +## Published outputs + +The Python train contains 13 distributions: + +- `sie-sdk`, `sie-server`, `sie-config`, and `sie-mcp`; +- `sie-langchain`, `sie-llamaindex`, `sie-haystack`, `sie-dspy`, + `sie-crewai`, `sie-chroma`, `sie-lancedb`, `sie-qdrant`, and + `sie-weaviate`. + +The npm train contains five packages under `@superlinked`: `sie-sdk`, +`sie-chroma`, `sie-langchain`, `sie-llamaindex`, and `sie-lancedb`. + +The GHCR image names are `sie-server`, `sie-gateway`, `sie-config`, `sie-mcp`, +`sie-server-sidecar`, and `sie-server-rust`, under `ghcr.io/superlinked`. +`.github/release-matrix.json` defines the supported server platform/bundle +combinations. Missing bundle source or build recipes are errors, not an +instruction to skip a release target. The Rust worker retains its explicit +CUDA/architecture image suffix. + +The Helm chart is published at +`oci://ghcr.io/superlinked/charts/sie-cluster` after its versioned images verify. + +Native audio is distributed as a GitHub Release asset, not a PyPI project: + +```text +sie_audio_prep--cp312-abi3-manylinux_2_28_x86_64.whl +``` + +A Linux amd64 sidecar executable and checksum are also provided for consumers +that embed the binary rather than run its container: + +```text +sie-server-sidecar-v-linux-amd64 +sie-server-sidecar-v-linux-amd64.sha256 +``` + +The executable is extracted from the already-verified sidecar image, not rebuilt. +Its attached provenance records the source revision, architecture, and ABI; +the compatibility check uses Debian 12. Rust library dependencies +can still be consumed using Cargo's Git support; no crates.io publication is +implied by an image or binary release. + +## Build, verify, publish + +The top-level `release.yml` has two automatic entrypoints: + +- A push to `main` runs the App-authored release-please and release-PR lock + refresh steps only when `PUBLIC_RELEASE_AUTOMATION_ENABLED` is exactly + `true`. An absent variable leaves authoring cleanly skipped. +- The App-created stable `release: published` event runs preparation, builds, + verification, and direct publisher fanout at the tagged commit. + +This separates the commit that triggers release-please from the commit it +releases. A later `main` push cannot cause packages from an earlier release to +be published with the later run's provenance. The App credential is required +because its events trigger workflows; a release created with `GITHUB_TOKEN` +does not provide that handoff. Publication does not depend on a tag-push event. + +Release work is serialized with GitHub's `queue: max` concurrency mode and no +in-progress cancellation, so newer events do not replace pending releases. The +queue has GitHub's documented bound of 100 pending runs. The current actionlint +schema does not recognize `queue`; CI exempts only that diagnostic and separately +checks the required queue setting. See [workflow +concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency). + +PR and candidate builds produce archives without publishing. They use the +actual package versions in that source tree. Release builds additionally +require the complete package set to match the release version. + +Build outputs are tested before upload. Publisher jobs consume those same +archives or images; they do not independently rebuild them. Before a release +upload, the run commit, release output commit, checked-out source, and stable +tag must identify the same revision. Versioned outputs are immutable: an +existing matching upload may be accepted, but different bytes at the same +version are a failure. + +Floating image aliases move only after the full versioned image set verifies. +An older release recovery keeps those aliases unchanged when a newer stable +release exists; it repairs only the requested versioned outputs. +The dependent chart waits for that verification. A GitHub Release can exist +while a publisher fails; the release-completion check, not the release page +alone, indicates that all expected outputs are available. + +Ordinary PR checks include lint, typechecking, unit tests, public integration, +packed-distribution checks and CPU/container verification. Full release image +builds include the declared CUDA variants. Building a CUDA image is not a claim +that GPU inference was exercised. + +## Publisher setup + +Finalize these identities before enabling uploads: + +| Publisher | Repository | Workflow identity | Environment | Authority | +| --- | --- | --- | --- | --- | +| PyPI distributions | `superlinked/sie` | `release.yml` | `pypi` | Trusted Publishing, upload-job `id-token: write` | +| npm packages | `superlinked/sie` | `release.yml` | `npm` | Trusted Publishing, upload-job `id-token: write` | +| Images | `superlinked/sie` | Release image workflow | `ghcr` | `GITHUB_TOKEN`, `packages: write` | +| Helm chart | `superlinked/sie` | Release chart workflow | `helm` | `GITHUB_TOKEN`, `packages: write` | +| Release assets | `superlinked/sie` | Native asset workflows | `github-release` | `GITHUB_TOKEN`, `contents: write` | + +PyPI/npm upload jobs live in the top-level workflow so the configured OIDC +identity is unambiguous. Existing package names use their existing registry +settings; only new PyPI projects need pending publishers. Register config and +MCP when their first upload is ready. Audio assets do not need another PyPI +registration. + +npm supports one trusted publisher per package. Its actual upload job uses a +GitHub-hosted runner and a pinned supported npm version. Ordinary builds and +tests use Blacksmith. Do not carry a long-lived npm token into the OIDC upload +job. See [npm trusted publishing](https://docs.npmjs.com/trusted-publishers/). + +For GHCR, explicitly grant this repository's Actions jobs access to each +existing package and confirm public visibility. A source label or successful +anonymous pull does not prove write permission. GHCR uses the repository token, +not an external OIDC registration. + +Protect `main` and the `v*` tag namespace before enabling publication. Restrict +stable tag creation to the release App, and forbid tag updates/deletions. A tag +pattern allowed by an environment is not a substitute for those repository +rules. Keep creation bypass and tag immutability in separate rulesets so the +App's permission to create does not also permit replacing or deleting tags. +The App does not need a bypass of main-branch review or CI requirements. +Preparation and every publisher verify the protected tag, exact source, +and ancestry in protected `main`. + +The App and manual recovery use the main-only `release-automation` environment. +The `pypi`, `npm`, `ghcr`, `helm`, and `github-release` environments allow only +the protected release tags, not pull-request branches. `release-automation` supplies +`PUBLIC_RELEASE_APP_ID` and `PUBLIC_RELEASE_APP_PRIVATE_KEY`. Install that App +only on this repository with Contents and Pull requests read/write permission. +These App credentials are not distribution-registry credentials. + +Keep the repository variable `PUBLIC_RELEASE_AUTOMATION_ENABLED` absent while +setting up the App and `release-automation` environment. After the App is +installed, its ID/private key are configured, protected `main` and stable-tag +rules are in force, and the release workflow has passed review, set the variable +to exactly `true` to enable release-PR authoring. This authoring gate is separate +from artifact publication and must not be enabled as a substitute for publisher +configuration. + +Actual publication additionally requires +`PUBLIC_RELEASE_PUBLISHING_ENABLED=true`. Keep it absent until the release +baseline, required checks, publisher identities, package access and build +capacity have been checked. Build-only paths need neither this setting nor +registry credentials. Do not enable two competing publishers for the same +artifact destination. + +## Recovery + +Retain release build artifacts for at least 30 days. Normal recovery reruns +failed publication jobs from the original release workflow run. This preserves +the original commit/ref and reuses the original outputs. + +The manual recovery entrypoint validates an existing version and its original +run before requesting those reruns. It is not an alternate uploader from newer +source. npm's automatic provenance uses the run's commit; checking out an older +tag inside a newer run would not change that identity. + +In **Actions → Release → Run workflow**, select `main` and provide `version` +(without `v`), the numeric `original_run` ID, and an optional `family` selection. +The tag is derived from the version. Select the original release-event +publication run, not the main-push authoring run. Its `prepare` job must have +succeeded, a selected publisher family must have failed, and its original +archives must still exist. A skipped-only family or a failure only in the +completion check is not treated as a successful retry request. Diagnose those +conditions explicitly instead of rerunning release-please and losing its +original release outputs. Failed builds can be rerun from the original run +before retrying publication. + +If the original run or artifacts are unavailable, or an existing version has +conflicting bytes, stop and resolve that release explicitly. Do not move a tag, +overwrite a package, or attach a newly rebuilt conflicting native asset. + +When filling a missing older npm version after a newer version is already +`latest`, recovery uses the version-scoped `release-vX.Y.Z` dist-tag. It does not +move `latest` backwards. Normal new releases use `latest`; identical existing +versions are verified and left untouched. + +## Candidate compatibility checks + +An unmerged public commit can be built without creating a stable release. +Consumers may select its exact archives/image digests for an isolated test run +while their stable dependency pins stay unchanged. Candidate artifacts are +code: approve the exact source before executing them in an environment with +private access. Candidate builds do not receive production or publishing +credentials. + +Local build, pack, and smoke checks cannot prove account-side OIDC trust, +registry permissions, or hosted-runner capacity. Those remain checks for the +first authorized publication from the final release source. diff --git a/deny.toml b/deny.toml index 6bbc5a9d3..5e36eadad 100644 --- a/deny.toml +++ b/deny.toml @@ -1,7 +1,7 @@ -# cargo-deny policy for the public Rust workspace. Standalone crates excluded -# from the root workspace are outside this single-lock audit. +# cargo-deny policy shared by the public Rust workspace and standalone worker. +# Each dependency graph is audited against its own committed lockfile. # -# Run with: `mise run gateway-deny` +# Workspace: `mise run gateway-deny`; standalone worker: the CI Rust audit lane. # Docs: https://embarkstudios.github.io/cargo-deny/ [graph] @@ -53,6 +53,9 @@ confidence-threshold = 0.9 # licenses. Scoped as per-crate exceptions rather than a global allow so the # gateway/sidecar keep their historical no-MPL posture for any future dep. exceptions = [ + # option-ext is consumed unmodified only by the standalone worker through + # hf-hub -> dirs -> dirs-sys. Keep its MPL allowance exact-version scoped. + { name = "option-ext", version = "=0.2.0", allow = ["MPL-2.0"] }, { name = "symphonia", allow = ["MPL-2.0"] }, { name = "symphonia-adapter-libopus", allow = ["MPL-2.0"] }, { name = "symphonia-bundle-flac", allow = ["MPL-2.0"] }, diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 90692a8d1..f0beb9ad1 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -10,6 +10,21 @@ helm install sie-cluster oci://ghcr.io/superlinked/charts/sie-cluster \ --create-namespace ``` +## Local validation + +Prepare dependencies from the checked-in `Chart.yaml` and `Chart.lock`, then +render with an explicit non-secret payload-store choice: + +```bash +mise run helm -- dependencies +mise run helm -- lint --set payloadStore.enabled=false +mise run helm -- template --set payloadStore.enabled=false +``` + +The task temporarily stages the public model and bundle YAML files into the +chart and removes them after each render. Helm's generated `charts/` directory +is ignored and must not be committed. + ## Architecture ``` diff --git a/examples/document-ocr/README.md b/examples/document-ocr/README.md index 481ecd538..914ff4016 100644 --- a/examples/document-ocr/README.md +++ b/examples/document-ocr/README.md @@ -205,9 +205,9 @@ add a `ModelOption` entry and a HuggingFace model ID; SIE handles the rest. The demo runs on the `latest-cpu-transformers5` SIE image; this is the bundle where `lighton_ocr` / `glm_ocr` / `paddle` adapters live. The Florence-2 family ships in SIE's `default` bundle (which pins -`transformers<5`) and is not loadable from this image. See -[sie-internal#828](https://github.com/superlinked/sie-internal/issues/828) -for the bundle-composition story. +`transformers<5`) and is not loadable from this image. This separation keeps +the incompatible tokenizer dependency sets reproducible; use the default +bundle when evaluating Florence-2. --- diff --git a/examples/document-ocr/compose.gpu.yml b/examples/document-ocr/compose.gpu.yml index 0fa064df3..2a9f2480f 100644 --- a/examples/document-ocr/compose.gpu.yml +++ b/examples/document-ocr/compose.gpu.yml @@ -3,8 +3,8 @@ services: # GPU compose uses the transformers5 bundle so we can serve LightOnOCR / # GLM-OCR / Paddle (their adapters need transformers>=5). Florence-2 is # in the catalog but its tokenizer relies on a transformers<5 API and is - # therefore unavailable on this image (sie-internal#828, #832). The CPU - # compose covers Florence-2. + # therefore unavailable on this image. The CPU compose covers Florence-2 + # through the default bundle. image: ghcr.io/superlinked/sie-server:latest-cuda12-transformers5 command: - serve diff --git a/examples/document-ocr/compose.yml b/examples/document-ocr/compose.yml index 6b090791b..4e1c99e01 100644 --- a/examples/document-ocr/compose.yml +++ b/examples/document-ocr/compose.yml @@ -2,8 +2,8 @@ services: sie: # transformers5 bundle so LightOnOCR-2-1B (the recognition default) loads. # Florence-2 family is incompatible with transformers>=5 (tokenizer API - # change), so the UI auto-disables those entries on this image - # (sie-internal#828 tracks unifying the bundles). + # change), so the UI auto-disables those entries on this image. Use the + # default bundle when testing Florence-2 models. image: ghcr.io/superlinked/sie-server:latest-cpu-transformers5 # linux/amd64 is a no-op on x86_64 Linux hosts; on Apple Silicon it forces # Rosetta translation (slower but works for local testing). diff --git a/integrations/sie_ts_chroma/tests/embedding.test.ts b/integrations/sie_ts_chroma/tests/embedding.test.ts index a167931ea..1a73c6e92 100644 --- a/integrations/sie_ts_chroma/tests/embedding.test.ts +++ b/integrations/sie_ts_chroma/tests/embedding.test.ts @@ -10,6 +10,12 @@ import { type SIESparseEmbeddingFunctionOptions, } from "../src/index.js"; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -19,9 +25,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -60,11 +64,11 @@ describe("SIEEmbeddingFunction", () => { { dense: new Float32Array([0.5, 0.25, 0.75]) }, { dense: new Float32Array([1.0, 0.5, 0.25]) }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIEEmbeddingFunction({ baseUrl: "http://localhost:8080", @@ -82,11 +86,11 @@ describe("SIEEmbeddingFunction", () => { it("calls encode with correct parameters", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{ dense: new Float32Array([0.5, 0.25, 0.75]) }]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIEEmbeddingFunction({ baseUrl: "http://localhost:8080", @@ -107,11 +111,11 @@ describe("SIEEmbeddingFunction", () => { .mockResolvedValue([ { sparse: { indices: new Int32Array([]), values: new Float32Array([]) } }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIEEmbeddingFunction(); @@ -162,11 +166,11 @@ describe("SIESparseEmbeddingFunction", () => { }, }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIESparseEmbeddingFunction({ baseUrl: "http://localhost:8080", @@ -197,11 +201,11 @@ describe("SIESparseEmbeddingFunction", () => { }, }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIESparseEmbeddingFunction({ baseUrl: "http://localhost:8080", @@ -218,11 +222,11 @@ describe("SIESparseEmbeddingFunction", () => { it("returns empty indices/values when sparse is missing", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{ dense: new Float32Array([0.5]) }]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIESparseEmbeddingFunction(); const result = await ef.generate(["test"]); @@ -240,11 +244,11 @@ describe("SIESparseEmbeddingFunction", () => { }, }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIESparseEmbeddingFunction({ baseUrl: "http://localhost:8080", @@ -278,11 +282,11 @@ describe("SIESparseEmbeddingFunction", () => { }, }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + }), + ); const ef = new SIESparseEmbeddingFunction(); diff --git a/integrations/sie_ts_lancedb/src/index.ts b/integrations/sie_ts_lancedb/src/index.ts index e3c5549bf..5370f5f9c 100644 --- a/integrations/sie_ts_lancedb/src/index.ts +++ b/integrations/sie_ts_lancedb/src/index.ts @@ -29,10 +29,9 @@ import { type DType, type EncodeOptions, type EncodeResult, - type ModelInfo, - type ScoreResult, SIEClient, type SIEClientOptions, + type ScoreResult, toNumberArray, } from "@superlinked/sie-sdk"; @@ -311,7 +310,7 @@ export class SIEReranker { columnArrays[field.name] = vals; } } - columnArrays["_relevance_score"] = Array.from(scores); + columnArrays._relevance_score = Array.from(scores); const newTable = arrow.tableFromArrays(columnArrays); const batch = newTable.batches[0]; @@ -384,7 +383,11 @@ export class SIEReranker { } const table = arrow.tableFromArrays(columnArrays); - return table.batches[0]!; + const batch = table.batches[0]; + if (!batch) { + throw new Error("Failed to merge result batches"); + } + return batch; } async close(): Promise { diff --git a/integrations/sie_ts_lancedb/tests/embedding.test.ts b/integrations/sie_ts_lancedb/tests/embedding.test.ts index f2b3750ed..72c0d7155 100644 --- a/integrations/sie_ts_lancedb/tests/embedding.test.ts +++ b/integrations/sie_ts_lancedb/tests/embedding.test.ts @@ -10,6 +10,12 @@ import { type SIERerankerOptions, } from "../src/index.js"; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", () => { const mockClient = { @@ -20,9 +26,7 @@ vi.mock("@superlinked/sie-sdk", () => { }; return { - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), toNumberArray: (arr: Float32Array | Int32Array | number[]) => Array.from(arr), }; }); @@ -64,13 +68,13 @@ describe("SIEEmbeddingFunction", () => { { dense: new Float32Array([0.5, 0.25, 0.75]) }, { dense: new Float32Array([1.0, 0.5, 0.25]) }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - getModel: vi.fn(), - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + getModel: vi.fn(), + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "BAAI/bge-m3" }); const embeddings = await func.generateEmbeddings(["Hello world", "Goodbye world"]); @@ -83,13 +87,13 @@ describe("SIEEmbeddingFunction", () => { it("calls encode with correct parameters", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{ dense: new Float32Array([0.5]) }]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - getModel: vi.fn(), - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + getModel: vi.fn(), + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "test-model", @@ -109,13 +113,13 @@ describe("SIEEmbeddingFunction", () => { it("embedQuery passes isQuery: true", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{ dense: new Float32Array([0.5, 0.25]) }]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - getModel: vi.fn(), - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + getModel: vi.fn(), + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "test-model" }); const result = await func.embedQuery("search text"); @@ -131,13 +135,13 @@ describe("SIEEmbeddingFunction", () => { it("embedDocuments does not pass isQuery", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{ dense: new Float32Array([0.5]) }]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - getModel: vi.fn(), - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + getModel: vi.fn(), + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "test-model" }); await func.embedDocuments(["doc text"]); @@ -149,13 +153,13 @@ describe("SIEEmbeddingFunction", () => { it("throws error when dense embedding is missing", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{ sparse: {} }]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - getModel: vi.fn(), - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + getModel: vi.fn(), + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction(); await expect(func.generateEmbeddings(["test"])).rejects.toThrow( @@ -165,16 +169,20 @@ describe("SIEEmbeddingFunction", () => { it("ndims queries server metadata", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); - const mockGetModel = vi.fn().mockResolvedValue( - { name: "BAAI/bge-m3", dims: { dense: 1024 }, loaded: true, inputs: ["text"], outputs: ["dense"] }, - ); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: vi.fn(), - getModel: mockGetModel, - close: vi.fn(), - }; + const mockGetModel = vi.fn().mockResolvedValue({ + name: "BAAI/bge-m3", + dims: { dense: 1024 }, + loaded: true, + inputs: ["text"], + outputs: ["dense"], }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: vi.fn(), + getModel: mockGetModel, + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "BAAI/bge-m3" }); const dims = await func.ndims(); @@ -185,16 +193,20 @@ describe("SIEEmbeddingFunction", () => { it("ndims caches after first call", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); - const mockGetModel = vi.fn().mockResolvedValue( - { name: "test-model", dims: { dense: 384 }, loaded: true, inputs: ["text"], outputs: ["dense"] }, - ); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: vi.fn(), - getModel: mockGetModel, - close: vi.fn(), - }; + const mockGetModel = vi.fn().mockResolvedValue({ + name: "test-model", + dims: { dense: 384 }, + loaded: true, + inputs: ["text"], + outputs: ["dense"], }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: vi.fn(), + getModel: mockGetModel, + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "test-model" }); await func.ndims(); @@ -205,16 +217,20 @@ describe("SIEEmbeddingFunction", () => { it("ndims throws for model without dense dims", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); - const mockGetModel = vi.fn().mockResolvedValue( - { name: "multivec-only", dims: { multivector: 128 }, loaded: true, inputs: ["text"], outputs: ["multivector"] }, - ); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: vi.fn(), - getModel: mockGetModel, - close: vi.fn(), - }; + const mockGetModel = vi.fn().mockResolvedValue({ + name: "multivec-only", + dims: { multivector: 128 }, + loaded: true, + inputs: ["text"], + outputs: ["multivector"], }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: vi.fn(), + getModel: mockGetModel, + close: vi.fn(), + }), + ); const func = new SIEEmbeddingFunction({ model: "multivec-only" }); await expect(func.ndims()).rejects.toThrow("does not support dense"); diff --git a/integrations/sie_ts_langchain/tests/embeddings.test.ts b/integrations/sie_ts_langchain/tests/embeddings.test.ts index 9809ac0de..12794eae9 100644 --- a/integrations/sie_ts_langchain/tests/embeddings.test.ts +++ b/integrations/sie_ts_langchain/tests/embeddings.test.ts @@ -5,6 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { SIEEmbeddings, SIESparseEncoder } from "../src/index.js"; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -15,9 +21,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -58,12 +62,12 @@ describe("SIEEmbeddings", () => { { dense: new Float32Array([0.5, 0.25, 0.75]) }, { dense: new Float32Array([1.0, 2.0, 3.0]) }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embeddings = new SIEEmbeddings({ model: "test-model" }); const result = await embeddings.embedDocuments(["Hello", "World"]); @@ -87,12 +91,12 @@ describe("SIEEmbeddings", () => { const mockEncode = vi.fn().mockResolvedValue({ dense: new Float32Array([0.5, 0.25, 0.125]), }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embeddings = new SIEEmbeddings({ model: "test-model" }); const result = await embeddings.embedQuery("What is this?"); @@ -112,12 +116,12 @@ describe("SIEEmbeddings", () => { it("embedQuery throws if dense is missing", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue({}); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embeddings = new SIEEmbeddings(); await expect(embeddings.embedQuery("test")).rejects.toThrow("missing dense embedding"); @@ -128,12 +132,12 @@ describe("SIEEmbeddings", () => { const mockEncode = vi.fn().mockResolvedValue({ dense: new Float32Array([0.5, 0.25]), }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embeddings = new SIEEmbeddings({ instruction: "Represent this for retrieval:", @@ -178,12 +182,12 @@ describe("SIESparseEncoder", () => { }, }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const encoder = new SIESparseEncoder({ model: "test-model" }); const result = await encoder.encodeQueries(["test query"]); @@ -209,12 +213,12 @@ describe("SIESparseEncoder", () => { .mockResolvedValue([ { sparse: { indices: new Int32Array([2, 4]), values: new Float32Array([0.5, 0.75]) } }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const encoder = new SIESparseEncoder(); const result = await encoder.encodeDocuments(["test doc"]); @@ -237,12 +241,12 @@ describe("SIESparseEncoder", () => { it("returns empty arrays when sparse is missing", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{}]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const encoder = new SIESparseEncoder(); const result = await encoder.encodeDocuments(["test"]); diff --git a/integrations/sie_ts_langchain/tests/extractors.test.ts b/integrations/sie_ts_langchain/tests/extractors.test.ts index 3aa8ecf76..7a23e28c6 100644 --- a/integrations/sie_ts_langchain/tests/extractors.test.ts +++ b/integrations/sie_ts_langchain/tests/extractors.test.ts @@ -8,6 +8,12 @@ import { SIEExtractor } from "../src/index.js"; // Default empty extract result const emptyExtractResult = { entities: [], relations: [], classifications: [], objects: [] }; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -18,9 +24,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -60,12 +64,12 @@ describe("SIEExtractor", () => { classifications: [], objects: [], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const extractor = new SIEExtractor({ model: "test-ner" }); const result = await extractor._call("John Smith works at Acme Corp"); @@ -93,12 +97,12 @@ describe("SIEExtractor", () => { it("passes custom labels to extract", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockExtract = vi.fn().mockResolvedValue(emptyExtractResult); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const extractor = new SIEExtractor({ labels: ["product", "date"], @@ -113,12 +117,12 @@ describe("SIEExtractor", () => { it("passes threshold when specified", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockExtract = vi.fn().mockResolvedValue(emptyExtractResult); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const extractor = new SIEExtractor({ threshold: 0.5, @@ -134,12 +138,12 @@ describe("SIEExtractor", () => { it("returns empty result for no extractions", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockExtract = vi.fn().mockResolvedValue(emptyExtractResult); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const extractor = new SIEExtractor(); const result = await extractor._call("no entities here"); @@ -159,12 +163,12 @@ describe("SIEExtractor", () => { classifications: [], objects: [], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const extractor = new SIEExtractor(); const result = await extractor._call("test"); diff --git a/integrations/sie_ts_langchain/tests/rerankers.test.ts b/integrations/sie_ts_langchain/tests/rerankers.test.ts index 8bce62002..422ee3922 100644 --- a/integrations/sie_ts_langchain/tests/rerankers.test.ts +++ b/integrations/sie_ts_langchain/tests/rerankers.test.ts @@ -5,6 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { SIEReranker } from "../src/index.js"; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -15,9 +21,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -57,12 +61,12 @@ describe("SIEReranker", () => { { itemId: "2", score: 0.31, rank: 2 }, ], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - score: mockScore, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + score: mockScore, + close: vi.fn(), + }), + ); const documents = [ { pageContent: "First document", metadata: { source: "a" } }, @@ -73,11 +77,11 @@ describe("SIEReranker", () => { const reranker = new SIEReranker({ model: "test-reranker" }); const result = await reranker.compressDocuments(documents, "search query"); - expect(mockScore).toHaveBeenCalledWith( - "test-reranker", - { text: "search query" }, - [{ text: "First document" }, { text: "Second document" }, { text: "Third document" }], - ); + expect(mockScore).toHaveBeenCalledWith("test-reranker", { text: "search query" }, [ + { text: "First document" }, + { text: "Second document" }, + { text: "Third document" }, + ]); expect(result).toHaveLength(3); // Sorted by score descending (server returns sorted) @@ -101,12 +105,12 @@ describe("SIEReranker", () => { { itemId: "2", score: 0.31, rank: 2 }, ], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - score: mockScore, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + score: mockScore, + close: vi.fn(), + }), + ); const documents = [ { pageContent: "Doc A", metadata: {} }, @@ -127,12 +131,12 @@ describe("SIEReranker", () => { const mockScore = vi.fn().mockResolvedValue({ scores: [{ itemId: "0", score: 0.9, rank: 0 }], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - score: mockScore, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + score: mockScore, + close: vi.fn(), + }), + ); const documents = [{ pageContent: "Doc", metadata: {}, id: "doc-123" }]; diff --git a/integrations/sie_ts_llamaindex/src/extractors.ts b/integrations/sie_ts_llamaindex/src/extractors.ts index ea8202684..05b1f1629 100644 --- a/integrations/sie_ts_llamaindex/src/extractors.ts +++ b/integrations/sie_ts_llamaindex/src/extractors.ts @@ -116,7 +116,11 @@ class _SIEExtractor { extractOptions.threshold = this.threshold; } - const result: ExtractResult = await this.client.extract(this.modelName, { text }, extractOptions); + const result: ExtractResult = await this.client.extract( + this.modelName, + { text }, + extractOptions, + ); return JSON.stringify({ entities: result.entities.map((e) => ({ diff --git a/integrations/sie_ts_llamaindex/tests/embedding.test.ts b/integrations/sie_ts_llamaindex/tests/embedding.test.ts index b13c670d5..4820893a0 100644 --- a/integrations/sie_ts_llamaindex/tests/embedding.test.ts +++ b/integrations/sie_ts_llamaindex/tests/embedding.test.ts @@ -5,6 +5,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { SIEEmbedding, SIESparseEmbeddingFunction } from "../src/index.js"; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -15,9 +21,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -61,12 +65,12 @@ describe("SIEEmbedding", () => { const mockEncode = vi.fn().mockResolvedValue({ dense: new Float32Array([0.5, 0.25, 0.75]), }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embedding = new SIEEmbedding(); const result = await embedding.getTextEmbedding("Document text"); @@ -97,12 +101,12 @@ describe("SIEEmbedding", () => { { dense: new Float32Array([0.5, 0.25]) }, { dense: new Float32Array([0.75, 0.125]) }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embedding = new SIEEmbedding(); const result = await embedding.getTextEmbeddings(["Hello", "World"]); @@ -124,12 +128,12 @@ describe("SIEEmbedding", () => { it("throws if dense is missing", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue({}); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embedding = new SIEEmbedding(); await expect(embedding.getTextEmbedding("test")).rejects.toThrow("missing dense embedding"); @@ -140,12 +144,12 @@ describe("SIEEmbedding", () => { const mockEncode = vi.fn().mockResolvedValue({ dense: new Float32Array([0.5, 0.25]), }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const embedding = new SIEEmbedding({ instruction: "Represent this for retrieval:", @@ -187,12 +191,12 @@ describe("SIESparseEmbeddingFunction", () => { .mockResolvedValue([ { sparse: { indices: new Int32Array([1, 5]), values: new Float32Array([0.5, 0.25]) } }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const fn = new SIESparseEmbeddingFunction({ modelName: "test-model" }); const [indices, values] = await fn.encodeQueries(["test query"]); @@ -217,12 +221,12 @@ describe("SIESparseEmbeddingFunction", () => { .mockResolvedValue([ { sparse: { indices: new Int32Array([2, 4]), values: new Float32Array([0.5, 0.75]) } }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const fn = new SIESparseEmbeddingFunction(); const [indices, values] = await fn.encodeDocuments(["test doc"]); @@ -243,12 +247,12 @@ describe("SIESparseEmbeddingFunction", () => { it("returns empty arrays when sparse is missing", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockEncode = vi.fn().mockResolvedValue([{}]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const fn = new SIESparseEmbeddingFunction(); const [indices, values] = await fn.encodeDocuments(["test"]); @@ -265,12 +269,12 @@ describe("SIESparseEmbeddingFunction", () => { { sparse: { indices: new Int32Array([1]), values: new Float32Array([0.5]) } }, { sparse: { indices: new Int32Array([2, 3]), values: new Float32Array([0.25, 0.75]) } }, ]); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - encode: mockEncode, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + encode: mockEncode, + close: vi.fn(), + }), + ); const fn = new SIESparseEmbeddingFunction(); const [indices, values] = await fn.encodeDocuments(["text1", "text2"]); diff --git a/integrations/sie_ts_llamaindex/tests/extractors.test.ts b/integrations/sie_ts_llamaindex/tests/extractors.test.ts index 7e739c8f6..5444c528f 100644 --- a/integrations/sie_ts_llamaindex/tests/extractors.test.ts +++ b/integrations/sie_ts_llamaindex/tests/extractors.test.ts @@ -8,6 +8,12 @@ import { createSIEExtractorTool } from "../src/index.js"; // Default empty extract result const emptyExtractResult = { entities: [], relations: [], classifications: [], objects: [] }; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -18,9 +24,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -57,12 +61,12 @@ describe("createSIEExtractorTool", () => { classifications: [], objects: [], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const tool = createSIEExtractorTool({ modelName: "test-ner" }); const result = await tool.call({ text: "John Smith works at Acme Corp" }); @@ -90,12 +94,12 @@ describe("createSIEExtractorTool", () => { it("passes custom labels to extract", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockExtract = vi.fn().mockResolvedValue(emptyExtractResult); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const tool = createSIEExtractorTool({ labels: ["product", "date"], @@ -110,12 +114,12 @@ describe("createSIEExtractorTool", () => { it("returns empty result for no extractions", async () => { const { SIEClient } = await import("@superlinked/sie-sdk"); const mockExtract = vi.fn().mockResolvedValue(emptyExtractResult); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - extract: mockExtract, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + extract: mockExtract, + close: vi.fn(), + }), + ); const tool = createSIEExtractorTool(); const result = await tool.call({ text: "no entities here" }); diff --git a/integrations/sie_ts_llamaindex/tests/rerankers.test.ts b/integrations/sie_ts_llamaindex/tests/rerankers.test.ts index 1bafff8dc..166f3d6f2 100644 --- a/integrations/sie_ts_llamaindex/tests/rerankers.test.ts +++ b/integrations/sie_ts_llamaindex/tests/rerankers.test.ts @@ -6,6 +6,12 @@ import type { MessageContent, NodeWithScore } from "llamaindex"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { SIENodePostprocessor } from "../src/index.js"; +function asConstructor(instance: T): () => T { + return function constructorMock() { + return instance; + }; +} + // Mock the SIEClient vi.mock("@superlinked/sie-sdk", async (importOriginal) => { const actual = await importOriginal(); @@ -16,9 +22,7 @@ vi.mock("@superlinked/sie-sdk", async (importOriginal) => { return { ...actual, - SIEClient: vi.fn().mockImplementation(function () { - return mockClient; - }), + SIEClient: vi.fn().mockImplementation(asConstructor(mockClient)), }; }); @@ -80,12 +84,12 @@ describe("SIENodePostprocessor", () => { { itemId: "2", score: 0.31, rank: 2 }, ], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - score: mockScore, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + score: mockScore, + close: vi.fn(), + }), + ); const nodes = [ mockNodeWithScore("First doc", 0.5), @@ -99,11 +103,11 @@ describe("SIENodePostprocessor", () => { "search query", ); - expect(mockScore).toHaveBeenCalledWith( - "test-reranker", - { text: "search query" }, - [{ text: "First doc" }, { text: "Second doc" }, { text: "Third doc" }], - ); + expect(mockScore).toHaveBeenCalledWith("test-reranker", { text: "search query" }, [ + { text: "First doc" }, + { text: "Second doc" }, + { text: "Third doc" }, + ]); expect(result).toHaveLength(3); // Sorted by score descending (server returns sorted) @@ -125,12 +129,12 @@ describe("SIENodePostprocessor", () => { { itemId: "0", score: 0.72, rank: 1 }, ], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - score: mockScore, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + score: mockScore, + close: vi.fn(), + }), + ); const nodes = [mockNodeWithScore("doc1"), mockNodeWithScore("doc2")]; @@ -150,12 +154,12 @@ describe("SIENodePostprocessor", () => { const mockScore = vi.fn().mockResolvedValue({ scores: [{ itemId: "0", score: 0.9, rank: 0 }], }); - (SIEClient as unknown as ReturnType).mockImplementation(function () { - return { - score: mockScore, - close: vi.fn(), - }; - }); + (SIEClient as unknown as ReturnType).mockImplementation( + asConstructor({ + score: mockScore, + close: vi.fn(), + }), + ); const nodes = [mockNodeWithScore("doc1")]; diff --git a/mise.toml b/mise.toml index 0a5f6dd74..6360245b8 100644 --- a/mise.toml +++ b/mise.toml @@ -10,22 +10,13 @@ rust = { version = "1.97.0", profile = "default", components = ["rustfmt", "clip node = ["24.12.0", "22.23.2"] pnpm = "9.15.9" helm = "3.16.4" +zig = "0.13.0" +"github:nats-io/nats-server" = "2.11.8" +"github:rhysd/actionlint" = "1.7.7" [env] -MISE_CACHE_DIR = "{{config_root}}/.cache/mise" -MISE_CONFIG_DIR = "{{config_root}}/.cache/mise-config" -MISE_DATA_DIR = "{{config_root}}/.cache/mise-data" -MISE_GLOBAL_CONFIG_FILE = "{{config_root}}/.cache/mise-config/config.toml" -MISE_GLOBAL_CONFIG_ROOT = "{{config_root}}" -MISE_PROJECT_ROOT = "{{config_root}}" -MISE_TRUSTED_CONFIG_PATHS = "{{config_root}}:{{env.HOME}}/.config/mise" -MISE_IGNORED_CONFIG_PATHS = "{{env.HOME}}/.config/mise" -MISE_STATE_DIR = "{{config_root}}/.cache/mise-state" -MISE_SYSTEM_CONFIG_FILE = "{{config_root}}/.cache/mise-config/system.toml" UV_CACHE_DIR = "{{env.HOME}}/.cache/sie/uv" XDG_CACHE_HOME = "{{env.HOME}}/.cache/sie/xdg-cache" -XDG_CONFIG_HOME = "{{config_root}}/.cache/xdg-config" -XDG_STATE_HOME = "{{config_root}}/.cache/xdg-state" [task_config] includes = ["tools/mise_tasks"] diff --git a/package.json b/package.json index 58cfc2a0d..25146d110 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "devDependencies": { "typescript": "^5.9.3" }, - "packageManager": "pnpm@9.15.0", + "packageManager": "pnpm@9.15.9", "workspaces": [ "packages/sie_ts_sdk", "integrations/sie_ts_langchain", @@ -23,8 +23,11 @@ "esbuild": ">=0.25.0", "form-data": ">=4.0.4", "langsmith": ">=0.6.0", + "picomatch": ">=4.0.4", + "postcss": ">=8.5.10", "protobufjs": ">=7.5.8", "qs": ">=6.15.2", + "rollup": ">=4.59.0", "tar": ">=7.5.11", "uuid": ">=11.1.1", "vite": "^6.4.2", diff --git a/packages/sie_sdk/tests/client/test_transport_error_retry.py b/packages/sie_sdk/tests/client/test_transport_error_retry.py index ecb58ee27..ffd391af6 100644 --- a/packages/sie_sdk/tests/client/test_transport_error_retry.py +++ b/packages/sie_sdk/tests/client/test_transport_error_retry.py @@ -8,7 +8,6 @@ import logging import threading -import time from concurrent.futures import ThreadPoolExecutor from unittest.mock import AsyncMock, MagicMock, patch @@ -23,7 +22,7 @@ @pytest.fixture(autouse=True) def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: - # No-op the retry sleeps; budget tests still use real time.monotonic. + # No-op ordinary retry sleeps; budget tests install a deterministic clock. monkeypatch.setattr("sie_sdk.client.sync.time.sleep", lambda _: None) async def _noop_async_sleep(_: float) -> None: @@ -32,6 +31,31 @@ async def _noop_async_sleep(_: float) -> None: monkeypatch.setattr("sie_sdk.client.async_.asyncio.sleep", _noop_async_sleep) +class _RetryClock: + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, delay: float) -> None: + self.now += delay + + async def async_sleep(self, delay: float) -> None: + self.sleep(delay) + + +@pytest.fixture +def retry_clock(monkeypatch: pytest.MonkeyPatch) -> _RetryClock: + clock = _RetryClock() + monkeypatch.setattr("sie_sdk.client.sync.time", clock) + monkeypatch.setattr("sie_sdk.client.async_.time", clock) + monkeypatch.setattr("sie_sdk.client._shared.time", clock) + monkeypatch.setattr("sie_sdk.client.async_.asyncio.sleep", clock.async_sleep) + monkeypatch.setattr("sie_sdk.client._shared.apply_jitter", lambda delay, **_kwargs: delay) + return clock + + def _logged_origin(message: str) -> str: """Extract the origin the retry WARNING reports. @@ -173,7 +197,7 @@ def test_transport_error_not_retried_when_wait_for_capacity_false(self) -> None: assert mock_client.return_value.post.call_count == 1 client.close() - def test_transport_error_retries_bounded_by_provision_timeout(self) -> None: + def test_transport_error_retries_bounded_by_provision_timeout(self, retry_clock: _RetryClock) -> None: from sie_sdk import ProvisioningError, SIEClient from sie_sdk.client.errors import SIEConnectionError @@ -182,7 +206,7 @@ def test_transport_error_retries_bounded_by_provision_timeout(self) -> None: mock_client.return_value.post = MagicMock(side_effect=exc) client = SIEClient("http://localhost:8080") - start = time.monotonic() + start = retry_clock.monotonic() # Either error type is valid: SIEConnectionError after budget # exhausted, or ProvisioningError if the pre-request budget # check caught a freshly-zeroed remaining timeout. @@ -193,10 +217,10 @@ def test_transport_error_retries_bounded_by_provision_timeout(self) -> None: wait_for_capacity=True, provision_timeout_s=0.05, ) - elapsed = time.monotonic() - start + elapsed = retry_clock.monotonic() - start - assert elapsed < 0.25, f"Retry loop did not honour provision_timeout_s: {elapsed:.2f}s" - assert mock_client.return_value.post.call_count >= 1 + assert elapsed == pytest.approx(0.05) + assert mock_client.return_value.post.call_count == 1 client.close() def test_connect_error_retried_when_wait_for_capacity_true_then_succeeds(self) -> None: @@ -238,7 +262,7 @@ def test_connect_error_not_retried_when_wait_for_capacity_false(self) -> None: assert mock_client.return_value.post.call_count == 1 client.close() - def test_connect_error_retries_bounded_by_provision_timeout(self) -> None: + def test_connect_error_retries_bounded_by_provision_timeout(self, retry_clock: _RetryClock) -> None: from sie_sdk import ProvisioningError, SIEClient from sie_sdk.client.errors import SIEConnectionError @@ -247,7 +271,7 @@ def test_connect_error_retries_bounded_by_provision_timeout(self) -> None: mock_client.return_value.post = MagicMock(side_effect=exc) client = SIEClient("http://localhost:8080") - start = time.monotonic() + start = retry_clock.monotonic() with pytest.raises((SIEConnectionError, ProvisioningError)): client.encode( "bge-m3", @@ -255,10 +279,10 @@ def test_connect_error_retries_bounded_by_provision_timeout(self) -> None: wait_for_capacity=True, provision_timeout_s=0.05, ) - elapsed = time.monotonic() - start + elapsed = retry_clock.monotonic() - start - assert elapsed < 0.25, f"Retry loop did not honour provision_timeout_s: {elapsed:.2f}s" - assert mock_client.return_value.post.call_count >= 1 + assert elapsed == pytest.approx(0.05) + assert mock_client.return_value.post.call_count == 1 client.close() @@ -371,7 +395,7 @@ async def test_connector_error_not_retried_when_wait_for_capacity_false(self) -> await client.close() @pytest.mark.asyncio - async def test_connector_error_retries_bounded_by_provision_timeout(self) -> None: + async def test_connector_error_retries_bounded_by_provision_timeout(self, retry_clock: _RetryClock) -> None: from sie_sdk import ProvisioningError, SIEAsyncClient from sie_sdk.client.errors import SIEConnectionError @@ -379,7 +403,7 @@ async def test_connector_error_retries_bounded_by_provision_timeout(self) -> Non client = SIEAsyncClient("http://localhost:8080") client._post = AsyncMock(side_effect=exc) # type: ignore - start = time.monotonic() + start = retry_clock.monotonic() with pytest.raises((SIEConnectionError, ProvisioningError)): await client.encode( "bge-m3", @@ -387,10 +411,10 @@ async def test_connector_error_retries_bounded_by_provision_timeout(self) -> Non wait_for_capacity=True, provision_timeout_s=0.05, ) - elapsed = time.monotonic() - start + elapsed = retry_clock.monotonic() - start - assert elapsed < 0.25, f"Retry loop did not honour provision_timeout_s: {elapsed:.2f}s" - assert client._post.call_count >= 1 + assert elapsed == pytest.approx(0.05) + assert client._post.call_count == 1 await client.close() diff --git a/packages/sie_sdk/tests/test_cache.py b/packages/sie_sdk/tests/test_cache.py index 77d668f22..512e882ad 100644 --- a/packages/sie_sdk/tests/test_cache.py +++ b/packages/sie_sdk/tests/test_cache.py @@ -21,6 +21,12 @@ from sie_sdk.storage import OSSBackend +@pytest.fixture(autouse=True) +def offline_hf_metadata(): + with patch("huggingface_hub.file_exists", return_value=False) as metadata: + yield metadata + + class FakeOSSCacheBucket: """Small hierarchical OSS fake backed by realistic HF cache objects.""" @@ -224,6 +230,7 @@ def test_model_from_cluster_cache(self, tmp_path: Path) -> None: config = CacheConfig( local_cache=local_cache, cluster_cache="s3://my-bucket/cache", + hf_fallback=False, ) # Mock the storage backend @@ -234,18 +241,17 @@ def test_model_from_cluster_cache(self, tmp_path: Path) -> None: # Simulate model exists in cluster cache (snapshots prefix non-empty) mock_backend.has_children.return_value = True - # list_dirs returns subdirectories - we need to simulate the HF cache structure - # First call lists snapshots dir contents, subsequent calls return empty - list_dirs_calls = [["abc123"], []] # abc123 is a snapshot, then no more subdirs - mock_backend.list_dirs.side_effect = lambda _: list_dirs_calls.pop(0) if list_dirs_calls else [] - mock_backend.list_files.return_value = iter(["model.bin"]) + cluster_root = "s3://my-bucket/cache/models--BAAI--bge-m3" + directories = {cluster_root: ["snapshots"], f"{cluster_root}/snapshots": ["abc123"]} + mock_backend.list_dirs.side_effect = lambda path: directories.get(path, []) + mock_backend.list_files.side_effect = lambda path: ["model.bin"] if path.endswith("/abc123") else [] + mock_backend.download_file.side_effect = lambda _source, destination: destination.write_bytes(b"weights") - ensure_model_cached("BAAI/bge-m3", config) + result = ensure_model_cached("BAAI/bge-m3", config) - # Should call backend methods + assert result == local_cache / "models--BAAI--bge-m3" + assert is_model_cached("BAAI/bge-m3", config) mock_backend.has_children.assert_called() - # Result depends on whether files were actually downloaded - # In mocked scenario, we just verify the logic flow def test_model_not_in_cluster_cache(self, tmp_path: Path) -> None: """Raises error if model not in cluster cache and HF fallback disabled.""" diff --git a/packages/sie_server/tests/adapters/test_docling_smoke.py b/packages/sie_server/tests/adapters/test_docling_smoke.py index 13e770456..696132cf3 100644 --- a/packages/sie_server/tests/adapters/test_docling_smoke.py +++ b/packages/sie_server/tests/adapters/test_docling_smoke.py @@ -51,7 +51,7 @@ def _make_html_bytes() -> bytes: @pytest.mark.parametrize( ("format_hint", "maker"), [ - ("pdf", _make_pdf_bytes), + pytest.param("pdf", _make_pdf_bytes, marks=pytest.mark.model), ("docx", _make_docx_bytes), ("html", _make_html_bytes), ], diff --git a/packages/sie_server_sidecar/Dockerfile b/packages/sie_server_sidecar/Dockerfile index 96eb27c44..5cb8dbaa8 100644 --- a/packages/sie_server_sidecar/Dockerfile +++ b/packages/sie_server_sidecar/Dockerfile @@ -30,7 +30,7 @@ ARG RUST_VERSION=1.97.0 # ============================================================================= # Dependency planner: capture the Cargo graph only. # ============================================================================= -FROM rust:${RUST_VERSION}-slim-bookworm AS chef +FROM rust:${RUST_VERSION}-slim-bookworm@sha256:6d220bf85c74e842a79da63997af8d2e74455c0b8847d8bb3a5888572334991d AS chef RUN cargo install cargo-chef --locked --version 0.1.77 WORKDIR /build @@ -108,7 +108,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ # ============================================================================= # `distroless/cc-debian12` ships glibc + common C runtime libs, matching what # the Rust build links against. `nonroot` gives us uid/gid 65532. -FROM gcr.io/distroless/cc-debian12:nonroot AS runtime +FROM gcr.io/distroless/cc-debian12:nonroot@sha256:9dac0a79194e45a7da0158a9c6da57b217585af0786db3845d1f0ec1a0dd182f AS runtime COPY --from=builder /usr/local/bin/sie-server-sidecar /sie-server-sidecar diff --git a/packages/sie_ts_sdk/package.json b/packages/sie_ts_sdk/package.json index bad70da5b..6d4e09f16 100644 --- a/packages/sie_ts_sdk/package.json +++ b/packages/sie_ts_sdk/package.json @@ -39,6 +39,7 @@ "test:integration": "vitest run --config vitest.integration.config.ts", "test:browser": "playwright test", "test:browser:serve": "npx serve -l 3456 -C .", + "prepare": "pnpm run build", "prepack": "pnpm run build" }, "engines": { @@ -65,14 +66,5 @@ "tsup": "^8.5.1", "typescript": "^5.9.3", "vitest": "^4.1.0" - }, - "pnpm": { - "overrides": { - "esbuild": ">=0.25.0", - "picomatch": ">=4.0.4", - "postcss": ">=8.5.10", - "rollup": ">=4.59.0", - "vite": "^6.4.2" - } } } diff --git a/packages/sie_ts_sdk/pnpm-lock.yaml b/packages/sie_ts_sdk/pnpm-lock.yaml deleted file mode 100644 index b29904a1c..000000000 --- a/packages/sie_ts_sdk/pnpm-lock.yaml +++ /dev/null @@ -1,1515 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -overrides: - esbuild: '>=0.25.0' - picomatch: '>=4.0.4' - postcss: '>=8.5.10' - rollup: '>=4.59.0' - vite: ^6.4.2 - -importers: - - .: - dependencies: - '@msgpack/msgpack': - specifier: ^3.1.3 - version: 3.1.3 - devDependencies: - '@biomejs/biome': - specifier: ^1.9.4 - version: 1.9.4 - '@playwright/test': - specifier: ^1.60.0 - version: 1.60.0 - '@types/node': - specifier: ^22.19.19 - version: 22.19.19 - tsup: - specifier: ^8.5.1 - version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) - typescript: - specifier: ^5.9.3 - version: 5.9.3 - vitest: - specifier: ^4.1.0 - version: 4.1.8(@types/node@22.19.19)(vite@6.4.2(@types/node@22.19.19)(lightningcss@1.32.0)) - -packages: - - '@biomejs/biome@1.9.4': - resolution: {integrity: sha512-1rkd7G70+o9KkTn5KLmDYXihGoTaIGO9PIIN2ZB7UJxFrWw04CZHPYiMRjYsaDvVV7hP1dYNRLxSANLaBFGpog==} - engines: {node: '>=14.21.3'} - hasBin: true - - '@biomejs/cli-darwin-arm64@1.9.4': - resolution: {integrity: sha512-bFBsPWrNvkdKrNCYeAp+xo2HecOGPAy9WyNyB/jKnnedgzl4W4Hb9ZMzYNbf8dMCGmUdSavlYHiR01QaYR58cw==} - engines: {node: '>=14.21.3'} - cpu: [arm64] - os: [darwin] - - '@biomejs/cli-darwin-x64@1.9.4': - resolution: {integrity: sha512-ngYBh/+bEedqkSevPVhLP4QfVPCpb+4BBe2p7Xs32dBgs7rh9nY2AIYUL6BgLw1JVXV8GlpKmb/hNiuIxfPfZg==} - engines: {node: '>=14.21.3'} - cpu: [x64] - os: [darwin] - - '@biomejs/cli-linux-arm64-musl@1.9.4': - resolution: {integrity: sha512-v665Ct9WCRjGa8+kTr0CzApU0+XXtRgwmzIf1SeKSGAv+2scAlW6JR5PMFo6FzqqZ64Po79cKODKf3/AAmECqA==} - engines: {node: '>=14.21.3'} - cpu: [arm64] - os: [linux] - - '@biomejs/cli-linux-arm64@1.9.4': - resolution: {integrity: sha512-fJIW0+LYujdjUgJJuwesP4EjIBl/N/TcOX3IvIHJQNsAqvV2CHIogsmA94BPG6jZATS4Hi+xv4SkBBQSt1N4/g==} - engines: {node: '>=14.21.3'} - cpu: [arm64] - os: [linux] - - '@biomejs/cli-linux-x64-musl@1.9.4': - resolution: {integrity: sha512-gEhi/jSBhZ2m6wjV530Yy8+fNqG8PAinM3oV7CyO+6c3CEh16Eizm21uHVsyVBEB6RIM8JHIl6AGYCv6Q6Q9Tg==} - engines: {node: '>=14.21.3'} - cpu: [x64] - os: [linux] - - '@biomejs/cli-linux-x64@1.9.4': - resolution: {integrity: sha512-lRCJv/Vi3Vlwmbd6K+oQ0KhLHMAysN8lXoCI7XeHlxaajk06u7G+UsFSO01NAs5iYuWKmVZjmiOzJ0OJmGsMwg==} - engines: {node: '>=14.21.3'} - cpu: [x64] - os: [linux] - - '@biomejs/cli-win32-arm64@1.9.4': - resolution: {integrity: sha512-tlbhLk+WXZmgwoIKwHIHEBZUwxml7bRJgk0X2sPyNR3S93cdRq6XulAZRQJ17FYGGzWne0fgrXBKpl7l4M87Hg==} - engines: {node: '>=14.21.3'} - cpu: [arm64] - os: [win32] - - '@biomejs/cli-win32-x64@1.9.4': - resolution: {integrity: sha512-8Y5wMhVIPaWe6jw2H+KlEm4wP/f7EW3810ZLmDlrEEy5KvBsb9ECEfu/kMWD484ijfQ8+nIi0giMgu9g1UAuuA==} - engines: {node: '>=14.21.3'} - cpu: [x64] - os: [win32] - - '@esbuild/aix-ppc64@0.27.2': - resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.27.2': - resolution: {integrity: sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm@0.27.2': - resolution: {integrity: sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-x64@0.27.2': - resolution: {integrity: sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/darwin-arm64@0.27.2': - resolution: {integrity: sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-x64@0.27.2': - resolution: {integrity: sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.27.2': - resolution: {integrity: sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.27.2': - resolution: {integrity: sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/linux-arm64@0.27.2': - resolution: {integrity: sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm@0.27.2': - resolution: {integrity: sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-ia32@0.27.2': - resolution: {integrity: sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.27.2': - resolution: {integrity: sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-mips64el@0.27.2': - resolution: {integrity: sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-ppc64@0.27.2': - resolution: {integrity: sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-riscv64@0.27.2': - resolution: {integrity: sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-s390x@0.27.2': - resolution: {integrity: sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.27.2': - resolution: {integrity: sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/netbsd-arm64@0.27.2': - resolution: {integrity: sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.27.2': - resolution: {integrity: sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/openbsd-arm64@0.27.2': - resolution: {integrity: sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.27.2': - resolution: {integrity: sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.27.2': - resolution: {integrity: sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/sunos-x64@0.27.2': - resolution: {integrity: sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/win32-arm64@0.27.2': - resolution: {integrity: sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-ia32@0.27.2': - resolution: {integrity: sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-x64@0.27.2': - resolution: {integrity: sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@jridgewell/gen-mapping@0.3.13': - resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} - - '@jridgewell/resolve-uri@3.1.2': - resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} - engines: {node: '>=6.0.0'} - - '@jridgewell/sourcemap-codec@1.5.5': - resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - - '@jridgewell/trace-mapping@0.3.31': - resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - - '@msgpack/msgpack@3.1.3': - resolution: {integrity: sha512-47XIizs9XZXvuJgoaJUIE2lFoID8ugvc0jzSHP+Ptfk8nTbnR8g788wv48N03Kx0UkAv559HWRQ3yzOgzlRNUA==} - engines: {node: '>= 18'} - - '@playwright/test@1.60.0': - resolution: {integrity: sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==} - engines: {node: '>=18'} - hasBin: true - - '@rollup/rollup-android-arm-eabi@4.60.4': - resolution: {integrity: sha512-F5QXMSiFebS9hKZj02XhWLLnRpJ3B3AROP0tWbFBSj+6kCbg5m9j5JoHKd4mmSVy5mS/IMQloYgYxCuJC0fxEQ==} - cpu: [arm] - os: [android] - - '@rollup/rollup-android-arm64@4.60.4': - resolution: {integrity: sha512-GxxTKApUpzRhof7poWvCJHRF51C67u1R7D6DiluBE8wKU1u5GWE8t+v81JvJYtbawoBFX1hLv5Ei4eVjkWokaw==} - cpu: [arm64] - os: [android] - - '@rollup/rollup-darwin-arm64@4.60.4': - resolution: {integrity: sha512-tua0TaJxMOB1R0V0RS1jFZ/RpURFDJIOR2A6jWwQeawuFyS4gBW+rntLRaQd0EQ4bd6Vp44Z2rXW+YYDBsj6IA==} - cpu: [arm64] - os: [darwin] - - '@rollup/rollup-darwin-x64@4.60.4': - resolution: {integrity: sha512-CSKq7MsP+5PFIcydhAiR1K0UhEI1A2jWXVKHPCBZ151yOutENwvnPocgVHkivu2kviURtCEB6zUQw0vs8RrhMg==} - cpu: [x64] - os: [darwin] - - '@rollup/rollup-freebsd-arm64@4.60.4': - resolution: {integrity: sha512-+O8OkVdyvXMtJEciu2wS/pzm1IxntEEQx3z5TAVy4l32G0etZn+RsA48ARRrFm6Ri8fvqPQfgrvNxSjKAbnd3g==} - cpu: [arm64] - os: [freebsd] - - '@rollup/rollup-freebsd-x64@4.60.4': - resolution: {integrity: sha512-Iw3oMskH3AfNuhU0MSN7vNbdi4me/NiYo2azqPz/Le16zHSa+3RRmliCMWWQmh4lcndccU40xcJuTYJZxNo/lw==} - cpu: [x64] - os: [freebsd] - - '@rollup/rollup-linux-arm-gnueabihf@4.60.4': - resolution: {integrity: sha512-EIPRXTVQpHyF8WOo219AD2yEltPehLTcTMz2fn6JsatLYSzQf00hj3rulF+yauOlF9/FtM2WpkT/hJh/KJFGhA==} - cpu: [arm] - os: [linux] - - '@rollup/rollup-linux-arm-musleabihf@4.60.4': - resolution: {integrity: sha512-J3Yh9PzzF1Ovah2At+lHiGQdsYgArxBbXv/zHfSyaiFQEqvNv7DcW98pCrmdjCZBrqBiKrKKe2V+aaSGWuBe/w==} - cpu: [arm] - os: [linux] - - '@rollup/rollup-linux-arm64-gnu@4.60.4': - resolution: {integrity: sha512-BFDEZMYfUvLn37ONE1yMBojPxnMlTFsdyNoqncT0qFq1mAfllL+ATMMJd8TeuVMiX84s1KbcxcZbXInmcO2mRg==} - cpu: [arm64] - os: [linux] - - '@rollup/rollup-linux-arm64-musl@4.60.4': - resolution: {integrity: sha512-pc9EYOSlOgdQ2uPl1o9PF6/kLSgaUosia7gOuS8mB69IxJvlclko1MECXysjs5ryez1/5zjYqx3+xYU0TU6R1A==} - cpu: [arm64] - os: [linux] - - '@rollup/rollup-linux-loong64-gnu@4.60.4': - resolution: {integrity: sha512-NxnomyxYerDh5n4iLrNa+sH+Z+U4BMEE46V2PgQ/hoB909i8gV1M5wPojWg9fk1jWpO3IQnOs20K4wyZuFLEFQ==} - cpu: [loong64] - os: [linux] - - '@rollup/rollup-linux-loong64-musl@4.60.4': - resolution: {integrity: sha512-nbJnQ8a3z1mtmrwImCYhc6BGpThAyYVRQxw9uKSKG4wR6aAYno9sVjJ0zaZcW9BPJX1GbrDPf+SvdWjgTuDmnw==} - cpu: [loong64] - os: [linux] - - '@rollup/rollup-linux-ppc64-gnu@4.60.4': - resolution: {integrity: sha512-2EU6acNrQLd8tYvo/LXW535wupT3m6fo7HKo6lr7ktQoItxTyOL1ZCR/GfGCuXl2vR+zmfI6eRXkSemafv+iVg==} - cpu: [ppc64] - os: [linux] - - '@rollup/rollup-linux-ppc64-musl@4.60.4': - resolution: {integrity: sha512-WeBtoMuaMxiiIrO2IYP3xs6GMWkJP2C0EoT8beTLkUPmzV1i/UcOSVw1d5r9KBODtHKilG5yFxsGRnBbK3wJ4A==} - cpu: [ppc64] - os: [linux] - - '@rollup/rollup-linux-riscv64-gnu@4.60.4': - resolution: {integrity: sha512-FJHFfqpKUI3A10WrWKiFbBZ7yVbGT4q4B5o1qKFFojqpaYoh9LrQgqWCmmcxQzVSXYtyB5bzkXrYzlHTs21MYA==} - cpu: [riscv64] - os: [linux] - - '@rollup/rollup-linux-riscv64-musl@4.60.4': - resolution: {integrity: sha512-mcEl6CUT5IAUmQf1m9FYSmVqCJlpQ8r8eyftFUHG8i9OhY7BkBXSUdnLH5DOf0wCOjcP9v/QO93zpmF1SptCCw==} - cpu: [riscv64] - os: [linux] - - '@rollup/rollup-linux-s390x-gnu@4.60.4': - resolution: {integrity: sha512-ynt3JxVd2w2buzoKDWIyiV1pJW93xlQic1THVLXilz429oijRpSHivZAgp65KBu+cMcgf1eVVjdnTLvPxgCuoQ==} - cpu: [s390x] - os: [linux] - - '@rollup/rollup-linux-x64-gnu@4.60.4': - resolution: {integrity: sha512-Boiz5+MsaROEWDf+GGEwF8VMHGhlUoQMtIPjOgA5fv4osupqTVnJteQNKJwUcnUog2G55jYXH7KZFFiJe0TEzQ==} - cpu: [x64] - os: [linux] - - '@rollup/rollup-linux-x64-musl@4.60.4': - resolution: {integrity: sha512-+qfSY27qIrFfI/Hom04KYFw3GKZSGU4lXus51wsb5EuySfFlWRwjkKWoE9emgRw/ukoT4Udsj4W/+xxG8VbPKg==} - cpu: [x64] - os: [linux] - - '@rollup/rollup-openbsd-x64@4.60.4': - resolution: {integrity: sha512-VpTfOPHgVXEBeeR8hZ2O0F3aSso+JDWqTWmTmzcQKted54IAdUVbxE+j/MVxUsKa8L20HJhv3vUezVPoquqWjA==} - cpu: [x64] - os: [openbsd] - - '@rollup/rollup-openharmony-arm64@4.60.4': - resolution: {integrity: sha512-IPOsh5aRYuLv/nkU51X10Bf75Bsf6+gZdx1X+QP5QM6lIJFHHqbHLG0uJn/hWthzo13UAc2umiUorqZy3axoZg==} - cpu: [arm64] - os: [openharmony] - - '@rollup/rollup-win32-arm64-msvc@4.60.4': - resolution: {integrity: sha512-4QzE9E81OohJ/HKzHhsqU+zcYYojVOXlFMs1DdyMT6qXl/niOH7AVElmmEdUNHHS/oRkc++d5k6Vy85zFs0DEw==} - cpu: [arm64] - os: [win32] - - '@rollup/rollup-win32-ia32-msvc@4.60.4': - resolution: {integrity: sha512-zTPgT1YuHHcd+Tmx7h8aml0FWFVelV5N54oHow9SLj+GfoDy/huQ+UV396N/C7KpMDMiPspRktzM1/0r1usYEA==} - cpu: [ia32] - os: [win32] - - '@rollup/rollup-win32-x64-gnu@4.60.4': - resolution: {integrity: sha512-DRS4G7mi9lJxqEDezIkKCaUIKCrLUUDCUaCsTPCi/rtqaC6D/jjwslMQyiDU50Ka0JKpeXeRBFBAXwArY52vBw==} - cpu: [x64] - os: [win32] - - '@rollup/rollup-win32-x64-msvc@4.60.4': - resolution: {integrity: sha512-QVTUovf40zgTqlFVrKA1uXMVvU2QWEFWfAH8Wdc48IxLvrJMQVMBRjuQyUpzZCDkakImib9eVazbWlC6ksWtJw==} - cpu: [x64] - os: [win32] - - '@standard-schema/spec@1.1.0': - resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - - '@types/chai@5.2.3': - resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} - - '@types/deep-eql@4.0.2': - resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - - '@types/estree@1.0.8': - resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} - - '@types/node@22.19.19': - resolution: {integrity: sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==} - - '@vitest/expect@4.1.8': - resolution: {integrity: sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==} - - '@vitest/mocker@4.1.8': - resolution: {integrity: sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==} - peerDependencies: - msw: ^2.4.9 - vite: ^6.4.2 - peerDependenciesMeta: - msw: - optional: true - vite: - optional: true - - '@vitest/pretty-format@4.1.8': - resolution: {integrity: sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==} - - '@vitest/runner@4.1.8': - resolution: {integrity: sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==} - - '@vitest/snapshot@4.1.8': - resolution: {integrity: sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==} - - '@vitest/spy@4.1.8': - resolution: {integrity: sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==} - - '@vitest/utils@4.1.8': - resolution: {integrity: sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==} - - acorn@8.15.0: - resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} - engines: {node: '>=0.4.0'} - hasBin: true - - any-promise@1.3.0: - resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} - - assertion-error@2.0.1: - resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} - engines: {node: '>=12'} - - bundle-require@5.1.0: - resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - peerDependencies: - esbuild: '>=0.25.0' - - cac@6.7.14: - resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} - engines: {node: '>=8'} - - chai@6.2.2: - resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} - engines: {node: '>=18'} - - chokidar@4.0.3: - resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} - engines: {node: '>= 14.16.0'} - - commander@4.1.1: - resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} - engines: {node: '>= 6'} - - confbox@0.1.8: - resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} - - consola@3.4.2: - resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} - engines: {node: ^14.18.0 || >=16.10.0} - - convert-source-map@2.0.0: - resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - - debug@4.4.3: - resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - es-module-lexer@2.1.0: - resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==} - - esbuild@0.27.2: - resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} - engines: {node: '>=18'} - hasBin: true - - estree-walker@3.0.3: - resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} - - expect-type@1.3.0: - resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} - engines: {node: '>=12.0.0'} - - fdir@6.5.0: - resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} - engines: {node: '>=12.0.0'} - peerDependencies: - picomatch: '>=4.0.4' - peerDependenciesMeta: - picomatch: - optional: true - - fix-dts-default-cjs-exports@1.0.1: - resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} - - fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - joycon@3.1.1: - resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} - engines: {node: '>=10'} - - lightningcss-android-arm64@1.32.0: - resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [android] - - lightningcss-darwin-arm64@1.32.0: - resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [darwin] - - lightningcss-darwin-x64@1.32.0: - resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [darwin] - - lightningcss-freebsd-x64@1.32.0: - resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [freebsd] - - lightningcss-linux-arm-gnueabihf@1.32.0: - resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} - engines: {node: '>= 12.0.0'} - cpu: [arm] - os: [linux] - - lightningcss-linux-arm64-gnu@1.32.0: - resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [linux] - - lightningcss-linux-arm64-musl@1.32.0: - resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [linux] - - lightningcss-linux-x64-gnu@1.32.0: - resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [linux] - - lightningcss-linux-x64-musl@1.32.0: - resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [linux] - - lightningcss-win32-arm64-msvc@1.32.0: - resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} - engines: {node: '>= 12.0.0'} - cpu: [arm64] - os: [win32] - - lightningcss-win32-x64-msvc@1.32.0: - resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} - engines: {node: '>= 12.0.0'} - cpu: [x64] - os: [win32] - - lightningcss@1.32.0: - resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} - engines: {node: '>= 12.0.0'} - - lilconfig@3.1.3: - resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} - engines: {node: '>=14'} - - lines-and-columns@1.2.4: - resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - - load-tsconfig@0.2.5: - resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - - magic-string@0.30.21: - resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - - mlly@1.8.0: - resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - - mz@2.7.0: - resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} - - nanoid@3.3.12: - resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - - object-assign@4.1.1: - resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} - engines: {node: '>=0.10.0'} - - obug@2.1.2: - resolution: {integrity: sha512-AWGB9WFcRXOQs48Z/udjI5ZcZMHXwX8XPByNpOydgcGsDLIzjGizhoMWJyKAWze7AVW/2W1i+/gPX4YtKe5cyg==} - engines: {node: '>=12.20.0'} - - pathe@2.0.3: - resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - - picocolors@1.1.1: - resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - - picomatch@4.0.4: - resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} - engines: {node: '>=12'} - - pirates@4.0.7: - resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} - engines: {node: '>= 6'} - - pkg-types@1.3.1: - resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} - - playwright-core@1.60.0: - resolution: {integrity: sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==} - engines: {node: '>=18'} - hasBin: true - - playwright@1.60.0: - resolution: {integrity: sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==} - engines: {node: '>=18'} - hasBin: true - - postcss-load-config@6.0.1: - resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} - engines: {node: '>= 18'} - peerDependencies: - jiti: '>=1.21.0' - postcss: '>=8.5.10' - tsx: ^4.8.1 - yaml: ^2.4.2 - peerDependenciesMeta: - jiti: - optional: true - postcss: - optional: true - tsx: - optional: true - yaml: - optional: true - - postcss@8.5.15: - resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} - engines: {node: ^10 || ^12 || >=14} - - readdirp@4.1.2: - resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} - engines: {node: '>= 14.18.0'} - - resolve-from@5.0.0: - resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} - engines: {node: '>=8'} - - rollup@4.60.4: - resolution: {integrity: sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - - siginfo@2.0.0: - resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} - - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} - engines: {node: '>=0.10.0'} - - source-map@0.7.6: - resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} - engines: {node: '>= 12'} - - stackback@0.0.2: - resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - - std-env@4.1.0: - resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} - - sucrase@3.35.1: - resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} - engines: {node: '>=16 || 14 >=14.17'} - hasBin: true - - thenify-all@1.6.0: - resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} - engines: {node: '>=0.8'} - - thenify@3.3.1: - resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} - - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} - - tinyexec@0.3.2: - resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} - - tinyexec@1.2.4: - resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} - engines: {node: '>=18'} - - tinyglobby@0.2.15: - resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} - engines: {node: '>=12.0.0'} - - tinyglobby@0.2.16: - resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} - engines: {node: '>=12.0.0'} - - tinyrainbow@3.1.0: - resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} - engines: {node: '>=14.0.0'} - - tree-kill@1.2.2: - resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} - hasBin: true - - ts-interface-checker@0.1.13: - resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} - - tsup@8.5.1: - resolution: {integrity: sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==} - engines: {node: '>=18'} - hasBin: true - peerDependencies: - '@microsoft/api-extractor': ^7.36.0 - '@swc/core': ^1 - postcss: '>=8.5.10' - typescript: '>=4.5.0' - peerDependenciesMeta: - '@microsoft/api-extractor': - optional: true - '@swc/core': - optional: true - postcss: - optional: true - typescript: - optional: true - - typescript@5.9.3: - resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} - engines: {node: '>=14.17'} - hasBin: true - - ufo@1.6.1: - resolution: {integrity: sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==} - - undici-types@6.21.0: - resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - - vite@6.4.2: - resolution: {integrity: sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} - hasBin: true - peerDependencies: - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 - jiti: '>=1.21.0' - less: '*' - lightningcss: ^1.21.0 - sass: '*' - sass-embedded: '*' - stylus: '*' - sugarss: '*' - terser: ^5.16.0 - tsx: ^4.8.1 - yaml: ^2.4.2 - peerDependenciesMeta: - '@types/node': - optional: true - jiti: - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - sass-embedded: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - tsx: - optional: true - yaml: - optional: true - - vitest@4.1.8: - resolution: {integrity: sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==} - engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} - hasBin: true - peerDependencies: - '@edge-runtime/vm': '*' - '@opentelemetry/api': ^1.9.0 - '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.8 - '@vitest/browser-preview': 4.1.8 - '@vitest/browser-webdriverio': 4.1.8 - '@vitest/coverage-istanbul': 4.1.8 - '@vitest/coverage-v8': 4.1.8 - '@vitest/ui': 4.1.8 - happy-dom: '*' - jsdom: '*' - vite: ^6.4.2 - peerDependenciesMeta: - '@edge-runtime/vm': - optional: true - '@opentelemetry/api': - optional: true - '@types/node': - optional: true - '@vitest/browser-playwright': - optional: true - '@vitest/browser-preview': - optional: true - '@vitest/browser-webdriverio': - optional: true - '@vitest/coverage-istanbul': - optional: true - '@vitest/coverage-v8': - optional: true - '@vitest/ui': - optional: true - happy-dom: - optional: true - jsdom: - optional: true - - why-is-node-running@2.3.0: - resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} - engines: {node: '>=8'} - hasBin: true - -snapshots: - - '@biomejs/biome@1.9.4': - optionalDependencies: - '@biomejs/cli-darwin-arm64': 1.9.4 - '@biomejs/cli-darwin-x64': 1.9.4 - '@biomejs/cli-linux-arm64': 1.9.4 - '@biomejs/cli-linux-arm64-musl': 1.9.4 - '@biomejs/cli-linux-x64': 1.9.4 - '@biomejs/cli-linux-x64-musl': 1.9.4 - '@biomejs/cli-win32-arm64': 1.9.4 - '@biomejs/cli-win32-x64': 1.9.4 - - '@biomejs/cli-darwin-arm64@1.9.4': - optional: true - - '@biomejs/cli-darwin-x64@1.9.4': - optional: true - - '@biomejs/cli-linux-arm64-musl@1.9.4': - optional: true - - '@biomejs/cli-linux-arm64@1.9.4': - optional: true - - '@biomejs/cli-linux-x64-musl@1.9.4': - optional: true - - '@biomejs/cli-linux-x64@1.9.4': - optional: true - - '@biomejs/cli-win32-arm64@1.9.4': - optional: true - - '@biomejs/cli-win32-x64@1.9.4': - optional: true - - '@esbuild/aix-ppc64@0.27.2': - optional: true - - '@esbuild/android-arm64@0.27.2': - optional: true - - '@esbuild/android-arm@0.27.2': - optional: true - - '@esbuild/android-x64@0.27.2': - optional: true - - '@esbuild/darwin-arm64@0.27.2': - optional: true - - '@esbuild/darwin-x64@0.27.2': - optional: true - - '@esbuild/freebsd-arm64@0.27.2': - optional: true - - '@esbuild/freebsd-x64@0.27.2': - optional: true - - '@esbuild/linux-arm64@0.27.2': - optional: true - - '@esbuild/linux-arm@0.27.2': - optional: true - - '@esbuild/linux-ia32@0.27.2': - optional: true - - '@esbuild/linux-loong64@0.27.2': - optional: true - - '@esbuild/linux-mips64el@0.27.2': - optional: true - - '@esbuild/linux-ppc64@0.27.2': - optional: true - - '@esbuild/linux-riscv64@0.27.2': - optional: true - - '@esbuild/linux-s390x@0.27.2': - optional: true - - '@esbuild/linux-x64@0.27.2': - optional: true - - '@esbuild/netbsd-arm64@0.27.2': - optional: true - - '@esbuild/netbsd-x64@0.27.2': - optional: true - - '@esbuild/openbsd-arm64@0.27.2': - optional: true - - '@esbuild/openbsd-x64@0.27.2': - optional: true - - '@esbuild/openharmony-arm64@0.27.2': - optional: true - - '@esbuild/sunos-x64@0.27.2': - optional: true - - '@esbuild/win32-arm64@0.27.2': - optional: true - - '@esbuild/win32-ia32@0.27.2': - optional: true - - '@esbuild/win32-x64@0.27.2': - optional: true - - '@jridgewell/gen-mapping@0.3.13': - dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - '@jridgewell/trace-mapping': 0.3.31 - - '@jridgewell/resolve-uri@3.1.2': {} - - '@jridgewell/sourcemap-codec@1.5.5': {} - - '@jridgewell/trace-mapping@0.3.31': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - - '@msgpack/msgpack@3.1.3': {} - - '@playwright/test@1.60.0': - dependencies: - playwright: 1.60.0 - - '@rollup/rollup-android-arm-eabi@4.60.4': - optional: true - - '@rollup/rollup-android-arm64@4.60.4': - optional: true - - '@rollup/rollup-darwin-arm64@4.60.4': - optional: true - - '@rollup/rollup-darwin-x64@4.60.4': - optional: true - - '@rollup/rollup-freebsd-arm64@4.60.4': - optional: true - - '@rollup/rollup-freebsd-x64@4.60.4': - optional: true - - '@rollup/rollup-linux-arm-gnueabihf@4.60.4': - optional: true - - '@rollup/rollup-linux-arm-musleabihf@4.60.4': - optional: true - - '@rollup/rollup-linux-arm64-gnu@4.60.4': - optional: true - - '@rollup/rollup-linux-arm64-musl@4.60.4': - optional: true - - '@rollup/rollup-linux-loong64-gnu@4.60.4': - optional: true - - '@rollup/rollup-linux-loong64-musl@4.60.4': - optional: true - - '@rollup/rollup-linux-ppc64-gnu@4.60.4': - optional: true - - '@rollup/rollup-linux-ppc64-musl@4.60.4': - optional: true - - '@rollup/rollup-linux-riscv64-gnu@4.60.4': - optional: true - - '@rollup/rollup-linux-riscv64-musl@4.60.4': - optional: true - - '@rollup/rollup-linux-s390x-gnu@4.60.4': - optional: true - - '@rollup/rollup-linux-x64-gnu@4.60.4': - optional: true - - '@rollup/rollup-linux-x64-musl@4.60.4': - optional: true - - '@rollup/rollup-openbsd-x64@4.60.4': - optional: true - - '@rollup/rollup-openharmony-arm64@4.60.4': - optional: true - - '@rollup/rollup-win32-arm64-msvc@4.60.4': - optional: true - - '@rollup/rollup-win32-ia32-msvc@4.60.4': - optional: true - - '@rollup/rollup-win32-x64-gnu@4.60.4': - optional: true - - '@rollup/rollup-win32-x64-msvc@4.60.4': - optional: true - - '@standard-schema/spec@1.1.0': {} - - '@types/chai@5.2.3': - dependencies: - '@types/deep-eql': 4.0.2 - assertion-error: 2.0.1 - - '@types/deep-eql@4.0.2': {} - - '@types/estree@1.0.8': {} - - '@types/node@22.19.19': - dependencies: - undici-types: 6.21.0 - - '@vitest/expect@4.1.8': - dependencies: - '@standard-schema/spec': 1.1.0 - '@types/chai': 5.2.3 - '@vitest/spy': 4.1.8 - '@vitest/utils': 4.1.8 - chai: 6.2.2 - tinyrainbow: 3.1.0 - - '@vitest/mocker@4.1.8(vite@6.4.2(@types/node@22.19.19)(lightningcss@1.32.0))': - dependencies: - '@vitest/spy': 4.1.8 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 6.4.2(@types/node@22.19.19)(lightningcss@1.32.0) - - '@vitest/pretty-format@4.1.8': - dependencies: - tinyrainbow: 3.1.0 - - '@vitest/runner@4.1.8': - dependencies: - '@vitest/utils': 4.1.8 - pathe: 2.0.3 - - '@vitest/snapshot@4.1.8': - dependencies: - '@vitest/pretty-format': 4.1.8 - '@vitest/utils': 4.1.8 - magic-string: 0.30.21 - pathe: 2.0.3 - - '@vitest/spy@4.1.8': {} - - '@vitest/utils@4.1.8': - dependencies: - '@vitest/pretty-format': 4.1.8 - convert-source-map: 2.0.0 - tinyrainbow: 3.1.0 - - acorn@8.15.0: {} - - any-promise@1.3.0: {} - - assertion-error@2.0.1: {} - - bundle-require@5.1.0(esbuild@0.27.2): - dependencies: - esbuild: 0.27.2 - load-tsconfig: 0.2.5 - - cac@6.7.14: {} - - chai@6.2.2: {} - - chokidar@4.0.3: - dependencies: - readdirp: 4.1.2 - - commander@4.1.1: {} - - confbox@0.1.8: {} - - consola@3.4.2: {} - - convert-source-map@2.0.0: {} - - debug@4.4.3: - dependencies: - ms: 2.1.3 - - detect-libc@2.1.2: - optional: true - - es-module-lexer@2.1.0: {} - - esbuild@0.27.2: - optionalDependencies: - '@esbuild/aix-ppc64': 0.27.2 - '@esbuild/android-arm': 0.27.2 - '@esbuild/android-arm64': 0.27.2 - '@esbuild/android-x64': 0.27.2 - '@esbuild/darwin-arm64': 0.27.2 - '@esbuild/darwin-x64': 0.27.2 - '@esbuild/freebsd-arm64': 0.27.2 - '@esbuild/freebsd-x64': 0.27.2 - '@esbuild/linux-arm': 0.27.2 - '@esbuild/linux-arm64': 0.27.2 - '@esbuild/linux-ia32': 0.27.2 - '@esbuild/linux-loong64': 0.27.2 - '@esbuild/linux-mips64el': 0.27.2 - '@esbuild/linux-ppc64': 0.27.2 - '@esbuild/linux-riscv64': 0.27.2 - '@esbuild/linux-s390x': 0.27.2 - '@esbuild/linux-x64': 0.27.2 - '@esbuild/netbsd-arm64': 0.27.2 - '@esbuild/netbsd-x64': 0.27.2 - '@esbuild/openbsd-arm64': 0.27.2 - '@esbuild/openbsd-x64': 0.27.2 - '@esbuild/openharmony-arm64': 0.27.2 - '@esbuild/sunos-x64': 0.27.2 - '@esbuild/win32-arm64': 0.27.2 - '@esbuild/win32-ia32': 0.27.2 - '@esbuild/win32-x64': 0.27.2 - - estree-walker@3.0.3: - dependencies: - '@types/estree': 1.0.8 - - expect-type@1.3.0: {} - - fdir@6.5.0(picomatch@4.0.4): - optionalDependencies: - picomatch: 4.0.4 - - fix-dts-default-cjs-exports@1.0.1: - dependencies: - magic-string: 0.30.21 - mlly: 1.8.0 - rollup: 4.60.4 - - fsevents@2.3.2: - optional: true - - fsevents@2.3.3: - optional: true - - joycon@3.1.1: {} - - lightningcss-android-arm64@1.32.0: - optional: true - - lightningcss-darwin-arm64@1.32.0: - optional: true - - lightningcss-darwin-x64@1.32.0: - optional: true - - lightningcss-freebsd-x64@1.32.0: - optional: true - - lightningcss-linux-arm-gnueabihf@1.32.0: - optional: true - - lightningcss-linux-arm64-gnu@1.32.0: - optional: true - - lightningcss-linux-arm64-musl@1.32.0: - optional: true - - lightningcss-linux-x64-gnu@1.32.0: - optional: true - - lightningcss-linux-x64-musl@1.32.0: - optional: true - - lightningcss-win32-arm64-msvc@1.32.0: - optional: true - - lightningcss-win32-x64-msvc@1.32.0: - optional: true - - lightningcss@1.32.0: - dependencies: - detect-libc: 2.1.2 - optionalDependencies: - lightningcss-android-arm64: 1.32.0 - lightningcss-darwin-arm64: 1.32.0 - lightningcss-darwin-x64: 1.32.0 - lightningcss-freebsd-x64: 1.32.0 - lightningcss-linux-arm-gnueabihf: 1.32.0 - lightningcss-linux-arm64-gnu: 1.32.0 - lightningcss-linux-arm64-musl: 1.32.0 - lightningcss-linux-x64-gnu: 1.32.0 - lightningcss-linux-x64-musl: 1.32.0 - lightningcss-win32-arm64-msvc: 1.32.0 - lightningcss-win32-x64-msvc: 1.32.0 - optional: true - - lilconfig@3.1.3: {} - - lines-and-columns@1.2.4: {} - - load-tsconfig@0.2.5: {} - - magic-string@0.30.21: - dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - - mlly@1.8.0: - dependencies: - acorn: 8.15.0 - pathe: 2.0.3 - pkg-types: 1.3.1 - ufo: 1.6.1 - - ms@2.1.3: {} - - mz@2.7.0: - dependencies: - any-promise: 1.3.0 - object-assign: 4.1.1 - thenify-all: 1.6.0 - - nanoid@3.3.12: {} - - object-assign@4.1.1: {} - - obug@2.1.2: {} - - pathe@2.0.3: {} - - picocolors@1.1.1: {} - - picomatch@4.0.4: {} - - pirates@4.0.7: {} - - pkg-types@1.3.1: - dependencies: - confbox: 0.1.8 - mlly: 1.8.0 - pathe: 2.0.3 - - playwright-core@1.60.0: {} - - playwright@1.60.0: - dependencies: - playwright-core: 1.60.0 - optionalDependencies: - fsevents: 2.3.2 - - postcss-load-config@6.0.1(postcss@8.5.15): - dependencies: - lilconfig: 3.1.3 - optionalDependencies: - postcss: 8.5.15 - - postcss@8.5.15: - dependencies: - nanoid: 3.3.12 - picocolors: 1.1.1 - source-map-js: 1.2.1 - - readdirp@4.1.2: {} - - resolve-from@5.0.0: {} - - rollup@4.60.4: - dependencies: - '@types/estree': 1.0.8 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.60.4 - '@rollup/rollup-android-arm64': 4.60.4 - '@rollup/rollup-darwin-arm64': 4.60.4 - '@rollup/rollup-darwin-x64': 4.60.4 - '@rollup/rollup-freebsd-arm64': 4.60.4 - '@rollup/rollup-freebsd-x64': 4.60.4 - '@rollup/rollup-linux-arm-gnueabihf': 4.60.4 - '@rollup/rollup-linux-arm-musleabihf': 4.60.4 - '@rollup/rollup-linux-arm64-gnu': 4.60.4 - '@rollup/rollup-linux-arm64-musl': 4.60.4 - '@rollup/rollup-linux-loong64-gnu': 4.60.4 - '@rollup/rollup-linux-loong64-musl': 4.60.4 - '@rollup/rollup-linux-ppc64-gnu': 4.60.4 - '@rollup/rollup-linux-ppc64-musl': 4.60.4 - '@rollup/rollup-linux-riscv64-gnu': 4.60.4 - '@rollup/rollup-linux-riscv64-musl': 4.60.4 - '@rollup/rollup-linux-s390x-gnu': 4.60.4 - '@rollup/rollup-linux-x64-gnu': 4.60.4 - '@rollup/rollup-linux-x64-musl': 4.60.4 - '@rollup/rollup-openbsd-x64': 4.60.4 - '@rollup/rollup-openharmony-arm64': 4.60.4 - '@rollup/rollup-win32-arm64-msvc': 4.60.4 - '@rollup/rollup-win32-ia32-msvc': 4.60.4 - '@rollup/rollup-win32-x64-gnu': 4.60.4 - '@rollup/rollup-win32-x64-msvc': 4.60.4 - fsevents: 2.3.3 - - siginfo@2.0.0: {} - - source-map-js@1.2.1: {} - - source-map@0.7.6: {} - - stackback@0.0.2: {} - - std-env@4.1.0: {} - - sucrase@3.35.1: - dependencies: - '@jridgewell/gen-mapping': 0.3.13 - commander: 4.1.1 - lines-and-columns: 1.2.4 - mz: 2.7.0 - pirates: 4.0.7 - tinyglobby: 0.2.15 - ts-interface-checker: 0.1.13 - - thenify-all@1.6.0: - dependencies: - thenify: 3.3.1 - - thenify@3.3.1: - dependencies: - any-promise: 1.3.0 - - tinybench@2.9.0: {} - - tinyexec@0.3.2: {} - - tinyexec@1.2.4: {} - - tinyglobby@0.2.15: - dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - - tinyglobby@0.2.16: - dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - - tinyrainbow@3.1.0: {} - - tree-kill@1.2.2: {} - - ts-interface-checker@0.1.13: {} - - tsup@8.5.1(postcss@8.5.15)(typescript@5.9.3): - dependencies: - bundle-require: 5.1.0(esbuild@0.27.2) - cac: 6.7.14 - chokidar: 4.0.3 - consola: 3.4.2 - debug: 4.4.3 - esbuild: 0.27.2 - fix-dts-default-cjs-exports: 1.0.1 - joycon: 3.1.1 - picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.15) - resolve-from: 5.0.0 - rollup: 4.60.4 - source-map: 0.7.6 - sucrase: 3.35.1 - tinyexec: 0.3.2 - tinyglobby: 0.2.15 - tree-kill: 1.2.2 - optionalDependencies: - postcss: 8.5.15 - typescript: 5.9.3 - transitivePeerDependencies: - - jiti - - supports-color - - tsx - - yaml - - typescript@5.9.3: {} - - ufo@1.6.1: {} - - undici-types@6.21.0: {} - - vite@6.4.2(@types/node@22.19.19)(lightningcss@1.32.0): - dependencies: - esbuild: 0.27.2 - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - postcss: 8.5.15 - rollup: 4.60.4 - tinyglobby: 0.2.16 - optionalDependencies: - '@types/node': 22.19.19 - fsevents: 2.3.3 - lightningcss: 1.32.0 - - vitest@4.1.8(@types/node@22.19.19)(vite@6.4.2(@types/node@22.19.19)(lightningcss@1.32.0)): - dependencies: - '@vitest/expect': 4.1.8 - '@vitest/mocker': 4.1.8(vite@6.4.2(@types/node@22.19.19)(lightningcss@1.32.0)) - '@vitest/pretty-format': 4.1.8 - '@vitest/runner': 4.1.8 - '@vitest/snapshot': 4.1.8 - '@vitest/spy': 4.1.8 - '@vitest/utils': 4.1.8 - es-module-lexer: 2.1.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.2 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.1.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 - tinyglobby: 0.2.16 - tinyrainbow: 3.1.0 - vite: 6.4.2(@types/node@22.19.19)(lightningcss@1.32.0) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/node': 22.19.19 - transitivePeerDependencies: - - msw - - why-is-node-running@2.3.0: - dependencies: - siginfo: 2.0.0 - stackback: 0.0.2 diff --git a/packages/sie_ts_sdk/src/encoding.ts b/packages/sie_ts_sdk/src/encoding.ts index 988b32fcd..60f4f6891 100644 --- a/packages/sie_ts_sdk/src/encoding.ts +++ b/packages/sie_ts_sdk/src/encoding.ts @@ -76,8 +76,11 @@ export function sparseEmbeddingMap(result: EncodeResult): Map { const indices = toNumberArray(sparse.indices); const values = toNumberArray(sparse.values); const map = new Map(); - for (let i = 0; i < indices.length; i++) { - map.set(indices[i]!, values[i]!); + for (const [index, tokenIndex] of indices.entries()) { + const weight = values[index]; + if (weight !== undefined) { + map.set(tokenIndex, weight); + } } return map; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96aafd6da..14f12f4e7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,8 +8,11 @@ overrides: esbuild: '>=0.25.0' form-data: '>=4.0.4' langsmith: '>=0.6.0' + picomatch: '>=4.0.4' + postcss: '>=8.5.10' protobufjs: '>=7.5.8' qs: '>=6.15.2' + rollup: '>=4.59.0' tar: '>=7.5.11' uuid: '>=11.1.1' vite: ^6.4.2 @@ -875,6 +878,7 @@ packages: '@lancedb/lancedb@0.17.0': resolution: {integrity: sha512-AgQ4dqHBjoEpVdND5tmzhy0OJxwP4gJnEjOz3jVh7ip+8EYhR44wO6XbEhpxvvGfXyyepDrulo3bpL1323eNSg==} engines: {node: '>= 18'} + cpu: [x64, arm64] os: [darwin, linux, win32] peerDependencies: apache-arrow: '>=15.0.0 <=18.1.0' @@ -1412,6 +1416,7 @@ packages: '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version '@xterm/xterm@5.5.0': resolution: {integrity: sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==} @@ -1419,6 +1424,7 @@ packages: '@zenfs/core@1.11.4': resolution: {integrity: sha512-jFihhedKitw1X9rlImtaca4KTs1Xk9awo4GxUemqt2ucE1jLi10lyRfFFbiYXZsFDkuCWGHwEQwv7BeDIEXMbQ==} engines: {node: '>= 18'} + deprecated: v1 is no longer supported. Please update ZenFS (or whatever ZenFS dependent you depend on). hasBin: true '@zilliz/milvus2-sdk-node@2.6.15': @@ -1991,7 +1997,7 @@ packages: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} peerDependencies: - picomatch: ^3 || ^4 + picomatch: '>=4.0.4' peerDependenciesMeta: picomatch: optional: true @@ -2832,7 +2838,7 @@ packages: engines: {node: '>= 18'} peerDependencies: jiti: '>=1.21.0' - postcss: '>=8.0.9' + postcss: '>=8.5.10' tsx: ^4.8.1 yaml: ^2.4.2 peerDependenciesMeta: @@ -3237,7 +3243,7 @@ packages: peerDependencies: '@microsoft/api-extractor': ^7.36.0 '@swc/core': ^1 - postcss: ^8.4.12 + postcss: '>=8.5.10' typescript: '>=4.5.0' peerDependenciesMeta: '@microsoft/api-extractor': diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 000000000..db5c88cf4 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "always-update": true, + "bootstrap-sha": "60996d9c30168e0f8e85b680295f147fdee87f61", + "packages": { + ".": { + "release-type": "simple", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md", + "extra-files": [ + { "type": "toml", "path": "packages/sie_sdk/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "packages/sie_server/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "packages/sie_config/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "packages/sie_mcp/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_langchain/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_llamaindex/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_haystack/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_dspy/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_crewai/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_chroma/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_lancedb/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_qdrant/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "toml", "path": "integrations/sie_weaviate/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "json", "path": "packages/sie_ts_sdk/package.json", "jsonpath": "$.version" }, + { "type": "json", "path": "integrations/sie_ts_chroma/package.json", "jsonpath": "$.version" }, + { "type": "json", "path": "integrations/sie_ts_langchain/package.json", "jsonpath": "$.version" }, + { "type": "json", "path": "integrations/sie_ts_llamaindex/package.json", "jsonpath": "$.version" }, + { "type": "json", "path": "integrations/sie_ts_lancedb/package.json", "jsonpath": "$.version" }, + { "type": "generic", "path": "packages/sie_ts_sdk/src/version.ts" }, + { "type": "toml", "path": "packages/sie_gateway/Cargo.toml", "jsonpath": "$.package.version" }, + { "type": "toml", "path": "packages/sie_server_sidecar/Cargo.toml", "jsonpath": "$.package.version" }, + { "type": "toml", "path": "packages/sie_audio_prep/Cargo.toml", "jsonpath": "$.package.version" }, + { "type": "toml", "path": "packages/sie_audio_prep/pyproject.toml", "jsonpath": "$.project.version" }, + { "type": "generic", "path": "packages/sie_audio_prep/build_wheel.py" }, + { "type": "generic", "path": "deploy/helm/sie-cluster/Chart.yaml" } + ] + } + } +} diff --git a/telemetry/README.md b/telemetry/README.md index b39de7b25..e1e9a926b 100644 --- a/telemetry/README.md +++ b/telemetry/README.md @@ -204,8 +204,8 @@ process lifecycles. The semantic methods and their emitted instruments are nevertheless governed by the one contract: - `sie_gateway` owns HTTP completion, admission, KEDA capacity state, and the - request span/log boundary. `sie_cloud/gateway` calls the gateway facade for - the final i6pn-or-Modal dispatch result. + request span/log boundary. Downstream deployments can reuse that facade for + their final dispatch result. - The Modal dispatcher owns actual substrate invocation attempts. - `sie_config` owns config HTTP and authoritative state changes. - `sie_server_sidecar` owns realtime queueing and batch formation plus its @@ -596,7 +596,7 @@ CI should reject a change unless it proves all of the following: fields, events, linked spans, trace-state and status text while the local Tempo branch remains unchanged; 9. median-of-three warmed telemetry-off/on benchmarks cover the gateway - facade/Tower path, the managed cloud-gateway final-dispatch wrapper, Python + facade/Tower path, the downstream final-dispatch facade integration, Python and Rust workers, config, sidecar, and dispatcher hot paths; a paired durability-disabled/enabled benchmark separately gates the gateway dispatch-durability lifecycle before the change is declared ready; diff --git a/telemetry/contract.yaml b/telemetry/contract.yaml index 98e4b0e88..643169dbb 100644 --- a/telemetry/contract.yaml +++ b/telemetry/contract.yaml @@ -470,12 +470,10 @@ services: managed_collector: regional runtimes: - packages/sie_gateway - - packages/sie_cloud/gateway dispatcher: service_name: sie-dispatcher managed_collector: regional - runtimes: - - packages/sie_cloud/dispatcher + runtimes: [] config: service_name: sie-config managed_collector: regional diff --git a/tests/parity/README.md b/tests/parity/README.md new file mode 100644 index 000000000..4bd3f4f5e --- /dev/null +++ b/tests/parity/README.md @@ -0,0 +1,49 @@ +# RunBatch IPC Fixtures + +This directory holds JSON fixtures for the `RunBatch` IPC contract between the +`worker-sidecar` container (`sie-server-sidecar` binary) and the Python +`sie_server` adapter process. + +The fixtures pin the small set of fields that must stay stable across +implementations: + +- work item and request identifiers +- item indexes and outcome ordering +- publish/ACK/NAK disposition +- error codes +- LoRA routing expectations + +Timing fields and raw inference bytes are intentionally excluded from the +canonical comparison. Each fixture lists those elided fields under +`notes.elided_fields`. + +## Running + +```bash +tests/parity/run_parity.sh +tests/parity/run_parity.sh -v +``` + +Today this script runs the Python-side fixture consumer. The same JSON shape is +kept language-neutral so Rust-side coverage can consume it without rewriting the +fixtures. + +## Fixtures + +| Fixture | Pins | +| --- | --- | +| `run_batch_empty.json` | Empty batch returns an empty outcome without handler calls | +| `run_batch_encode_no_lora.json` | Encode base path, empty `lora_key` | +| `run_batch_encode_lora.json` | Encode LoRA plumbing plus invalid-before-valid ordering | +| `run_batch_extract_lora.json` | Extract LoRA plumbing plus invalid-before-valid ordering | +| `run_batch_mixed_op.json` | Mixed op rejection | +| `run_batch_score_basic.json` | Score dispatch path | +| `run_batch_score_lora_warns.json` | Score drops non-empty `lora_key` and serves base | +| `run_batch_unknown_op.json` | Unknown op rejection | + +## Adding A Fixture + +1. Add a `run_batch_*.json` file here. +2. Add the filename to `PARITY_FIXTURES` in + `packages/sie_server/tests/test_parity_run_batch.py`. +3. Run `tests/parity/run_parity.sh`. diff --git a/tests/parity/run_batch_empty.json b/tests/parity/run_batch_empty.json new file mode 100644 index 000000000..5937610ea --- /dev/null +++ b/tests/parity/run_batch_empty.json @@ -0,0 +1,15 @@ +{ + "description": "RunBatch with items=[]. The sidecar scheduler is constructed to never flush zero-item batches, so this path indicates a protocol drift on the sidecar side. Both backends must (1) log a WARN, (2) return BatchOutcome with outcomes=[], (3) NOT call any process_*_batch method, (4) NOT propagate an error to the wire (an empty outcome is benign — the sidecar's publisher treats it as a no-op).", + "input": { + "model_id": "test/parity-empty", + "batch_id": 300, + "lora_key": "", + "total_cost": 0, + "items": [] + }, + "expected_canonical_outcomes": [], + "notes": { + "elided_fields": [], + "rationale": "An empty outcomes array is the only correct response. A NAK would loop (the empty batch will keep redelivering); a non-empty response would invent identifiers the publisher has no work-item to correlate with. Both backends must agree that empty-in == empty-out." + } +} diff --git a/tests/parity/run_batch_encode_lora.json b/tests/parity/run_batch_encode_lora.json new file mode 100644 index 000000000..ca55c6e79 --- /dev/null +++ b/tests/parity/run_batch_encode_lora.json @@ -0,0 +1,78 @@ +{ + "description": "RunBatch with op=encode, two valid items + one invalid (encode payload missing) + a non-empty lora_key. Pins Python adapter_call_loop invariants for the sie-worker sidecar contract: invalid-before-valid ordering, preserved identity for invalid payloads, and lora_key plumbing into per-item options['lora'].", + "input": { + "model_id": "test/parity-encode", + "batch_id": 42, + "lora_key": "shared-lora", + "total_cost": 100, + "items": [ + { + "op": "encode", + "encode": { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "total_items": 3, + "timestamp": 0.0, + "item": {"text": "hello"}, + "options": null + } + }, + { + "op": "encode", + "work_item_id": "w1", + "request_id": "r1", + "item_index": 1, + "encode": null + }, + { + "op": "encode", + "encode": { + "work_item_id": "w2", + "request_id": "r2", + "item_index": 2, + "total_items": 3, + "timestamp": 0.0, + "item": {"text": "world"}, + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w1", + "request_id": "r1", + "item_index": 1, + "disposition": "publish_error_and_ack", + "error_code": "run_batch_invalid_item" + }, + { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "disposition": "publish_and_ack", + "error_code": null + }, + { + "work_item_id": "w2", + "request_id": "r2", + "item_index": 2, + "disposition": "publish_and_ack", + "error_code": null + } + ], + "expected_lora_in_options": "shared-lora", + "notes": { + "elided_fields": [ + "inference_ms", + "tokenization_ms", + "postprocessing_ms", + "result_msgpack", + "raw_output", + "error", + "nak_delay_ms" + ], + "rationale": "inference_ms / tokenization_ms / postprocessing_ms are wall-clock dependent. result_msgpack / raw_output are backend-specific (different model weights produce different bytes). error text is human-readable and may legitimately differ. nak_delay_ms is set only on retry paths which this fixture doesn't exercise." + } +} diff --git a/tests/parity/run_batch_encode_no_lora.json b/tests/parity/run_batch_encode_no_lora.json new file mode 100644 index 000000000..d4907b1d1 --- /dev/null +++ b/tests/parity/run_batch_encode_no_lora.json @@ -0,0 +1,64 @@ +{ + "description": "RunBatch with op=encode, all valid items, lora_key=\"\" (the empty-string base alias). Pins the Python adapter_call_loop base-model fast path: NO injection into options['lora']. Empty lora_key must be byte-identical to a base-only deploy that never set lora_key in the first place.", + "input": { + "model_id": "test/parity-encode-base", + "batch_id": 7, + "lora_key": "", + "total_cost": 50, + "items": [ + { + "op": "encode", + "encode": { + "work_item_id": "w0", + "request_id": "r", + "item_index": 0, + "total_items": 2, + "timestamp": 0.0, + "item": {"text": "alpha"}, + "options": null + } + }, + { + "op": "encode", + "encode": { + "work_item_id": "w1", + "request_id": "r", + "item_index": 1, + "total_items": 2, + "timestamp": 0.0, + "item": {"text": "beta"}, + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w0", + "request_id": "r", + "item_index": 0, + "disposition": "publish_and_ack", + "error_code": null + }, + { + "work_item_id": "w1", + "request_id": "r", + "item_index": 1, + "disposition": "publish_and_ack", + "error_code": null + } + ], + "expected_lora_in_options": null, + "notes": { + "elided_fields": [ + "inference_ms", + "tokenization_ms", + "postprocessing_ms", + "result_msgpack", + "raw_output", + "error", + "nak_delay_ms" + ], + "rationale": "expected_lora_in_options=null means the per-item assertion in the echo executor checks the COMPLEMENT: items must arrive WITHOUT options['lora'] set. The base path is required to be allocation-free + indistinguishable from a non-LoRA deploy." + } +} diff --git a/tests/parity/run_batch_extract_lora.json b/tests/parity/run_batch_extract_lora.json new file mode 100644 index 000000000..df8aa2261 --- /dev/null +++ b/tests/parity/run_batch_extract_lora.json @@ -0,0 +1,52 @@ +{ + "description": "RunBatch with op=extract + non-empty lora_key + one valid item + one invalid (extract payload missing). Pins Python adapter_call_loop invariants for the sie-worker sidecar contract: lora_key injection into options['lora'], invalid-before-valid ordering, and preserved identity for invalid payloads.", + "input": { + "model_id": "test/parity-extract", + "batch_id": 500, + "lora_key": "extract-lora-v1", + "total_cost": 20, + "items": [ + { + "op": "extract", + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "extract": null + }, + { + "op": "extract", + "extract": { + "work_item_id": "w1", + "request_id": "r1", + "item_index": 1, + "total_items": 2, + "timestamp": 0.0, + "item": {"text": "Acme Corp filed in 2023."}, + "labels": ["organization", "year"], + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "disposition": "publish_error_and_ack", + "error_code": "run_batch_invalid_item" + }, + { + "work_item_id": "w1", + "request_id": "r1", + "item_index": 1, + "disposition": "publish_and_ack", + "error_code": null + } + ], + "expected_lora_in_options": "extract-lora-v1", + "notes": { + "elided_fields": ["inference_ms", "tokenization_ms", "postprocessing_ms", "result_msgpack", "raw_output", "error", "nak_delay_ms"], + "rationale": "expected_lora_in_options=\"extract-lora-v1\" pins the Python-side options['lora'] injection. The invalid-item appears at item_index=0 and keeps its identity so the Rust dispatcher can publish and ACK the original work item." + } +} diff --git a/tests/parity/run_batch_mixed_op.json b/tests/parity/run_batch_mixed_op.json new file mode 100644 index 000000000..5c7710603 --- /dev/null +++ b/tests/parity/run_batch_mixed_op.json @@ -0,0 +1,56 @@ +{ + "description": "RunBatch with mixed-op items (encode + score) — the sidecar scheduler MUST emit homogeneous-op batches; this fixture is defence-in-depth that pins Python adapter_call_loop._reject_all: (1) emit one publish_error_and_ack per input item, (2) tag each with error_code=\"run_batch_mixed_op\", (3) preserve the original payload item_index, and (4) extract (work_item_id, request_id) from whichever payload variant is populated.", + "input": { + "model_id": "test/parity-mixed", + "batch_id": 100, + "lora_key": "", + "total_cost": 30, + "items": [ + { + "op": "encode", + "encode": { + "work_item_id": "w-enc", + "request_id": "r-enc", + "item_index": 5, + "total_items": 1, + "timestamp": 0.0, + "item": {"text": "should be rejected"}, + "options": null + } + }, + { + "op": "score", + "score": { + "work_item_id": "w-scr", + "request_id": "r-scr", + "item_index": 9, + "total_items": 1, + "timestamp": 0.0, + "query_item": {"text": "Q"}, + "score_items": [{"text": "D"}], + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w-enc", + "request_id": "r-enc", + "item_index": 5, + "disposition": "publish_error_and_ack", + "error_code": "run_batch_mixed_op" + }, + { + "work_item_id": "w-scr", + "request_id": "r-scr", + "item_index": 9, + "disposition": "publish_error_and_ack", + "error_code": "run_batch_mixed_op" + } + ], + "notes": { + "elided_fields": ["error", "nak_delay_ms", "result_msgpack", "raw_output", "inference_ms", "tokenization_ms", "postprocessing_ms"], + "rationale": "item_index for the rejection MUST preserve the payload's item_index (5, 9) so wholesale protocol rejections use the same per-request index contract as successful outcomes — see reject_all_outcomes in run_batch.rs and _reject_all in adapter_call_loop.py." + } +} diff --git a/tests/parity/run_batch_score_basic.json b/tests/parity/run_batch_score_basic.json new file mode 100644 index 000000000..0b59f856c --- /dev/null +++ b/tests/parity/run_batch_score_basic.json @@ -0,0 +1,41 @@ +{ + "description": "RunBatch with op=score, valid items, lora_key=\"\". Pins the Python score dispatch path: adapter_call_loop._dispatch_score must hand the inner ProcessScoreBatchRequest to the executor with item identifiers preserved. Score is base-only on the Python adapter path, so this fixture exercises the simple base path.", + "input": { + "model_id": "test/parity-score", + "batch_id": 400, + "lora_key": "", + "total_cost": 10, + "items": [ + { + "op": "score", + "score": { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "total_items": 1, + "timestamp": 0.0, + "query_item": {"text": "is python a snake?"}, + "score_items": [ + {"text": "yes, it is a kind of snake"}, + {"text": "no, it is a programming language"} + ], + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "disposition": "publish_and_ack", + "error_code": null + } + ], + "expected_lora_in_options": null, + "notes": { + "elided_fields": ["inference_ms", "tokenization_ms", "postprocessing_ms", "result_msgpack", "raw_output", "error", "nak_delay_ms"], + "rationale": "Score has no LoRA support on either backend — empty lora_key is the only valid score-with-LoRA-aware-test we can write without exercising the WARN path. The score-with-non-empty-lora WARN path is covered by run_batch_score_lora_warns.json." + } +} diff --git a/tests/parity/run_batch_score_lora_warns.json b/tests/parity/run_batch_score_lora_warns.json new file mode 100644 index 000000000..09e1ec205 --- /dev/null +++ b/tests/parity/run_batch_score_lora_warns.json @@ -0,0 +1,38 @@ +{ + "description": "RunBatch with op=score AND non-empty lora_key. Score is base-only on the Python adapter path: ModelWorker.submit_score uses _batchers[None] only. Python must (1) log a WARN, (2) NOT inject options['lora'], and (3) serve the base path successfully — NOT NAK, NOT reject. The pinned invariant: score is unconditionally base-served regardless of lora_key.", + "input": { + "model_id": "test/parity-score-warn", + "batch_id": 401, + "lora_key": "ignored-on-score-path", + "total_cost": 10, + "items": [ + { + "op": "score", + "score": { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "total_items": 1, + "timestamp": 0.0, + "query_item": {"text": "Q"}, + "score_items": [{"text": "D1"}, {"text": "D2"}], + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w0", + "request_id": "r0", + "item_index": 0, + "disposition": "publish_and_ack", + "error_code": null + } + ], + "expected_lora_in_options": null, + "notes": { + "elided_fields": ["inference_ms", "tokenization_ms", "postprocessing_ms", "result_msgpack", "raw_output", "error", "nak_delay_ms"], + "rationale": "expected_lora_in_options=null even though input.lora_key is non-empty: this is the WARN-and-strip-LoRA contract. The matching log assertion (per side) belongs in language-specific log tests; the parity contract is the wire shape, which must be base-output regardless of the WARN." + } +} diff --git a/tests/parity/run_batch_unknown_op.json b/tests/parity/run_batch_unknown_op.json new file mode 100644 index 000000000..9e3817f10 --- /dev/null +++ b/tests/parity/run_batch_unknown_op.json @@ -0,0 +1,36 @@ +{ + "description": "RunBatch with op=\"future_op\" — an op a forward-compat sidecar might emit before this adapter image knows about it. Python adapter_call_loop must reject the WHOLE batch with error_code=\"run_batch_unknown_op\", NOT crash, and NOT silently drop. The payload is a real encode payload to confirm it extracts identifiers from whichever variant is set even when the op tag is unrecognised.", + "input": { + "model_id": "test/parity-future", + "batch_id": 200, + "lora_key": "", + "total_cost": 1, + "items": [ + { + "op": "future_op", + "encode": { + "work_item_id": "w-fut", + "request_id": "r-fut", + "item_index": 0, + "total_items": 1, + "timestamp": 0.0, + "item": {"text": "hello from the future"}, + "options": null + } + } + ] + }, + "expected_canonical_outcomes": [ + { + "work_item_id": "w-fut", + "request_id": "r-fut", + "item_index": 0, + "disposition": "publish_error_and_ack", + "error_code": "run_batch_unknown_op" + } + ], + "notes": { + "elided_fields": ["error", "nak_delay_ms", "result_msgpack", "raw_output", "inference_ms", "tokenization_ms", "postprocessing_ms"], + "rationale": "Forward-compat: a Rust scheduler that learns op=embed_image before this adapter image is rebuilt should surface a clean run_batch_unknown_op rather than crash or silently drop. The per-payload (work_item_id, request_id) extraction proves the rejection still threads the customer-facing identifiers through to the sidecar publisher." + } +} diff --git a/tests/parity/run_parity.sh b/tests/parity/run_parity.sh new file mode 100755 index 000000000..85c0f4739 --- /dev/null +++ b/tests/parity/run_parity.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Run the RunBatch IPC fixture suite against the Python adapter process. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +VERBOSE="" +for arg in "$@"; do + case "$arg" in + -v|--verbose) VERBOSE="-v" ;; + -h|--help) + sed -n '2,8p' "$0" | sed 's/^# \?//' + exit 0 + ;; + py|python|both|all|rs|rust) + echo "run_parity.sh: language selectors are not supported; this runner executes the Python fixture consumer." >&2 + exit 64 + ;; + *) + echo "run_parity.sh: unknown argument: $arg" >&2 + echo " usage: run_parity.sh [-v]" >&2 + exit 64 + ;; + esac +done + +fixture_count="$(find "$SCRIPT_DIR" -maxdepth 1 -name 'run_batch_*.json' | wc -l | tr -d ' ')" + +echo "== RunBatch IPC fixture parity ==" +echo "fixtures dir: $SCRIPT_DIR" +echo "fixtures: $fixture_count" +echo + +cd "$REPO_ROOT" +args=(packages/sie_server/tests/test_parity_run_batch.py) +if [[ -n "$VERBOSE" ]]; then + args+=("$VERBOSE") +fi + +if mise run test -- "${args[@]}"; then + echo + echo "== parity PASSED ($fixture_count fixtures, python) ==" +else + status=$? + echo + echo "== parity FAILED (python exit=$status) ==" >&2 + exit "$status" +fi diff --git a/tools/ci/build_audio_prep_release_asset.py b/tools/ci/build_audio_prep_release_asset.py new file mode 100755 index 000000000..5cc1c0528 --- /dev/null +++ b/tools/ci/build_audio_prep_release_asset.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Build and stage the exact native audio wheel used as a release asset.""" + +from __future__ import annotations + +import argparse +import importlib.util +import shutil +import sys +from pathlib import Path +from types import ModuleType + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] + + +def load_build_wheel(project_root: Path) -> ModuleType: + build_wheel_path = project_root / "packages/sie_audio_prep/build_wheel.py" + spec = importlib.util.spec_from_file_location("sie_audio_prep_build_wheel", build_wheel_path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load {build_wheel_path}") + module = importlib.util.module_from_spec(spec) + sys.modules.setdefault("sie_audio_prep_build_wheel", module) + spec.loader.exec_module(module) + return module + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--project-root", type=Path, default=REPOSITORY_ROOT) + args = parser.parse_args(argv) + + project_root = args.project_root.resolve() + build_wheel = load_build_wheel(project_root) + wheel = build_wheel.build_audio_prep_wheel(project_root, required=True) + if wheel is None: + raise RuntimeError("required audio wheel build returned no artifact") + args.out.mkdir(parents=True, exist_ok=True) + destination = args.out / wheel.name + if wheel.resolve() != destination.resolve(): + if destination.exists() and destination.read_bytes() != wheel.read_bytes(): + raise ValueError("refusing to replace a different retained audio wheel") + shutil.copyfile(wheel, destination) + build_wheel._validate_wheel(destination) + print(destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/build_sidecar_release_asset.py b/tools/ci/build_sidecar_release_asset.py new file mode 100755 index 000000000..b175e77d7 --- /dev/null +++ b/tools/ci/build_sidecar_release_asset.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Extract the Linux sidecar asset from its tested release image archive.""" + +from __future__ import annotations + +import argparse +import json +import re +import struct +from pathlib import Path + +from tools.ci.release_artifact import create_manifest, file_digest +from tools.mise_tasks.docker_task import capture, load_image_archive, run, singleton_image + +ELF_HEADER_PREFIX_SIZE = 20 +ELF_MACHINE_X86_64 = 62 + + +def inspect_abi(binary: Path) -> dict[str, object]: + with binary.open("rb") as stream: + header = stream.read(ELF_HEADER_PREFIX_SIZE) + if ( + len(header) != ELF_HEADER_PREFIX_SIZE + or header[:6] != b"\x7fELF\x02\x01" + or struct.unpack(" (2, 36): + raise ValueError("sidecar glibc requirement must be present and no newer than Debian 12 glibc 2.36") + dynamic = capture(["readelf", "--dynamic", str(binary)]) + libraries = sorted(re.findall(r"\(NEEDED\).*?\[([^\]]+)\]", dynamic)) + return { + "format": "ELF64", + "os": "linux", + "architecture": "amd64", + "glibc_minimum": ".".join(map(str, glibc[-1])), + "needed_libraries": libraries, + "runtime_baseline": "Debian 12 (glibc 2.36)", + } + + +def build(directory: Path, out: Path, *, version: str, source_revision: str, run_id: str) -> None: + image = singleton_image("ghcr.io/superlinked", version, "sie-server-sidecar") + source = load_image_archive(image, directory, version=version, source_revision=source_revision, run_id=run_id) + out.mkdir(parents=True, exist_ok=True) + if any(out.iterdir()): + raise ValueError("refusing to replace an existing native asset directory") + binary = out / f"sie-server-sidecar-v{version}-linux-amd64" + container = capture(["docker", "create", image]) + try: + run(["docker", "cp", f"{container}:/sie-server-sidecar", str(binary)]) + finally: + run(["docker", "rm", container]) + binary.chmod(0o755) + abi = inspect_abi(binary) + run( + [ + "docker", + "run", + "--rm", + "--pull", + "never", + "--network", + "none", + "--mount", + f"type=bind,src={binary.resolve()},dst=/asset,readonly", + "--entrypoint", + "/asset", + image, + "--help", + ] + ) + (out / f"{binary.name}.sha256").write_text(f"{file_digest(binary)} {binary.name}\n") + metadata = { + **abi, + "source_image": image, + "source_image_id": source["metadata"]["image_id"], + "source_revision": source_revision, + "version": version, + "tag_name": f"v{version}", + "run_id": run_id, + "sha256": file_digest(binary), + } + (out / f"{binary.name}.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + create_manifest( + out, + kind="native-sidecar", + version=version, + tag_name=f"v{version}", + source_revision=source_revision, + run_id=run_id, + metadata=metadata, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--run-id", required=True) + build(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/check_public_tree.py b/tools/ci/check_public_tree.py new file mode 100755 index 000000000..54cd13a4b --- /dev/null +++ b/tools/ci/check_public_tree.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Reject references that cannot be resolved from the public repository.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +FORBIDDEN = ( + b"superlinked/" + b"sie-" + b"internal", + b"sie-" + b"internal#", + b"packages/" + b"sie_admin", + b"packages/" + b"sie_bench", + b"packages/" + b"sie_tools", + b"packages/" + b"sie_cloud", + b"tools/" + b"internal_python", +) +ARCHIVE_GENERATED_DIRS = { + ".cache", + ".pytest_cache", + ".venv", + "__pycache__", + "build", + "dist", + "node_modules", + "target", +} + + +def candidate_paths() -> list[Path]: + """Return tracked and non-ignored untracked files for local and CI checks.""" + result = subprocess.run( + ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], # noqa: S607 + cwd=REPOSITORY_ROOT, + check=False, + capture_output=True, + ) + if result.returncode == 0: + return [REPOSITORY_ROOT / item.decode() for item in result.stdout.split(b"\0") if item] + return sorted( + path + for path in REPOSITORY_ROOT.rglob("*") + if path.is_file() + and not ARCHIVE_GENERATED_DIRS.intersection(path.relative_to(REPOSITORY_ROOT).parts) + and "deploy/helm/sie-cluster/charts" not in path.as_posix() + ) + + +def violations(paths: list[Path]) -> list[str]: + findings: list[str] = [] + for path in paths: + if not path.is_file(): + continue + data = path.read_bytes() + if b"\0" in data: + continue + for line_number, line in enumerate(data.splitlines(), start=1): + for forbidden in FORBIDDEN: + if forbidden in line: + findings.append( + f"{path.relative_to(REPOSITORY_ROOT)}:{line_number}: forbidden public-tree reference" + ) + break + return findings + + +def main() -> int: + findings = violations(candidate_paths()) + if findings: + print("\n".join(findings)) + return 1 + print("Public-tree reference check passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/check_release_contract.py b/tools/ci/check_release_contract.py new file mode 100755 index 000000000..7691dffac --- /dev/null +++ b/tools/ci/check_release_contract.py @@ -0,0 +1,763 @@ +#!/usr/bin/env python3 +"""Validate the public version, artifact, and fail-closed publication contract.""" + +from __future__ import annotations + +import json +import re +import runpy +import shlex +import subprocess +import tomllib +from pathlib import Path +from typing import Any + +from tools.ci import distributions +from tools.ci.release_guard import SEED_VERSION, stable_version + +ROOT = Path(__file__).resolve().parents[2] +PYTHON_DISTRIBUTIONS = ( + "sie-sdk", + "sie-server", + "sie-config", + "sie-mcp", + "sie-langchain", + "sie-llamaindex", + "sie-haystack", + "sie-dspy", + "sie-crewai", + "sie-chroma", + "sie-lancedb", + "sie-qdrant", + "sie-weaviate", +) +NPM_PACKAGES = ( + "@superlinked/sie-sdk", + "@superlinked/sie-chroma", + "@superlinked/sie-langchain", + "@superlinked/sie-llamaindex", + "@superlinked/sie-lancedb", +) +PUBLIC_IMAGE_NAMES = { + "sie-server", + "sie-gateway", + "sie-config", + "sie-mcp", + "sie-server-sidecar", + "sie-server-rust", +} +EXTRA_VERSION_PATHS = { + "packages/sie_sdk/pyproject.toml", + "packages/sie_server/pyproject.toml", + "packages/sie_config/pyproject.toml", + "packages/sie_mcp/pyproject.toml", + "integrations/sie_langchain/pyproject.toml", + "integrations/sie_llamaindex/pyproject.toml", + "integrations/sie_haystack/pyproject.toml", + "integrations/sie_dspy/pyproject.toml", + "integrations/sie_crewai/pyproject.toml", + "integrations/sie_chroma/pyproject.toml", + "integrations/sie_lancedb/pyproject.toml", + "integrations/sie_qdrant/pyproject.toml", + "integrations/sie_weaviate/pyproject.toml", + "packages/sie_ts_sdk/package.json", + "integrations/sie_ts_chroma/package.json", + "integrations/sie_ts_langchain/package.json", + "integrations/sie_ts_llamaindex/package.json", + "integrations/sie_ts_lancedb/package.json", + "packages/sie_ts_sdk/src/version.ts", + "packages/sie_gateway/Cargo.toml", + "packages/sie_server_sidecar/Cargo.toml", + "packages/sie_audio_prep/Cargo.toml", + "packages/sie_audio_prep/pyproject.toml", + "packages/sie_audio_prep/build_wheel.py", + "deploy/helm/sie-cluster/Chart.yaml", +} +ACTION_PIN = re.compile(r"^[^@\s]+@[0-9a-f]{40}$") +JOB_FIELD_INDENT = 4 +ARTIFACT_RETENTION_DAYS = 30 +REQUIRED_OCI_LABEL_OCCURRENCES = 2 +RELEASE_BOOTSTRAP_SHA = "60996d9c30168e0f8e85b680295f147fdee87f61" +MPL_LICENSE_EXCEPTION_NAMES = frozenset( + { + "option-ext", + "symphonia", + "symphonia-adapter-libopus", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-common", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", + } +) +AUDIO_MANYLINUX_IMAGE = ( + "quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb" +) +TRUSTED_WRITE_CONDITION_TERMS = ( + "inputs.publish == true", + "vars.PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true'", + "github.event_name == 'release'", + "github.event.action == 'published'", + "github.event.release.draft == false", + "github.event.release.prerelease == false", + "github.ref == format('refs/tags/{0}', inputs.tag_name)", + "github.ref_protected == true", + "github.repository == 'superlinked/sie'", + "github.sha == inputs.sha", +) +PUBLISHER_JOBS = ( + (".github/workflows/release.yml", "python-publish", "pypi", {"contents": "read", "id-token": "write"}), + (".github/workflows/release.yml", "npm-publish", "npm", {"contents": "read", "id-token": "write"}), + (".github/workflows/release-audio.yml", "publish", "github-release", {"contents": "write"}), + (".github/workflows/release-native.yml", "publish", "github-release", {"contents": "write"}), + (".github/workflows/release-docker.yml", "push-server", "ghcr", {"contents": "read", "packages": "write"}), + (".github/workflows/release-docker.yml", "push-service", "ghcr", {"contents": "read", "packages": "write"}), + (".github/workflows/release-docker.yml", "alias", "ghcr", {"contents": "read", "packages": "write"}), + (".github/workflows/release-helm.yml", "publish", "helm", {"contents": "read", "packages": "write"}), +) + +CANDLE_PATHS = ( + "packages/sie_server_rust/Cargo.lock", + "packages/sie_server_rust/Cargo.toml", + "packages/sie_server_rust/Dockerfile", + "packages/sie_server_rust/Dockerfile.candle", + "packages/sie_server_rust/src/candle_backend.rs", + "packages/sie_server_rust/src/candle_bert_flash.rs", + "packages/sie_server_rust/src/candle_embedding.rs", + "packages/sie_server_rust/src/candle_gte_rope.rs", + "packages/sie_server_rust/src/candle_layers.rs", + "packages/sie_server_rust/src/candle_modernbert.rs", + "packages/sie_server_rust/src/candle_residency.rs", + "packages/sie_server_rust/src/candle_rope.rs", + "packages/sie_server_rust/src/candle_splade.rs", + "packages/sie_server_rust/src/candle_xlm_roberta.rs", + "packages/sie_server_rust/src/ipc.rs", + "packages/sie_server_rust/src/ipc_types.rs", + "packages/sie_server_rust/src/lib.rs", + "packages/sie_server_rust/src/main.rs", + "packages/sie_server_rust/src/native_backend.rs", + "packages/sie_server_rust/src/observability/metrics.rs", + "packages/sie_server_rust/src/observability/mod.rs", + "packages/sie_server_rust/src/observability/propagation.rs", + "packages/sie_server_rust/src/observability/resource.rs", + "packages/sie_server_rust/src/observability/tracing.rs", + "packages/sie_server_rust/src/observability/transport.rs", + "packages/sie_server_rust/src/text_prep.rs", + "packages/sie_server_rust/vendor/candle-cublaslt/Cargo.toml", + "packages/sie_server_rust/vendor/candle-cublaslt/LICENSE-APACHE", + "packages/sie_server_rust/vendor/candle-cublaslt/LICENSE-MIT", + "packages/sie_server_rust/vendor/candle-cublaslt/README.md", + "packages/sie_server_rust/vendor/candle-cublaslt/src/lib.rs", + "packages/sie_server_rust/vendor/candle-gated-activation/Cargo.toml", + "packages/sie_server_rust/vendor/candle-gated-activation/build.rs", + "packages/sie_server_rust/vendor/candle-gated-activation/kernels/gated_activation.cu", + "packages/sie_server_rust/vendor/candle-gated-activation/kernels/gelu_erf_gate.cu", + "packages/sie_server_rust/vendor/candle-gated-activation/src/ffi.rs", + "packages/sie_server_rust/vendor/candle-gated-activation/src/lib.rs", + "packages/sie_server_rust/vendor/candle-layer-norm/Cargo.toml", + "packages/sie_server_rust/vendor/candle-layer-norm/LICENSE", + "packages/sie_server_rust/vendor/candle-layer-norm/LICENSE-APACHE", + "packages/sie_server_rust/vendor/candle-layer-norm/LICENSE-MIT", + "packages/sie_server_rust/vendor/candle-layer-norm/README.md", + "packages/sie_server_rust/vendor/candle-layer-norm/build.rs", + "packages/sie_server_rust/vendor/candle-layer-norm/kernels/ln.h", + "packages/sie_server_rust/vendor/candle-layer-norm/kernels/ln_api.cu", + "packages/sie_server_rust/vendor/candle-layer-norm/kernels/ln_fwd_kernels.cuh", + "packages/sie_server_rust/vendor/candle-layer-norm/kernels/ln_kernel_traits.h", + "packages/sie_server_rust/vendor/candle-layer-norm/kernels/ln_utils.cuh", + "packages/sie_server_rust/vendor/candle-layer-norm/kernels/static_switch.h", + "packages/sie_server_rust/vendor/candle-layer-norm/src/ffi.rs", + "packages/sie_server_rust/vendor/candle-layer-norm/src/lib.rs", + "packages/sie_server_rust/vendor/candle-rotary/Cargo.toml", + "packages/sie_server_rust/vendor/candle-rotary/LICENSE-APACHE", + "packages/sie_server_rust/vendor/candle-rotary/LICENSE-MIT", + "packages/sie_server_rust/vendor/candle-rotary/README.md", + "packages/sie_server_rust/vendor/candle-rotary/build.rs", + "packages/sie_server_rust/vendor/candle-rotary/kernels/cuda_compat.h", + "packages/sie_server_rust/vendor/candle-rotary/kernels/rotary.cu", + "packages/sie_server_rust/vendor/candle-rotary/src/ffi.rs", + "packages/sie_server_rust/vendor/candle-rotary/src/lib.rs", + "packages/sie_server_rust/vendor/candle-rotary/tests/rotary_tests.rs", + "packages/sie_server_rust/vendor/candle-splade-pool/Cargo.toml", + "packages/sie_server_rust/vendor/candle-splade-pool/build.rs", + "packages/sie_server_rust/vendor/candle-splade-pool/kernels/splade_pool.cu", + "packages/sie_server_rust/vendor/candle-splade-pool/src/ffi.rs", + "packages/sie_server_rust/vendor/candle-splade-pool/src/lib.rs", +) + + +def load_json(path: str) -> Any: + return json.loads((ROOT / path).read_text()) + + +def workflow_job_blocks(path: str) -> dict[str, str]: + """Extract top-level job blocks without requiring a YAML dependency.""" + lines = (ROOT / path).read_text().splitlines() + try: + jobs_index = lines.index("jobs:") + except ValueError: + return {} + starts: list[tuple[str, int]] = [] + for index, line in enumerate(lines[jobs_index + 1 :], start=jobs_index + 1): + match = re.fullmatch(r" ([A-Za-z0-9_-]+):", line) + if match: + starts.append((match.group(1), index)) + blocks: dict[str, str] = {} + for offset, (name, start) in enumerate(starts): + end = starts[offset + 1][1] if offset + 1 < len(starts) else len(lines) + blocks[name] = "\n".join(lines[start:end]) + return blocks + + +def job_scalar(block: str, key: str) -> str | None: + lines = block.splitlines() + prefix = f" {key}:" + for index, line in enumerate(lines): + if not line.startswith(prefix): + continue + value = line.removeprefix(prefix).strip() + if value not in {">", ">-", "|", "|-"}: + return value + parts: list[str] = [] + for continuation in lines[index + 1 :]: + if continuation and len(continuation) - len(continuation.lstrip()) <= JOB_FIELD_INDENT: + break + if continuation.strip(): + parts.append(continuation.strip()) + return " ".join(parts) + return None + + +def job_permissions(block: str) -> dict[str, str]: + lines = block.splitlines() + for index, line in enumerate(lines): + if line != " permissions:": + continue + permissions: dict[str, str] = {} + for continuation in lines[index + 1 :]: + match = re.fullmatch(r" ([A-Za-z0-9_-]+):\s*([^\s]+)", continuation) + if not match: + break + permissions[match.group(1)] = match.group(2) + return permissions + return {} + + +def publisher_job_errors() -> list[str]: + errors: list[str] = [] + for path, job_name, environment, expected_permissions in PUBLISHER_JOBS: + if not (ROOT / path).exists(): + errors.append(f"publisher workflow is missing: {path}") + continue + block = workflow_job_blocks(path).get(job_name, "") + condition = job_scalar(block, "if") or "" + terms = TRUSTED_WRITE_CONDITION_TERMS + if path.endswith("/release.yml"): + terms = tuple( + term.replace("inputs.sha", "needs.prepare.outputs.sha").replace( + "inputs.tag_name", "needs.prepare.outputs.tag_name" + ) + for term in terms + if term != "inputs.publish == true" + ) + missing = [term for term in terms if term not in condition] + if missing: + errors.append(f"{path}:{job_name} lacks write guards: {missing}") + if job_scalar(block, "environment") != environment or job_permissions(block) != expected_permissions: + errors.append(f"{path}:{job_name} environment/permissions mismatch") + if ( + "release_guard.py publish" not in block + and "Validate trusted publication context and tag binding" not in block + ): + errors.append(f"{path}:{job_name} lacks runtime tag/SHA verification") + return errors + + +def python_matrices() -> tuple[tuple[str, ...], tuple[str, ...]]: + names = tuple(distributions.manifests("python")) + return names, names + + +def npm_matrix() -> tuple[str, ...]: + return tuple(distributions.manifests("npm")) + + +def workflow_pin_errors() -> list[str]: + errors: list[str] = [] + for path in sorted((ROOT / ".github/workflows").glob("*.yml")): + for line_number, line in enumerate(path.read_text().splitlines(), start=1): + match = re.search(r"\buses:\s*([^\s#]+)", line) + if not match or match.group(1).startswith("./"): + continue + if not ACTION_PIN.fullmatch(match.group(1)): + errors.append(f"{path.relative_to(ROOT)}:{line_number}: action is not pinned by full SHA") + return errors + + +def release_config_errors() -> list[str]: + errors: list[str] = [] + manifest = load_json(".release-please-manifest.json") + try: + if set(manifest) != {"."} or stable_version(manifest["."], new=False) < stable_version(SEED_VERSION, new=False): + errors.append("release-please manifest is below the public 0.7.3 seed") + except (ValueError, TypeError): + errors.append("release-please manifest must contain one stable root version") + config = load_json("release-please-config.json") + packages = config.get("packages", {}) + if set(packages) != {"."} or packages["."].get("release-type") != "simple": + errors.append("release-please must define one simple root release") + return errors + for flag in ("bump-minor-pre-major", "bump-patch-for-minor-pre-major", "include-v-in-tag"): + if packages["."].get(flag) is not True: + errors.append(f"release-please must preserve {flag}") + if config.get("bootstrap-sha") != RELEASE_BOOTSTRAP_SHA or "bootstrap-sha" in packages["."]: + errors.append("release-please bootstrap-sha must be the exact public v0.7.3 commit") + for override in ("release-as", "last-release-sha"): + if override in config or override in packages["."]: + errors.append(f"release-please must derive its native release boundary, not {override}") + extra_paths = {item["path"] for item in packages["."]["extra-files"]} + if extra_paths != EXTRA_VERSION_PATHS: + errors.append("release-please extra-file version surface differs from the public contract") + for path in extra_paths: + if not (ROOT / path).is_file(): + errors.append(f"release-please extra file does not exist: {path}") + return errors + + +def _license_policy_errors(policy: object) -> list[str]: + if not isinstance(policy, dict) or not isinstance(policy.get("licenses"), dict): + return ["cargo-deny license policy is malformed"] + licenses = policy["licenses"] + allow = licenses.get("allow") + exceptions = licenses.get("exceptions") + if not isinstance(allow, list) or not isinstance(exceptions, list): + return ["cargo-deny license allowlists are malformed"] + + errors: list[str] = [] + if "MPL-2.0" in allow: + errors.append("MPL-2.0 must not be globally allowed") + option_ext = [entry for entry in exceptions if isinstance(entry, dict) and entry.get("name") == "option-ext"] + expected_option_ext = {"name": "option-ext", "version": "=0.2.0", "allow": ["MPL-2.0"]} + if option_ext != [expected_option_ext]: + errors.append("option-ext MPL-2.0 allowance must be confined to exact version 0.2.0") + mpl_names = { + entry.get("name") + for entry in exceptions + if isinstance(entry, dict) and isinstance(entry.get("allow"), list) and "MPL-2.0" in entry["allow"] + } + if mpl_names != MPL_LICENSE_EXCEPTION_NAMES: + errors.append("cargo-deny MPL-2.0 exception surface differs from the reviewed crate set") + return errors + + +def license_policy_errors() -> list[str]: + return _license_policy_errors(tomllib.loads((ROOT / "deny.toml").read_text())) + + +QUEUE_SCHEMA_DIAGNOSTIC = ( + r'^unexpected key "queue" for "concurrency" section\. ' + r'expected one of "cancel-in-progress", "group"$' +) + + +def release_queue_errors(top: str, ci: str) -> list[str]: + errors = [] + concurrency = top.partition("concurrency:\n")[2].partition("\njobs:")[0] + if re.findall(r"^ queue: (.+)$", concurrency, re.MULTILINE) != ["max"]: + errors.append("release concurrency must preserve all pending runs with queue: max") + if re.findall(r"^ cancel-in-progress: (.+)$", concurrency, re.MULTILINE) != ["false"]: + errors.append("release concurrency must not cancel original publication runs") + if re.findall(r"-ignore '([^']*)'", ci) != [QUEUE_SCHEMA_DIAGNOSTIC]: + errors.append("actionlint may ignore only the exact unsupported concurrency.queue diagnostic") + return errors + + +def release_workflow_errors() -> list[str]: + errors: list[str] = [] + top = (ROOT / ".github/workflows/release.yml").read_text() + for family in ("python", "npm", "audio", "docker", "helm", "native"): + if f"uses: ./.github/workflows/release-{family}.yml" not in top: + errors.append(f"top-level release does not call {family} directly") + for family in ("python", "npm"): + text = (ROOT / f".github/workflows/release-{family}.yml").read_text() + if "id-token:" in text or "environment:" in text or " publish:" in text: + errors.append(f"{family} reusable must only build and test") + if f"distributions.py build {family}" not in text or "inputs.source_ref" not in text: + errors.append(f"{family} reusable lacks exact-source distribution checks") + for path in sorted((ROOT / ".github/workflows").glob("release*.yml")): + text = path.read_text() + for token_name in ("PYPI_TOKEN", "NPM_TOKEN", "NODE_AUTH_TOKEN"): + if token_name in text: + errors.append(f"{path.name} references forbidden publication token {token_name}") + for retention in re.findall(r"retention-days:\s*(\d+)", text): + if int(retention) < ARTIFACT_RETENTION_DAYS: + errors.append(f"{path.name} expires retry artifacts before 30 days") + if path.name != "release.yml" and "workflow_dispatch" in text: + errors.append(f"{path.name} must not expose a separate recovery writer") + blocks = workflow_job_blocks(".github/workflows/release.yml") + prepare = blocks.get("prepare", "") + if "release_guard.py prepare" not in prepare or "ref: ${{ github.sha }}" not in prepare: + errors.append("publication must prepare the exact published tag event") + for family in ("python", "npm", "docker", "helm", "audio", "native", "python-publish", "npm-publish"): + if "needs.prepare" not in blocks.get(family, "") or "needs.release-please" in blocks.get(family, ""): + errors.append(f"{family} must consume the published-release prepare identity") + errors.extend(release_queue_errors(top, (ROOT / ".github/workflows/ci.yml").read_text())) + recover = blocks.get("recover", "") + if "release_recovery" not in recover or "id-token:" in recover or "publish-" in recover: + errors.append("manual recovery must only rerun the original jobs") + complete = blocks.get("complete", "") + if "always()" not in complete or 'job["result"] != "success"' not in complete: + errors.append("release completion must reject every non-success result") + if job_scalar(complete, "needs") != "[prepare, python-publish, npm-publish, docker, helm, audio, native]": + errors.append("release completion must include every artifact family") + if (ROOT / ".github/workflows/repair-audio-asset.yml").exists(): + errors.append("obsolete audio repair workflow must be removed") + npm = blocks.get("npm-publish", "") + for term in ("runs-on: ubuntu-24.04", "node-version: '24.12.0'", "npm@11.6.2"): + if term not in npm: + errors.append(f"npm trusted publication is missing {term}") + return errors + + +def release_automation_gate_errors(condition: str) -> list[str]: + gate = "vars.PUBLIC_RELEASE_AUTOMATION_ENABLED == 'true'" + if condition.count(gate) != 1 or condition.count("PUBLIC_RELEASE_AUTOMATION_ENABLED") != 1 or "||" in condition: + return ["release-please authoring must use one exact default-off activation gate"] + return [] + + +def release_app_errors() -> list[str]: + errors: list[str] = [] + path = ".github/workflows/release.yml" + text = (ROOT / path).read_text() + block = workflow_job_blocks(path).get("release-please", "") + if job_scalar(block, "environment") != "release-automation": + errors.append("release-please must use the protected release-automation environment") + if job_permissions(block) != {"contents": "read"}: + errors.append("release-please GITHUB_TOKEN permissions must remain read-only") + release_condition = job_scalar(block, "if") or "" + errors.extend(release_automation_gate_errors(release_condition)) + for term in ( + "github.event_name == 'push'", + "github.ref == 'refs/heads/main'", + "github.ref_protected == true", + "github.repository == 'superlinked/sie'", + ): + if term not in release_condition: + errors.append(f"release-please is missing trusted context condition: {term}") + required = ( + "actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349", + "vars.PUBLIC_RELEASE_APP_ID", + "secrets.PUBLIC_RELEASE_APP_PRIVATE_KEY", + "permission-contents: write", + "permission-pull-requests: write", + "token: ${{ steps.app-token.outputs.token }}", + "git push origin", + "Verify the release PR final head after the App-authored push", + 'gh api "repos/$GITHUB_REPOSITORY/pulls/$pr_number" --jq .head.sha', + 'test "$remote_sha" = "$expected_sha"', + "release_pr_head: ${{ steps.release-pr-head.outputs.sha }}", + ) + missing = [item for item in required if item not in text] + if missing: + errors.append(f"public release GitHub App exact-head handoff is incomplete: {missing}") + forbidden = ("secrets.GITHUB_TOKEN", "token: ${{ github.token }}") + leaked = [item for item in forbidden if item in block] + if leaked: + errors.append(f"release-please or lock pushes fall back to GITHUB_TOKEN: {leaked}") + ci = (ROOT / ".github/workflows/ci.yml").read_text() + if " pull_request:" not in ci: + errors.append("App-authored release PR changes do not trigger public pull-request CI") + if ( + "release_guard.py seed" not in block + or "googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7" not in block + or "skip-github-release: ${{ steps.seed.outputs.at_seed }}" not in block + ): + errors.append("stock release-please must preflight the genuine 0.7.3 seed") + return errors + + +def audio_release_contract() -> tuple[str, str, str]: + values = runpy.run_path(str(ROOT / "packages/sie_audio_prep/build_wheel.py")) + version = values["AUDIO_PREP_VERSION"] + filename = values["AUDIO_WHEEL_FILENAME"] + url = f"https://github.com/superlinked/sie/releases/download/v{version}/{filename}" + return version, filename, url + + +def audio_release_errors() -> list[str]: + errors: list[str] = [] + version, filename, url = audio_release_contract() + expected_filename = f"sie_audio_prep-{version}-cp312-abi3-manylinux_2_28_x86_64.whl" + if filename != expected_filename: + errors.append(f"native audio asset filename changed unexpectedly: {filename}") + mise_config = tomllib.loads((ROOT / "mise.toml").read_text()) + mise_tools = mise_config.get("tools", {}) + if mise_tools.get("zig") != "0.13.0": + errors.append("native audio release must pin Zig 0.13.0") + rust = mise_tools.get("rust", {}) + if not isinstance(rust, dict) or rust.get("version") != "1.97.0": + errors.append("native audio release must use the repository Rust 1.97.0 pin") + workflow = (ROOT / ".github/workflows/release-audio.yml").read_text() + required = ( + "ref: ${{ inputs.sha }}", + AUDIO_MANYLINUX_IMAGE, + "version: 2026.7.11", + "mise --no-config install python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0", + "rust@1.97.0 -- rustc --version", + "rust@1.97.0 -- cargo --version", + "python tools/ci/build_audio_prep_release_asset.py --out dist", + expected_filename.replace(version, "$RELEASE_VERSION"), + "tools/ci/upload_audio_prep_release_asset.bash", + "environment: github-release", + ) + missing = [item for item in required if item not in workflow] + if missing: + errors.append(f"native audio release workflow is incomplete: {missing}") + if not (ROOT / "tools/ci/build_audio_prep_release_asset.py").is_file(): + errors.append("native audio release asset builder is missing") + uploader_path = ROOT / "tools/ci/upload_native_release_asset.bash" + uploader = "" + if not uploader_path.is_file(): + errors.append("native audio immutable release uploader is missing") + else: + uploader = uploader_path.read_text() + for item in ("gh release upload", "sha256sum", "browser_download_url", "identical"): + if item not in uploader: + errors.append(f"native audio uploader is missing immutable check: {item}") + if "--clobber" in uploader: + errors.append("native audio uploader must never clobber a versioned asset") + top = (ROOT / ".github/workflows/release.yml").read_text() + if "uses: ./.github/workflows/release-audio.yml" not in top or "contents: write" not in workflow: + errors.append("top-level release does not fan out the native audio asset writer") + expected_url_fragment = ( + "https://github.com/$GITHUB_REPOSITORY/releases/download/$RELEASE_TAG/$RELEASE_ASSET_FILENAME" + ) + if expected_url_fragment not in uploader: + errors.append(f"native audio workflow does not verify browser URL contract {url}") + return errors + + +def candle_source_errors() -> list[str]: + errors: list[str] = [] + root = ROOT / "packages/sie_server_rust" + if (ROOT / ".git").exists(): + listed = subprocess.run( + [ + "/usr/bin/git", + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "packages/sie_server_rust", + ], + cwd=ROOT, + check=True, + capture_output=True, + ) + actual = {item.decode() for item in listed.stdout.split(b"\0") if item} + else: + actual = { + str(path.relative_to(ROOT)) + for path in root.rglob("*") + if path.is_file() and "target" not in path.relative_to(root).parts + } + expected = set(CANDLE_PATHS) + if actual != expected: + errors.append( + "Candle source closure differs from the reviewed 66-path allowlist: " + f"missing={sorted(expected - actual)}, unexpected={sorted(actual - expected)}" + ) + return errors + for relative in sorted(expected): + path = ROOT / relative + if path.is_symlink(): + errors.append(f"Candle source closure must not contain symlinks: {relative}") + try: + path.read_text() + except UnicodeDecodeError: + errors.append(f"Candle source closure contains non-text content: {relative}") + + required_licenses = { + "packages/sie_server_rust/vendor/candle-cublaslt/LICENSE-APACHE", + "packages/sie_server_rust/vendor/candle-cublaslt/LICENSE-MIT", + "packages/sie_server_rust/vendor/candle-layer-norm/LICENSE", + "packages/sie_server_rust/vendor/candle-layer-norm/LICENSE-APACHE", + "packages/sie_server_rust/vendor/candle-layer-norm/LICENSE-MIT", + "packages/sie_server_rust/vendor/candle-rotary/LICENSE-APACHE", + "packages/sie_server_rust/vendor/candle-rotary/LICENSE-MIT", + } + if not required_licenses.issubset(actual): + errors.append("Candle vendored license set is incomplete") + + def walk_paths(value: Any) -> list[str]: + found: list[str] = [] + if isinstance(value, dict): + for key, item in value.items(): + if key == "path" and isinstance(item, str): + found.append(item) + else: + found.extend(walk_paths(item)) + elif isinstance(value, list): + for item in value: + found.extend(walk_paths(item)) + return found + + for manifest in root.rglob("Cargo.toml"): + data = tomllib.loads(manifest.read_text()) + for dependency_path in walk_paths(data): + if not (manifest.parent / dependency_path).resolve().exists(): + errors.append( + f"Candle manifest path does not resolve: {manifest.relative_to(ROOT)} -> {dependency_path}" + ) + return errors + + +def docker_copy_errors() -> list[str]: + errors: list[str] = [] + dockerfiles = [ + ROOT / "packages/sie_server/Dockerfile.cpu", + ROOT / "packages/sie_server/Dockerfile.cuda12", + ROOT / "packages/sie_server/Dockerfile.cuda13", + ROOT / "packages/sie_gateway/Dockerfile", + ROOT / "packages/sie_config/Dockerfile", + ROOT / "packages/sie_mcp/Dockerfile", + ROOT / "packages/sie_server_sidecar/Dockerfile", + ROOT / "packages/sie_server_rust/Dockerfile", + ROOT / "packages/sie_server_rust/Dockerfile.candle", + ] + for dockerfile in dockerfiles: + logical_text = dockerfile.read_text().replace("\\\n", " ") + for line_number, line in enumerate(logical_text.splitlines(), start=1): + stripped = line.strip() + if not stripped.startswith("COPY "): + continue + tokens = shlex.split(stripped) + if any(token.startswith("--from=") for token in tokens[1:]): + continue + arguments = [token for token in tokens[1:] if not token.startswith("--")] + for source in arguments[:-1]: + if source.startswith("/") or "$" in source: + errors.append( + f"{dockerfile.relative_to(ROOT)}:{line_number}: unsupported release COPY source {source}" + ) + continue + matches = list(ROOT.glob(source)) + if not matches: + errors.append(f"{dockerfile.relative_to(ROOT)}:{line_number}: missing release COPY source {source}") + for line in logical_text.splitlines(): + if ( + "org.opencontainers.image.source=" in line + and 'org.opencontainers.image.source="https://github.com/superlinked/sie"' not in line + ): + errors.append(f"{dockerfile.relative_to(ROOT)} has a non-public OCI source label") + return errors + + +def docker_release_errors() -> list[str]: + errors = [*candle_source_errors(), *docker_copy_errors()] + matrix = load_json(".github/release-matrix.json") + pairs = {(platform, bundle) for platform in matrix.get("platforms", []) for bundle in matrix.get("bundles", [])} + pairs.update((item.get("platform"), item.get("bundle")) for item in matrix.get("include", [])) + expected_pairs = { + (platform, bundle) + for platform in ("cuda12", "cpu") + for bundle in ("default", "ctranslate2", "sglang", "transformers5") + } | {("cuda13", "sglang-cu130"), ("cuda13", "tensorrt-llm")} + if pairs != expected_pairs: + errors.append("Docker release matrix differs from the supported server pairs") + + values = (ROOT / "deploy/helm/sie-cluster/values.yaml").read_text() + chart_images = set(re.findall(r"repository:\s*ghcr\.io/superlinked/(sie-[a-z-]+)", values)) + if chart_images != PUBLIC_IMAGE_NAMES: + errors.append(f"chart-advertised SIE repositories differ from release set: {sorted(chart_images)}") + + workflow = (ROOT / ".github/workflows/release-docker.yml").read_text() + if "inputs.publish == true" not in workflow or "PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true'" not in workflow: + errors.append("Docker release is missing its dual publication latch") + if "needs: [matrix, verify]" not in workflow: + errors.append("Docker latest aliases are not ordered after full-set verification") + docker_task = (ROOT / "tools/mise_tasks/docker_task.py").read_text() + for label in ( + "org.opencontainers.image.revision={revision}", + "org.opencontainers.image.source=https://github.com/superlinked/sie", + ): + if docker_task.count(label) < REQUIRED_OCI_LABEL_OCCURRENCES: + errors.append(f"server and singleton builds must both carry OCI label {label}") + return errors + + +def helm_release_errors() -> list[str]: + errors: list[str] = [] + chart = (ROOT / "deploy/helm/sie-cluster/Chart.yaml").read_text() + version_match = re.search(r"^version:\s*([^\s#]+)", chart, re.MULTILINE) + app_match = re.search(r"^appVersion:\s*([^\s#]+)", chart, re.MULTILINE) + if not version_match or not app_match or app_match.group(1) != f"v{version_match.group(1)}": + errors.append("Helm Chart version and appVersion must share the vX.Y.Z release identity") + + workflow = (ROOT / ".github/workflows/release-helm.yml").read_text() + required = ( + "ref: ${{ inputs.sha }}", + "mise run helm -- dependencies", + "mise run helm -- lint --set payloadStore.enabled=false", + "mise run helm -- template --set payloadStore.enabled=false", + "helm package deploy/helm/sie-cluster", + "needs: build", + "inputs.publish == true", + "PUBLIC_RELEASE_PUBLISHING_ENABLED == 'true'", + "packages: write", + "tools.ci.publish_helm_archive", + ) + missing = [item for item in required if item not in workflow] + if missing: + errors.append(f"Helm release workflow is missing contract elements: {missing}") + if "latest" in workflow or "alias" in workflow: + errors.append("Helm release must not move a floating chart alias") + + docker_task = (ROOT / "tools/mise_tasks/docker_task.py").read_text() + if ":v{validate_version(version)}" not in docker_task: + errors.append("Docker versioned tags must match the chart's v-prefixed appVersion") + return errors + + +def validate() -> list[str]: + errors = [ + *release_config_errors(), + *license_policy_errors(), + *release_workflow_errors(), + *release_app_errors(), + *publisher_job_errors(), + *audio_release_errors(), + *docker_release_errors(), + *helm_release_errors(), + *workflow_pin_errors(), + ] + if python_matrices() != (PYTHON_DISTRIBUTIONS, PYTHON_DISTRIBUTIONS): + errors.append("Python build/publish matrices differ from the exact 13-package contract") + if npm_matrix() != NPM_PACKAGES: + errors.append("npm publish matrix differs from the exact five-package contract") + if not (ROOT / "pnpm-lock.yaml").is_file() or (ROOT / "packages/sie_ts_sdk/pnpm-lock.yaml").exists(): + errors.append("root pnpm lock must be the only TypeScript lock authority") + return errors + + +def main() -> int: + errors = validate() + if errors: + print("\n".join(f"ERROR: {error}" for error in errors)) + return 1 + print("Public release contract passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/cpu_stack_smoke.py b/tools/ci/cpu_stack_smoke.py new file mode 100644 index 000000000..fd460cd32 --- /dev/null +++ b/tools/ci/cpu_stack_smoke.py @@ -0,0 +1,213 @@ +"""Build and exercise the public CPU containers without registry credentials.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +from tools.ci.live_sdk import smoke_python, wait_for_api + +SERVICES = ("sie-config", "sie-gateway", "sie-server-sidecar", "sie-mcp", "sie-server-rust-cpu") + + +def docker(*args: str, check: bool = True) -> str: + result = subprocess.run( + ["docker", *args], check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120 + ) + if check and result.returncode: + raise RuntimeError(result.stdout) + return result.stdout.strip() + + +def require_local_docker() -> None: + context = json.loads(docker("context", "inspect"))[0] + endpoint = context["Endpoints"]["docker"]["Host"] + if not os.environ.get("DOCKER_CONTEXT"): + endpoint = os.environ.get("DOCKER_HOST") or endpoint + if not endpoint.startswith("unix://"): + raise RuntimeError(f"CPU smoke requires a local Unix Docker endpoint, got {endpoint}") + system = json.loads(docker("info", "--format", "{{json .}}")) + if system["OSType"] != "linux": + raise RuntimeError("CPU smoke requires a Linux Docker daemon") + print(f"Local Docker: {endpoint}; Linux {system['Architecture']}") + + +def wait_health(url: str, timeout: float = 180) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as response: + if response.status == 200: + return + except (OSError, urllib.error.URLError): + pass + time.sleep(1) + raise RuntimeError(f"Container health did not become ready: {url}") + + +def build_images(registry: str, revision: str) -> None: + common = ["--registry", registry, "--version", "0.0.0", "--source-revision", revision] + commands = [["build-server", "--platform", "cpu", "--bundle", "default", *common]] + commands.extend(["build-service", "--service", service, *common] for service in SERVICES) + for args in commands: + subprocess.run(["mise", "run", "docker", "--", *args], check=True, timeout=3600) + + +def main() -> None: + require_local_docker() + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + registry = f"local/sie-ci-{uuid.uuid4().hex[:8]}" + build_images(registry, revision) + network = f"sie-ci-{uuid.uuid4().hex[:8]}" + containers: list[str] = [] + logs = Path(".cache/ci-logs") + logs.mkdir(parents=True, exist_ok=True) + docker("network", "create", network) + with tempfile.TemporaryDirectory(prefix="sie-cpu-ipc-", dir="/tmp") as ipc: + + def start( + name: str, + image: str, + *, + env: dict[str, str] | None = None, + command: tuple[str, ...] = (), + port: int | None = None, + shared_ipc: bool = False, + ) -> str: + container = f"{network}-{name}" + args = ["run", "--detach", "--name", container, "--network", network, "--network-alias", name] + if port is not None: + args += ["-p", f"127.0.0.1::{port}"] + if shared_ipc: + args += ["--user", "0:0", "-v", f"{ipc}:/var/run/sie"] + for key, value in (env or {}).items(): + args += ["-e", f"{key}={value}"] + containers.append(container) + docker(*args, image, *command) + if port is None: + return container + host_port = docker("port", container, f"{port}/tcp").rsplit(":", 1)[1] + return f"http://127.0.0.1:{host_port}" + + def image(service: str) -> str: + return f"{registry}/{service}:v0.0.0" + + try: + start("nats", "nats:2.11.8-alpine", command=("-js",)) + config_url = start("config", image("sie-config"), env={"SIE_NATS_URL": "nats://nats:4222"}, port=8080) + wait_health(f"{config_url}/healthz") + worker_env = { + "SIE_POOL": "default", + "SIE_BUNDLE": "fake", + "SIE_MACHINE_PROFILE": "cpu", + "SIE_IPC_SOCKET_PATH": "/var/run/sie/ipc.sock", + "SIE_TELEMETRY_DISABLED": "1", + "HF_HUB_OFFLINE": "1", + "SIE_FAKE_MEMORY_BUDGET": "4GiB", + } + worker_url = start( + "worker", + f"{registry}/sie-server:v0.0.0-cpu-default", + env=worker_env, + command=("serve", "--host", "0.0.0.0", "--port", "8080", "--device", "cpu", "-b", "fake"), + port=8080, + shared_ipc=True, + ) + wait_for_api(worker_url) + gateway_env = { + "SIE_NATS_URL": "nats://nats:4222", + "SIE_CONFIG_SERVICE_URL": "http://config:8080", + "SIE_GATEWAY_HEALTH_MODE": "nats", + "SIE_GATEWAY_ENABLE_POOLS": "1", + "SIE_GATEWAY_REQUEST_TIMEOUT": "60", + "SIE_GATEWAY_CONFIGURED_GPUS": "cpu", + "SIE_GATEWAY_CONFIGURED_PHYSICAL_LANES": '[{"pool":"default","machineProfile":"cpu","bundle":"fake"}]', + } + gateway_url = start( + "gateway", + image("sie-gateway"), + env=gateway_env, + command=("--port", "8080", "--host", "0.0.0.0"), + port=8080, + ) + sidecar_url = start( + "sidecar", + image("sie-server-sidecar"), + shared_ipc=True, + port=9095, + env={ + **worker_env, + "SIE_NATS_URL": "nats://nats:4222", + "SIE_WORKER_ID": "cpu-smoke", + "SIE_GATEWAY_URL": "http://gateway:8080", + }, + ) + wait_health(f"{sidecar_url}/readyz") + wait_for_api(gateway_url) + smoke_python(gateway_url) + mcp_url = start( + "mcp", + image("sie-mcp"), + env={ + "SIE_BASE_URL": "http://gateway:8080", + "SIE_MCP_ALLOW_ANONYMOUS": "true", + "SIE_MCP_OAUTH_ENABLED": "false", + }, + port=8088, + ) + wait_health(f"{mcp_url}/healthz") + rust_url = start( + "rust", + f"{registry}/sie-server-rust:v0.0.0-cpu", + shared_ipc=True, + port=8080, + env={"SIE_IPC_SOCKET_PATH": "/var/run/sie/rust.sock", "SIE_DEVICE": "cpu"}, + ) + wait_health(f"{rust_url}/healthz") + docker("exec", f"{network}-worker", "python", "-c", IPC_SMOKE) + print("CPU gateway/config/worker/sidecar queue requests, MCP health and Rust worker IPC passed.") + finally: + for container in reversed(containers): + (logs / f"{container}.log").write_text(docker("logs", container, check=False)) + docker("rm", "--force", container, check=False) + docker("network", "rm", network, check=False) + + +IPC_SMOKE = """ +import socket, struct, msgpack +def read_exact(stream, size): + data = b'' + while len(data) < size: + part = stream.recv(size - len(data)) + assert part, 'IPC closed before the response completed' + data += part + return data +with socket.socket(socket.AF_UNIX) as stream: + stream.settimeout(15) + stream.connect('/var/run/sie/rust.sock') + for method in ('Ping', 'WorkerCapabilities'): + request = msgpack.packb({'version': 1, 'method': method, 'request_id': method, + 'body': {'timestamp_ms': 1.0} if method == 'Ping' else {}}, use_bin_type=True) + stream.sendall(struct.pack('>I', len(request)) + request) + length = struct.unpack('>I', read_exact(stream, 4))[0] + assert 0 < length < 1048576 + result = msgpack.unpackb(read_exact(stream, length), raw=False) + assert result['version'] == 1 and result['request_id'] == method and result['ok'], result + if method == 'Ping': + assert isinstance(result['body']['ready'], bool), result + assert result['body']['worker_id'] == 'sie-server-rust', result + else: + assert isinstance(result['body']['supported_models'], list), result +print('CPU Rust worker Ping and WorkerCapabilities IPC passed; no inference model configured') +""" + + +if __name__ == "__main__": + main() diff --git a/tools/ci/distributions.py b/tools/ci/distributions.py new file mode 100755 index 000000000..88f955cd2 --- /dev/null +++ b/tools/ci/distributions.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Build, inspect and consume the public distributions without editable installs.""" + +from __future__ import annotations + +import argparse +import base64 +import email +import hashlib +import http +import json +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +import tomllib +import urllib.error +import urllib.request +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PYTHON_PATHS = ( + "packages/sie_sdk", + "packages/sie_server", + "packages/sie_config", + "packages/sie_mcp", + "integrations/sie_langchain", + "integrations/sie_llamaindex", + "integrations/sie_haystack", + "integrations/sie_dspy", + "integrations/sie_crewai", + "integrations/sie_chroma", + "integrations/sie_lancedb", + "integrations/sie_qdrant", + "integrations/sie_weaviate", +) +NPM_PATHS = ( + "packages/sie_ts_sdk", + "integrations/sie_ts_chroma", + "integrations/sie_ts_langchain", + "integrations/sie_ts_llamaindex", + "integrations/sie_ts_lancedb", +) + + +def run(*args: str, cwd: Path = ROOT) -> None: + subprocess.run(args, cwd=cwd, check=True) # noqa: S603 + + +def manifests(family: str, version: str = "") -> dict[str, tuple[str, Path]]: + result = {} + for relative in PYTHON_PATHS if family == "python" else NPM_PATHS: + path = ROOT / relative + data = ( + tomllib.loads((path / "pyproject.toml").read_text())["project"] + if family == "python" + else json.loads((path / "package.json").read_text()) + ) + if version and data["version"] != version: + raise ValueError(f"{relative}: actual {data['version']} != release {version}") + result[data["name"]] = (data["version"], path) + return result + + +def archive_metadata(path: Path) -> tuple[str, str]: + if path.suffix == ".whl": + with zipfile.ZipFile(path) as archive: + entries = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] + if len(entries) != 1: + raise ValueError(f"{path.name}: expected one wheel metadata record") + data = email.message_from_bytes(archive.read(entries[0])) + return data["Name"], data["Version"] + with tarfile.open(path, "r:gz") as archive: + if path.suffix == ".tgz": + member = archive.extractfile("package/package.json") + if member is None: + raise ValueError("npm archive has no package manifest") + data = json.load(member) + names = set(archive.getnames()) + for entry in (data.get("main"), data.get("module"), data.get("types")): + if entry and "package/" + entry.removeprefix("./") not in names: + raise ValueError(f"{path.name}: missing packed entrypoint {entry}") + if "workspace:" in json.dumps(data.get("dependencies", {})): + raise ValueError("packed dependencies still reference the workspace") + return data["name"], data["version"] + entries = [ + entry for entry in archive.getmembers() if entry.name.count("/") == 1 and entry.name.endswith("/PKG-INFO") + ] + if len(entries) != 1: + raise ValueError(f"{path.name}: expected one sdist metadata record") + member = archive.extractfile(entries[0]) + if member is None: + raise ValueError("sdist has no package metadata") + data = email.message_from_bytes(member.read()) + return data["Name"], data["Version"] + + +def normalize(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def verify(family: str, directory: Path, version: str = "") -> list[Path]: + expected = {normalize(name): value[0] for name, value in manifests(family, version).items()} + found: dict[tuple[str, str], Path] = {} + for path in sorted(directory.iterdir()): + if path.name == "provenance.json": + continue + suffix = ".whl" if path.suffix == ".whl" else ".tar.gz" if path.name.endswith(".tar.gz") else path.suffix + if suffix not in ({".whl", ".tar.gz"} if family == "python" else {".tgz"}) or path.is_symlink(): + raise ValueError(f"unexpected package output: {path.name}") + name, actual = archive_metadata(path) + name = normalize(name) + if expected.get(name) != actual or (name, suffix) in found: + raise ValueError(f"{path.name}: wrong/duplicate distribution name or version") + found[name, suffix] = path + wanted = { + (name, suffix) for name in expected for suffix in ({".whl", ".tar.gz"} if family == "python" else {".tgz"}) + } + if set(found) != wanted: + raise ValueError(f"incomplete {family} archives: missing {sorted(wanted - set(found))}") + return list(found.values()) + + +def clean_python(archives: list[Path]) -> None: + sdk = next(path for path in archives if path.name.startswith("sie_sdk-") and path.suffix == ".whl") + for archive in archives: + name, version = archive_metadata(archive) + module = name.replace("-", "_") + module = { + "sie_config": "sie_config.cli", + "sie_mcp": "sie_mcp.cli", + "sie_server": "sie_server.bundle_requirements", + }.get(module, module) + script = ( + "import importlib,importlib.metadata,pathlib,sys; " + f"m=importlib.import_module({module!r}); " + f"assert importlib.metadata.version({name!r})=={version!r}; " + "assert pathlib.Path(m.__file__).resolve().is_relative_to(pathlib.Path(sys.prefix).resolve())" + ) + requirements = ["--with", str(archive)] + if normalize(name) != "sie-sdk": + requirements += ["--with", str(sdk)] + with tempfile.TemporaryDirectory(prefix="sie-packed-python-") as temporary: + run( + "uv", + "--no-config", + "run", + "--no-project", + "--isolated", + "--python", + "3.12", + *requirements, + "python", + "-I", + "-c", + script, + cwd=Path(temporary), + ) + + +def clean_npm(archives: list[Path]) -> None: + with tempfile.TemporaryDirectory(prefix="sie-packed-npm-") as temporary: + path = Path(temporary) + (path / "package.json").write_text(json.dumps({"name": "sie-packed-consumer", "private": True})) + run("npm", "install", "--ignore-scripts", "--no-audit", "--no-fund", *map(str, archives), cwd=path) + for archive in archives: + name, version = archive_metadata(archive) + run("node", "--input-type=module", "-e", f"await import({json.dumps(name)})", cwd=path) + run("node", "-e", f"require({json.dumps(name)})", cwd=path) + installed = json.loads((path / "node_modules" / name / "package.json").read_text()) + if installed["version"] != version: + raise ValueError(f"{name}: consumer loaded a different version") + + +def build(family: str, directory: Path, version: str) -> None: + packages = manifests(family, version) + directory.mkdir(parents=True, exist_ok=False) + if family == "python": + run("uv", "lock", "--check", "--project", str(ROOT)) + for name in packages: + with tempfile.TemporaryDirectory(prefix="sie-package-build-") as temporary: + run("uv", "build", "--package", name, "--out-dir", temporary) + for archive in Path(temporary).iterdir(): + if archive.name.endswith((".whl", ".tar.gz")): + destination = directory / archive.name + if destination.exists(): + raise ValueError(f"duplicate built archive: {archive.name}") + shutil.copyfile(archive, destination) + else: + run("pnpm", "install", "--frozen-lockfile") + run("pnpm", "-r", "build") + for _, path in packages.values(): + run("pnpm", "--dir", str(path), "pack", "--pack-destination", str(directory)) + archives = verify(family, directory, version) + (clean_python if family == "python" else clean_npm)(archives) + + +def prepare_pypi(directory: Path, destination: Path, version: str) -> None: + archives = verify("python", directory, version) + destination.mkdir(parents=True, exist_ok=False) + releases = {} + for archive in archives: + name, _ = archive_metadata(archive) + if name not in releases: + try: + with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=30) as response: + releases[name] = json.load(response)["urls"] + except urllib.error.HTTPError as error: + if error.code != http.HTTPStatus.NOT_FOUND: + raise + releases[name] = [] + existing = next((item for item in releases[name] if item["filename"] == archive.name), None) + if existing: + if existing["digests"]["sha256"] != hashlib.sha256(archive.read_bytes()).hexdigest(): + raise ValueError(f"PyPI already has different bytes for {archive.name}") + else: + shutil.copyfile(archive, destination / archive.name) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"pending={'true' if any(destination.iterdir()) else 'false'}\n") + + +def npm_view(spec: str, field: str) -> str | None: + result = subprocess.run( # noqa: S603 + ["npm", "view", spec, field, "--json", "--registry=https://registry.npmjs.org"], # noqa: S607 + capture_output=True, + text=True, + check=False, + ) + try: + reply = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise ValueError(f"malformed npm reply for {spec} {field}") from error + if result.returncode: + if isinstance(reply, dict) and isinstance(reply.get("error"), dict) and reply["error"].get("code") == "E404": + return None + raise ValueError(f"cannot check npm {spec} {field}: {result.stderr}") + if not isinstance(reply, str): + raise ValueError(f"malformed npm reply for {spec} {field}") + return reply + + +def publish_npm(directory: Path, version: str) -> None: + stable = r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" + if not re.fullmatch(stable, version): + raise ValueError("npm publication requires a stable version") + archives = verify("npm", directory, version) + pending = [] + for archive in archives: + name, _ = archive_metadata(archive) + published = npm_view(f"{name}@{version}", "dist.integrity") + if published is None: + latest = npm_view(name, "dist-tags.latest") + if latest is not None and not re.fullmatch(stable, latest): + raise ValueError(f"malformed npm latest version for {name}: {latest}") + older = latest is not None and tuple(map(int, latest.split("."))) > tuple(map(int, version.split("."))) + pending.append((archive, f"release-v{version}" if older else "latest")) + else: + integrity = "sha512-" + base64.b64encode(hashlib.sha512(archive.read_bytes()).digest()).decode() + if published != integrity: + raise ValueError(f"npm already has different bytes for {name}@{version}") + for archive, tag in pending: + run( + "npm", + "publish", + str(archive), + "--tag", + tag, + "--access", + "public", + "--provenance", + "--ignore-scripts", + "--registry=https://registry.npmjs.org", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["build", "verify", "prepare-pypi", "publish-npm"]) + parser.add_argument("family", choices=["python", "npm"]) + parser.add_argument("--directory", required=True, type=Path) + parser.add_argument("--version", default="") + parser.add_argument("--destination", type=Path) + args = parser.parse_args() + directory = args.directory.resolve() + if args.mode == "build": + build(args.family, directory, args.version) + elif args.mode == "verify": + verify(args.family, directory, args.version) + elif args.mode == "prepare-pypi": + prepare_pypi(directory, args.destination, args.version) + else: + publish_npm(directory, args.version) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/fresh_bootstrap.bash b/tools/ci/fresh_bootstrap.bash new file mode 100644 index 000000000..9bd291cb9 --- /dev/null +++ b/tools/ci/fresh_bootstrap.bash @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +test ! -e .venv +test ! -e node_modules +bootstrap_checksums=$(mktemp) +trap 'rm -f "$bootstrap_checksums"' EXIT +git ls-files -z '*uv.lock' '*pnpm-lock.yaml' '*Cargo.lock' | xargs -0 sha256sum > "$bootstrap_checksums" +./tools/init.sh +sha256sum --check "$bootstrap_checksums" +git diff --exit-code -- '*uv.lock' '*pnpm-lock.yaml' '*Cargo.lock' diff --git a/tools/ci/live_sdk.py b/tools/ci/live_sdk.py new file mode 100644 index 000000000..b438681a8 --- /dev/null +++ b/tools/ci/live_sdk.py @@ -0,0 +1,126 @@ +"""Real CPU server and SDK transport smoke with the checked-in weightless bundle.""" + +from __future__ import annotations + +import os +import signal +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import numpy as np +from sie_sdk import SIEClient + +MODEL = "sie-fake" + + +def free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def wait_for_api(url: str, process: subprocess.Popen | None = None, timeout: float = 300) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process is not None and process.poll() is not None: + raise RuntimeError("SIE server exited before becoming ready") + try: + with SIEClient(url, timeout_s=2) as client: + if client.list_models(): + return + except Exception: # noqa: BLE001 — bounded readiness polling + pass + time.sleep(1) + raise RuntimeError(f"SIE API did not become ready at {url}") + + +def smoke_python(url: str) -> None: + with SIEClient(url, timeout_s=300) as client: + assert client.list_models() + result = client.encode(MODEL, {"id": "cpu-smoke", "text": "Hello world"}, output_types=["dense"]) + assert result["id"] == "cpu-smoke" + assert result["dense"].shape == (384,) + assert np.isfinite(result["dense"]).all() + batch = client.encode( + MODEL, [{"id": "one", "text": "Hello"}, {"id": "two", "text": "World"}], output_types=["dense"] + ) + assert [item["id"] for item in batch] == ["one", "two"] + scored = client.score(MODEL, {"text": "query"}, [{"text": "one"}, {"text": "two"}]) + assert len(scored["scores"]) == 2 + generated = client.generate(MODEL, "a prompt", max_new_tokens=16) + assert generated["text"] + assert generated["usage"]["completion_tokens"] == 16 + print("Python SDK CPU encode/batch/score/generate passed.") + + +def smoke_typescript(url: str) -> None: + subprocess.run( + ["mise", "exec", "--", "node", "tools/ci/live_typescript.mjs"], + env={**os.environ, "SIE_SERVER_URL": url}, + check=True, + timeout=300, + ) + + +def main() -> None: + logs = Path(".cache/ci-logs") + logs.mkdir(parents=True, exist_ok=True) + port = free_port() + url = f"http://127.0.0.1:{port}" + with ( + tempfile.TemporaryDirectory(prefix="sie-live-", dir="/tmp") as runtime, + (logs / "live-sdk.log").open("w") as log, + ): + env = { + **os.environ, + "PYTHONUNBUFFERED": "1", + "SIE_IPC_SOCKET_PATH": f"{runtime}/worker.sock", + "SIE_TELEMETRY_DISABLED": "1", + "HF_HUB_OFFLINE": "1", + "SIE_FAKE_MEMORY_BUDGET": "4GiB", + } + for key in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "SIE_API_KEY", "SIE_GATEWAY_URL", "SIE_NATS_URL"): + env.pop(key, None) + process = subprocess.Popen( + [ + sys.executable, + "-m", + "sie_server.cli", + "serve", + "--host", + "127.0.0.1", + "-p", + str(port), + "-d", + "cpu", + "-b", + "fake", + ], + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + wait_for_api(url, process) + smoke_python(url) + smoke_typescript(url) + finally: + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + except ProcessLookupError: + pass + log.flush() + print((logs / "live-sdk.log").read_text()[-12000:], file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/live_typescript.mjs b/tools/ci/live_typescript.mjs new file mode 100644 index 000000000..0627c5047 --- /dev/null +++ b/tools/ci/live_typescript.mjs @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { SIEClient } from "../../packages/sie_ts_sdk/dist/index.js"; + +const client = new SIEClient(process.env.SIE_SERVER_URL, { timeout: 60_000 }); +try { + assert.ok((await client.listModels()).length > 0); + const model = "sie-fake"; + const result = await client.encode(model, { id: "ts-cpu-smoke", text: "Hello world" }); + assert.equal(result.id, "ts-cpu-smoke"); + assert.ok(result.dense instanceof Float32Array); + assert.equal(result.dense.length, 384); + assert.ok(Array.from(result.dense).every(Number.isFinite)); + const batch = await client.encode(model, [ + { id: "one", text: "Hello" }, + { id: "two", text: "World" }, + ]); + assert.deepEqual(batch.map((item) => item.id), ["one", "two"]); + const scored = await client.score(model, { text: "query" }, [{ text: "one" }, { text: "two" }]); + assert.equal(scored.scores.length, 2); + console.log("Built TypeScript SDK CPU encode/batch/score passed."); +} finally { + await client.close(); +} diff --git a/tools/ci/publish_helm_archive.py b/tools/ci/publish_helm_archive.py new file mode 100755 index 000000000..1b7d64c9a --- /dev/null +++ b/tools/ci/publish_helm_archive.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Publish the retained chart without replacing a different immutable version.""" + +from __future__ import annotations + +import argparse +import subprocess +import tempfile +from pathlib import Path + +from tools.ci.release_artifact import file_digest, validate_manifest +from tools.ci.release_guard import stable_version + +REGISTRY = "oci://ghcr.io/superlinked/charts" + + +def publish(directory: Path, *, version: str, source_revision: str, run_id: str) -> None: + stable_version(version) + manifest = validate_manifest( + directory, + kind="helm", + version=version, + tag_name=f"v{version}", + source_revision=source_revision, + run_id=run_id, + ) + filename = f"sie-cluster-{version}.tgz" + if [item["name"] for item in manifest["files"]] != [filename]: + raise ValueError("chart archive must contain exactly the versioned package") + chart = directory / filename + with tempfile.TemporaryDirectory(prefix="sie-chart-verify-") as temporary: + pull = ["helm", "pull", f"{REGISTRY}/sie-cluster", "--version", version, "--destination", temporary] + result = subprocess.run(pull, check=False, capture_output=True, text=True) # noqa: S603 + remote = Path(temporary) / filename + if result.returncode: + if not any(marker in result.stderr.lower() for marker in ("not found", "manifest unknown")): + raise RuntimeError(f"cannot inspect existing chart: {result.stderr.strip()}") + subprocess.run(["helm", "push", str(chart), REGISTRY], check=True) # noqa: S603, S607 + subprocess.run(pull, check=True) # noqa: S603 + if file_digest(chart) != file_digest(remote): + raise ValueError("remote chart differs from the retained tested chart; refusing to replace it") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--run-id", required=True) + publish(**vars(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/release_artifact.py b/tools/ci/release_artifact.py new file mode 100755 index 000000000..439f07669 --- /dev/null +++ b/tools/ci/release_artifact.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Bind retained release outputs to their original source and Actions run.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +MANIFEST = "provenance.json" + + +def file_digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _identity(version: str, tag_name: str, source_revision: str, run_id: str) -> dict[str, Any]: + if re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) is None or tag_name != f"v{version}": + raise ValueError("artifact requires an exact stable version/tag") + if re.fullmatch(r"[0-9a-f]{40}", source_revision) is None: + raise ValueError("artifact requires a full source revision") + if re.fullmatch(r"[1-9][0-9]*", str(run_id)) is None: + raise ValueError("artifact requires the original Actions run ID") + return { + "schema": 1, + "repository": "superlinked/sie", + "version": version, + "tag_name": tag_name, + "source_revision": source_revision, + "run_id": str(run_id), + } + + +def _files(directory: Path) -> list[dict[str, Any]]: + if directory.is_symlink() or not directory.is_dir(): + raise ValueError("artifact directory must be a real directory") + files = [] + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError("artifact must not contain symlinks") + if path.is_dir(): + continue + if not path.is_file(): + raise ValueError("artifact must contain only regular files") + name = path.relative_to(directory).as_posix() + if name != MANIFEST: + files.append({"name": name, "sha256": file_digest(path), "size": path.stat().st_size}) + if not files: + raise ValueError("artifact contains no payload files") + return files + + +def create_manifest( + directory: Path, + *, + kind: str, + version: str, + tag_name: str, + source_revision: str, + run_id: str, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + manifest = { + **_identity(version, tag_name, source_revision, run_id), + "kind": kind, + "files": _files(directory), + "metadata": metadata or {}, + } + encoded = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + path = directory / MANIFEST + if path.is_symlink() or (path.exists() and path.read_text() != encoded): + raise ValueError("refusing to replace different artifact provenance") + path.write_text(encoded) + return manifest + + +def validate_manifest( + directory: Path, + *, + version: str, + tag_name: str, + source_revision: str, + run_id: str, + kind: str | None = None, +) -> dict[str, Any]: + path = directory / MANIFEST + if path.is_symlink(): + raise ValueError("artifact provenance must not be a symlink") + manifest = json.loads(path.read_text()) + for key, value in _identity(version, tag_name, source_revision, run_id).items(): + if manifest.get(key) != value: + raise ValueError(f"artifact provenance mismatch: {key}") + if kind is not None and manifest.get("kind") != kind: + raise ValueError("artifact provenance mismatch: kind") + if not isinstance(manifest.get("metadata"), dict) or not isinstance(manifest.get("kind"), str): + raise ValueError("artifact metadata/kind is malformed") + if manifest.get("files") != _files(directory): + raise ValueError("artifact file set, size, or SHA256 mismatch") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("stamp", "check")) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--kind") + parser.add_argument("--version", required=True) + parser.add_argument("--tag-name", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--run-id", required=True) + args = parser.parse_args() + kwargs = vars(args).copy() + command = kwargs.pop("command") + if command == "stamp" and not args.kind: + parser.error("stamp requires --kind") + operation = create_manifest if command == "stamp" else validate_manifest + operation(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/release_guard.py b/tools/ci/release_guard.py new file mode 100755 index 000000000..0f77373b1 --- /dev/null +++ b/tools/ci/release_guard.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Verify the real public release boundary before invoking any writer.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + +REPOSITORY = "superlinked/sie" +SEED_VERSION = "0.7.3" +SHA = re.compile(r"[0-9a-f]{40}") + + +def stable_version(version: str, *, new: bool = True) -> tuple[int, ...]: + if not re.fullmatch(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)", version): + raise ValueError("release version must be stable X.Y.Z") + parts = tuple(map(int, version.split("."))) + if new and parts <= tuple(map(int, SEED_VERSION.split("."))): + raise ValueError("new publication must be newer than the 0.7.3 seed") + return parts + + +def seed_manifest(manifest: dict) -> bool: + if set(manifest) != {"."}: + raise ValueError("release manifest must contain one coordinated root version") + version = stable_version(manifest["."], new=False) + seed = stable_version(SEED_VERSION, new=False) + if version < seed: + raise ValueError("release manifest must not move below the 0.7.3 seed") + return version == seed + + +def command(*args: str) -> str: + return subprocess.check_output(args, text=True).strip() # noqa: S603 + + +def api(path: str) -> dict: + return json.loads(command("gh", "api", f"repos/{REPOSITORY}/{path}")) + + +def stable_release(version: str, source_sha: str | None = None) -> str: + stable_version(version, new=False) + tag = f"v{version}" + release = api(f"releases/tags/{tag}") + if release.get("tag_name") != tag or release.get("draft") is not False or release.get("prerelease") is not False: + raise ValueError(f"{tag} must have a genuine stable GitHub Release") + obj = api(f"git/ref/tags/{tag}")["object"] + for _ in range(5): + if obj["type"] == "commit": + break + if obj["type"] != "tag" or not SHA.fullmatch(obj["sha"]): + raise ValueError("release tag does not resolve to a commit") + obj = api(f"git/tags/{obj['sha']}")["object"] + sha = obj["sha"] + if obj["type"] != "commit" or not SHA.fullmatch(sha) or (source_sha is not None and sha != source_sha): + raise ValueError("release tag commit differs from the original release SHA") + command("git", "fetch", "--no-tags", "origin", f"refs/tags/{tag}") + if command("git", "rev-parse", "FETCH_HEAD^{commit}") != sha: + raise ValueError("fetched release tag differs from GitHub tag identity") + command("git", "merge-base", "--is-ancestor", sha, "HEAD") + return sha + + +def published_event(environment: dict[str, str], event: dict, source_sha: str) -> str: + release = event.get("release", {}) + tag = release.get("tag_name", "") + version = tag.removeprefix("v") + stable_version(version) + if ( + environment.get("GITHUB_REPOSITORY") != REPOSITORY + or event.get("repository", {}).get("full_name") != REPOSITORY + or environment.get("GITHUB_EVENT_NAME") != "release" + or environment.get("GITHUB_REF_PROTECTED") != "true" + or event.get("action") != "published" + or release.get("draft") is not False + or release.get("prerelease") is not False + or tag != f"v{version}" + or environment.get("GITHUB_REF") != f"refs/tags/{tag}" + or not SHA.fullmatch(source_sha) + or environment.get("GITHUB_SHA") != source_sha + ): + raise ValueError("publication requires the exact stable published release event and tag SHA") + return version + + +def protected_main_ancestor(source_sha: str) -> None: + if api("branches/main").get("protected") is not True: + raise ValueError("publication requires independently verified protected main") + command("git", "fetch", "--no-tags", "origin", "refs/heads/main") + command("git", "merge-base", "--is-ancestor", source_sha, "FETCH_HEAD") + + +def event_payload(environment: dict[str, str]) -> dict: + return json.loads(Path(environment["GITHUB_EVENT_PATH"]).read_text()) + + +def trusted_context( + environment: dict[str, str], source_sha: str, *, recovery: bool = False, event: dict | None = None +) -> None: + expected = {"GITHUB_REPOSITORY": REPOSITORY, "PUBLIC_RELEASE_PUBLISHING_ENABLED": "true"} + if recovery: + expected.update( + GITHUB_EVENT_NAME="workflow_dispatch", GITHUB_REF="refs/heads/main", GITHUB_REF_PROTECTED="true" + ) + if any(environment.get(key) != value for key, value in expected.items()): + raise ValueError("publication requires activated protected public main and the correct event") + if not SHA.fullmatch(source_sha) or environment.get("GITHUB_SHA") != source_sha: + raise ValueError("publication SHA must be the original workflow SHA") + if not recovery: + published_event(environment, event if event is not None else event_payload(environment), source_sha) + + +def prepare_release(environment: dict[str, str], event: dict) -> dict[str, str]: + source_sha = environment["GITHUB_SHA"] + version = published_event(environment, event, source_sha) + if command("git", "rev-parse", "HEAD") != source_sha: + raise ValueError("release event checkout differs from the tagged commit") + stable_release(SEED_VERSION) + stable_release(version, source_sha) + protected_main_ancestor(source_sha) + return {"sha": source_sha, "version": version, "tag_name": f"v{version}"} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["seed", "prepare", "build", "publish"]) + parser.add_argument("--version", default="") + parser.add_argument("--tag-name", default="") + parser.add_argument("--source-ref", default="") + args = parser.parse_args() + if args.mode == "seed": + at_seed = seed_manifest(json.loads(Path(".release-please-manifest.json").read_text())) + stable_release(SEED_VERSION) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"at_seed={str(at_seed).lower()}\n") + return + environment = dict(os.environ) + if args.mode == "prepare": + identity = prepare_release(environment, event_payload(environment)) + with Path(environment["GITHUB_OUTPUT"]).open("a") as output: + output.writelines(f"{key}={value}\n" for key, value in identity.items()) + return + if not SHA.fullmatch(args.source_ref) or command("git", "rev-parse", "HEAD") != args.source_ref: + raise ValueError("checkout is not the exact requested source SHA") + if args.version: + stable_version(args.version) + if args.tag_name != f"v{args.version}": + raise ValueError("release tag/version mismatch") + stable_release(args.version, args.source_ref) + elif args.tag_name or args.mode == "publish": + raise ValueError("publication requires an exact stable release version") + if args.mode == "publish": + event = event_payload(environment) + trusted_context(environment, args.source_ref, event=event) + if published_event(environment, event, args.source_ref) != args.version: + raise ValueError("publisher version differs from the original release event") + protected_main_ancestor(args.source_ref) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/release_recovery.py b/tools/ci/release_recovery.py new file mode 100755 index 000000000..97eb45dc6 --- /dev/null +++ b/tools/ci/release_recovery.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Request a retry on the original release run; never publish from this dispatch.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +from datetime import UTC, datetime, timedelta + +from tools.ci.release_guard import ( + REPOSITORY, + api, + command, + protected_main_ancestor, + stable_release, + stable_version, + trusted_context, +) + +FAMILIES = ("python", "npm", "docker", "helm", "audio", "native") +FAILED = {"failure", "timed_out", "cancelled", "action_required"} +PUBLISH_JOB = re.compile( + r"^(python-publish|npm-publish)$|^(docker|helm|audio|native) / " + r"(push-server|push-service|publish|verify|alias)(?:\s|\(|$)" +) + + +def artifact_names(family: str, version: str) -> set[str]: + if family in {"python", "npm"}: + return {f"{family}-distributions"} + if family == "docker": + pairs = [ + (platform, bundle) + for platform in ("cpu", "cuda12") + for bundle in ("default", "ctranslate2", "sglang", "transformers5") + ] + pairs += [("cuda13", "sglang-cu130"), ("cuda13", "tensorrt-llm")] + return {f"docker-server-{platform}-{bundle}-{version}" for platform, bundle in pairs} | { + f"docker-service-{service}-{version}" + for service in ("sie-gateway", "sie-config", "sie-mcp", "sie-server-sidecar", "sie-server-rust") + } + return {f"{ {'helm': 'helm-sie-cluster', 'audio': 'audio-prep', 'native': 'native-sidecar'}[family] }-{version}"} + + +def validate_run(run: dict, *, original_run: int, source_sha: str, tag_name: str, now: datetime) -> None: + expected = { + "id": original_run, + "event": "release", + "head_branch": tag_name, + "head_sha": source_sha, + "path": ".github/workflows/release.yml", + "status": "completed", + } + if any(run.get(key) != value for key, value in expected.items()): + raise ValueError("original run must be the completed release.yml release event for the exact tag and SHA") + if ( + run.get("repository", {}).get("full_name") != REPOSITORY + or run.get("head_repository", {}).get("full_name") != REPOSITORY + ): + raise ValueError("original run must belong to the public repository, not a fork") + age = now - datetime.fromisoformat(run["created_at"]) + if not timedelta(0) <= age < timedelta(days=30): + raise ValueError("original run is outside GitHub's 30-day rerun window") + if run.get("conclusion") == "success": + raise ValueError("successful releases do not need recovery") + + +def validate_artifacts( + artifacts: list[dict], wanted: set[str], *, original_run: int, source_sha: str, tag_name: str, now: datetime +) -> None: + for name in wanted: + matches = [artifact for artifact in artifacts if artifact.get("name") == name] + if len(matches) != 1: + raise ValueError(f"missing or ambiguous original archive: {name}") + artifact = matches[0] + linkage = artifact.get("workflow_run", {}) + if ( + linkage.get("id") != original_run + or linkage.get("head_sha") != source_sha + or linkage.get("head_branch") != tag_name + ): + raise ValueError(f"archive is not bound to the original release run: {name}") + if artifact.get("expired") is not False or datetime.fromisoformat(artifact["expires_at"]) <= now: + raise ValueError(f"original archive has expired: {name}") + if ( + not re.fullmatch(r"sha256:[0-9a-f]{64}", artifact.get("digest", "")) + or artifact.get("size_in_bytes", 0) <= 0 + ): + raise ValueError(f"original archive has no immutable digest: {name}") + + +def selected_jobs(jobs: list[dict], family: str) -> list[int]: + release_jobs = [job for job in jobs if job.get("name") == "prepare"] + if len(release_jobs) != 1 or release_jobs[0].get("conclusion") != "success": + raise ValueError("original prepare must remain successful; it must not be rerun") + prefixes = ("python-publish",) if family == "python" else ("npm-publish",) if family == "npm" else (f"{family} /",) + if family == "all": + prefixes = ("python-publish", "npm-publish", "docker /", "helm /", "audio /", "native /") + selected = [ + job["id"] + for job in jobs + if job.get("conclusion") in FAILED + and PUBLISH_JOB.match(job.get("name", "")) + and job.get("name", "").startswith(prefixes) + ] + if not selected: + raise ValueError( + "no failed original family jobs to rerun; skipped-only publication requires operator diagnosis" + ) + return selected + + +def retry_endpoint(jobs: list[dict], family: str, original_run: int) -> str: + selected = selected_jobs(jobs, family) + if family != "all" and len(selected) == 1: + return f"actions/jobs/{selected[0]}/rerun" + other_failures = [ + job + for job in jobs + if job.get("conclusion") in FAILED + and job["id"] not in selected + and job.get("name", "").split(" / ")[-1] != "complete" + ] + if other_failures: + raise ValueError("retry would also rerun other failed jobs; use all only after resolving failed builders") + return f"actions/runs/{original_run}/rerun-failed-jobs" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--original-run", required=True, type=int) + parser.add_argument("--family", choices=["all", *FAMILIES], default="all") + args = parser.parse_args() + stable_version(args.version) + if args.original_run <= 0 or str(args.original_run) == os.environ.get("GITHUB_RUN_ID"): + raise ValueError("recovery requires a different original run") + trusted_context(dict(os.environ), os.environ["GITHUB_SHA"], recovery=True) + if command("git", "rev-parse", "HEAD") != os.environ["GITHUB_SHA"]: + raise ValueError("recovery must execute the reviewed dispatch commit") + source_sha = stable_release(args.version) + protected_main_ancestor(source_sha) + now = datetime.now(UTC) + run = api(f"actions/runs/{args.original_run}") + tag_name = f"v{args.version}" + validate_run(run, original_run=args.original_run, source_sha=source_sha, tag_name=tag_name, now=now) + pages = json.loads( + command( + "gh", + "api", + "--paginate", + "--slurp", + f"repos/{REPOSITORY}/actions/runs/{args.original_run}/artifacts?per_page=100", + ) + ) + artifacts = [artifact for page in pages for artifact in page["artifacts"]] + families = FAMILIES if args.family == "all" else (args.family,) + wanted = set().union(*(artifact_names(family, args.version) for family in families)) + validate_artifacts( + artifacts, wanted, original_run=args.original_run, source_sha=source_sha, tag_name=tag_name, now=now + ) + pages = json.loads( + command( + "gh", + "api", + "--paginate", + "--slurp", + f"repos/{REPOSITORY}/actions/runs/{args.original_run}/jobs?filter=latest&per_page=100", + ) + ) + endpoint = retry_endpoint([job for page in pages for job in page["jobs"]], args.family, args.original_run) + command("gh", "api", "--method", "POST", f"repos/{REPOSITORY}/{endpoint}") + print( + f"Requested {args.family} recovery on original run {args.original_run}; " + "no artifacts published by this dispatch." + ) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/required_ci.py b/tools/ci/required_ci.py new file mode 100644 index 000000000..47808ef75 --- /dev/null +++ b/tools/ci/required_ci.py @@ -0,0 +1,42 @@ +"""Fail the required check unless every mandatory lane succeeded.""" + +from __future__ import annotations + +import json +import os +import sys + +MANDATORY_JOBS = ( + "policy", + "bootstrap", + "python", + "typescript", + "rust", + "contracts", + "helm", + "live-sdk", + "cpu-stack", + "python-distributions", + "npm-distributions", +) + + +def failures(needs: dict[str, dict[str, object]]) -> list[str]: + return [ + f"{name}: {needs.get(name, {}).get('result', 'missing')}" + for name in MANDATORY_JOBS + if needs.get(name, {}).get("result") != "success" + ] + + +def main() -> int: + failed = failures(json.loads(os.environ["NEEDS"])) + if failed: + print("Mandatory CI lanes did not succeed:\n" + "\n".join(failed), file=sys.stderr) + return 1 + print("Every mandatory CI lane succeeded.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/restore_release_artifact.py b/tools/ci/restore_release_artifact.py new file mode 100755 index 000000000..6931f2834 --- /dev/null +++ b/tools/ci/restore_release_artifact.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Reuse completed outputs from the original run when a release job is retried.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +from pathlib import Path + +from tools.ci.release_artifact import validate_manifest + + +def restore( + directory: Path, + *, + name: str, + kind: str, + version: str, + tag_name: str, + source_revision: str, + run_id: str, + evidence_of: Path | None = None, +) -> bool: + pages = json.loads( + subprocess.check_output( # noqa: S603 + ["gh", "api", "--paginate", "--slurp", f"repos/superlinked/sie/actions/runs/{run_id}/artifacts"], # noqa: S607 + text=True, + ) + ) + matches = [item for page in pages for item in page["artifacts"] if item["name"] == name] + if not matches: + return False + if len(matches) != 1 or matches[0].get("expired") is not False: + raise ValueError("original release artifact is duplicated or expired") + original = matches[0].get("workflow_run", {}) + if str(original.get("id")) != str(run_id) or original.get("head_sha") != source_revision: + raise ValueError("original release artifact source/run mismatch") + subprocess.run( # noqa: S603 + [ # noqa: S607 + "gh", + "run", + "download", + run_id, + "--repo", + "superlinked/sie", + "--name", + name, + "--dir", + str(directory), + ], + check=True, + ) + identity = { + "kind": kind, + "version": version, + "tag_name": tag_name, + "source_revision": source_revision, + "run_id": run_id, + } + if evidence_of is None: + validate_manifest(directory, **identity) + else: + validate_manifest(evidence_of, **identity) + if ( + sorted(path.name for path in directory.iterdir()) != ["provenance.json"] + or (directory / "provenance.json").is_symlink() + or (directory / "provenance.json").read_bytes() != (evidence_of / "provenance.json").read_bytes() + ): + raise ValueError("retained image evidence differs from its retained archive") + return True + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--name", required=True) + parser.add_argument("--kind", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--tag-name", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--evidence-of", type=Path) + restored = restore(**vars(parser.parse_args())) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write(f"restored={str(restored).lower()}\n") + + +if __name__ == "__main__": + main() diff --git a/tools/ci/rust_tests.py b/tools/ci/rust_tests.py new file mode 100644 index 000000000..444618d6a --- /dev/null +++ b/tools/ci/rust_tests.py @@ -0,0 +1,75 @@ +"""Run functional Rust tests with mandatory local JetStream coverage.""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + + +def free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def wait_for_jetstream(url: str, process: subprocess.Popen, timeout: float = 30) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("NATS exited before JetStream became ready") + try: + with urllib.request.urlopen(url, timeout=1) as response: + if "streams" in json.load(response): + return + except (OSError, urllib.error.URLError): + pass + time.sleep(0.2) + raise RuntimeError("Mandatory JetStream broker did not become ready") + + +def main() -> None: + if shutil.which("nats-server") is None: + raise RuntimeError("nats-server is required; run mise install") + if shutil.which("mise") is None: + raise RuntimeError("mise is required by the sidecar integration harness") + port, monitor = free_port(), free_port() + logs = Path(".cache/ci-logs") + logs.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="sie-ci-nats-") as storage, (logs / "nats.log").open("w") as log: + process = subprocess.Popen( + ["nats-server", "-js", "-a", "127.0.0.1", "-p", str(port), "-m", str(monitor), "-sd", storage], + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + wait_for_jetstream(f"http://127.0.0.1:{monitor}/jsz", process) + env = {**os.environ, "NATS_URL": f"nats://127.0.0.1:{port}", "SIE_RUN_NATS_PUBLISHER_TEST": "1"} + for key in tuple(env): + if "BENCHMARK" in key: + env.pop(key) + for args in ( + ["test", "--locked", "--workspace"], + ["test", "--locked", "-p", "sie-server-sidecar", "--features", "cloud-storage"], + ["test", "--locked", "--manifest-path", "packages/sie_server_rust/Cargo.toml"], + ): + subprocess.run(["cargo", *args], env=env, check=True, timeout=2700) + wait_for_jetstream(f"http://127.0.0.1:{monitor}/jsz", process) + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +if __name__ == "__main__": + main() diff --git a/tools/ci/tests/test_cpu_checks.py b/tools/ci/tests/test_cpu_checks.py new file mode 100644 index 000000000..a82c6cc5a --- /dev/null +++ b/tools/ci/tests/test_cpu_checks.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json +from unittest.mock import Mock + +import pytest + +from tools.ci import cpu_stack_smoke, live_sdk, rust_tests + + +def test_rust_fails_without_nats(monkeypatch): + monkeypatch.setattr(rust_tests.shutil, "which", lambda _: None) + with pytest.raises(RuntimeError, match="nats-server is required"): + rust_tests.main() + + +def test_rust_fails_when_broker_exits(): + process = Mock() + process.poll.return_value = 1 + with pytest.raises(RuntimeError, match="NATS exited"): + rust_tests.wait_for_jetstream("http://127.0.0.1:1/jsz", process) + + +def test_rust_runs_real_nats_opt_in_and_cloud_feature_without_benchmarks(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("SIE_RUN_TELEMETRY_BENCHMARK", "1") + monkeypatch.setattr(rust_tests.shutil, "which", lambda _: "/test/bin") + monkeypatch.setattr(rust_tests, "wait_for_jetstream", Mock()) + process = Mock() + monkeypatch.setattr(rust_tests.subprocess, "Popen", Mock(return_value=process)) + run = Mock() + monkeypatch.setattr(rust_tests.subprocess, "run", run) + rust_tests.main() + commands = [call.args[0] for call in run.call_args_list] + assert commands == [ + ["cargo", "test", "--locked", "--workspace"], + ["cargo", "test", "--locked", "-p", "sie-server-sidecar", "--features", "cloud-storage"], + ["cargo", "test", "--locked", "--manifest-path", "packages/sie_server_rust/Cargo.toml"], + ] + for call in run.call_args_list: + assert call.kwargs["env"]["NATS_URL"].startswith("nats://127.0.0.1:") + assert call.kwargs["env"]["SIE_RUN_NATS_PUBLISHER_TEST"] == "1" + assert not any("BENCHMARK" in key for key in call.kwargs["env"]) + process.terminate.assert_called_once() + + +def test_live_sdk_fails_when_server_exits(): + process = Mock() + process.poll.return_value = 1 + with pytest.raises(RuntimeError, match="SIE server exited"): + live_sdk.wait_for_api("http://127.0.0.1:1", process) + + +@pytest.mark.parametrize("endpoint", ["tcp://remote.example:2375", "ssh://remote.example"]) +def test_cpu_smoke_rejects_remote_docker(monkeypatch, endpoint): + monkeypatch.delenv("DOCKER_CONTEXT", raising=False) + monkeypatch.delenv("DOCKER_HOST", raising=False) + monkeypatch.setattr( + cpu_stack_smoke, "docker", lambda *args: json.dumps([{"Endpoints": {"docker": {"Host": endpoint}}}]) + ) + with pytest.raises(RuntimeError, match="local Unix"): + cpu_stack_smoke.require_local_docker() + + +def test_docker_host_override_cannot_hide_remote_endpoint(monkeypatch): + monkeypatch.delenv("DOCKER_CONTEXT", raising=False) + monkeypatch.setenv("DOCKER_HOST", "tcp://remote.example:2375") + monkeypatch.setattr( + cpu_stack_smoke, + "docker", + lambda *args: json.dumps([{"Endpoints": {"docker": {"Host": "unix:///var/run/docker.sock"}}}]), + ) + with pytest.raises(RuntimeError, match="local Unix"): + cpu_stack_smoke.require_local_docker() + + +def test_cpu_builds_all_six_images_without_publish(monkeypatch): + run = Mock() + monkeypatch.setattr(cpu_stack_smoke.subprocess, "run", run) + cpu_stack_smoke.build_images("local/test", "a" * 40) + assert len(run.call_args_list) == 6 + commands = [call.args[0] for call in run.call_args_list] + assert all(command[:4] == ["mise", "run", "docker", "--"] for command in commands) + assert all("--push" not in command for command in commands) + assert [command[command.index("--service") + 1] for command in commands[1:]] == list(cpu_stack_smoke.SERVICES) diff --git a/tools/ci/tests/test_distributions.py b/tools/ci/tests/test_distributions.py new file mode 100644 index 000000000..e6a144216 --- /dev/null +++ b/tools/ci/tests/test_distributions.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import subprocess +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from tools.ci import distributions as packages + + +def python_archives(tmp_path, name="sie-sdk", version="0.7.2"): + wheel = tmp_path / f"{name.replace('-', '_')}-{version}-py3-none-any.whl" + metadata = f"Name: {name}\nVersion: {version}\n" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr(f"{name}-{version}.dist-info/METADATA", metadata) + sdist = tmp_path / f"{name}-{version}.tar.gz" + with tarfile.open(sdist, "w:gz") as archive: + member = tarfile.TarInfo(f"{name}-{version}/PKG-INFO") + member.size = len(metadata) + archive.addfile(member, io.BytesIO(metadata.encode())) + return wheel, sdist + + +def test_pr_checks_actual_versions_and_release_checks_coordinated_versions(): + actual = packages.manifests("python") + assert len(actual) == 13 + assert "sie-config" in actual + assert "sie-mcp" in actual + assert len(packages.manifests("npm")) == 5 + with pytest.raises(ValueError, match="actual"): + packages.manifests("python", "999.0.0") + + +def test_archives_require_both_formats_and_exact_metadata(tmp_path, monkeypatch): + monkeypatch.setattr(packages, "manifests", lambda *args: {"sie-sdk": ("0.7.2", tmp_path)}) + wheel, sdist = python_archives(tmp_path) + assert set(packages.verify("python", tmp_path)) == {wheel, sdist} + sdist.unlink() + with pytest.raises(ValueError, match="incomplete"): + packages.verify("python", tmp_path) + + +def test_wrong_distribution_version_and_unexpected_files_fail(tmp_path, monkeypatch): + monkeypatch.setattr(packages, "manifests", lambda *args: {"sie-sdk": ("0.7.4", tmp_path)}) + python_archives(tmp_path) + with pytest.raises(ValueError, match="version"): + packages.verify("python", tmp_path) + + +def test_clean_python_consumes_wheel_and_sdist_outside_source(tmp_path, monkeypatch): + archives = python_archives(tmp_path) + calls = [] + monkeypatch.setattr(packages, "run", lambda *args, **kwargs: calls.append((args, kwargs))) + packages.clean_python(list(archives)) + assert len(calls) == 2 + for args, kwargs in calls: + assert "--isolated" in args + assert "--no-project" in args + assert "-I" in args + assert str(kwargs["cwd"]) != str(packages.ROOT) + assert "sys.prefix" in args[-1] + assert "--no-deps" not in args + + +def test_npm_metadata_requires_shipped_entrypoints_and_resolved_workspace(tmp_path): + package = tmp_path / "package.tgz" + metadata = json.dumps({"name": "@superlinked/sie-sdk", "version": "0.7.2", "main": "dist/index.cjs"}).encode() + with tarfile.open(package, "w:gz") as archive: + member = tarfile.TarInfo("package/package.json") + member.size = len(metadata) + archive.addfile(member, io.BytesIO(metadata)) + with pytest.raises(ValueError, match="missing packed entrypoint"): + packages.archive_metadata(package) + + +def test_python_build_retains_only_package_archives_not_uv_gitignore(tmp_path, monkeypatch): + monkeypatch.setattr(packages, "manifests", lambda *args: {"sie-sdk": ("0.7.2", tmp_path)}) + + def run(*args, **kwargs): + if "build" in args: + directory = Path(args[-1]) + python_archives(directory) + (directory / ".gitignore").write_text("*\n") + + monkeypatch.setattr(packages, "run", run) + monkeypatch.setattr(packages, "clean_python", lambda archives: None) + output = tmp_path / "artifacts" + packages.build("python", output, "") + assert len(list(output.iterdir())) == 2 + assert not (output / ".gitignore").exists() + + +@pytest.mark.parametrize("conflicting", [False, True]) +def test_pypi_retry_accepts_only_exact_existing_bytes(tmp_path, monkeypatch, conflicting): + source = tmp_path / "artifacts" + source.mkdir() + wheel, sdist = python_archives(source) + monkeypatch.setattr(packages, "manifests", lambda *args: {"sie-sdk": ("0.7.2", tmp_path)}) + records = [ + { + "filename": wheel.name, + "digests": {"sha256": "0" * 64 if conflicting else hashlib.sha256(wheel.read_bytes()).hexdigest()}, + } + ] + monkeypatch.setattr( + packages.urllib.request, "urlopen", lambda *args, **kwargs: io.BytesIO(json.dumps({"urls": records}).encode()) + ) + monkeypatch.setenv("GITHUB_OUTPUT", str(tmp_path / "outputs")) + if conflicting: + with pytest.raises(ValueError, match="different bytes"): + packages.prepare_pypi(source, tmp_path / "pending", "0.7.2") + else: + packages.prepare_pypi(source, tmp_path / "pending", "0.7.2") + assert (tmp_path / "pending" / sdist.name).read_bytes() == sdist.read_bytes() + assert not (tmp_path / "pending" / wheel.name).exists() + + +def test_npm_retry_checks_all_versions_before_any_upload(tmp_path, monkeypatch): + archive = tmp_path / "package.tgz" + archive.write_bytes(b"tested archive") + monkeypatch.setattr(packages, "verify", lambda *args: [archive]) + monkeypatch.setattr(packages, "archive_metadata", lambda path: ("@superlinked/sie-sdk", "0.7.4")) + uploads = [] + monkeypatch.setattr(packages, "run", lambda *args: uploads.append(args)) + monkeypatch.setattr( + packages.subprocess, "run", lambda *args, **kwargs: subprocess.CompletedProcess([], 0, '"sha512-different"', "") + ) + with pytest.raises(ValueError, match="different bytes"): + packages.publish_npm(tmp_path, "0.7.4") + assert uploads == [] + + +def npm_publish_fixture(tmp_path, monkeypatch, replies, *, count=1): + archives = [tmp_path / f"package-{index}.tgz" for index in range(count)] + for archive in archives: + archive.write_bytes(b"tested archive") + monkeypatch.setattr(packages, "verify", lambda *args: archives) + monkeypatch.setattr(packages, "archive_metadata", lambda path: (f"@superlinked/{path.stem}", "0.7.4")) + calls, uploads = [], [] + results = iter(replies) + + def query(args, **kwargs): + calls.append(args) + return next(results) + + monkeypatch.setattr(packages.subprocess, "run", query) + monkeypatch.setattr(packages, "run", lambda *args: uploads.append(args)) + return calls, uploads + + +def npm_missing(): + return subprocess.CompletedProcess([], 1, '{"error":{"code":"E404"}}', "npm error code E404") + + +@pytest.mark.parametrize( + ("latest", "tag"), [("0.7.5", "release-v0.7.4"), ("0.7.3", "latest"), ("0.7.10", "release-v0.7.4")] +) +def test_npm_publish_preserves_newer_latest_on_historical_repair(tmp_path, monkeypatch, latest, tag): + calls, uploads = npm_publish_fixture( + tmp_path, monkeypatch, [npm_missing(), subprocess.CompletedProcess([], 0, json.dumps(latest), "")] + ) + packages.publish_npm(tmp_path, "0.7.4") + assert calls[0][2:4] == ["@superlinked/package-0@0.7.4", "dist.integrity"] + assert calls[1][2:4] == ["@superlinked/package-0", "dist-tags.latest"] + assert len(uploads) == 1 + assert uploads[0][3:5] == ("--tag", tag) + assert uploads[0][:2] == ("npm", "publish") + + +def test_first_npm_publication_uses_latest(tmp_path, monkeypatch): + _, uploads = npm_publish_fixture(tmp_path, monkeypatch, [npm_missing(), npm_missing()]) + packages.publish_npm(tmp_path, "0.7.4") + assert uploads[0][3:5] == ("--tag", "latest") + + +@pytest.mark.parametrize("reply", ["not-json", "null", "{}", "[]", '"0.7.5-rc.1"', '"bogus"', '"01.2.3"', '""']) +def test_malformed_latest_blocks_all_pending_uploads(tmp_path, monkeypatch, reply): + _, uploads = npm_publish_fixture( + tmp_path, + monkeypatch, + [ + npm_missing(), + subprocess.CompletedProcess([], 0, '"0.7.3"', ""), + npm_missing(), + subprocess.CompletedProcess([], 0, reply, ""), + ], + count=2, + ) + with pytest.raises(ValueError, match="malformed npm"): + packages.publish_npm(tmp_path, "0.7.4") + assert uploads == [] + + +@pytest.mark.parametrize("code", ["E500", "E401", "ETIMEDOUT"]) +def test_latest_query_failure_does_not_publish(tmp_path, monkeypatch, code): + _, uploads = npm_publish_fixture( + tmp_path, + monkeypatch, + [ + npm_missing(), + subprocess.CompletedProcess([], 1, json.dumps({"error": {"code": code}}), code), + ], + ) + with pytest.raises(ValueError, match="cannot check npm"): + packages.publish_npm(tmp_path, "0.7.4") + assert uploads == [] + + +def test_identical_existing_npm_version_never_touches_latest(tmp_path, monkeypatch): + integrity = "sha512-" + base64.b64encode(hashlib.sha512(b"tested archive").digest()).decode() + calls, uploads = npm_publish_fixture( + tmp_path, monkeypatch, [subprocess.CompletedProcess([], 0, json.dumps(integrity), "")] + ) + packages.publish_npm(tmp_path, "0.7.4") + assert len(calls) == 1 + assert uploads == [] diff --git a/tools/ci/tests/test_docker_task.py b/tools/ci/tests/test_docker_task.py new file mode 100644 index 000000000..5ee4e612d --- /dev/null +++ b/tools/ci/tests/test_docker_task.py @@ -0,0 +1,393 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from tools.ci.release_artifact import create_manifest +from tools.mise_tasks import docker_task + +FULL_SHA = "a" * 40 +IMAGE_ID = "sha256:" + "b" * 64 +VERSION = "0.7.4" +IMAGE = f"ghcr.io/superlinked/sie-config:v{VERSION}" +MATRIX = docker_task.DEFAULT_MATRIX + + +@pytest.fixture +def complete_source(tmp_path: Path, monkeypatch): + bundles = tmp_path / "packages/sie_server/bundles" + bundles.mkdir(parents=True) + for name in ("default", "ctranslate2", "sglang", "transformers5", "sglang-cu130", "tensorrt-llm"): + platform = "cuda13" if name in {"sglang-cu130", "tensorrt-llm"} else "cuda12" + (bundles / f"{name}.yaml").write_text(f"name: {name}\nplatform: {platform}\n") + monkeypatch.setattr(docker_task, "ROOT", tmp_path) + return docker_task.load_release_matrix(MATRIX) + + +def metadata(image: str = IMAGE): + return {"image": image, "image_id": IMAGE_ID, "os": "linux", "architecture": "amd64"} + + +def archive(tmp_path: Path, image: str = IMAGE): + (tmp_path / "image.tar").write_bytes(b"same tested image bytes") + return create_manifest( + tmp_path, + kind="docker", + version=VERSION, + tag_name=f"v{VERSION}", + source_revision=FULL_SHA, + run_id="1234", + metadata=metadata(image), + ) + + +def test_release_matrix_resolves_exact_ten_pairs(complete_source): + assert {(target.platform, target.bundle) for target in complete_source} == { + (platform, bundle) + for platform in ("cuda12", "cpu") + for bundle in ("default", "ctranslate2", "sglang", "transformers5") + } | {("cuda13", "sglang-cu130"), ("cuda13", "tensorrt-llm")} + assert len(complete_source) == 10 + + +def test_release_matrix_fails_closed_for_absent_bundle(complete_source, tmp_path): + (tmp_path / "packages/sie_server/bundles/ctranslate2.yaml").unlink() + with pytest.raises(ValueError, match="release bundle does not exist: ctranslate2"): + docker_task.load_release_matrix(MATRIX) + + +def test_bundle_requires_its_adapter_source(complete_source, tmp_path): + (tmp_path / "packages/sie_server/bundles/ctranslate2.yaml").write_text( + "adapters: [sie_server.adapters.ctranslate2]\n" + ) + with pytest.raises(ValueError, match="adapter source is missing"): + docker_task.load_release_matrix(MATRIX) + + +def test_release_matrix_rejects_duplicates(tmp_path): + path = tmp_path / "matrix.json" + path.write_text(json.dumps({"platforms": ["cpu", "cpu"], "bundles": ["default"]})) + with pytest.raises(ValueError, match="duplicate"): + docker_task.load_release_matrix(path) + + +def test_release_matrix_rejects_bundle_platform_disagreement(complete_source): + with pytest.raises(ValueError, match="disagrees"): + docker_task.validate_target(docker_task.ServerTarget("cuda13", "default")) + + +def test_build_commands_only_load_source_bound_images(): + for command in ( + docker_task.build_server_command( + registry="ghcr.io/superlinked", + version=VERSION, + target=docker_task.ServerTarget("cpu", "default"), + source_revision=FULL_SHA, + ), + docker_task.build_service_command( + registry="ghcr.io/superlinked", + version=VERSION, + service="sie-config", + source_revision=FULL_SHA, + ), + ): + assert "--load" in command + assert "--push" not in command + assert f"org.opencontainers.image.revision={FULL_SHA}" in command + assert "org.opencontainers.image.source=https://github.com/superlinked/sie" in command + with pytest.raises(ValueError, match="full 40-character"): + docker_task.validate_source_revision("abc123") + with pytest.raises(SystemExit): + docker_task.parser().parse_args( + [ + "build-service", + "--registry", + "local", + "--version", + "0.0.0", + "--service", + "sie-config", + "--source-revision", + FULL_SHA, + "--push", + ] + ) + + +def test_complete_set_verified_before_alias_commands(complete_source, monkeypatch, tmp_path): + commands = [] + + def fail_verification(*_args, **_kwargs): + raise RuntimeError("incomplete") + + monkeypatch.setattr(docker_task, "verify_release", fail_verification) + monkeypatch.setattr(docker_task, "run", commands.append) + with pytest.raises(RuntimeError, match="incomplete"): + docker_task.move_aliases( + "ghcr.io/superlinked", + VERSION, + complete_source, + evidence_dir=tmp_path, + source_revision=FULL_SHA, + run_id="1234", + ) + assert commands == [] + + +def test_expected_release_set_has_fifteen_tags_and_six_names(complete_source): + images = docker_task.expected_versioned_images("ghcr.io/superlinked", VERSION, complete_source) + assert len(images) == len(set(images)) == 15 + assert len({image.split(":")[0] for image in images}) == 6 + assert f"ghcr.io/superlinked/sie-server-rust:v{VERSION}-cuda12-sm89" in images + + +def test_export_saves_inspected_image_and_records_same_bytes(tmp_path, monkeypatch): + monkeypatch.setattr(docker_task, "inspect_loaded", lambda *_: metadata()) + commands = [] + + def save(command): + commands.append(command) + Path(command[4]).write_bytes(b"tested bytes") + + monkeypatch.setattr(docker_task, "run", save) + manifest = docker_task.export_image(IMAGE, tmp_path, version=VERSION, source_revision=FULL_SHA, run_id="1234") + assert commands == [["docker", "image", "save", "--output", str(tmp_path / "image.tar"), IMAGE]] + assert manifest["metadata"] == metadata() + assert manifest["files"][0]["size"] == len(b"tested bytes") + + +@pytest.mark.parametrize("change", ["revision", "archive", "image", "digest"]) +def test_archive_rejects_mismatched_binding_before_push(tmp_path, monkeypatch, change): + data = archive(tmp_path) + if change == "revision": + data["source_revision"] = "c" * 40 + if change == "image": + data["metadata"]["image"] = IMAGE + "-other" + if change == "digest": + data["metadata"]["image_id"] = "sha256:" + "c" * 64 + (tmp_path / "provenance.json").write_text(json.dumps(data)) + if change == "archive": + (tmp_path / "image.tar").write_bytes(b"different") + commands = [] + monkeypatch.setattr(docker_task, "run", commands.append) + monkeypatch.setattr(docker_task, "inspect_loaded", lambda *_: metadata()) + with pytest.raises(ValueError, match="mismatch"): + docker_task.publish_archive(IMAGE, tmp_path, version=VERSION, source_revision=FULL_SHA, run_id="1234") + assert not any(command[1] == "push" for command in commands) + + +@pytest.mark.parametrize("existing", [None, IMAGE_ID, "sha256:" + "c" * 64]) +def test_publication_loads_same_archive_and_never_rebuilds_or_clobbers(tmp_path, monkeypatch, existing): + archive(tmp_path) + commands = [] + monkeypatch.setattr(docker_task, "run", commands.append) + monkeypatch.setattr(docker_task, "inspect_loaded", lambda *_: metadata()) + monkeypatch.setattr( + docker_task, "remote_image_id", lambda _image, **kwargs: existing if kwargs.get("allow_missing") else IMAGE_ID + ) + if existing is not None and existing != IMAGE_ID: + with pytest.raises(ValueError, match="overwrite"): + docker_task.publish_archive(IMAGE, tmp_path, version=VERSION, source_revision=FULL_SHA, run_id="1234") + else: + docker_task.publish_archive(IMAGE, tmp_path, version=VERSION, source_revision=FULL_SHA, run_id="1234") + assert commands[0] == ["docker", "image", "load", "--input", str(tmp_path / "image.tar")] + assert all("build" not in command and "buildx" not in command for command in commands) + assert (["docker", "push", IMAGE] in commands) == (existing is None) + + +def test_remote_inspection_does_not_treat_authorization_failure_as_absent(monkeypatch): + monkeypatch.setattr( + docker_task.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 1, "", "unauthorized"), + ) + with pytest.raises(RuntimeError, match="cannot inspect"): + docker_task.remote_image_id(IMAGE, allow_missing=True) + + +def test_full_set_verifier_rejects_source_or_remote_digest_mismatch(complete_source, tmp_path, monkeypatch): + evidence = tmp_path / "evidence" + evidence.mkdir() + for offset, image in enumerate( + docker_task.expected_versioned_images("ghcr.io/superlinked", VERSION, complete_source) + ): + data = { + "schema": 1, + "repository": "superlinked/sie", + "kind": "docker", + "version": VERSION, + "tag_name": f"v{VERSION}", + "source_revision": FULL_SHA, + "run_id": "1234", + "metadata": metadata(image), + } + (evidence / f"{offset}.json").write_text(json.dumps(data)) + monkeypatch.setattr(docker_task, "remote_image_id", lambda *_: IMAGE_ID) + kwargs = {"evidence_dir": evidence, "source_revision": FULL_SHA, "run_id": "1234"} + docker_task.verify_release("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + monkeypatch.setattr(docker_task, "remote_image_id", lambda *_: "sha256:" + "c" * 64) + with pytest.raises(ValueError, match="differs"): + docker_task.verify_release("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + (evidence / "0.json").unlink() + with pytest.raises(ValueError, match="exact complete"): + docker_task.verify_release("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + + +def published_release(version): + return {"tag_name": f"v{version}", "draft": False, "prerelease": False, "published_at": "2026-09-03T12:00:00Z"} + + +@pytest.fixture +def public_releases(monkeypatch): + state = {"pages": [[published_release(VERSION)]], "revisions": {f"v{VERSION}": FULL_SHA}, "status": "ahead"} + + def capture(command): + assert command == ["gh", "api", "--paginate", "--slurp", "repos/superlinked/sie/releases?per_page=100"] + return json.dumps(state["pages"]) + + def api(path): + if path.startswith("git/ref/tags/"): + tag = path.removeprefix("git/ref/tags/") + return {"ref": f"refs/tags/{tag}", "object": {"type": "commit", "sha": state["revisions"][tag]}} + assert path.startswith("compare/") + return {"status": state["status"]} + + monkeypatch.setattr(docker_task, "capture", capture) + monkeypatch.setattr(docker_task, "api", api) + monkeypatch.setattr(docker_task, "verify_release", lambda *_args, **_kwargs: None) + return state + + +def test_failed_old_alias_retry_never_rolls_back_newer_success(complete_source, public_releases, monkeypatch, tmp_path): + writes = [] + + def interrupted_write(command): + if len(writes) == 1: + raise RuntimeError("interrupted old alias job") + writes.append(command) + + monkeypatch.setattr(docker_task, "run", interrupted_write) + kwargs = {"evidence_dir": tmp_path, "source_revision": FULL_SHA, "run_id": "1234"} + with pytest.raises(RuntimeError, match="interrupted old alias job"): + docker_task.move_aliases("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + public_releases["pages"].append([published_release("0.7.5")]) + public_releases["revisions"]["v0.7.5"] = "c" * 40 + monkeypatch.setattr(docker_task, "run", writes.append) + docker_task.move_aliases( + "ghcr.io/superlinked", + "0.7.5", + complete_source, + evidence_dir=tmp_path, + source_revision="c" * 40, + run_id="1235", + ) + newer_writes = writes[1:].copy() + assert len(newer_writes) == 15 + assert all(":v0.7.5" in command[-1] for command in newer_writes) + docker_task.move_aliases("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + assert writes[1:] == newer_writes + + +def test_same_release_alias_retry_is_idempotent(complete_source, public_releases, monkeypatch, tmp_path): + writes = [] + monkeypatch.setattr(docker_task, "run", writes.append) + kwargs = {"evidence_dir": tmp_path, "source_revision": FULL_SHA, "run_id": "1234"} + docker_task.move_aliases("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + first = writes.copy() + docker_task.move_aliases("ghcr.io/superlinked", VERSION, complete_source, **kwargs) + assert len(first) == 15 + assert writes == first + first + + +@pytest.mark.parametrize("state", ["invalid_version", "missing_release", "wrong_sha", "missing_tag", "diverged"]) +def test_unverifiable_published_release_never_writes_aliases( + complete_source, public_releases, monkeypatch, tmp_path, state +): + public_releases["pages"].append([published_release("0.7.5")]) + public_releases["revisions"]["v0.7.5"] = "c" * 40 + if state == "invalid_version": + public_releases["pages"][1][0]["tag_name"] = "vnext" + elif state == "missing_release": + public_releases["pages"].pop(0) + elif state == "wrong_sha": + public_releases["revisions"][f"v{VERSION}"] = "b" * 40 + elif state == "missing_tag": + public_releases["revisions"]["v0.7.5"] = "invalid" + else: + public_releases["status"] = "diverged" + writes = [] + monkeypatch.setattr(docker_task, "run", writes.append) + with pytest.raises(ValueError, match=r"release|revision|version"): + docker_task.move_aliases( + "ghcr.io/superlinked", + VERSION, + complete_source, + evidence_dir=tmp_path, + source_revision=FULL_SHA, + run_id="1234", + ) + assert writes == [] + + +def test_failed_public_release_read_never_writes_aliases(complete_source, monkeypatch, tmp_path): + def unavailable(_command): + raise subprocess.CalledProcessError(1, ["gh", "api"]) + + monkeypatch.setattr(docker_task, "capture", unavailable) + monkeypatch.setattr(docker_task, "verify_release", lambda *_args, **_kwargs: None) + writes = [] + monkeypatch.setattr(docker_task, "run", writes.append) + with pytest.raises(subprocess.CalledProcessError): + docker_task.move_aliases( + "ghcr.io/superlinked", + VERSION, + complete_source, + evidence_dir=tmp_path, + source_revision=FULL_SHA, + run_id="1234", + ) + assert writes == [] + + +def test_published_tag_rejects_contradictory_reference_and_annotated_identity(monkeypatch): + tag = f"v{VERSION}" + monkeypatch.setattr(docker_task, "api", lambda _path: {"ref": "refs/tags/v0.7.5"}) + with pytest.raises(ValueError, match="reference mismatch"): + docker_task.published_tag_revision(tag) + reference = {"ref": f"refs/tags/{tag}", "object": {"type": "tag", "sha": "d" * 40}} + annotated = {"tag": tag, "sha": "d" * 40, "object": {"type": "commit", "sha": FULL_SHA}} + monkeypatch.setattr(docker_task, "api", lambda path: reference if path.startswith("git/ref/") else annotated) + assert docker_task.published_tag_revision(tag) == FULL_SHA + annotated["sha"] = "e" * 40 + with pytest.raises(ValueError, match="annotated tag identity mismatch"): + docker_task.published_tag_revision(tag) + + +def test_draft_and_prerelease_do_not_block_current_stable_aliases(public_releases): + public_releases["pages"].append( + [ + {**published_release("0.7.5"), "draft": True}, + {**published_release("0.7.6"), "prerelease": True}, + ] + ) + assert docker_task.alias_release_is_current(VERSION, FULL_SHA) + + +@pytest.mark.parametrize( + "pages", + [ + {"not": "paginated releases"}, + [[{**published_release(VERSION), "draft": "false"}]], + [[{**published_release(VERSION), "published_at": True}]], + [[{**published_release(VERSION), "published_at": "not a timestamp"}]], + [[{**published_release(VERSION), "published_at": "2026-02-30T12:00:00Z"}]], + [[published_release(VERSION), published_release(VERSION)]], + ], +) +def test_malformed_or_duplicate_release_listing_fails_closed(public_releases, pages): + public_releases["pages"] = pages + with pytest.raises(ValueError, match=r"malformed|duplicated"): + docker_task.alias_release_is_current(VERSION, FULL_SHA) diff --git a/tools/ci/tests/test_helm_task.py b/tools/ci/tests/test_helm_task.py new file mode 100644 index 000000000..7da5f6498 --- /dev/null +++ b/tools/ci/tests/test_helm_task.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path + +from tools.mise_tasks import helm + + +def test_dependency_command_uses_checked_in_chart() -> None: + assert helm.dependency_command() == ["dependency", "build", "deploy/helm/sie-cluster"] + + +def test_dependency_repositories_are_derived_and_idempotent() -> None: + commands = helm.dependency_repository_commands() + assert commands + assert all(command[:2] == ["repo", "add"] for command in commands) + assert all(command[-1] == "--force-update" for command in commands) + urls = [command[-2] for command in commands] + assert len(urls) == len(set(urls)) + assert "https://kedacore.github.io/charts" in urls + assert all(not url.startswith("oci://") for url in urls) + + +def test_validation_defaults_disable_payload_store(monkeypatch) -> None: + calls: list[list[str]] = [] + monkeypatch.setattr(helm, "run_helm", lambda args: calls.append(args) or 0) + assert helm.cmd_lint([]) == 0 + assert calls == [["lint", "deploy/helm/sie-cluster", "--set", "payloadStore.enabled=false"]] + + +def test_config_staging_is_removed_after_render(tmp_path: Path, monkeypatch) -> None: + root = tmp_path + bundles = root / "packages/sie_server/bundles" + models = root / "packages/sie_server/models" + bundles.mkdir(parents=True) + models.mkdir(parents=True) + (bundles / "default.yaml").write_text("name: default\n", encoding="utf-8") + (models / "model.yaml").write_text("id: model\n", encoding="utf-8") + monkeypatch.setattr(helm, "resolve_project_root", lambda: root) + + helm._sync_configs_to_helm() + staged = root / "deploy/helm/sie-cluster/files" + assert (staged / "bundles/default.yaml").is_file() + assert (staged / "models/model.yaml").is_file() + + helm._cleanup_helm_configs() + assert not (staged / "bundles").exists() + assert not (staged / "models").exists() diff --git a/tools/ci/tests/test_public_tree.py b/tools/ci/tests/test_public_tree.py new file mode 100644 index 000000000..729c12f4f --- /dev/null +++ b/tools/ci/tests/test_public_tree.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import re +from pathlib import Path + +from tools.ci import check_public_tree + +REPOSITORY_ROOT = check_public_tree.REPOSITORY_ROOT + + +def test_reference_guard_reports_forbidden_text(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(check_public_tree, "REPOSITORY_ROOT", tmp_path) + clean = tmp_path / "clean.txt" + clean.write_text("https://github.com/superlinked/sie\n", encoding="utf-8") + bad = tmp_path / "bad.txt" + bad.write_bytes(b"packages/" + b"sie_cloud" + b"/gateway\n") + findings = check_public_tree.violations([clean, bad]) + assert len(findings) == 1 + assert "forbidden public-tree reference" in findings[0] + + +def test_exported_tree_fallback_excludes_generated_dependencies(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(check_public_tree, "REPOSITORY_ROOT", tmp_path) + source = tmp_path / "source.py" + source.write_text("public\n") + generated = tmp_path / "node_modules/dependency.txt" + generated.parent.mkdir() + generated.write_bytes(b"packages/" + b"sie_cloud" + b"/gateway\n") + assert check_public_tree.candidate_paths() == [source] + + +def test_bootstrap_uses_root_locks_even_in_ci() -> None: + full_sync = (REPOSITORY_ROOT / "tools/mise_tasks/full-sync.bash").read_text() + init = (REPOSITORY_ROOT / "tools/init.sh").read_text() + package = (REPOSITORY_ROOT / "package.json").read_text() + mise_config = (REPOSITORY_ROOT / "mise.toml").read_text() + assert "mise run full-sync" in init + assert "mise run sync" in full_sync + assert "pnpm install --frozen-lockfile" in full_sync + assert "packages/sie_ts_sdk" not in full_sync + assert "CI:-" not in full_sync + assert '"packageManager": "pnpm@9.15.9"' in package + assert '"prepare": "pnpm run build"' in (REPOSITORY_ROOT / "packages/sie_ts_sdk/package.json").read_text() + assert "MISE_" not in mise_config + assert "XDG_CONFIG_HOME" not in mise_config + assert "XDG_STATE_HOME" not in mise_config + assert not (REPOSITORY_ROOT / ".npmrc").exists() + assert not (REPOSITORY_ROOT / "packages/sie_ts_sdk/pnpm-lock.yaml").exists() + + +def test_ci_is_fork_safe_and_actions_are_immutable() -> None: + workflow = (REPOSITORY_ROOT / ".github/workflows/ci.yml").read_text() + assert "pull_request_target" not in workflow + assert "secrets." not in workflow + assert "self-hosted" not in workflow + assert "permissions:\n contents: read" in workflow + for name in ( + "CI / Policy", + "CI / Python", + "CI / TypeScript", + "CI / Rust", + "CI / Contracts", + "CI / Helm", + "CI / Required", + ): + assert f"name: {name}" in workflow + action_lines = [line.strip() for line in workflow.splitlines() if "uses:" in line] + assert action_lines + assert all( + "uses: ./.github/workflows/" in line or re.search(r"@[0-9a-f]{40}(?:\s|$)", line) for line in action_lines + ) + + +def test_ci_rust_job_owns_the_standalone_candle_workspace() -> None: + workflow = (REPOSITORY_ROOT / ".github/workflows/ci.yml").read_text() + manifest = "--manifest-path packages/sie_server_rust/Cargo.toml" + assert f"cargo fmt {manifest} -- --check" in workflow + assert f"cargo check {manifest} --locked --all-targets" in workflow + assert f"cargo clippy {manifest} --locked --all-targets -- -D warnings" in workflow + assert "python tools/ci/rust_tests.py" in workflow diff --git a/tools/ci/tests/test_release_artifact.py b/tools/ci/tests/test_release_artifact.py new file mode 100644 index 000000000..c6dbfac19 --- /dev/null +++ b/tools/ci/tests/test_release_artifact.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tools.ci import build_audio_prep_release_asset as audio +from tools.ci import build_sidecar_release_asset as sidecar +from tools.ci import publish_helm_archive as helm +from tools.ci import release_artifact as artifact +from tools.ci import restore_release_artifact as restore + +IDENTITY = {"version": "0.7.4", "tag_name": "v0.7.4", "source_revision": "a" * 40, "run_id": "1234"} + + +def test_provenance_roundtrip_and_identical_restamp(tmp_path): + (tmp_path / "tested.whl").write_bytes(b"tested wheel") + first = artifact.create_manifest(tmp_path, kind="python", **IDENTITY) + assert artifact.create_manifest(tmp_path, kind="python", **IDENTITY) == first + assert artifact.validate_manifest(tmp_path, kind="python", **IDENTITY) == first + + +@pytest.mark.parametrize("field", ["version", "tag_name", "source_revision", "run_id", "repository"]) +def test_wrong_provenance_identity_is_rejected(tmp_path, field): + (tmp_path / "tested.whl").write_bytes(b"tested wheel") + data = artifact.create_manifest(tmp_path, kind="python", **IDENTITY) + data[field] = "wrong" + (tmp_path / artifact.MANIFEST).write_text(json.dumps(data)) + with pytest.raises(ValueError, match="provenance mismatch"): + artifact.validate_manifest(tmp_path, **IDENTITY) + + +@pytest.mark.parametrize("change", ["bytes", "missing", "extra", "symlink"]) +def test_provenance_requires_exact_payload_file_set(tmp_path, change): + payload = tmp_path / "image.tar" + payload.write_bytes(b"tested") + artifact.create_manifest(tmp_path, kind="docker", **IDENTITY) + if change == "bytes": + payload.write_bytes(b"unseen") + elif change == "missing": + payload.unlink() + elif change == "extra": + (tmp_path / "extra").write_text("unexpected") + else: + (tmp_path / "extra").symlink_to(payload) + with pytest.raises(ValueError, match="artifact"): + artifact.validate_manifest(tmp_path, **IDENTITY) + + +def test_provenance_cannot_replace_different_existing_output(tmp_path): + payload = tmp_path / "image.tar" + payload.write_bytes(b"tested") + artifact.create_manifest(tmp_path, kind="docker", **IDENTITY) + payload.write_bytes(b"unseen") + with pytest.raises(ValueError, match="replace different"): + artifact.create_manifest(tmp_path, kind="docker", **IDENTITY) + + +def test_audio_builder_supports_same_source_and_output_path(tmp_path, monkeypatch): + wheel = tmp_path / "audio.whl" + wheel.write_bytes(b"native wheel") + validated = [] + module = SimpleNamespace( + build_audio_prep_wheel=lambda *_args, **_kwargs: wheel, + _validate_wheel=validated.append, + ) + monkeypatch.setattr(audio, "load_build_wheel", lambda *_: module) + assert audio.main(["--out", str(tmp_path)]) == 0 + assert validated == [wheel] + assert wheel.read_bytes() == b"native wheel" + + +@pytest.mark.parametrize("glibc", ["2.17", "2.36", "2.38"]) +def test_sidecar_binary_checks_architecture_and_glibc_floor(tmp_path, monkeypatch, glibc): + binary = tmp_path / "sidecar" + binary.write_bytes(b"\x7fELF\x02\x01" + b"\0" * 12 + b"\x3e\0") + monkeypatch.setattr( + sidecar, + "capture", + lambda command: f"GLIBC_{glibc}" if "--version-info" in command else "(NEEDED) Shared library: [libc.so.6]", + ) + if glibc == "2.38": + with pytest.raises(ValueError, match="glibc requirement"): + sidecar.inspect_abi(binary) + else: + assert sidecar.inspect_abi(binary)["glibc_minimum"] == glibc + binary.write_bytes(b"\x7fELF\x02\x01" + b"\0" * 12 + b"\xb7\0") + with pytest.raises(ValueError, match="x86_64"): + sidecar.inspect_abi(binary) + + +@pytest.mark.parametrize("remote", [b"tested chart", b"different chart", None]) +def test_chart_publisher_only_uploads_missing_exact_archive(tmp_path, monkeypatch, remote): + chart = tmp_path / "sie-cluster-0.7.4.tgz" + chart.write_bytes(b"tested chart") + artifact.create_manifest(tmp_path, kind="helm", **IDENTITY) + calls = [] + uploaded = False + + def execute(command, **_kwargs): + nonlocal uploaded + calls.append(command) + if command[1] == "push": + uploaded = True + if command[1] == "pull": + if remote is None and not uploaded: + return subprocess.CompletedProcess(command, 1, "", "manifest unknown") + destination = Path(command[-1]) / chart.name + destination.write_bytes(remote if remote is not None else chart.read_bytes()) + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(helm.subprocess, "run", execute) + kwargs = {key: value for key, value in IDENTITY.items() if key != "tag_name"} + if remote == b"different chart": + with pytest.raises(ValueError, match="remote chart differs"): + helm.publish(tmp_path, **kwargs) + else: + helm.publish(tmp_path, **kwargs) + assert uploaded == (remote is None) + assert all(command[1] in {"pull", "push"} for command in calls) + + +@pytest.mark.parametrize("mode", ["missing", "valid", "wrong_sha", "expired"]) +def test_retry_restores_only_original_retained_bytes(tmp_path, monkeypatch, mode): + (tmp_path / "image.tar").write_bytes(b"retained tested bytes") + artifact.create_manifest(tmp_path, kind="docker", **IDENTITY) + record = { + "name": "docker-service-sie-config-0.7.4", + "expired": mode == "expired", + "workflow_run": {"id": 1234, "head_sha": "b" * 40 if mode == "wrong_sha" else IDENTITY["source_revision"]}, + } + pages = [{"artifacts": [] if mode == "missing" else [record]}] + monkeypatch.setattr(restore.subprocess, "check_output", lambda *_args, **_kwargs: json.dumps(pages)) + commands = [] + monkeypatch.setattr(restore.subprocess, "run", lambda command, **_kwargs: commands.append(command)) + if mode in {"wrong_sha", "expired"}: + with pytest.raises(ValueError, match="original release artifact"): + restore.restore(tmp_path, name=record["name"], kind="docker", **IDENTITY) + assert commands == [] + else: + assert restore.restore(tmp_path, name=record["name"], kind="docker", **IDENTITY) == (mode == "valid") + assert all(command[:3] == ["gh", "run", "download"] for command in commands) + + +def test_retry_evidence_must_match_retained_image_archive(tmp_path, monkeypatch): + archive_dir = tmp_path / "archive" + archive_dir.mkdir() + (archive_dir / "image.tar").write_bytes(b"retained tested bytes") + artifact.create_manifest(archive_dir, kind="docker", **IDENTITY) + evidence = tmp_path / "evidence" + evidence.mkdir() + record = { + "name": "docker-evidence-service-sie-config-0.7.4", + "expired": False, + "workflow_run": {"id": 1234, "head_sha": IDENTITY["source_revision"]}, + } + monkeypatch.setattr( + restore.subprocess, "check_output", lambda *_args, **_kwargs: json.dumps([{"artifacts": [record]}]) + ) + monkeypatch.setattr(restore.subprocess, "run", lambda *_args, **_kwargs: None) + provenance = evidence / "provenance.json" + provenance.write_bytes((archive_dir / "provenance.json").read_bytes()) + kwargs = {"name": record["name"], "kind": "docker", "evidence_of": archive_dir, **IDENTITY} + assert restore.restore(evidence, **kwargs) + provenance.write_text("different image evidence") + with pytest.raises(ValueError, match="differs from its retained archive"): + restore.restore(evidence, **kwargs) diff --git a/tools/ci/tests/test_release_contract.py b/tools/ci/tests/test_release_contract.py new file mode 100644 index 000000000..87f2c1b3f --- /dev/null +++ b/tools/ci/tests/test_release_contract.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import textwrap +import tomllib +from copy import deepcopy +from pathlib import Path + +import pytest + +from tools.ci import check_release_contract as contract + + +def run_audio_uploader( + tmp_path: Path, *, asset_mode: str, remote_sha: str +) -> tuple[subprocess.CompletedProcess[str], Path, Path]: + tmp_path.mkdir() + filename = "sie_audio_prep-0.7.4-cp312-abi3-manylinux_2_28_x86_64.whl" + wheel = tmp_path / filename + wheel.write_bytes(b"validated native wheel bytes") + marker = tmp_path / "uploaded" + calls = tmp_path / "calls" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + """#!/usr/bin/env bash +set -euo pipefail +emit_asset() { + printf '{"assets":[{"id":1,"name":"%s","size":%s,"digest":"sha256:%s","browser_download_url":"%s"}]}\\n' \\ + "$AUDIO_WHEEL_FILENAME" "$FAKE_REMOTE_SIZE" "$FAKE_REMOTE_SHA" "$FAKE_BROWSER_URL" +} +if [[ "$1" == api && "$2" == repos/*/releases/tags/* ]]; then + if [[ "$FAKE_ASSET_MODE" == missing && ! -f "$FAKE_UPLOAD_MARKER" ]]; then + printf '{"assets":[]}\\n' + else + emit_asset + fi +elif [[ "$1" == release && "$2" == upload ]]; then + printf '%s\\n' "$*" >> "$FAKE_CALLS" + touch "$FAKE_UPLOAD_MARKER" +else + printf 'unexpected fake gh arguments: %s\\n' "$*" >&2 + exit 2 +fi +""" + ) + gh.chmod(0o755) + environment = { + **os.environ, + "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}", + "AUDIO_WHEEL_FILENAME": filename, + "GITHUB_REPOSITORY": "superlinked/sie", + "RELEASE_TAG": "v0.7.4", + "GH_TOKEN": str(tmp_path), + "FAKE_ASSET_MODE": asset_mode, + "FAKE_REMOTE_SIZE": str(wheel.stat().st_size), + "FAKE_REMOTE_SHA": remote_sha, + "FAKE_BROWSER_URL": f"https://github.com/superlinked/sie/releases/download/v0.7.4/{filename}", + "FAKE_UPLOAD_MARKER": str(marker), + "FAKE_CALLS": str(calls), + } + result = subprocess.run( # noqa: S603 + ["/bin/bash", str(contract.ROOT / "tools/ci/upload_audio_prep_release_asset.bash"), str(wheel)], + cwd=tmp_path, + env=environment, + check=False, + capture_output=True, + text=True, + ) + return result, marker, calls + + +def test_exact_package_matrices() -> None: + assert contract.python_matrices() == ( + contract.PYTHON_DISTRIBUTIONS, + contract.PYTHON_DISTRIBUTIONS, + ) + assert contract.npm_matrix() == contract.NPM_PACKAGES + + +def test_release_please_surface_is_exact() -> None: + assert contract.release_config_errors() == [] + + +def test_release_please_bootstrap_is_the_exact_public_v073_commit() -> None: + config = contract.load_json("release-please-config.json") + assert config["bootstrap-sha"] == contract.RELEASE_BOOTSTRAP_SHA + assert config["bootstrap-sha"] == "60996d9c30168e0f8e85b680295f147fdee87f61" + assert "bootstrap-sha" not in config["packages"]["."] + + +@pytest.mark.parametrize("bootstrap_sha", [None, "60996d9", "b" * 40]) +def test_release_please_rejects_any_other_bootstrap_boundary(monkeypatch, bootstrap_sha) -> None: + real_load_json = contract.load_json + + def load_json(path): + document = real_load_json(path) + if path == "release-please-config.json": + document = deepcopy(document) + if bootstrap_sha is None: + document.pop("bootstrap-sha") + else: + document["bootstrap-sha"] = bootstrap_sha + return document + + monkeypatch.setattr(contract, "load_json", load_json) + assert "release-please bootstrap-sha must be the exact public v0.7.3 commit" in contract.release_config_errors() + + +def test_option_ext_mpl_exception_is_exact_and_cannot_broaden() -> None: + policy = tomllib.loads((contract.ROOT / "deny.toml").read_text()) + assert contract._license_policy_errors(policy) == [] + + globally_allowed = deepcopy(policy) + globally_allowed["licenses"]["allow"].append("MPL-2.0") + assert "MPL-2.0 must not be globally allowed" in contract._license_policy_errors(globally_allowed) + + wildcard_version = deepcopy(policy) + option_ext = next(entry for entry in wildcard_version["licenses"]["exceptions"] if entry["name"] == "option-ext") + option_ext["version"] = "*" + assert "option-ext MPL-2.0 allowance must be confined to exact version 0.2.0" in contract._license_policy_errors( + wildcard_version + ) + + extra_crate = deepcopy(policy) + extra_crate["licenses"]["exceptions"].append({"name": "unreviewed", "allow": ["MPL-2.0"]}) + assert ( + "cargo-deny MPL-2.0 exception surface differs from the reviewed crate set" + in contract._license_policy_errors(extra_crate) + ) + + +def test_release_workflows_are_pinned_and_fail_closed() -> None: + assert contract.workflow_pin_errors() == [] + assert contract.release_workflow_errors() == [] + assert contract.publisher_job_errors() == [] + + +@pytest.mark.parametrize("replacement", [" queue: single", " queue: arbitrary", "", " queue: max\n queue: single"]) +def test_release_queue_must_retain_pending_runs(replacement): + top = (contract.ROOT / ".github/workflows/release.yml").read_text() + ci = (contract.ROOT / ".github/workflows/ci.yml").read_text() + assert contract.release_queue_errors(top, ci) == [] + assert contract.release_queue_errors(top.replace(" queue: max", replacement), ci) + + +def test_release_queue_cancellation_and_linter_exception_are_exact(): + top = (contract.ROOT / ".github/workflows/release.yml").read_text() + ci = (contract.ROOT / ".github/workflows/ci.yml").read_text() + assert contract.release_queue_errors(top.replace("cancel-in-progress: false", "cancel-in-progress: true"), ci) + assert contract.release_queue_errors(top, ci.replace(contract.QUEUE_SCHEMA_DIAGNOSTIC, ".*")) + diagnostic = 'unexpected key "queue" for "concurrency" section. expected one of "cancel-in-progress", "group"' + assert re.fullmatch(contract.QUEUE_SCHEMA_DIAGNOSTIC, diagnostic) + assert not re.fullmatch(contract.QUEUE_SCHEMA_DIAGNOSTIC, diagnostic.replace('"queue"', '"bogus"')) + + +def test_authoring_no_longer_assumes_release_sha_is_push_sha(): + jobs = contract.workflow_job_blocks(".github/workflows/release.yml") + assert 'test "$RELEASE_SHA" = "$EXPECTED_SHA"' not in jobs["release-please"] + assert "needs.release-please.outputs" not in (contract.ROOT / ".github/workflows/release.yml").read_text() + assert "release_guard.py prepare" in jobs["prepare"] + + +def test_prereleases_are_ignored_by_prepare_and_completion(): + jobs = contract.workflow_job_blocks(".github/workflows/release.yml") + for job in ("prepare", "complete"): + condition = contract.job_scalar(jobs[job], "if") + assert "github.event_name == 'release'" in condition + assert "github.event.action == 'published'" in condition + assert "github.event.release.draft == false" in condition + assert "github.event.release.prerelease == false" in condition + + +@pytest.mark.parametrize("result", ["success", "failure", "cancelled", "skipped"]) +def test_actual_release_completion_script_rejects_non_success(monkeypatch, result): + block = contract.workflow_job_blocks(".github/workflows/release.yml")["complete"] + script = re.search(r"python3 - <<'PY'\n(.*?)\n\s+PY", block, re.DOTALL) + assert script is not None + results = { + family: {"result": "success"} + for family in ("prepare", "python-publish", "npm-publish", "docker", "helm", "audio", "native") + } + results["native"]["result"] = result + monkeypatch.setenv("RESULTS", json.dumps(results)) + code = compile(textwrap.dedent(script.group(1)), "release-complete", "exec") + if result == "success": + exec(code, {}) # noqa: S102 + else: + with pytest.raises(SystemExit, match="Incomplete release"): + exec(code, {}) # noqa: S102 + + +def test_candle_and_docker_release_source_closure() -> None: + assert contract.candle_source_errors() == [] + assert contract.docker_copy_errors() == [] + assert contract.docker_release_errors() == [] + + +def test_helm_release_follows_verified_images() -> None: + assert contract.helm_release_errors() == [] + + +def test_public_release_app_hands_final_pr_head_to_ci() -> None: + assert contract.release_app_errors() == [] + + +def test_release_authoring_is_default_off_until_its_app_is_configured() -> None: + block = contract.workflow_job_blocks(".github/workflows/release.yml")["release-please"] + condition = contract.job_scalar(block, "if") + assert condition is not None + assert "vars.PUBLIC_RELEASE_AUTOMATION_ENABLED == 'true'" in condition + assert "vars.PUBLIC_RELEASE_AUTOMATION_ENABLED != 'false'" not in condition + assert "PUBLIC_RELEASE_PUBLISHING_ENABLED" not in condition + assert contract.release_automation_gate_errors(condition) == [] + + gate = "vars.PUBLIC_RELEASE_AUTOMATION_ENABLED == 'true'" + for unsafe in ( + condition.replace(gate, "vars.PUBLIC_RELEASE_AUTOMATION_ENABLED != 'false'"), + f"{condition} || true", + condition.replace(gate, f"({gate} || vars.PUBLIC_RELEASE_AUTOMATION_ENABLED == '')"), + ): + assert contract.release_automation_gate_errors(unsafe) + + +def test_native_audio_asset_matches_downstream_browser_download_contract() -> None: + version, filename, url = contract.audio_release_contract() + assert filename == f"sie_audio_prep-{version}-cp312-abi3-manylinux_2_28_x86_64.whl" + assert url == f"https://github.com/superlinked/sie/releases/download/v{version}/{filename}" + assert contract.audio_release_errors() == [] + + +def test_native_audio_builder_installs_exact_rust_and_never_clobbers() -> None: + workflow = (contract.ROOT / ".github/workflows/release-audio.yml").read_text() + uploader = (contract.ROOT / "tools/ci/upload_native_release_asset.bash").read_text() + assert contract.AUDIO_MANYLINUX_IMAGE in workflow + assert "version: 2026.7.11" in workflow + assert "mise --no-config install python@3.12.12 uv@0.5.31 zig@0.13.0 rust@1.97.0" in workflow + assert "rust@1.97.0 -- rustc --version" in workflow + assert "rust@1.97.0 -- cargo --version" in workflow + assert "--clobber" not in workflow + assert "--clobber" not in uploader + assert "sha256sum" in uploader + + +def test_native_audio_uploader_accepts_only_missing_or_identical_asset(tmp_path: Path) -> None: + local_sha = hashlib.sha256(b"validated native wheel bytes").hexdigest() + + identical, identical_marker, _ = run_audio_uploader( + tmp_path / "identical", asset_mode="present", remote_sha=local_sha + ) + assert identical.returncode == 0, identical.stderr + assert not identical_marker.exists() + + conflicting, conflicting_marker, _ = run_audio_uploader( + tmp_path / "conflicting", asset_mode="present", remote_sha="0" * 64 + ) + assert conflicting.returncode != 0 + assert not conflicting_marker.exists() + + missing, missing_marker, calls = run_audio_uploader( + tmp_path / "missing", asset_mode="missing", remote_sha=local_sha + ) + assert missing.returncode == 0, missing.stderr + assert missing_marker.is_file() + assert calls.read_text().startswith("release upload --repo superlinked/sie v0.7.4 ") + assert "--clobber" not in calls.read_text() diff --git a/tools/ci/tests/test_release_guard.py b/tools/ci/tests/test_release_guard.py new file mode 100644 index 000000000..4bf66c68d --- /dev/null +++ b/tools/ci/tests/test_release_guard.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import subprocess +from datetime import UTC, datetime, timedelta + +import pytest + +from tools.ci import release_guard as guard +from tools.ci import release_recovery as recovery + +SHA = "a" * 40 +NOW = datetime(2026, 9, 3, tzinfo=UTC) + + +@pytest.mark.parametrize("version", ["0.7.2", "0.7.3", "0.0.1", "0.7.4-rc.1", "v0.7.4", "01.2.3", "bad"]) +def test_new_publication_rejects_seed_and_nonstable_versions(version): + with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + guard.stable_version(version) + + +def published(): + return { + "action": "published", + "repository": {"full_name": "superlinked/sie"}, + "release": {"tag_name": "v0.7.4", "draft": False, "prerelease": False}, + } + + +def context(): + return { + "GITHUB_REPOSITORY": "superlinked/sie", + "GITHUB_EVENT_NAME": "release", + "GITHUB_REF": "refs/tags/v0.7.4", + "GITHUB_REF_PROTECTED": "true", + "PUBLIC_RELEASE_PUBLISHING_ENABLED": "true", + "GITHUB_SHA": SHA, + } + + +@pytest.mark.parametrize( + ("key", "value"), + [ + ("GITHUB_REPOSITORY", "someone/fork"), + ("GITHUB_EVENT_NAME", "push"), + ("GITHUB_EVENT_NAME", "workflow_dispatch"), + ("GITHUB_EVENT_NAME", "pull_request"), + ("GITHUB_REF", "refs/heads/feature"), + ("GITHUB_REF_PROTECTED", "false"), + ("PUBLIC_RELEASE_PUBLISHING_ENABLED", "false"), + ("GITHUB_SHA", "b" * 40), + ], +) +def test_actual_writer_guard_rejects_untrusted_context(key, value): + guard.trusted_context(context(), SHA, event=published()) + with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + guard.trusted_context({**context(), key: value}, SHA, event=published()) + + +@pytest.mark.parametrize(("key", "value"), [("draft", True), ("prerelease", True), ("tag_name", "v0.7.5")]) +def test_publisher_rejects_unstable_or_wrong_release_event(key, value): + event = published() + event["release"][key] = value + with pytest.raises(ValueError, match="exact stable published"): + guard.trusted_context(context(), SHA, event=event) + + +def test_source_a_release_created_during_push_b_publishes_only_from_event_a(monkeypatch): + main_sha = "b" * 40 + calls = [] + monkeypatch.setattr(guard, "stable_release", lambda version, source_sha=None: source_sha or "c" * 40) + monkeypatch.setattr(guard, "api", lambda path: {"protected": True, "commit": {"sha": main_sha}}) + + def command(*args): + calls.append(args) + return SHA if args == ("git", "rev-parse", "HEAD") else "" + + monkeypatch.setattr(guard, "command", command) + identity = guard.prepare_release(context(), published()) + assert identity == {"sha": SHA, "tag_name": "v0.7.4", "version": "0.7.4"} + assert ("git", "merge-base", "--is-ancestor", SHA, "FETCH_HEAD") in calls + guard.trusted_context(context(), SHA, event=published()) + push_b = {**context(), "GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/main", "GITHUB_SHA": main_sha} + with pytest.raises(ValueError, match="original workflow SHA"): + guard.trusted_context(push_b, SHA, event=published()) + with pytest.raises(ValueError, match="exact stable published"): + guard.trusted_context(push_b, main_sha, event=published()) + + +def test_tag_protection_is_not_a_substitute_for_protected_main(monkeypatch): + monkeypatch.setattr(guard, "api", lambda path: {"protected": False}) + with pytest.raises(ValueError, match="independently verified protected main"): + guard.protected_main_ancestor(SHA) + + +def test_non_main_ancestor_is_rejected_even_with_protected_tag(monkeypatch): + monkeypatch.setattr(guard, "api", lambda path: {"protected": True}) + + def command(*args): + if "merge-base" in args: + raise subprocess.CalledProcessError(1, args) + return "" + + monkeypatch.setattr(guard, "command", command) + with pytest.raises(subprocess.CalledProcessError): + guard.protected_main_ancestor(SHA) + + +def test_original_authoring_run_is_not_recovery_evidence(): + jobs = [ + {"id": 0, "name": "release-please", "conclusion": "success"}, + {"id": 1, "name": "python-publish", "conclusion": "failure"}, + ] + with pytest.raises(ValueError, match="original prepare"): + recovery.selected_jobs(jobs, "all") + + +def test_seed_checks_fixed_real_release_and_commit(monkeypatch): + calls = [] + + def api(path): + calls.append(path) + return ( + {"tag_name": "v0.7.3", "draft": False, "prerelease": False} + if path.startswith("releases/") + else {"object": {"sha": SHA, "type": "commit"}} + ) + + monkeypatch.setattr(guard, "api", api) + monkeypatch.setattr(guard, "command", lambda *args: SHA if args[1] == "rev-parse" else "") + assert guard.stable_release(guard.SEED_VERSION) == SHA + assert calls == ["releases/tags/v0.7.3", "git/ref/tags/v0.7.3"] + + +def test_seed_manifest_is_not_a_new_publication_candidate(): + assert guard.seed_manifest({".": "0.7.3"}) is True + assert guard.seed_manifest({".": "0.7.4"}) is False + with pytest.raises(ValueError, match="below"): + guard.seed_manifest({".": "0.7.2"}) + with pytest.raises(ValueError, match="one coordinated"): + guard.seed_manifest({".": "0.7.3", "other": "0.7.4"}) + + +def test_seed_requires_actual_tag_ancestry_in_a_controlled_repository(tmp_path, monkeypatch): + def command(*args): + return subprocess.check_output(args, cwd=tmp_path, text=True, stderr=subprocess.DEVNULL).strip() # noqa: S603 + + command("git", "init", "-b", "main") + command("git", "config", "user.name", "Release test") + command("git", "config", "user.email", "release-test@example.invalid") + command("git", "config", "commit.gpgsign", "false") + command("git", "config", "core.hooksPath", "/dev/null") + command("git", "commit", "--allow-empty", "-m", "test seed") + command("git", "-c", "tag.gpgsign=false", "tag", "v0.7.3") + source = command("git", "rev-parse", "HEAD") + command("git", "remote", "add", "origin", str(tmp_path)) + command("git", "commit", "--allow-empty", "-m", "after seed") + monkeypatch.setattr(guard, "command", command) + monkeypatch.setattr( + guard, + "api", + lambda path: ( + {"tag_name": "v0.7.3", "draft": False, "prerelease": False} + if path.startswith("releases/") + else {"object": {"sha": source, "type": "commit"}} + ), + ) + assert guard.stable_release("0.7.3") == source + with pytest.raises(ValueError, match="original release SHA"): + guard.stable_release("0.7.3", "b" * 40) + command("git", "checkout", "--orphan", "unrelated") + command("git", "commit", "--allow-empty", "-m", "unrelated") + with pytest.raises(subprocess.CalledProcessError): + guard.stable_release("0.7.3") + + +@pytest.mark.parametrize( + "release", + [ + {}, + {"tag_name": "v0.7.3", "draft": True, "prerelease": False}, + {"tag_name": "v0.7.3", "draft": False, "prerelease": True}, + {"tag_name": "v0.7.2", "draft": False, "prerelease": False}, + ], +) +def test_missing_or_unstable_seed_cannot_bootstrap(monkeypatch, release): + monkeypatch.setattr(guard, "api", lambda path: release) + with pytest.raises(ValueError, match="genuine stable GitHub Release"): + guard.stable_release(guard.SEED_VERSION) + + +def run_record(): + return { + "id": 123, + "event": "release", + "head_branch": "v0.7.4", + "head_sha": SHA, + "path": ".github/workflows/release.yml", + "status": "completed", + "conclusion": "failure", + "repository": {"full_name": "superlinked/sie"}, + "head_repository": {"full_name": "superlinked/sie"}, + "created_at": (NOW - timedelta(days=1)).isoformat(), + } + + +@pytest.mark.parametrize( + ("key", "value"), + [ + ("id", 999), + ("event", "workflow_dispatch"), + ("head_branch", "main"), + ("head_branch", "v0.7.5"), + ("head_sha", "b" * 40), + ("path", ".github/workflows/other.yml"), + ("status", "in_progress"), + ("conclusion", "success"), + ("repository", {"full_name": "someone/fork"}), + ("head_repository", {"full_name": "someone/fork"}), + ("created_at", (NOW - timedelta(days=30)).isoformat()), + ("created_at", (NOW + timedelta(days=1)).isoformat()), + ], +) +def test_recovery_cannot_change_original_provenance(key, value): + recovery.validate_run(run_record(), original_run=123, source_sha=SHA, tag_name="v0.7.4", now=NOW) + with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + recovery.validate_run( + {**run_record(), key: value}, original_run=123, source_sha=SHA, tag_name="v0.7.4", now=NOW + ) + + +def artifact(): + return { + "name": "python-distributions", + "workflow_run": {"id": 123, "head_sha": SHA, "head_branch": "v0.7.4"}, + "expired": False, + "expires_at": (NOW + timedelta(days=1)).isoformat(), + "digest": "sha256:" + "b" * 64, + "size_in_bytes": 500, + } + + +@pytest.mark.parametrize( + ("key", "value"), + [ + ("name", "wrong"), + ("expired", True), + ("expires_at", NOW.isoformat()), + ("digest", ""), + ("size_in_bytes", 0), + ("workflow_run", {"id": 999, "head_sha": SHA, "head_branch": "v0.7.4"}), + ("workflow_run", {"id": 123, "head_sha": "b" * 40, "head_branch": "v0.7.4"}), + ], +) +def test_recovery_rejects_missing_expired_or_unbound_archives(key, value): + kwargs = {"original_run": 123, "source_sha": SHA, "tag_name": "v0.7.4", "now": NOW} + recovery.validate_artifacts([artifact()], {"python-distributions"}, **kwargs) + with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + recovery.validate_artifacts([{**artifact(), key: value}], {"python-distributions"}, **kwargs) + + +def test_retry_selector_only_reruns_failed_original_jobs(): + jobs = [ + {"id": 0, "name": "prepare", "conclusion": "success"}, + {"id": 1, "name": "python-publish", "conclusion": "failure"}, + {"id": 2, "name": "npm-publish", "conclusion": "success"}, + {"id": 3, "name": "docker / push-server", "conclusion": "failure"}, + ] + assert recovery.selected_jobs(jobs, "python") == [1] + assert recovery.selected_jobs(jobs, "docker") == [3] + assert recovery.selected_jobs(jobs, "all") == [1, 3] + with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + recovery.selected_jobs(jobs, "npm") + + +def test_archive_recovery_scope_contains_all_families(): + assert len(recovery.artifact_names("docker", "0.7.4")) == 15 + assert recovery.artifact_names("native", "0.7.4") == {"native-sidecar-0.7.4"} + + +def test_retry_never_replays_prepare_or_only_completion(): + jobs = [ + {"id": 0, "name": "prepare", "conclusion": "success"}, + {"id": 1, "name": "complete", "conclusion": "failure"}, + {"id": 2, "name": "python-publish", "conclusion": "skipped"}, + ] + with pytest.raises(ValueError, match="skipped-only"): + recovery.selected_jobs(jobs, "all") + jobs[0]["conclusion"] = "failure" + with pytest.raises(ValueError, match="must not be rerun"): + recovery.selected_jobs(jobs, "all") + + +def test_retry_only_makes_one_api_call_for_matrix_failures(): + jobs = [ + {"id": 0, "name": "prepare", "conclusion": "success"}, + {"id": 1, "name": "docker / push-server (cpu)", "conclusion": "failure"}, + {"id": 2, "name": "docker / push-service (gateway)", "conclusion": "failure"}, + {"id": 3, "name": "docker / complete", "conclusion": "failure"}, + ] + assert recovery.retry_endpoint(jobs, "docker", 123) == "actions/runs/123/rerun-failed-jobs" + jobs.append({"id": 4, "name": "npm-publish", "conclusion": "failure"}) + with pytest.raises(ValueError, match="other failed jobs"): + recovery.retry_endpoint(jobs, "docker", 123) + assert recovery.retry_endpoint(jobs, "npm", 123) == "actions/jobs/4/rerun" + assert recovery.retry_endpoint(jobs, "all", 123) == "actions/runs/123/rerun-failed-jobs" + + +def test_failed_builders_and_skipped_publisher_completion_are_not_recovery(): + jobs = [ + {"id": 0, "name": "prepare", "conclusion": "success"}, + {"id": 1, "name": "docker / complete", "conclusion": "failure"}, + {"id": 2, "name": "docker / publish", "conclusion": "skipped"}, + ] + with pytest.raises(ValueError, match="skipped-only"): + recovery.retry_endpoint(jobs, "all", 123) + jobs += [ + {"id": 3, "name": "python / build", "conclusion": "failure"}, + {"id": 4, "name": "npm-publish", "conclusion": "failure"}, + ] + with pytest.raises(ValueError, match="failed builders"): + recovery.retry_endpoint(jobs, "all", 123) diff --git a/tools/ci/tests/test_required_ci.py b/tools/ci/tests/test_required_ci.py new file mode 100644 index 000000000..e338ed0a6 --- /dev/null +++ b/tools/ci/tests/test_required_ci.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +import os +import re +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from tools.ci import required_ci + +ROOT = Path(__file__).resolve().parents[3] + + +def successful_needs(): + return {name: {"result": "success"} for name in required_ci.MANDATORY_JOBS} + + +def test_every_mandatory_lane_must_succeed(): + assert required_ci.failures(successful_needs()) == [] + + +@pytest.mark.parametrize("name", required_ci.MANDATORY_JOBS) +@pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "neutral", "timed_out", "", None]) +def test_any_non_success_result_is_rejected(name, result): + needs = successful_needs() + needs[name]["result"] = result + assert required_ci.failures(needs) == [f"{name}: {result}"] + + +@pytest.mark.parametrize("name", required_ci.MANDATORY_JOBS) +def test_missing_lane_is_rejected(name): + needs = successful_needs() + del needs[name] + assert required_ci.failures(needs) == [f"{name}: missing"] + + +@pytest.mark.parametrize(("result", "code"), [("success", 0), ("skipped", 1), ("cancelled", 1), ("failure", 1)]) +def test_gate_process_exit_status(result, code): + needs = successful_needs() + needs["python"]["result"] = result + completed = subprocess.run( + [sys.executable, str(ROOT / "tools/ci/required_ci.py")], + env={**os.environ, "NEEDS": json.dumps(needs)}, + check=False, + capture_output=True, + ) + assert completed.returncode == code + + +def test_ci_mandatory_graph_and_permissions(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + jobs = workflow["jobs"] + assert set(jobs) == {*required_ci.MANDATORY_JOBS, "required"} + assert set(jobs["required"]["needs"]) == set(required_ci.MANDATORY_JOBS) + assert jobs["required"]["if"] == "${{ always() }}" + assert workflow["permissions"] == {"contents": "read"} + for name in required_ci.MANDATORY_JOBS: + assert "if" not in jobs[name] + assert "environment" not in jobs[name] + assert "secrets" not in jobs[name] + if "runs-on" in jobs[name]: + assert re.fullmatch(r"blacksmith-[248]vcpu-ubuntu-2404", jobs[name]["runs-on"]) + for step in jobs[name].get("steps", []): + if "uses" in step: + assert re.fullmatch(r"[\w/-]+@[a-f0-9]{40}", step["uses"]) + serialized = json.dumps(workflow) + for forbidden in ("pull_request_target", "id-token", "secrets.", "classify_paths", "BENCHMARK"): + assert forbidden not in serialized + + +def test_bootstrap_is_uncached_and_checks_all_locks(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + bootstrap = workflow["jobs"]["bootstrap"] + setup = next(step for step in bootstrap["steps"] if step.get("uses", "").startswith("jdx/mise-action@")) + assert setup["with"] == {"cache": False, "install": False} + script = (ROOT / "tools/ci/fresh_bootstrap.bash").read_text() + assert "./tools/init.sh" in script + assert "test ! -e .venv" in script + assert "test ! -e node_modules" in script + for lock in ("uv.lock", "pnpm-lock.yaml", "Cargo.lock"): + assert lock in script + assert "sha256sum --check" in script + + +@pytest.mark.parametrize(("mutate_lock", "old_venv", "code"), [(False, False, 0), (True, False, 1), (False, True, 1)]) +def test_bootstrap_rejects_reused_environment_and_changed_lock(tmp_path, mutate_lock, old_venv, code): + subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True) + for lock in ("uv.lock", "pnpm-lock.yaml", "Cargo.lock"): + (tmp_path / lock).write_text("committed-lock\n") + subprocess.run(["git", "-C", str(tmp_path), "add", "uv.lock", "pnpm-lock.yaml", "Cargo.lock"], check=True) + (tmp_path / "tools").mkdir() + init = tmp_path / "tools/init.sh" + init.write_text("#!/bin/sh\n" + ("printf changed > uv.lock\n" if mutate_lock else ":\n")) + init.chmod(0o755) + if old_venv: + (tmp_path / ".venv").mkdir() + result = subprocess.run( + ["bash", str(ROOT / "tools/ci/fresh_bootstrap.bash")], cwd=tmp_path, capture_output=True, check=False + ) + assert result.returncode == code + + +def test_typescript_build_precedes_typecheck(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + commands = [step.get("run") for step in workflow["jobs"]["typescript"]["steps"]] + assert commands.index("mise run ts -- build") < commands.index("mise run ts -- typecheck") + + +def test_rust_audits_both_committed_dependency_graphs(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + commands = [shlex.split(step.get("run", "")) for step in workflow["jobs"]["rust"]["steps"]] + assert ["mise", "run", "gateway-deny"] in commands + assert [ + "mise", + "exec", + "--", + "cargo-deny", + "--locked", + "--manifest-path", + "packages/sie_server_rust/Cargo.toml", + "--all-features", + "--config", + "deny.toml", + "check", + ] in commands diff --git a/tools/ci/upload_audio_prep_release_asset.bash b/tools/ci/upload_audio_prep_release_asset.bash new file mode 100755 index 000000000..af457d934 --- /dev/null +++ b/tools/ci/upload_audio_prep_release_asset.bash @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +export RELEASE_ASSET_FILENAME="${AUDIO_WHEEL_FILENAME:?AUDIO_WHEEL_FILENAME is required}" +exec bash "$(dirname "${BASH_SOURCE[0]}")/upload_native_release_asset.bash" "$@" diff --git a/tools/ci/upload_native_release_asset.bash b/tools/ci/upload_native_release_asset.bash new file mode 100755 index 000000000..42695939c --- /dev/null +++ b/tools/ci/upload_native_release_asset.bash @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${RELEASE_ASSET_FILENAME:?RELEASE_ASSET_FILENAME is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${RELEASE_TAG:?RELEASE_TAG is required}" +: "${GH_TOKEN:?GH_TOKEN is required}" + +wheel="${1:?wheel path is required}" +if [[ ! -f "$wheel" || -L "$wheel" || "$(basename "$wheel")" != "$RELEASE_ASSET_FILENAME" ]]; then + echo "invalid native release asset path: $wheel" >&2 + exit 2 +fi + +local_size="$(wc -c < "$wheel" | tr -d ' ')" +local_sha="$(sha256sum "$wheel" | cut -d' ' -f1)" +expected_url="https://github.com/$GITHUB_REPOSITORY/releases/download/$RELEASE_TAG/$RELEASE_ASSET_FILENAME" + +release_json="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG")" +asset_count="$(jq --arg name "$RELEASE_ASSET_FILENAME" '[.assets[] | select(.name == $name)] | length' <<<"$release_json")" +if [[ "$asset_count" -gt 1 ]]; then + echo "release contains duplicate native asset names: $RELEASE_ASSET_FILENAME" >&2 + exit 1 +fi + +verify_asset_bytes() { + local json="$1" + local asset_id remote_size remote_digest remote_sha downloaded + asset_id="$(jq -r --arg name "$RELEASE_ASSET_FILENAME" '.assets[] | select(.name == $name) | .id' <<<"$json")" + remote_size="$(jq -r --arg name "$RELEASE_ASSET_FILENAME" '.assets[] | select(.name == $name) | .size' <<<"$json")" + remote_digest="$(jq -r --arg name "$RELEASE_ASSET_FILENAME" '.assets[] | select(.name == $name) | .digest // empty' <<<"$json")" + test "$remote_size" = "$local_size" + if [[ -n "$remote_digest" ]]; then + test "$remote_digest" = "sha256:$local_sha" + return + fi + downloaded="$(mktemp)" + gh api -H 'Accept: application/octet-stream' \ + "repos/$GITHUB_REPOSITORY/releases/assets/$asset_id" > "$downloaded" + remote_sha="$(sha256sum "$downloaded" | cut -d' ' -f1)" + rm -f "$downloaded" + test "$remote_sha" = "$local_sha" +} + +if [[ "$asset_count" -eq 1 ]]; then + verify_asset_bytes "$release_json" + echo "identical native release asset already exists: $RELEASE_ASSET_FILENAME" +else + gh release upload --repo "$GITHUB_REPOSITORY" "$RELEASE_TAG" "$wheel" +fi + +verified_json="" +for _ in 1 2 3 4 5 6; do + verified_json="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG")" + verified_count="$(jq --arg name "$RELEASE_ASSET_FILENAME" '[.assets[] | select(.name == $name)] | length' <<<"$verified_json")" + if [[ "$verified_count" -eq 1 ]]; then + break + fi + sleep 2 +done +test "$verified_count" -eq 1 +actual_url="$(jq -r --arg name "$RELEASE_ASSET_FILENAME" '.assets[] | select(.name == $name) | .browser_download_url' <<<"$verified_json")" +test "$actual_url" = "$expected_url" +verify_asset_bytes "$verified_json" diff --git a/tools/mise_tasks/common/device.py b/tools/mise_tasks/common/device.py new file mode 100644 index 000000000..793880921 --- /dev/null +++ b/tools/mise_tasks/common/device.py @@ -0,0 +1,52 @@ +"""Device detection utilities for GPU/accelerator selection.""" + +from __future__ import annotations + +import platform +import subprocess + + +def detect_gpu() -> str | None: + """Detect available GPU type. + + Returns: + "cuda" if NVIDIA GPU is available (nvidia-smi works), + "mps" if Apple Silicon is detected, + None if no GPU is available. + """ + system = platform.system() + + # Check for Apple Silicon (MPS) + if system == "Darwin" and platform.machine() == "arm64": + return "mps" + + # Check for NVIDIA GPU (CUDA) + if system in ("Linux", "Windows"): + try: + result = subprocess.run( + ["nvidia-smi"], # noqa: S607 — intentional partial path + capture_output=True, + check=False, + ) + if result.returncode == 0: + return "cuda" + except FileNotFoundError: + pass + + return None + + +def default_device() -> str: + """Get the default device string for PyTorch. + + Returns: + "cuda:0" if NVIDIA GPU is available, + "mps:0" if Apple Silicon is detected, + "cpu" otherwise. + """ + gpu = detect_gpu() + if gpu == "cuda": + return "cuda:0" + if gpu == "mps": + return "mps:0" + return "cpu" diff --git a/tools/mise_tasks/docker-push-loaded.bash b/tools/mise_tasks/docker-push-loaded.bash new file mode 100755 index 000000000..ec5e11d15 --- /dev/null +++ b/tools/mise_tasks/docker-push-loaded.bash @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +#MISE description="Load, validate, and publish one retained release image archive" +#USAGE flag "--image " help="Complete immutable image reference" +#USAGE flag "--archive-dir " help="Retained image archive and provenance directory" +#USAGE flag "--version " help="Stable release version without v" +#USAGE flag "--source-revision " help="Exact release source commit" +#USAGE flag "--run-id " help="Original Actions run ID" + +set -euo pipefail + +exec mise run docker -- publish \ + --image "${usage_image:?--image is required}" \ + --archive-dir "${usage_archive_dir:?--archive-dir is required}" \ + --version "${usage_version:?--version is required}" \ + --source-revision "${usage_source_revision:?--source-revision is required}" \ + --run-id "${usage_run_id:?--run-id is required}" diff --git a/tools/mise_tasks/docker.bash b/tools/mise_tasks/docker.bash new file mode 100755 index 000000000..a35a615d3 --- /dev/null +++ b/tools/mise_tasks/docker.bash @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +#MISE description="Build and verify public SIE release images" +#USAGE arg "" help="Command: matrix, expected, build-server, build-service, load, publish, verify, alias" +#USAGE arg "[args]..." help="Arguments for the selected Docker command" + +set -euo pipefail + +mise exec -- uv lock --check --project . +exec mise exec -- uv run --frozen --project . python -m tools.mise_tasks.docker_task "$@" diff --git a/tools/mise_tasks/docker_task.py b/tools/mise_tasks/docker_task.py new file mode 100755 index 000000000..b7006b957 --- /dev/null +++ b/tools/mise_tasks/docker_task.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Public-only Docker build, release-matrix, verification, and alias logic.""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +import tomllib +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +from tools.ci.release_artifact import create_manifest, validate_manifest +from tools.ci.release_guard import api, stable_version + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_MATRIX = ROOT / ".github/release-matrix.json" +DEFAULT_DOCKER_PLATFORM = "linux/amd64" +SOURCE_REVISION = re.compile(r"[0-9a-f]{40}") +VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?") +SERVER_RUST_SUFFIX = "cuda12-sm89" +SINGLETON_DOCKERFILES = { + "sie-gateway": ROOT / "packages/sie_gateway/Dockerfile", + "sie-config": ROOT / "packages/sie_config/Dockerfile", + "sie-mcp": ROOT / "packages/sie_mcp/Dockerfile", + "sie-server-sidecar": ROOT / "packages/sie_server_sidecar/Dockerfile", + "sie-server-rust": ROOT / "packages/sie_server_rust/Dockerfile.candle", + "sie-server-rust-cpu": ROOT / "packages/sie_server_rust/Dockerfile", +} +RELEASE_SINGLETONS = ( + "sie-gateway", + "sie-config", + "sie-mcp", + "sie-server-sidecar", + "sie-server-rust", +) + + +@dataclass(frozen=True, order=True) +class ServerTarget: + platform: str + bundle: str + target: str | None = None + + def as_json(self) -> dict[str, str]: + result = {"platform": self.platform, "bundle": self.bundle} + if self.target: + result["target"] = self.target + return result + + +def normalize_registry(registry: str) -> str: + value = registry.strip().rstrip("/") + if not value or "://" in value: + raise ValueError("registry must be a non-empty Docker registry/repository prefix") + return f"{value}/" + + +def validate_version(version: str) -> str: + if VERSION.fullmatch(version) is None: + raise ValueError("version must use an unprefixed semantic version") + return version + + +def validate_source_revision(source_revision: str) -> str: + if SOURCE_REVISION.fullmatch(source_revision) is None: + raise ValueError("source revision must be a lowercase full 40-character Git SHA") + return source_revision + + +def validate_release_versions(version: str) -> None: + for package in ("sie_server", "sie_sdk", "sie_config", "sie_mcp"): + path = ROOT / "packages" / package / "pyproject.toml" + if tomllib.loads(path.read_text())["project"]["version"] != version: + raise ValueError(f"release version mismatch: {path.relative_to(ROOT)}") + for package in ("sie_gateway", "sie_server_sidecar", "sie_audio_prep"): + path = ROOT / "packages" / package / "Cargo.toml" + if tomllib.loads(path.read_text())["package"]["version"] != version: + raise ValueError(f"release version mismatch: {path.relative_to(ROOT)}") + + +def bundle_platform(bundle: str) -> str: + if re.fullmatch(r"[a-z0-9][a-z0-9-]*", bundle) is None: + raise ValueError("invalid release bundle name") + path = ROOT / "packages/sie_server/bundles" / f"{bundle}.yaml" + if not path.is_file(): + raise ValueError(f"release bundle does not exist: {bundle}") + data = yaml.safe_load(path.read_text()) or {} + for adapter in data.get("adapters", []): + if not isinstance(adapter, str) or not adapter.startswith("sie_server.adapters."): + raise ValueError(f"bundle {bundle} contains an invalid adapter path") + module = ROOT / "packages/sie_server/src" / adapter.replace(".", "/") + if not module.with_suffix(".py").is_file() and not (module / "__init__.py").is_file(): + raise ValueError(f"release bundle {bundle} adapter source is missing: {adapter}") + declared = data.get("platform", "cuda12") + if not isinstance(declared, str) or declared not in {"cuda12", "cuda13"}: + raise ValueError(f"bundle {bundle} declares unsupported platform {declared!r}") + return declared + + +def validate_target(target: ServerTarget) -> None: + declared = bundle_platform(target.bundle) + if target.platform == "cpu": + if declared != "cuda12": + raise ValueError(f"specialized {declared} bundle {target.bundle} has no CPU image") + elif target.platform != declared: + raise ValueError( + f"release target {target.platform}/{target.bundle} disagrees with declared platform {declared}" + ) + + +def load_release_matrix(path: Path = DEFAULT_MATRIX) -> tuple[ServerTarget, ...]: + data: dict[str, Any] = json.loads(path.read_text()) + platforms = data.get("platforms") + bundles = data.get("bundles") + includes = data.get("include", []) + if not isinstance(platforms, list) or not isinstance(bundles, list) or not isinstance(includes, list): + raise ValueError("release matrix must define platforms, bundles, and include lists") + targets = [ServerTarget(str(platform), str(bundle)) for platform in platforms for bundle in bundles] + for item in includes: + if not isinstance(item, dict) or "platform" not in item or "bundle" not in item: + raise ValueError("release matrix include entries require platform and bundle") + target_name = item.get("target") + targets.append( + ServerTarget( + str(item["platform"]), + str(item["bundle"]), + str(target_name) if target_name is not None else None, + ) + ) + identities = [(target.platform, target.bundle) for target in targets] + if len(identities) != len(set(identities)): + raise ValueError("release matrix contains duplicate platform/bundle targets") + for target in targets: + validate_target(target) + return tuple(targets) + + +def server_image(registry: str, version: str, target: ServerTarget) -> str: + return f"{normalize_registry(registry)}sie-server:v{validate_version(version)}-{target.platform}-{target.bundle}" + + +def singleton_image(registry: str, version: str, service: str) -> str: + if service not in SINGLETON_DOCKERFILES: + raise ValueError(f"unknown public image service: {service}") + suffix = f"-{SERVER_RUST_SUFFIX}" if service == "sie-server-rust" else "" + image_name = "sie-server-rust" if service == "sie-server-rust-cpu" else service + if service == "sie-server-rust-cpu": + suffix = "-cpu" + return f"{normalize_registry(registry)}{image_name}:v{validate_version(version)}{suffix}" + + +def expected_versioned_images( + registry: str, + version: str, + targets: tuple[ServerTarget, ...], +) -> tuple[str, ...]: + images = [server_image(registry, version, target) for target in targets] + images.extend(singleton_image(registry, version, service) for service in RELEASE_SINGLETONS) + if len(images) != len(set(images)): + raise ValueError("release image set contains duplicate tags") + return tuple(images) + + +def alias_plan( + registry: str, + version: str, + targets: tuple[ServerTarget, ...], +) -> tuple[tuple[str, str], ...]: + plan = [] + for target in targets: + source = server_image(registry, version, target) + alias = f"{normalize_registry(registry)}sie-server:latest-{target.platform}-{target.bundle}" + plan.append((source, alias)) + for service in RELEASE_SINGLETONS: + source = singleton_image(registry, version, service) + suffix = f"-{SERVER_RUST_SUFFIX}" if service == "sie-server-rust" else "" + alias = f"{normalize_registry(registry)}{service}:latest{suffix}" + plan.append((source, alias)) + return tuple(plan) + + +def build_server_command( + *, + registry: str, + version: str, + target: ServerTarget, + source_revision: str, +) -> list[str]: + validate_target(target) + revision = validate_source_revision(source_revision) + dockerfile = ROOT / "packages/sie_server" / f"Dockerfile.{target.platform}" + if not dockerfile.is_file(): + raise ValueError(f"server Dockerfile does not exist: {dockerfile.relative_to(ROOT)}") + command = [ + "docker", + "buildx", + "build", + "--platform", + DEFAULT_DOCKER_PLATFORM, + "--file", + str(dockerfile.relative_to(ROOT)), + "--build-arg", + f"BUNDLE={target.bundle}", + "--build-arg", + f"SIE_SRC_REV={revision}", + "--label", + f"org.opencontainers.image.revision={revision}", + "--label", + "org.opencontainers.image.source=https://github.com/superlinked/sie", + "--tag", + server_image(registry, version, target), + "--load", + ".", + ] + return command + + +def build_service_command( + *, + registry: str, + version: str, + service: str, + source_revision: str, +) -> list[str]: + revision = validate_source_revision(source_revision) + dockerfile = SINGLETON_DOCKERFILES.get(service) + if dockerfile is None or not dockerfile.is_file(): + raise ValueError(f"Dockerfile is unavailable for {service}") + command = [ + "docker", + "buildx", + "build", + "--platform", + DEFAULT_DOCKER_PLATFORM, + "--file", + str(dockerfile.relative_to(ROOT)), + ] + if service == "sie-server-rust": + command.extend(["--build-arg", "CUDA_COMPUTE_CAP=89"]) + command.extend( + [ + "--label", + f"org.opencontainers.image.revision={revision}", + "--label", + "org.opencontainers.image.source=https://github.com/superlinked/sie", + ] + ) + command.extend( + [ + "--tag", + singleton_image(registry, version, service), + "--load", + ".", + ] + ) + return command + + +def run(command: list[str]) -> None: + subprocess.run(command, cwd=ROOT, check=True) # noqa: S603 + + +def capture(command: list[str]) -> str: + return subprocess.check_output(command, cwd=ROOT, text=True).strip() # noqa: S603 + + +def inspect_loaded(image: str, source_revision: str) -> dict[str, str]: + records = json.loads(capture(["docker", "image", "inspect", image])) + if len(records) != 1: + raise ValueError("expected exactly one loaded image") + record = records[0] + labels = record.get("Config", {}).get("Labels", {}) + if labels.get("org.opencontainers.image.revision") != validate_source_revision(source_revision): + raise ValueError("loaded image source revision mismatch") + if labels.get("org.opencontainers.image.source") != "https://github.com/superlinked/sie": + raise ValueError("loaded image source repository mismatch") + if record.get("Os") != "linux" or record.get("Architecture") != "amd64": + raise ValueError("release image must be linux/amd64") + if re.fullmatch(r"sha256:[0-9a-f]{64}", record.get("Id", "")) is None: + raise ValueError("loaded image has no valid configuration digest") + return {"image": image, "image_id": record["Id"], "os": "linux", "architecture": "amd64"} + + +def smoke_image(image: str, *, bundle: str | None = None) -> None: + command = ["docker", "run", "--rm", "--pull", "never", "--network", "none"] + if bundle is not None: + imports = "import sie_server, sie_sdk, sie_audio_prep, torch, transformers; " + if bundle == "ctranslate2": + imports += "import ctranslate2; " + elif bundle in {"sglang", "sglang-cu130"}: + imports += "import sglang; " + elif bundle == "tensorrt-llm": + imports += "import tensorrt_llm; " + if bundle in {"sglang-cu130", "tensorrt-llm"}: + imports += "assert torch.version.cuda.startswith('13.'); assert transformers.__version__.startswith('5.'); " + imports += "print('release image imports passed')" + command.extend(["--entrypoint", "python", image, "-c", imports]) + else: + command.extend([image, "--help"]) + run(command) + + +def export_image(image: str, directory: Path, *, version: str, source_revision: str, run_id: str) -> dict[str, Any]: + metadata = inspect_loaded(image, source_revision) + directory.mkdir(parents=True, exist_ok=True) + if any(directory.iterdir()): + raise ValueError("refusing to overwrite an existing image archive") + run(["docker", "image", "save", "--output", str(directory / "image.tar"), image]) + return create_manifest( + directory, + kind="docker", + version=version, + tag_name=f"v{version}", + source_revision=source_revision, + run_id=run_id, + metadata=metadata, + ) + + +def load_image_archive( + image: str, directory: Path, *, version: str, source_revision: str, run_id: str +) -> dict[str, Any]: + manifest = validate_manifest( + directory, + kind="docker", + version=version, + tag_name=f"v{version}", + source_revision=source_revision, + run_id=run_id, + ) + if [item["name"] for item in manifest["files"]] != ["image.tar"]: + raise ValueError("image archive must contain exactly image.tar") + if manifest["metadata"].get("image") != image: + raise ValueError("image archive reference mismatch") + run(["docker", "image", "load", "--input", str(directory / "image.tar")]) + if inspect_loaded(image, source_revision) != manifest["metadata"]: + raise ValueError("loaded image configuration digest mismatch") + return manifest + + +def remote_image_id(image: str, *, allow_missing: bool = False) -> str | None: + result = subprocess.run( # noqa: S603 + ["docker", "buildx", "imagetools", "inspect", "--raw", image], # noqa: S607 + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + if result.returncode: + if allow_missing and any( + marker in result.stderr.lower() for marker in ("manifest unknown", "not found", "name unknown") + ): + return None + raise RuntimeError(f"cannot inspect remote image {image}: {result.stderr.strip()}") + manifest = json.loads(result.stdout) + if "manifests" in manifest: + descriptors = [ + item for item in manifest["manifests"] if item.get("platform") == {"architecture": "amd64", "os": "linux"} + ] + if len(descriptors) != 1: + raise ValueError("remote index must contain exactly one linux/amd64 image") + repository = image.split("@", 1)[0].rsplit(":", 1)[0] + return remote_image_id(f"{repository}@{descriptors[0]['digest']}") + digest = manifest.get("config", {}).get("digest", "") + if re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None: + raise ValueError("remote image configuration digest is missing") + return digest + + +def publish_archive(image: str, directory: Path, *, version: str, source_revision: str, run_id: str) -> None: + stable_version(version) + if not image.startswith("ghcr.io/superlinked/") or f":v{version}" not in image: + raise ValueError("publication requires a versioned public image reference") + manifest = load_image_archive(image, directory, version=version, source_revision=source_revision, run_id=run_id) + expected = manifest["metadata"]["image_id"] + existing = remote_image_id(image, allow_missing=True) + if existing is not None and existing != expected: + raise ValueError("refusing to overwrite a different versioned image") + if existing is None: + run(["docker", "push", image]) + if remote_image_id(image) != expected: + raise ValueError("remote image does not match the tested image configuration digest") + + +def verify_release( + registry: str, + version: str, + targets: tuple[ServerTarget, ...], + *, + evidence_dir: Path, + source_revision: str, + run_id: str, +) -> None: + images = set(expected_versioned_images(registry, version, targets)) + evidence = {} + for path in evidence_dir.rglob("*.json"): + manifest = json.loads(path.read_text()) + expected_identity = { + "schema": 1, + "repository": "superlinked/sie", + "kind": "docker", + "version": version, + "tag_name": f"v{version}", + "source_revision": validate_source_revision(source_revision), + "run_id": str(run_id), + } + if any(manifest.get(key) != value for key, value in expected_identity.items()): + raise ValueError("image evidence source/run identity mismatch") + metadata = manifest["metadata"] + image = metadata["image"] + if metadata.get("architecture") != "amd64" or metadata.get("os") != "linux" or image in evidence: + raise ValueError("duplicate or wrong-platform image evidence") + evidence[image] = metadata["image_id"] + if set(evidence) != images: + raise ValueError("image evidence does not cover the exact complete release set") + for image, image_id in evidence.items(): + if remote_image_id(image) != image_id: + raise ValueError(f"remote image differs from tested source-bound image: {image}") + + +def published_tag_revision(tag: str) -> str: + reference = api(f"git/ref/tags/{tag}") + if not isinstance(reference, dict) or reference.get("ref") != f"refs/tags/{tag}": + raise ValueError("published release tag reference mismatch") + obj = reference.get("object", {}) + for _ in range(5): + if not isinstance(obj, dict) or not isinstance(obj.get("sha"), str): + raise ValueError("published release tag object is malformed") + revision = validate_source_revision(obj.get("sha", "")) + if obj.get("type") == "commit": + return revision + if obj.get("type") != "tag": + raise ValueError("published release tag does not resolve to a commit") + annotated = api(f"git/tags/{revision}") + if not isinstance(annotated, dict) or annotated.get("sha") != revision or annotated.get("tag") != tag: + raise ValueError("published annotated tag identity mismatch") + obj = annotated.get("object", {}) + raise ValueError("published release tag chain does not resolve to a commit") + + +def alias_release_is_current(version: str, source_revision: str) -> bool: + requested = stable_version(version) + revision = validate_source_revision(source_revision) + pages = json.loads(capture(["gh", "api", "--paginate", "--slurp", "repos/superlinked/sie/releases?per_page=100"])) + if not isinstance(pages, list) or any(not isinstance(page, list) for page in pages): + raise ValueError("published release listing is malformed") + releases = {} + for release in (release for page in pages for release in page): + if ( + not isinstance(release, dict) + or not isinstance(release.get("draft"), bool) + or not isinstance(release.get("prerelease"), bool) + ): + raise ValueError("published release state is malformed") + if release["draft"] or release["prerelease"]: + continue + tag = release.get("tag_name") + if not isinstance(tag, str) or not tag.startswith("v") or not release.get("published_at"): + raise ValueError("published stable release identity is malformed") + try: + datetime.strptime(release["published_at"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC) + except (TypeError, ValueError) as error: + raise ValueError("published stable release timestamp is malformed") from error + released = stable_version(tag[1:], new=False) + if released in releases: + raise ValueError("published stable release identity is duplicated") + releases[released] = tag + if requested not in releases: + raise ValueError("requested release is absent from published stable releases") + if published_tag_revision(releases[requested]) != revision: + raise ValueError("requested published release tag differs from its source revision") + latest = max(releases) + if latest == requested: + return True + latest_revision = published_tag_revision(releases[latest]) + comparison = api(f"compare/{revision}...{latest_revision}") + if not isinstance(comparison, dict) or comparison.get("status") != "ahead": + raise ValueError("newer published release does not descend from the requested release") + print(f"Leaving floating aliases unchanged: newer stable release {releases[latest]} is published") + return False + + +def move_aliases(registry: str, version: str, targets: tuple[ServerTarget, ...], **kwargs: Any) -> None: + verify_release(registry, version, targets, **kwargs) + if not alias_release_is_current(version, kwargs["source_revision"]): + return + for source, alias in alias_plan(registry, version, targets): + run(["docker", "buildx", "imagetools", "create", "--tag", alias, source]) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + subparsers = result.add_subparsers(dest="command", required=True) + matrix = subparsers.add_parser("matrix") + matrix.add_argument("--file", type=Path, default=DEFAULT_MATRIX) + matrix.add_argument("--version") + + for name in ("expected", "verify", "alias"): + command = subparsers.add_parser(name) + command.add_argument("--registry", required=True) + command.add_argument("--version", required=True) + command.add_argument("--matrix-file", type=Path, default=DEFAULT_MATRIX) + if name != "expected": + command.add_argument("--evidence-dir", type=Path, required=True) + command.add_argument("--source-revision", required=True) + command.add_argument("--run-id", required=True) + + server = subparsers.add_parser("build-server") + server.add_argument("--registry", required=True) + server.add_argument("--version", required=True) + server.add_argument("--platform", required=True) + server.add_argument("--bundle", required=True) + server.add_argument("--source-revision", required=True) + + service = subparsers.add_parser("build-service") + service.add_argument("--registry", required=True) + service.add_argument("--version", required=True) + service.add_argument("--service", choices=sorted(SINGLETON_DOCKERFILES), required=True) + service.add_argument("--source-revision", required=True) + for command in (server, service): + command.add_argument("--archive-dir", type=Path) + command.add_argument("--evidence-dir", type=Path) + command.add_argument("--run-id") + + for name in ("load", "publish"): + command = subparsers.add_parser(name) + command.add_argument("--image", required=True) + command.add_argument("--archive-dir", type=Path, required=True) + command.add_argument("--version", required=True) + command.add_argument("--source-revision", required=True) + command.add_argument("--run-id", required=True) + return result + + +def main() -> int: + args = parser().parse_args() + try: + if args.command in {"build-server", "build-service"} and args.archive_dir and not args.run_id: + raise ValueError("archiving requires the original Actions run ID") + if args.command == "matrix": + print(json.dumps({"include": [item.as_json() for item in load_release_matrix(args.file)]})) + if args.version: + validate_release_versions(args.version) + elif args.command == "expected": + targets = load_release_matrix(args.matrix_file) + print("\n".join(expected_versioned_images(args.registry, args.version, targets))) + elif args.command == "build-server": + run( + build_server_command( + registry=args.registry, + version=args.version, + target=ServerTarget(args.platform, args.bundle), + source_revision=args.source_revision, + ) + ) + elif args.command == "build-service": + run( + build_service_command( + registry=args.registry, + version=args.version, + service=args.service, + source_revision=args.source_revision, + ) + ) + elif args.command in {"load", "publish"}: + operation = load_image_archive if args.command == "load" else publish_archive + operation( + args.image, + args.archive_dir, + version=args.version, + source_revision=args.source_revision, + run_id=args.run_id, + ) + elif args.command == "verify": + verify_release( + args.registry, + args.version, + load_release_matrix(args.matrix_file), + evidence_dir=args.evidence_dir, + source_revision=args.source_revision, + run_id=args.run_id, + ) + elif args.command == "alias": + move_aliases( + args.registry, + args.version, + load_release_matrix(args.matrix_file), + evidence_dir=args.evidence_dir, + source_revision=args.source_revision, + run_id=args.run_id, + ) + if args.command in {"build-server", "build-service"} and args.archive_dir: + image = ( + server_image(args.registry, args.version, ServerTarget(args.platform, args.bundle)) + if args.command == "build-server" + else singleton_image(args.registry, args.version, args.service) + ) + smoke_image(image, bundle=args.bundle if args.command == "build-server" else None) + export_image( + image, args.archive_dir, version=args.version, source_revision=args.source_revision, run_id=args.run_id + ) + if args.evidence_dir: + args.evidence_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(args.archive_dir / "provenance.json", args.evidence_dir / "provenance.json") + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"docker task failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/mise_tasks/full-sync.bash b/tools/mise_tasks/full-sync.bash index b68a6be25..09220405c 100755 --- a/tools/mise_tasks/full-sync.bash +++ b/tools/mise_tasks/full-sync.bash @@ -3,12 +3,5 @@ set -eu -o pipefail -[ -n "${CI:-}" ] && exit 0 || true - mise run sync -if [ -d packages/sie_ts_sdk ]; then - ( - cd packages/sie_ts_sdk - mise exec -- pnpm install --frozen-lockfile - ) -fi +mise exec -- pnpm install --frozen-lockfile diff --git a/tools/mise_tasks/helm.py b/tools/mise_tasks/helm.py index edecfafc1..84283a819 100755 --- a/tools/mise_tasks/helm.py +++ b/tools/mise_tasks/helm.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # fmt: off #MISE description="Helm chart operations for SIE cluster deployment" -#USAGE arg "[command]" help="Command: lint, template, install, upgrade, uninstall, status" +#USAGE arg "[command]" help="Command: dependencies, lint, template, install, upgrade, uninstall, status" #USAGE arg "[args]..." help="Additional arguments to pass to helm" # fmt: on @@ -27,6 +27,8 @@ from contextlib import contextmanager from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) + from common.colors import ( log, log_error, @@ -40,6 +42,7 @@ CHART_DIR = Path("deploy/helm/sie-cluster") RELEASE_NAME = os.environ.get("SIE_HELM_RELEASE", "sie") NAMESPACE = os.environ.get("SIE_HELM_NAMESPACE", "sie") +DEFAULT_VALIDATION_ARGS = ["--set", "payloadStore.enabled=false"] @contextmanager @@ -117,6 +120,40 @@ def run_helm(args: list[str]) -> int: return result.returncode +def dependency_command() -> list[str]: + """Return the locked chart dependency preparation command.""" + return ["dependency", "build", str(CHART_DIR)] + + +def dependency_repository_commands() -> list[list[str]]: + """Return idempotent Helm repo setup commands derived from Chart.yaml.""" + root = resolve_project_root() + chart = root / CHART_DIR / "Chart.yaml" + repositories: list[str] = [] + for line in chart.read_text().splitlines(): + stripped = line.strip() + if not stripped.startswith("repository:"): + continue + repository = stripped.split(":", maxsplit=1)[1].strip().strip("\"'") + if repository.startswith(("https://", "http://")) and repository not in repositories: + repositories.append(repository) + return [ + [ + "repo", + "add", + f"sie-dependency-{hashlib.sha256(repository.encode()).hexdigest()[:12]}", + repository, + "--force-update", + ] + for repository in repositories + ] + + +def validation_args(extra_args: list[str]) -> list[str]: + """Provide deterministic, non-secret values for local chart validation.""" + return [*DEFAULT_VALIDATION_ARGS, *extra_args] + + def show_help() -> None: """Show usage information.""" log("SIE Helm Chart Operations") @@ -124,6 +161,7 @@ def show_help() -> None: log("Usage: mise run helm -- [options]") log("") log("Commands:") + log(" dependencies Build dependencies from Chart.yaml and Chart.lock") log(" lint Lint the Helm chart") log(" template Render templates locally") log(" install Install to cluster (dry-run by default)") @@ -137,15 +175,28 @@ def show_help() -> None: log(f" SIE_HELM_NAMESPACE Namespace (default: {NAMESPACE})") log("") log("Examples:") + log(" mise run helm -- dependencies") log(" mise run helm -- lint") log(" mise run helm -- template --set gateway.replicas=3") log(" mise run helm -- install --apply --set workers.pools.l4.enabled=true") +def cmd_dependencies(extra_args: list[str]) -> int: + """Prepare locked chart dependencies.""" + if extra_args: + log_error("dependencies does not accept additional arguments") + return 1 + log(f"[helm] Preparing locked dependencies for: {CHART_DIR}") + for command in dependency_repository_commands(): + if run_helm(command) != 0: + return 1 + return run_helm(dependency_command()) + + def cmd_lint(extra_args: list[str]) -> int: """Lint the Helm chart.""" log(f"[helm] Linting chart: {CHART_DIR}") - if run_helm(["lint", str(CHART_DIR), *extra_args]) != 0: + if run_helm(["lint", str(CHART_DIR), *validation_args(extra_args)]) != 0: return 1 log_success("Lint passed!") return 0 @@ -161,7 +212,7 @@ def cmd_template(extra_args: list[str]) -> int: str(CHART_DIR), "--namespace", NAMESPACE, - *extra_args, + *validation_args(extra_args), ] ) @@ -280,6 +331,7 @@ def main() -> int: return 0 commands = { + "dependencies": cmd_dependencies, "lint": cmd_lint, "template": cmd_template, "install": cmd_install, @@ -297,6 +349,8 @@ def main() -> int: # Commands that render templates need bundle/model configs in files/ needs_configs = command in ("lint", "template", "install", "upgrade") if needs_configs: + if cmd_dependencies([]) != 0: + return 1 with _helm_config_sync_lock(): _sync_configs_to_helm() try: diff --git a/tools/mise_tasks/test-integrations.bash b/tools/mise_tasks/test-integrations.bash index db825b9c9..ae1fbb99a 100755 --- a/tools/mise_tasks/test-integrations.bash +++ b/tools/mise_tasks/test-integrations.bash @@ -1,11 +1,12 @@ #!/usr/bin/env bash #MISE description="Run integration SDK surface tests (Python + TypeScript, no server required)" +#USAGE flag "--python-only" help="Run only the nine Python framework suites" set -euo pipefail rc=0 mise exec -- uv lock --check --project . -mise exec -- uv sync --frozen --project . --all-packages --no-install-package sie-audio-prep +mise exec -- uv sync --frozen --project . --all-packages --all-extras --no-install-package sie-audio-prep echo "## Python integration SDK tests" echo "" @@ -17,12 +18,16 @@ echo "" for dir in integrations/sie_{chroma,crewai,dspy,haystack,lancedb,langchain,llamaindex,qdrant,weaviate}/tests; do name=$(echo "$dir" | cut -d/ -f2) echo "--- ${name} ---" - if ! mise exec -- uv run --frozen --project . --no-sync pytest -c pyproject.toml "${dir}" -q; then + if ! mise exec -- uv run --frozen --project "${dir%/tests}" --no-sync pytest -c pyproject.toml "${dir}" -q; then rc=1 fi echo "" done +if [[ "${usage_python_only:-}" == "true" ]]; then + exit "$rc" +fi + echo "## TypeScript integration SDK tests" echo "" From e488fccfef8cfd21bcd57de8c6fbf255e2af797d Mon Sep 17 00:00:00 2001 From: Marton Mayer Date: Fri, 4 Sep 2026 10:30:24 +0200 Subject: [PATCH 2/4] docs: simplify contributor wording --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f560182c5..4ade397f7 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ SIE is an open-source inference engine that runs the models behind every agent t ## Development -Read [CONTRIBUTING.md](CONTRIBUTING.md) for the complete public development +Read [CONTRIBUTING.md](CONTRIBUTING.md) for the complete development workflow and [AGENTS.md](AGENTS.md) for repository automation boundaries. Install [mise](https://mise.jdx.dev/getting-started.html), then bootstrap the From f726c5997f51caf12aea2228e99e70971cb5658d Mon Sep 17 00:00:00 2001 From: Marton Mayer Date: Fri, 4 Sep 2026 10:58:42 +0200 Subject: [PATCH 3/4] fix(ci): validate release tooling inputs --- tools/ci/check_public_tree.py | 1 + tools/ci/distributions.py | 12 ++ tools/ci/tests/test_device.py | 29 +++++ tools/ci/tests/test_distributions.py | 22 ++++ tools/ci/tests/test_public_tree.py | 10 +- tools/ci/tests/test_release_guard.py | 172 ++++++++++++++++++++------- tools/mise_tasks/common/device.py | 3 +- 7 files changed, 207 insertions(+), 42 deletions(-) create mode 100644 tools/ci/tests/test_device.py diff --git a/tools/ci/check_public_tree.py b/tools/ci/check_public_tree.py index 54cd13a4b..6025ca267 100755 --- a/tools/ci/check_public_tree.py +++ b/tools/ci/check_public_tree.py @@ -17,6 +17,7 @@ b"tools/" + b"internal_python", ) ARCHIVE_GENERATED_DIRS = { + ".git", ".cache", ".pytest_cache", ".venv", diff --git a/tools/ci/distributions.py b/tools/ci/distributions.py index 88f955cd2..65d0bc54b 100755 --- a/tools/ci/distributions.py +++ b/tools/ci/distributions.py @@ -284,6 +284,18 @@ def main() -> None: parser.add_argument("--version", default="") parser.add_argument("--destination", type=Path) args = parser.parse_args() + if args.mode == "prepare-pypi": + if args.family != "python": + parser.error("prepare-pypi requires the python family") + if args.destination is None: + parser.error("prepare-pypi requires --destination") + if not args.version: + parser.error("prepare-pypi requires --version") + elif args.mode == "publish-npm": + if args.family != "npm": + parser.error("publish-npm requires the npm family") + if not args.version: + parser.error("publish-npm requires --version") directory = args.directory.resolve() if args.mode == "build": build(args.family, directory, args.version) diff --git a/tools/ci/tests/test_device.py b/tools/ci/tests/test_device.py new file mode 100644 index 000000000..4c176f1aa --- /dev/null +++ b/tools/ci/tests/test_device.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import subprocess + +from tools.mise_tasks.common import device + + +def test_nvidia_detection_has_a_finite_timeout(monkeypatch) -> None: + calls = [] + monkeypatch.setattr(device.platform, "system", lambda: "Linux") + + def run(*args, **kwargs): + calls.append((args, kwargs)) + return subprocess.CompletedProcess(args, 0) + + monkeypatch.setattr(device.subprocess, "run", run) + assert device.detect_gpu() == "cuda" + assert calls[0][1]["timeout"] == 5 + + +def test_nvidia_detection_timeout_falls_back_to_cpu(monkeypatch) -> None: + monkeypatch.setattr(device.platform, "system", lambda: "Linux") + + def run(*args, **kwargs): + raise subprocess.TimeoutExpired(args[0], kwargs["timeout"]) + + monkeypatch.setattr(device.subprocess, "run", run) + assert device.detect_gpu() is None + assert device.default_device() == "cpu" diff --git a/tools/ci/tests/test_distributions.py b/tools/ci/tests/test_distributions.py index e6a144216..4a2e2cdc2 100644 --- a/tools/ci/tests/test_distributions.py +++ b/tools/ci/tests/test_distributions.py @@ -5,6 +5,7 @@ import io import json import subprocess +import sys import tarfile import zipfile from pathlib import Path @@ -14,6 +15,27 @@ from tools.ci import distributions as packages +@pytest.mark.parametrize( + ("arguments", "message"), + [ + ( + ["prepare-pypi", "npm", "--destination", "pending", "--version", "0.7.4"], + "prepare-pypi requires the python family", + ), + (["prepare-pypi", "python", "--version", "0.7.4"], "prepare-pypi requires --destination"), + (["prepare-pypi", "python", "--destination", "pending"], "prepare-pypi requires --version"), + (["publish-npm", "python", "--version", "0.7.4"], "publish-npm requires the npm family"), + (["publish-npm", "npm"], "publish-npm requires --version"), + ], +) +def test_mode_specific_cli_arguments_are_required(arguments, message, tmp_path, monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", ["distributions.py", *arguments, "--directory", str(tmp_path)]) + with pytest.raises(SystemExit) as error: + packages.main() + assert error.value.code == 2 + assert capsys.readouterr().err.endswith(f"distributions.py: error: {message}\n") + + def python_archives(tmp_path, name="sie-sdk", version="0.7.2"): wheel = tmp_path / f"{name.replace('-', '_')}-{version}-py3-none-any.whl" metadata = f"Name: {name}\nVersion: {version}\n" diff --git a/tools/ci/tests/test_public_tree.py b/tools/ci/tests/test_public_tree.py index 729c12f4f..38c143b27 100644 --- a/tools/ci/tests/test_public_tree.py +++ b/tools/ci/tests/test_public_tree.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import subprocess from pathlib import Path from tools.ci import check_public_tree @@ -19,13 +20,20 @@ def test_reference_guard_reports_forbidden_text(tmp_path: Path, monkeypatch) -> assert "forbidden public-tree reference" in findings[0] -def test_exported_tree_fallback_excludes_generated_dependencies(tmp_path: Path, monkeypatch) -> None: +def test_exported_tree_fallback_excludes_repository_metadata_and_generated_dependencies( + tmp_path: Path, monkeypatch +) -> None: monkeypatch.setattr(check_public_tree, "REPOSITORY_ROOT", tmp_path) + failed_git = subprocess.CompletedProcess([], 1) + monkeypatch.setattr(check_public_tree.subprocess, "run", lambda *_args, **_kwargs: failed_git) source = tmp_path / "source.py" source.write_text("public\n") generated = tmp_path / "node_modules/dependency.txt" generated.parent.mkdir() generated.write_bytes(b"packages/" + b"sie_cloud" + b"/gateway\n") + metadata = tmp_path / ".git/config" + metadata.parent.mkdir() + metadata.write_bytes(b"packages/" + b"sie_cloud" + b"/gateway\n") assert check_public_tree.candidate_paths() == [source] diff --git a/tools/ci/tests/test_release_guard.py b/tools/ci/tests/test_release_guard.py index 4bf66c68d..7d0245c90 100644 --- a/tools/ci/tests/test_release_guard.py +++ b/tools/ci/tests/test_release_guard.py @@ -12,10 +12,22 @@ NOW = datetime(2026, 9, 3, tzinfo=UTC) -@pytest.mark.parametrize("version", ["0.7.2", "0.7.3", "0.0.1", "0.7.4-rc.1", "v0.7.4", "01.2.3", "bad"]) -def test_new_publication_rejects_seed_and_nonstable_versions(version): - with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): +@pytest.mark.parametrize( + ("version", "message"), + [ + ("0.7.2", "new publication must be newer than the 0.7.3 seed"), + ("0.7.3", "new publication must be newer than the 0.7.3 seed"), + ("0.0.1", "new publication must be newer than the 0.7.3 seed"), + ("0.7.4-rc.1", "release version must be stable X.Y.Z"), + ("v0.7.4", "release version must be stable X.Y.Z"), + ("01.2.3", "release version must be stable X.Y.Z"), + ("bad", "release version must be stable X.Y.Z"), + ], +) +def test_new_publication_rejects_seed_and_nonstable_versions(version, message): + with pytest.raises(ValueError) as error: guard.stable_version(version) + assert str(error.value) == message def published(): @@ -38,22 +50,51 @@ def context(): @pytest.mark.parametrize( - ("key", "value"), + ("key", "value", "message"), [ - ("GITHUB_REPOSITORY", "someone/fork"), - ("GITHUB_EVENT_NAME", "push"), - ("GITHUB_EVENT_NAME", "workflow_dispatch"), - ("GITHUB_EVENT_NAME", "pull_request"), - ("GITHUB_REF", "refs/heads/feature"), - ("GITHUB_REF_PROTECTED", "false"), - ("PUBLIC_RELEASE_PUBLISHING_ENABLED", "false"), - ("GITHUB_SHA", "b" * 40), + ( + "GITHUB_REPOSITORY", + "someone/fork", + "publication requires activated protected public main and the correct event", + ), + ( + "GITHUB_EVENT_NAME", + "push", + "publication requires the exact stable published release event and tag SHA", + ), + ( + "GITHUB_EVENT_NAME", + "workflow_dispatch", + "publication requires the exact stable published release event and tag SHA", + ), + ( + "GITHUB_EVENT_NAME", + "pull_request", + "publication requires the exact stable published release event and tag SHA", + ), + ( + "GITHUB_REF", + "refs/heads/feature", + "publication requires the exact stable published release event and tag SHA", + ), + ( + "GITHUB_REF_PROTECTED", + "false", + "publication requires the exact stable published release event and tag SHA", + ), + ( + "PUBLIC_RELEASE_PUBLISHING_ENABLED", + "false", + "publication requires activated protected public main and the correct event", + ), + ("GITHUB_SHA", "b" * 40, "publication SHA must be the original workflow SHA"), ], ) -def test_actual_writer_guard_rejects_untrusted_context(key, value): +def test_actual_writer_guard_rejects_untrusted_context(key, value, message): guard.trusted_context(context(), SHA, event=published()) - with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + with pytest.raises(ValueError) as error: guard.trusted_context({**context(), key: value}, SHA, event=published()) + assert str(error.value) == message @pytest.mark.parametrize(("key", "value"), [("draft", True), ("prerelease", True), ("tag_name", "v0.7.5")]) @@ -204,28 +245,69 @@ def run_record(): @pytest.mark.parametrize( - ("key", "value"), + ("key", "value", "message"), [ - ("id", 999), - ("event", "workflow_dispatch"), - ("head_branch", "main"), - ("head_branch", "v0.7.5"), - ("head_sha", "b" * 40), - ("path", ".github/workflows/other.yml"), - ("status", "in_progress"), - ("conclusion", "success"), - ("repository", {"full_name": "someone/fork"}), - ("head_repository", {"full_name": "someone/fork"}), - ("created_at", (NOW - timedelta(days=30)).isoformat()), - ("created_at", (NOW + timedelta(days=1)).isoformat()), + ("id", 999, "original run must be the completed release.yml release event for the exact tag and SHA"), + ( + "event", + "workflow_dispatch", + "original run must be the completed release.yml release event for the exact tag and SHA", + ), + ( + "head_branch", + "main", + "original run must be the completed release.yml release event for the exact tag and SHA", + ), + ( + "head_branch", + "v0.7.5", + "original run must be the completed release.yml release event for the exact tag and SHA", + ), + ( + "head_sha", + "b" * 40, + "original run must be the completed release.yml release event for the exact tag and SHA", + ), + ( + "path", + ".github/workflows/other.yml", + "original run must be the completed release.yml release event for the exact tag and SHA", + ), + ( + "status", + "in_progress", + "original run must be the completed release.yml release event for the exact tag and SHA", + ), + ("conclusion", "success", "successful releases do not need recovery"), + ( + "repository", + {"full_name": "someone/fork"}, + "original run must belong to the public repository, not a fork", + ), + ( + "head_repository", + {"full_name": "someone/fork"}, + "original run must belong to the public repository, not a fork", + ), + ( + "created_at", + (NOW - timedelta(days=30)).isoformat(), + "original run is outside GitHub's 30-day rerun window", + ), + ( + "created_at", + (NOW + timedelta(days=1)).isoformat(), + "original run is outside GitHub's 30-day rerun window", + ), ], ) -def test_recovery_cannot_change_original_provenance(key, value): +def test_recovery_cannot_change_original_provenance(key, value, message): recovery.validate_run(run_record(), original_run=123, source_sha=SHA, tag_name="v0.7.4", now=NOW) - with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + with pytest.raises(ValueError) as error: recovery.validate_run( {**run_record(), key: value}, original_run=123, source_sha=SHA, tag_name="v0.7.4", now=NOW ) + assert str(error.value) == message def artifact(): @@ -240,22 +322,31 @@ def artifact(): @pytest.mark.parametrize( - ("key", "value"), + ("key", "value", "message"), [ - ("name", "wrong"), - ("expired", True), - ("expires_at", NOW.isoformat()), - ("digest", ""), - ("size_in_bytes", 0), - ("workflow_run", {"id": 999, "head_sha": SHA, "head_branch": "v0.7.4"}), - ("workflow_run", {"id": 123, "head_sha": "b" * 40, "head_branch": "v0.7.4"}), + ("name", "wrong", "missing or ambiguous original archive: python-distributions"), + ("expired", True, "original archive has expired: python-distributions"), + ("expires_at", NOW.isoformat(), "original archive has expired: python-distributions"), + ("digest", "", "original archive has no immutable digest: python-distributions"), + ("size_in_bytes", 0, "original archive has no immutable digest: python-distributions"), + ( + "workflow_run", + {"id": 999, "head_sha": SHA, "head_branch": "v0.7.4"}, + "archive is not bound to the original release run: python-distributions", + ), + ( + "workflow_run", + {"id": 123, "head_sha": "b" * 40, "head_branch": "v0.7.4"}, + "archive is not bound to the original release run: python-distributions", + ), ], ) -def test_recovery_rejects_missing_expired_or_unbound_archives(key, value): +def test_recovery_rejects_missing_expired_or_unbound_archives(key, value, message): kwargs = {"original_run": 123, "source_sha": SHA, "tag_name": "v0.7.4", "now": NOW} recovery.validate_artifacts([artifact()], {"python-distributions"}, **kwargs) - with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + with pytest.raises(ValueError) as error: recovery.validate_artifacts([{**artifact(), key: value}], {"python-distributions"}, **kwargs) + assert str(error.value) == message def test_retry_selector_only_reruns_failed_original_jobs(): @@ -268,8 +359,9 @@ def test_retry_selector_only_reruns_failed_original_jobs(): assert recovery.selected_jobs(jobs, "python") == [1] assert recovery.selected_jobs(jobs, "docker") == [3] assert recovery.selected_jobs(jobs, "all") == [1, 3] - with pytest.raises(ValueError, match=r"release|publication|original|archive|successful"): + with pytest.raises(ValueError) as error: recovery.selected_jobs(jobs, "npm") + assert str(error.value) == "no failed original family jobs to rerun; skipped-only publication requires operator diagnosis" def test_archive_recovery_scope_contains_all_families(): diff --git a/tools/mise_tasks/common/device.py b/tools/mise_tasks/common/device.py index 793880921..bdf824952 100644 --- a/tools/mise_tasks/common/device.py +++ b/tools/mise_tasks/common/device.py @@ -27,10 +27,11 @@ def detect_gpu() -> str | None: ["nvidia-smi"], # noqa: S607 — intentional partial path capture_output=True, check=False, + timeout=5, ) if result.returncode == 0: return "cuda" - except FileNotFoundError: + except (FileNotFoundError, subprocess.TimeoutExpired): pass return None From b8c145f26b4da431d80a59def1395290daef93ea Mon Sep 17 00:00:00 2001 From: Marton Mayer Date: Fri, 4 Sep 2026 11:01:33 +0200 Subject: [PATCH 4/4] test(ci): anchor release guard error matches --- tools/ci/tests/test_release_guard.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tools/ci/tests/test_release_guard.py b/tools/ci/tests/test_release_guard.py index 7d0245c90..f05791af5 100644 --- a/tools/ci/tests/test_release_guard.py +++ b/tools/ci/tests/test_release_guard.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import subprocess from datetime import UTC, datetime, timedelta @@ -25,9 +26,8 @@ ], ) def test_new_publication_rejects_seed_and_nonstable_versions(version, message): - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match=rf"^{re.escape(message)}$"): guard.stable_version(version) - assert str(error.value) == message def published(): @@ -92,9 +92,8 @@ def context(): ) def test_actual_writer_guard_rejects_untrusted_context(key, value, message): guard.trusted_context(context(), SHA, event=published()) - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match=rf"^{re.escape(message)}$"): guard.trusted_context({**context(), key: value}, SHA, event=published()) - assert str(error.value) == message @pytest.mark.parametrize(("key", "value"), [("draft", True), ("prerelease", True), ("tag_name", "v0.7.5")]) @@ -303,11 +302,10 @@ def run_record(): ) def test_recovery_cannot_change_original_provenance(key, value, message): recovery.validate_run(run_record(), original_run=123, source_sha=SHA, tag_name="v0.7.4", now=NOW) - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match=rf"^{re.escape(message)}$"): recovery.validate_run( {**run_record(), key: value}, original_run=123, source_sha=SHA, tag_name="v0.7.4", now=NOW ) - assert str(error.value) == message def artifact(): @@ -344,9 +342,8 @@ def artifact(): def test_recovery_rejects_missing_expired_or_unbound_archives(key, value, message): kwargs = {"original_run": 123, "source_sha": SHA, "tag_name": "v0.7.4", "now": NOW} recovery.validate_artifacts([artifact()], {"python-distributions"}, **kwargs) - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match=rf"^{re.escape(message)}$"): recovery.validate_artifacts([{**artifact(), key: value}], {"python-distributions"}, **kwargs) - assert str(error.value) == message def test_retry_selector_only_reruns_failed_original_jobs(): @@ -359,9 +356,9 @@ def test_retry_selector_only_reruns_failed_original_jobs(): assert recovery.selected_jobs(jobs, "python") == [1] assert recovery.selected_jobs(jobs, "docker") == [3] assert recovery.selected_jobs(jobs, "all") == [1, 3] - with pytest.raises(ValueError) as error: + message = "no failed original family jobs to rerun; skipped-only publication requires operator diagnosis" + with pytest.raises(ValueError, match=rf"^{re.escape(message)}$"): recovery.selected_jobs(jobs, "npm") - assert str(error.value) == "no failed original family jobs to rerun; skipped-only publication requires operator diagnosis" def test_archive_recovery_scope_contains_all_families():