diff --git a/.env.example b/.env.example index 202b0db..11d725f 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,10 @@ SERVER_HOST_PORT=8880 CLIENT_HOST_PORT=5180 +# Use the same exact server/client version. Alpha deployments must pin an +# immutable version such as 2.0.0-alpha.1 rather than using latest. +COPICK_WEB_VERSION=latest + # Path to your copick config JSON on the host machine. # This gets mounted read-only into the container at /data/copick_config.json. COPICK_CONFIG_PATH=./copick_config.json diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml index 2d342c3..8ec003a 100644 --- a/.github/workflows/conventional-commits.yml +++ b/.github/workflows/conventional-commits.yml @@ -2,7 +2,7 @@ name: conventional-commits on: pull_request: - branches: [main] + branches: [main, v2.0] types: - edited - opened diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9ba747f..0cb52b4 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -8,10 +8,7 @@ name: Docker Build Check on: pull_request: - paths: - - 'client/**' - - 'server/**' - - '.github/workflows/docker-build.yml' + branches: [main, v2.0] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -121,3 +118,14 @@ jobs: - name: Cleanup if: always() run: docker rm -f copick-client 2>/dev/null || true + + container-behavior-gate: + name: Container behavior gate + if: always() + needs: [build-server, build-client] + runs-on: ubuntu-latest + steps: + - name: Require both production container smoke tests + run: | + test '${{ needs.build-server.result }}' = success + test '${{ needs.build-client.result }}' = success diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2d35d2b..b3cd3ed 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,9 +1,35 @@ name: Build and Publish Docker Images on: - push: - branches: [main] + workflow_call: + inputs: + ref: + description: Immutable commit to build + required: true + type: string + version: + description: Exact application version + required: true + type: string + prerelease: + description: Publish the moving alpha tag instead of latest + required: true + type: boolean workflow_dispatch: + inputs: + ref: + description: Commit or tag to build + required: true + type: string + version: + description: Exact application version + required: true + type: string + prerelease: + description: Publish as an alpha without moving latest + required: true + default: true + type: boolean env: REGISTRY: ghcr.io @@ -17,6 +43,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Resolve immutable commit + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@v3 - uses: docker/setup-buildx-action@v3 @@ -32,8 +64,10 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}-server tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha + type=raw,value=${{ inputs.version }} + type=raw,value=sha-${{ steps.commit.outputs.sha }} + type=raw,value=alpha,enable=${{ inputs.prerelease }} + type=raw,value=latest,enable=${{ !inputs.prerelease }} - uses: docker/build-push-action@v6 with: @@ -47,6 +81,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Resolve immutable commit + id: commit + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: docker/setup-qemu-action@v3 - uses: docker/setup-buildx-action@v3 @@ -62,8 +102,10 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ github.repository }}-client tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha + type=raw,value=${{ inputs.version }} + type=raw,value=sha-${{ steps.commit.outputs.sha }} + type=raw,value=alpha,enable=${{ inputs.prerelease }} + type=raw,value=latest,enable=${{ !inputs.prerelease }} - uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/js-lint.yml b/.github/workflows/js-lint.yml index 72f54fe..b6f7956 100644 --- a/.github/workflows/js-lint.yml +++ b/.github/workflows/js-lint.yml @@ -2,10 +2,11 @@ name: JavaScript/TypeScript Linting on: pull_request: + branches: [main, v2.0] paths: - 'client/**' push: - branches: [main] + branches: [main, v2.0] paths: - 'client/**' diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml new file mode 100644 index 0000000..980b81f --- /dev/null +++ b/.github/workflows/migration-tests.yml @@ -0,0 +1,93 @@ +name: Migration Behavior + +on: + pull_request: + branches: [main, v2.0] + push: + branches: [main, v2.0] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + server-behavior: + name: Server behavior (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install server and test dependencies + run: python -m pip install -e './server[dev]' + + - name: Run server behavior tests + run: pytest -q server/tests + + client-behavior: + name: Client behavior (Node ${{ matrix.node-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ["20.19", "22"] + defaults: + run: + working-directory: client + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + cache-dependency-path: client/package-lock.json + + - name: Install locked client dependencies + run: npm ci + + - name: Run OME-Zarr behavior tests + run: npm test + + - name: Build client + run: npm run build + + server-behavior-gate: + name: Server behavior gate + if: always() + needs: server-behavior + runs-on: ubuntu-latest + steps: + - name: Require every supported Python version + run: test '${{ needs.server-behavior.result }}' = success + + client-behavior-gate: + name: Client behavior gate + if: always() + needs: client-behavior + runs-on: ubuntu-latest + steps: + - name: Require every supported Node version + run: test '${{ needs.client-behavior.result }}' = success + + migration-behavior-gate: + name: Migration behavior gate + if: always() + needs: [server-behavior-gate, client-behavior-gate] + runs-on: ubuntu-latest + steps: + - name: Require server and client behavior + run: | + test '${{ needs.server-behavior-gate.result }}' = success + test '${{ needs.client-behavior-gate.result }}' = success diff --git a/.github/workflows/py-lint.yml b/.github/workflows/py-lint.yml index 5862a80..fc03855 100644 --- a/.github/workflows/py-lint.yml +++ b/.github/workflows/py-lint.yml @@ -2,11 +2,12 @@ name: Python Linting on: pull_request: + branches: [main, v2.0] paths: - 'server/**' - '.pre-commit-config.yaml' push: - branches: [main] + branches: [main, v2.0] paths: - 'server/**' - '.pre-commit-config.yaml' diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 2a8c015..71fc32c 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -2,101 +2,84 @@ name: Create Release PRs on: workflow_dispatch: + inputs: + target_branch: + description: Branch to release + required: true + default: main + type: choice + options: + - main + - v2.0 push: branches: - main + - v2.0 + +env: + RELEASE_TARGET_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch || github.ref_name }} permissions: contents: write issues: write + packages: write pull-requests: write jobs: release-please: + if: github.event_name == 'workflow_dispatch' || github.event.created == false concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch || github.ref_name }} cancel-in-progress: true runs-on: ubuntu-latest steps: - - name: release please - uses: googleapis/release-please-action@v4 + - name: Create or update the linked release PR + uses: googleapis/release-please-action@v5 id: release with: manifest-file: ".release-please.manifest.json" - config-file: "release-please.config.json" - target-branch: "main" + config-file: ${{ env.RELEASE_TARGET_BRANCH == 'v2.0' && 'release-please.prerelease.config.json' || 'release-please.config.json' }} + target-branch: ${{ env.RELEASE_TARGET_BRANCH }} token: ${{ secrets.GITHUB_TOKEN }} outputs: - paths_released: ${{ steps.release.outputs.paths_released }} server_release_created: ${{ steps.release.outputs['server--release_created'] }} + server_sha: ${{ steps.release.outputs['server--sha'] }} + server_version: ${{ steps.release.outputs['server--version'] }} client_release_created: ${{ steps.release.outputs['client--release_created'] }} + client_sha: ${{ steps.release.outputs['client--sha'] }} + client_version: ${{ steps.release.outputs['client--version'] }} - # publish-pypi-package: - # name: Build and publish Python package to PyPI - # runs-on: ubuntu-latest - # needs: release-please - # if: needs.release-please.outputs.server_release_created == 'true' - # environment: - # name: pypi - # url: https://pypi.org/p/copick-web - # permissions: - # id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - # steps: - # - name: Checkout ref branch - # uses: actions/checkout@v6 - # with: - # ref: ${{ github.ref }} - # fetch-depth: 0 - # - # - name: Install uv - # uses: astral-sh/setup-uv@v7 - # with: - # version: "0.7.13" - # python-version: "3.12" - # - # - name: build - # run: | - # cd server - # uv build - # - # - name: Publish distribution to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # packages-dir: server/dist + validate-linked-release: + name: Validate linked release + needs: release-please + if: needs.release-please.outputs.server_release_created == 'true' || needs.release-please.outputs.client_release_created == 'true' + runs-on: ubuntu-latest + steps: + - name: Require the server and client to release together + env: + SERVER_RELEASE_CREATED: ${{ needs.release-please.outputs.server_release_created }} + CLIENT_RELEASE_CREATED: ${{ needs.release-please.outputs.client_release_created }} + SERVER_SHA: ${{ needs.release-please.outputs.server_sha }} + CLIENT_SHA: ${{ needs.release-please.outputs.client_sha }} + SERVER_VERSION: ${{ needs.release-please.outputs.server_version }} + CLIENT_VERSION: ${{ needs.release-please.outputs.client_version }} + run: | + test "$SERVER_RELEASE_CREATED" = true + test "$CLIENT_RELEASE_CREATED" = true + test "$SERVER_SHA" = "$CLIENT_SHA" + test "$SERVER_VERSION" = "$CLIENT_VERSION" - # publish-npm-package: - # name: Build and publish client to npm - # runs-on: ubuntu-latest - # needs: release-please - # if: needs.release-please.outputs.client_release_created == 'true' - # steps: - # - name: Checkout ref branch - # uses: actions/checkout@v6 - # with: - # ref: ${{ github.ref }} - # fetch-depth: 0 - # - # - name: Setup Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: '20' - # registry-url: 'https://registry.npmjs.org' - # - # - name: Install dependencies - # run: | - # cd client - # npm ci - # - # - name: Build - # run: | - # cd client - # npm run build - # - # - name: Publish to npm - # run: | - # cd client - # npm publish - # env: - # NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + publish-images: + name: Publish linked application images + needs: [release-please, validate-linked-release] + if: needs.validate-linked-release.result == 'success' + permissions: + contents: read + packages: write + uses: ./.github/workflows/docker-publish.yml + with: + ref: ${{ needs.release-please.outputs.server_sha }} + version: ${{ needs.release-please.outputs.server_version }} + prerelease: ${{ contains(needs.release-please.outputs.server_version, '-') }} diff --git a/README.md b/README.md index 08dc4a5..f83a359 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Web-based visualization tool for cryoET data using the copick data model. copick-web consists of two components: -- **Server**: Python FastAPI server that provides a REST API for copick metadata and proxies zarr data +- **Server**: Python FastAPI server that provides a REST API for copick metadata and proxies Zarr data - **Client**: React/TypeScript application using idetik-react for OME-Zarr visualization ## Features @@ -20,7 +20,7 @@ copick-web consists of two components: ## Prerequisites -- Python 3.9+ +- Python 3.11+ - Node.js 20.19.0+ - A copick configuration file pointing to your data @@ -71,7 +71,7 @@ We offer dev containers for ease of use or manual dev setups. ### Docker/Podman Compose Pre-requisites: Podman (recommended) or Docker installed with Compose extension. Check if installed with `docker-compose version` or `podman compose version` -- create .env file using .env.example as template. +- create .env file using .env.example as template. - obtain or use a copick project. Modify config.json's `overlay_root` parameter to `local:/data/copick_data/` ``` # Example .env @@ -140,10 +140,12 @@ When developing the client, run the server separately and the Vite dev server wi cd server black src/ ruff check src/ +pytest -q tests/ # Client cd client npm run lint +npm test npm run format ``` @@ -174,8 +176,14 @@ Create a `.env` file in the `server/` directory to set these values. ### Zarr Proxy -- `GET /zarr/tomo/{run}/{vs}/{type}/{path}` - Tomogram zarr chunks -- `GET /zarr/seg/{run}/{name}/{user}/{session}/{vs}/{path}` - Segmentation zarr chunks +- `GET|HEAD /zarr/tomo/{run}/{vs}/{type}/{path}` - Tomogram Zarr objects +- `GET|HEAD /zarr/seg/{run}/{name}/{user}/{session}/{vs}/{path}` - Segmentation Zarr objects + +The proxy reads the asynchronous Zarr 3 Store returned by copick 2.0. It +supports full responses and one `bytes` range (`start-end`, `start-`, or +`-suffix`) without downloading and slicing an entire shard in the web server. +Both legacy OME-Zarr 0.4 / Zarr v2 and OME-Zarr 0.5 / Zarr v3 are supported. +See [format and release compatibility](docs/zarr-v3-compatibility.md). ## Architecture diff --git a/client/.prettierignore b/client/.prettierignore index 007ea8a..dce36d9 100644 --- a/client/.prettierignore +++ b/client/.prettierignore @@ -1,3 +1,4 @@ dist node_modules coverage +tests/fixtures diff --git a/client/nginx.conf b/client/nginx.conf index d44c219..c472c8a 100644 --- a/client/nginx.conf +++ b/client/nginx.conf @@ -16,6 +16,8 @@ server { # Zarr proxy to server container location /zarr/ { proxy_pass http://server:8000; + proxy_set_header Range $http_range; + proxy_set_header If-Range $http_if_range; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/client/package-lock.json b/client/package-lock.json index 621a563..d03adba 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -33,7 +33,8 @@ "prettier": "^3.3.2", "tailwindcss": "^3.4.17", "typescript": "^5.2.2", - "vite": "^6.3.6" + "vite": "^6.3.6", + "vitest": "^4.1.11" }, "engines": { "node": ">=20.19.0" @@ -1930,6 +1931,13 @@ "win32" ] }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@tanstack/query-core": { "version": "5.90.20", "license": "MIT", @@ -1989,6 +1997,24 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "dev": true, @@ -2268,6 +2294,126 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, + "node_modules/@vitest/expect": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.11", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.11", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/@zarrita/storage": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.2.0.tgz", @@ -2364,6 +2510,16 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/autoprefixer": { "version": "10.4.24", "dev": true, @@ -2521,6 +2677,16 @@ ], "license": "CC-BY-4.0" }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "dev": true, @@ -2714,6 +2880,13 @@ "is-arrayish": "^0.2.1" } }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.25.12", "dev": true, @@ -2971,6 +3144,16 @@ "node": ">=4.0" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/esutils": { "version": "2.0.3", "dev": true, @@ -2979,6 +3162,16 @@ "node": ">=0.10.0" } }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "dev": true, @@ -3183,7 +3376,9 @@ } }, "node_modules/hasown": { - "version": "2.0.2", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -3428,6 +3623,16 @@ "yallist": "^3.0.2" } }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, "node_modules/merge2": { "version": "1.4.1", "dev": true, @@ -3545,6 +3750,20 @@ "node": ">= 6" } }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/optionator": { "version": "0.9.4", "dev": true, @@ -3642,6 +3861,13 @@ "node": ">=8" } }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "license": "ISC" @@ -4104,6 +4330,13 @@ "node": ">=8" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/source-map": { "version": "0.5.7", "license": "BSD-3-Clause", @@ -4119,6 +4352,20 @@ "node": ">=0.10.0" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, "node_modules/strip-json-comments": { "version": "3.1.1", "dev": true, @@ -4231,6 +4478,23 @@ "node": ">=0.8" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "dev": true, @@ -4246,6 +4510,16 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinyrainbow": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "dev": true, @@ -4424,6 +4698,96 @@ } } }, + "node_modules/vitest": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, "node_modules/webgpu-utils": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/webgpu-utils/-/webgpu-utils-2.1.1.tgz", @@ -4443,6 +4807,23 @@ "node": ">= 8" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "dev": true, @@ -4456,23 +4837,6 @@ "dev": true, "license": "ISC" }, - "node_modules/yaml": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", - "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", - "dev": true, - "optional": true, - "peer": true, - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "dev": true, diff --git a/client/package.json b/client/package.json index 53089d1..a7a530a 100644 --- a/client/package.json +++ b/client/package.json @@ -8,6 +8,7 @@ "build": "tsc && vite build", "build:server": "tsc && vite build --outDir ../server/src/copick_web/static --emptyOutDir", "lint": "eslint .", + "test": "vitest run", "preview": "vite preview", "format": "prettier --write './**/*.{ts,tsx,html,json,css}'" }, @@ -37,7 +38,8 @@ "prettier": "^3.3.2", "tailwindcss": "^3.4.17", "typescript": "^5.2.2", - "vite": "^6.3.6" + "vite": "^6.3.6", + "vitest": "^4.1.11" }, "engines": { "node": ">=20.19.0" diff --git a/client/tests/fixtures/generate.py b/client/tests/fixtures/generate.py new file mode 100644 index 0000000..9867fec --- /dev/null +++ b/client/tests/fixtures/generate.py @@ -0,0 +1,93 @@ +"""Generate the small behavioral OME-Zarr fixtures used by the client tests.""" + +import shutil +from pathlib import Path + +import numpy as np +import zarr + +from copick.util.ome import write_ome_zarr_3d + +ROOT = Path(__file__).parent +SHAPE = (4, 5, 6) +CHUNKS = (2, 3, 4) + + +def _reset(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + + +def _axes() -> list[dict[str, str]]: + return [ + {"name": "z", "type": "space", "unit": "angstrom"}, + {"name": "y", "type": "space", "unit": "angstrom"}, + {"name": "x", "type": "space", "unit": "angstrom"}, + ] + + +def _datasets(level_path: str) -> list[dict[str, object]]: + return [ + { + "path": level_path, + "coordinateTransformations": [ + {"type": "scale", "scale": [10.0, 10.0, 10.0]} + ], + } + ] + + +def _write_v2(name: str, level_path: str) -> None: + path = ROOT / name + _reset(path) + values = np.arange(np.prod(SHAPE), dtype=np.uint16).reshape(SHAPE) + group = zarr.open_group(path, mode="w", zarr_format=2) + group.create_array(level_path, data=values, chunks=CHUNKS, compressors=None) + group.attrs["multiscales"] = [ + { + "version": "0.4", + "axes": _axes(), + "datasets": _datasets(level_path), + } + ] + + +def _write_unsharded_v3() -> None: + path = ROOT / "v3-unsharded.zarr" + _reset(path) + values = np.arange(np.prod(SHAPE), dtype=np.uint16).reshape(SHAPE) + group = zarr.open_group(path, mode="w", zarr_format=3) + group.create_array("0", data=values, chunks=CHUNKS) + group.attrs["ome"] = { + "version": "0.5", + "multiscales": [{"axes": _axes(), "datasets": _datasets("0")}], + } + + +def _write_v3(name: str, values: np.ndarray) -> None: + path = ROOT / name + _reset(path) + write_ome_zarr_3d(str(path), {10.0: values}, chunk_size=CHUNKS) + + +def _normalize_metadata_newlines() -> None: + metadata_names = {".zarray", ".zattrs", ".zgroup", "zarr.json"} + for path in ROOT.rglob("*"): + if path.name in metadata_names: + path.write_text(path.read_text(encoding="utf-8").rstrip() + "\n", encoding="utf-8") + + +if __name__ == "__main__": + indices = np.indices(SHAPE, sparse=True) + _reset(ROOT / "v2-legacy.zarr") + _write_v2("v2-numeric.zarr", "0") + _write_v2("v2-nonnumeric.zarr", "scale0") + _write_unsharded_v3() + _write_v3( + "v3-integer.zarr", np.arange(np.prod(SHAPE), dtype=np.uint16).reshape(SHAPE) + ) + _write_v3( + "v3-floating.zarr", + (0.5 * indices[0] - 0.25 * indices[1] + 0.125 * indices[2]).astype(np.float32), + ) + _normalize_metadata_newlines() diff --git a/client/tests/fixtures/v2-nonnumeric.zarr/.zattrs b/client/tests/fixtures/v2-nonnumeric.zarr/.zattrs new file mode 100644 index 0000000..5cf67b3 --- /dev/null +++ b/client/tests/fixtures/v2-nonnumeric.zarr/.zattrs @@ -0,0 +1,39 @@ +{ + "multiscales": [ + { + "version": "0.4", + "axes": [ + { + "name": "z", + "type": "space", + "unit": "angstrom" + }, + { + "name": "y", + "type": "space", + "unit": "angstrom" + }, + { + "name": "x", + "type": "space", + "unit": "angstrom" + } + ], + "datasets": [ + { + "path": "scale0", + "coordinateTransformations": [ + { + "type": "scale", + "scale": [ + 10.0, + 10.0, + 10.0 + ] + } + ] + } + ] + } + ] +} diff --git a/client/tests/fixtures/v2-nonnumeric.zarr/.zgroup b/client/tests/fixtures/v2-nonnumeric.zarr/.zgroup new file mode 100644 index 0000000..ea02389 --- /dev/null +++ b/client/tests/fixtures/v2-nonnumeric.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} diff --git a/client/tests/fixtures/v2-nonnumeric.zarr/scale0/.zarray b/client/tests/fixtures/v2-nonnumeric.zarr/scale0/.zarray new file mode 100644 index 0000000..8758268 --- /dev/null +++ b/client/tests/fixtures/v2-nonnumeric.zarr/scale0/.zarray @@ -0,0 +1,19 @@ +{ + "shape": [ + 4, + 5, + 6 + ], + "chunks": [ + 2, + 3, + 4 + ], + "dtype": "[0]; + +function asArrayBuffer(value: Uint8Array): ArrayBuffer { + const buffer = new ArrayBuffer(value.byteLength); + new Uint8Array(buffer).set(value); + return buffer; +} + +function fixturePath(url: URL): string | undefined { + const relative = decodeURIComponent(url.pathname) + .replace(/^\/+/, "") + .replace("v3-truncated.zarr", "v3-floating.zarr"); + const candidate = resolve(fixtureRoot, relative); + if ( + candidate !== fixtureRoot && + !candidate.startsWith(`${fixtureRoot}${sep}`) + ) + return undefined; + return candidate; +} + +async function fixtureFetch( + input: RequestInfo | URL, + init?: RequestInit, +): Promise { + const request = new Request(input, init); + requests.push(request.clone()); + const requestUrl = new URL(request.url); + + if ( + requestUrl.pathname === "/invalid.zarr/zarr.json" && + request.method === "GET" + ) { + return new Response( + JSON.stringify({ + zarr_format: 3, + node_type: "group", + attributes: { ome: { version: "0.5", multiscales: [] } }, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + const path = fixturePath(requestUrl); + if (!path) return new Response(null, { status: 404 }); + + let contents: Uint8Array; + try { + contents = await readFile(path); + } catch { + return new Response(null, { status: 404 }); + } + if ( + requestUrl.pathname.includes("v3-truncated.zarr") && + requestUrl.pathname.endsWith("/0/0/0/0") + ) { + contents = contents.slice(0, -16); + } + + const commonHeaders = { + "Accept-Ranges": "bytes", + "Content-Length": String(contents.byteLength), + }; + if (request.method === "HEAD") { + return new Response(null, { status: 200, headers: commonHeaders }); + } + + const range = request.headers.get("Range"); + if (!range) { + return new Response(asArrayBuffer(contents), { + status: 200, + headers: commonHeaders, + }); + } + + const match = /^bytes=(\d*)-(\d*)$/.exec(range); + if (!match) return new Response(null, { status: 416 }); + const start = match[1] + ? Number(match[1]) + : Math.max(contents.byteLength - Number(match[2]), 0); + const end = match[2] ? Number(match[2]) : contents.byteLength - 1; + const selected = contents.slice(start, Math.min(end + 1, contents.length)); + return new Response(asArrayBuffer(selected), { + status: 206, + headers: { + ...commonHeaders, + "Content-Length": String(selected.byteLength), + "Content-Range": `bytes ${start}-${start + selected.byteLength - 1}/${contents.byteLength}`, + }, + }); +} + +function createFirstChunk(source: OmeZarrImageSource): LoaderChunk { + const dimensions = source.getDimensions(); + return { + state: "unloaded" as const, + lod: 0, + shape: { + x: dimensions.x.lods[0].chunkSize, + y: dimensions.y.lods[0].chunkSize, + z: dimensions.z?.lods[0].chunkSize ?? 1, + c: 1, + }, + rowAlignmentBytes: 1 as const, + chunkIndex: { x: 0, y: 0, z: 0, c: 0, t: 0 }, + scale: { x: 1, y: 1, z: 1 }, + offset: { x: 0, y: 0, z: 0 }, + visible: true, + prefetch: false, + priority: null, + orderKey: null, + }; +} + +async function loadFirstChunk(fixture: string) { + const source = await OmeZarrImageSource.fromHttp({ + url: `http://fixtures.test/${fixture}`, + }); + const chunk = createFirstChunk(source); + await source.loader.loadChunkData(chunk, new AbortController().signal); + return { source, chunk }; +} + +const firstIntegerChunk = [ + 0, 1, 2, 3, 6, 7, 8, 9, 12, 13, 14, 15, 30, 31, 32, 33, 36, 37, 38, 39, 42, + 43, 44, 45, +]; + +beforeEach(() => { + requests = []; + vi.stubGlobal("fetch", fixtureFetch); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("OME-Zarr format compatibility", () => { + it.each(["v2-numeric.zarr", "v2-nonnumeric.zarr"])( + "discovers and decodes legacy OME-Zarr 0.4 / Zarr v2 paths from %s", + async (fixture) => { + const { source, chunk } = await loadFirstChunk(fixture); + + expect(source.getDimensions()).toMatchObject({ + numLods: 1, + x: { lods: [{ size: 6, chunkSize: 4, scale: 10 }] }, + y: { lods: [{ size: 5, chunkSize: 3, scale: 10 }] }, + z: { lods: [{ size: 4, chunkSize: 2, scale: 10 }] }, + }); + expect(Array.from(chunk.data ?? [])).toEqual(firstIntegerChunk); + }, + ); + + it("decodes an unsharded OME-Zarr 0.5 / Zarr v3 chunk", async () => { + const { chunk } = await loadFirstChunk("v3-unsharded.zarr"); + expect(Array.from(chunk.data ?? [])).toEqual(firstIntegerChunk); + }); + + it("decodes a canonical sharded integer OME-Zarr 0.5 / Zarr v3 chunk", async () => { + const { source, chunk } = await loadFirstChunk("v3-integer.zarr"); + + expect(source.getDimensions()).toMatchObject({ + numLods: 1, + x: { lods: [{ size: 6, chunkSize: 4, scale: 10 }] }, + y: { lods: [{ size: 5, chunkSize: 3, scale: 10 }] }, + z: { lods: [{ size: 4, chunkSize: 2, scale: 10 }] }, + }); + expect(Array.from(chunk.data ?? [])).toEqual(firstIntegerChunk); + }); + + it("decodes shuffled floating-point chunks and uses bounded shard reads", async () => { + const { chunk } = await loadFirstChunk("v3-floating.zarr"); + + expect(Array.from(chunk.data ?? [])).toEqual([ + 0, 0.125, 0.25, 0.375, -0.25, -0.125, 0, 0.125, -0.5, -0.375, -0.25, + -0.125, 0.5, 0.625, 0.75, 0.875, 0.25, 0.375, 0.5, 0.625, 0, 0.125, 0.25, + 0.375, + ]); + + const shardRequests = requests.filter((request) => + new URL(request.url).pathname.endsWith("/0/0/0/0"), + ); + expect(shardRequests.some((request) => request.method === "HEAD")).toBe( + true, + ); + const shardGets = shardRequests.filter( + (request) => request.method === "GET", + ); + expect(shardGets.length).toBeGreaterThan(0); + expect(shardGets.every((request) => request.headers.has("Range"))).toBe( + true, + ); + }); + + it("rejects invalid metadata and truncated shards", async () => { + await expect( + OmeZarrImageSource.fromHttp({ url: "http://fixtures.test/invalid.zarr" }), + ).rejects.toThrow("Failed to parse OME-Zarr image"); + await expect(loadFirstChunk("v3-truncated.zarr")).rejects.toThrow(); + }); +}); diff --git a/client/tests/setup.ts b/client/tests/setup.ts new file mode 100644 index 0000000..488f49f --- /dev/null +++ b/client/tests/setup.ts @@ -0,0 +1,15 @@ +Object.defineProperty(globalThis, "navigator", { + configurable: true, + value: { hardwareConcurrency: 1 }, +}); + +// webgpu-utils references this constructor while its module is initialized. +// Node 20 and 22 do not provide it, and these reader tests do not exercise +// half-float GPU buffers, so a distinct typed-array stand-in is sufficient. +if (!("Float16Array" in globalThis)) { + class Float16ArrayShim extends Uint16Array {} + Object.defineProperty(globalThis, "Float16Array", { + configurable: true, + value: Float16ArrayShim, + }); +} diff --git a/client/tsconfig.json b/client/tsconfig.json index 2a27e96..f620b8f 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -19,6 +19,6 @@ "@/*": ["./src/*"] } }, - "include": ["src"], + "include": ["src", "tests", "vitest.config.ts"], "references": [{ "path": "./tsconfig.node.json" }] } diff --git a/client/vitest.config.ts b/client/vitest.config.ts new file mode 100644 index 0000000..a5b3412 --- /dev/null +++ b/client/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + setupFiles: ["./tests/setup.ts"], + }, +}); diff --git a/compose-prod.yml b/compose-prod.yml index 60fe6c3..2c39a0e 100644 --- a/compose-prod.yml +++ b/compose-prod.yml @@ -1,6 +1,6 @@ services: server: - image: ghcr.io/copick/copick-web-server:latest + image: ghcr.io/copick/copick-web-server:${COPICK_WEB_VERSION:-latest} volumes: - ${COPICK_CONFIG_PATH:?Set COPICK_CONFIG_PATH in .env}:/data/copick_config.json:ro - ${COPICK_DATA_DIR:-./data}:/data/copick_data @@ -12,7 +12,7 @@ services: - copick-net client: - image: ghcr.io/copick/copick-web-client:latest + image: ghcr.io/copick/copick-web-client:${COPICK_WEB_VERSION:-latest} ports: - "${SERVER_HOST_PORT:-8880}:80" environment: diff --git a/docs/deployment.md b/docs/deployment.md index e263705..a03d376 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -30,6 +30,7 @@ Edit `.env` to point to your copick config and data: ```bash # .env SERVER_HOST_PORT=8880 +COPICK_WEB_VERSION=2.0.0-alpha.1 COPICK_CONFIG_PATH=/path/to/your/copick_config.json COPICK_DATA_DIR=/path/to/your/copick/data BASE_PATH= @@ -38,6 +39,11 @@ BASE_PATH= > **Important:** Paths inside your `copick_config.json` must reference `/data/copick_data` > since that is where `COPICK_DATA_DIR` is mounted inside the container. +Use the same exact `COPICK_WEB_VERSION` for the server and client images. +Alpha deployments must use an immutable `2.0.0-alpha.N` tag. The moving +`alpha` tag is convenient for disposable testing, but is not reproducible; +`latest` is reserved for stable releases. + ### 4. Start the service ```bash @@ -62,6 +68,24 @@ podman compose -f compose-prod.yml pull podman compose -f compose-prod.yml up -d ``` +For a controlled upgrade, edit `COPICK_WEB_VERSION` to the new exact version, +then run the same two commands. Server and client versions must never differ. + +## Rollback + +Restore the previously tested exact `COPICK_WEB_VERSION`, then pull and recreate +the pair: + +```bash +podman compose -f compose-prod.yml pull +podman compose -f compose-prod.yml up -d +podman compose -f compose-prod.yml ps +``` + +The application is read-only at its Zarr boundary, so rolling back copick-web +does not rewrite datasets. A rollback must target a version that supports the +formats present in the configured project. + ## Running behind a reverse proxy Set `BASE_PATH` in `.env` to match your reverse proxy sub-path. The client container diff --git a/docs/zarr-v3-compatibility.md b/docs/zarr-v3-compatibility.md new file mode 100644 index 0000000..8519387 --- /dev/null +++ b/docs/zarr-v3-compatibility.md @@ -0,0 +1,59 @@ +# OME-Zarr 0.5 and Zarr v3 compatibility + +The copick-web 2.0 line consumes the public storage contract introduced by +copick 2.0. It reads both existing OME-Zarr 0.4 / Zarr v2 datasets and newly +created OME-Zarr 0.5 / Zarr v3 datasets in the same application build. + +## Runtime support + +- Server: Python 3.11, 3.12, 3.13, and 3.14. +- Client development and CI: Node.js 20.19 or Node.js 22. +- Core: `copick>=2.0.0a1,<2.1`. +- Store API: `zarr>=3.1.6,<4` asynchronous `Store` contract. +- Browser reader: the locked `@idetik/core` 0.35 / Zarrita 0.7 line. + +Behavioral fixtures cover legacy unsharded integer arrays and canonical +copick-produced sharded integer and shuffled floating-point arrays. The client +tests decode exact values and confirm that shard access uses size probes plus +bounded byte ranges. + +## HTTP proxy contract + +The tomogram and segmentation resource URLs are unchanged. Both implement: + +| Request | Result | +| --- | --- | +| Full `GET` | `200` with the complete object | +| `HEAD` | `200` with object length and no body | +| `Range: bytes=start-end` | `206`; the inclusive HTTP end is translated to an exclusive Zarr end | +| `Range: bytes=start-` | `206` from the offset through the last byte | +| `Range: bytes=-suffix` | `206` with up to the requested trailing bytes | +| Missing key | `404` | +| Malformed, multiple, or unsatisfiable range | `416` with `Content-Range: bytes */size` | + +Successful responses advertise `Accept-Ranges: bytes`. CORS exposes +`Accept-Ranges`, `Content-Length`, and `Content-Range`. Partial reads are sent +to the copick-owned Zarr Store and are never implemented by fetching a full +shard and slicing it in the FastAPI process. + +Authentication, credentials, retries, reconnection, and filesystem creation +remain owned by copick, Zarr, and fsspec. The web layer does not reconstruct +backend URLs. Local behavior is exercised directly in this repository; S3, +SSH, and ML Croissant deployments use the same Store boundary and remain +release-environment validation gates. SMB is optional until it is promoted by +copick core. + +## Releases and images + +Server and client versions are linked and released together. The `v2.0` branch +uses `2.0.0-alpha.N` GitHub prereleases and publishes: + +- `ghcr.io/copick/copick-web-server:2.0.0-alpha.N`; +- `ghcr.io/copick/copick-web-client:2.0.0-alpha.N`; +- matching immutable `sha-` tags; +- the moving `alpha` tag for prereleases. + +Alpha publication never moves `latest`. Stable publication adds the exact +stable version and moves `latest` only after the stable release is created. +Set one `COPICK_WEB_VERSION` in `compose-prod.yml` to pin both images. No PyPI +or npm package is published by this release path. diff --git a/release-please.config.json b/release-please.config.json index c415e4e..2629dc6 100644 --- a/release-please.config.json +++ b/release-please.config.json @@ -12,6 +12,13 @@ { "type": "build", "section": "Build System" }, { "type": "ci", "section": "Continuous Integration" } ], + "plugins": [ + { + "type": "linked-versions", + "groupName": "copick-web", + "components": ["server", "client"] + } + ], "packages": { "server": { "package-name": "copick-web", diff --git a/release-please.prerelease.config.json b/release-please.prerelease.config.json new file mode 100644 index 0000000..f300456 --- /dev/null +++ b/release-please.prerelease.config.json @@ -0,0 +1,50 @@ +{ + "changelog-sections": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "perf", "section": "Performance Improvements" }, + { "type": "revert", "section": "Reverts" }, + { "type": "docs", "section": "Documentation" }, + { "type": "style", "section": "Styles" }, + { "type": "chore", "section": "Miscellaneous Chores" }, + { "type": "refactor", "section": "Code Refactoring" }, + { "type": "test", "section": "Tests" }, + { "type": "build", "section": "Build System" }, + { "type": "ci", "section": "Continuous Integration" } + ], + "plugins": [ + { + "type": "linked-versions", + "groupName": "copick-web", + "components": ["server", "client"] + } + ], + "packages": { + "server": { + "package-name": "copick-web", + "release-type": "python", + "component": "server", + "versioning": "prerelease", + "prerelease-type": "alpha", + "changelog-path": "CHANGELOG.md", + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "draft": false, + "prerelease": true, + "pull-request-footer": "Merging this PR will create the copick-web server alpha prerelease." + }, + "client": { + "package-name": "copick-web-client", + "release-type": "node", + "component": "client", + "versioning": "prerelease", + "prerelease-type": "alpha", + "changelog-path": "CHANGELOG.md", + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "draft": false, + "prerelease": true, + "pull-request-footer": "Merging this PR will create the copick-web client alpha prerelease." + } + } +} diff --git a/server/pyproject.toml b/server/pyproject.toml index ac06723..b505207 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -6,13 +6,16 @@ build-backend = "hatchling.build" name = "copick-web" version = "0.1.2" description = "Web visualization for copick datasets" -requires-python = ">=3.9" +requires-python = ">=3.11" dependencies = [ "fastapi>=0.109.0", "uvicorn>=0.25.0", "pydantic>=2.5.0", "pydantic-settings>=2.0.0", - "copick>=0.8.0", + "copick>=2.0.0a1,<2.1", + # The proxy constructs native byte-range requests and response buffers + # against the public asynchronous Store API. + "zarr>=3.1.6,<4", "python-dotenv>=1.0.0", "click>=8.1.8", ] @@ -22,6 +25,7 @@ dev = [ "black>=25.1.0", "ruff>=0.12.0", "pytest>=8.0.0", + "httpx>=0.27.0", "pre-commit>=4.0.0", ] diff --git a/server/src/copick_web/app/main.py b/server/src/copick_web/app/main.py index 9a2fa9d..0ec9efe 100644 --- a/server/src/copick_web/app/main.py +++ b/server/src/copick_web/app/main.py @@ -24,7 +24,7 @@ async def lifespan(app: FastAPI): try: init_copick_service(settings.copick_config_path) logger.info("Copick service initialized successfully") - except FileNotFoundError as e: + except FileNotFoundError: logger.error(f"Config file not found: {settings.copick_config_path}") raise except OSError as e: @@ -35,7 +35,7 @@ async def lifespan(app: FastAPI): raise RuntimeError( f"Storage connection failed. Check that any required services (SSH tunnel, etc.) are running. " f"Error: {e}" - ) + ) from e except Exception as e: logger.error(f"Failed to initialize copick service: {e}") raise @@ -61,6 +61,7 @@ async def lifespan(app: FastAPI): allow_credentials=True, allow_methods=["*"], allow_headers=["*"], + expose_headers=["Accept-Ranges", "Content-Length", "Content-Range"], ) # Include API routers diff --git a/server/src/copick_web/app/routes/config.py b/server/src/copick_web/app/routes/config.py index c16d375..0d2ecb3 100644 --- a/server/src/copick_web/app/routes/config.py +++ b/server/src/copick_web/app/routes/config.py @@ -1,15 +1,18 @@ """Configuration and objects routes.""" +from typing import Annotated + from fastapi import APIRouter, Depends from ..models import ConfigResponse, PickableObjectResponse from ..services.copick_service import CopickService, get_copick_service router = APIRouter(prefix="/api", tags=["config"]) +ServiceDependency = Annotated[CopickService, Depends(get_copick_service)] @router.get("/config", response_model=ConfigResponse) -def get_config(service: CopickService = Depends(get_copick_service)) -> ConfigResponse: +def get_config(service: ServiceDependency) -> ConfigResponse: """Get project configuration.""" config = service.config return ConfigResponse( @@ -22,7 +25,7 @@ def get_config(service: CopickService = Depends(get_copick_service)) -> ConfigRe @router.get("/objects", response_model=list[PickableObjectResponse]) -def get_objects(service: CopickService = Depends(get_copick_service)) -> list[PickableObjectResponse]: +def get_objects(service: ServiceDependency) -> list[PickableObjectResponse]: """Get all pickable objects.""" return [ PickableObjectResponse( diff --git a/server/src/copick_web/app/routes/runs.py b/server/src/copick_web/app/routes/runs.py index 7f92c0d..97e6951 100644 --- a/server/src/copick_web/app/routes/runs.py +++ b/server/src/copick_web/app/routes/runs.py @@ -1,10 +1,9 @@ """Runs, tomograms, picks, and segmentations routes.""" -from typing import Optional +from typing import Annotated, Optional from fastapi import APIRouter, Depends, HTTPException, Query, Request -from ..validation import validate_copick_name from ..models import ( CreatePicksRequest, CreatePicksResponse, @@ -20,18 +19,20 @@ VoxelSpacingSummaryResponse, ) from ..services.copick_service import CopickService, get_copick_service +from ..validation import validate_copick_name router = APIRouter(prefix="/api", tags=["runs"]) +ServiceDependency = Annotated[CopickService, Depends(get_copick_service)] @router.get("/runs", response_model=list[RunSummaryResponse]) -def get_runs(service: CopickService = Depends(get_copick_service)) -> list[RunSummaryResponse]: +def get_runs(service: ServiceDependency) -> list[RunSummaryResponse]: """Get all runs.""" return [RunSummaryResponse(name=name) for name in service.get_runs()] @router.get("/runs/{run_name}", response_model=RunDetailResponse) -def get_run(run_name: str, service: CopickService = Depends(get_copick_service)) -> RunDetailResponse: +def get_run(run_name: str, service: ServiceDependency) -> RunDetailResponse: """Get run details including voxel spacings and tomograms.""" run = service.get_run(run_name) if not run: @@ -51,7 +52,7 @@ def get_tomogram( voxel_size: float, tomo_type: str, request: Request, - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, ) -> TomogramResponse: """Get tomogram details with zarr URL.""" tomo = service.get_tomogram(run_name, voxel_size, tomo_type) @@ -73,10 +74,10 @@ def get_tomogram( @router.get("/runs/{run_name}/picks", response_model=list[PicksSummaryResponse]) def get_picks( run_name: str, - object_name: Optional[str] = Query(None), - user_id: Optional[str] = Query(None), - session_id: Optional[str] = Query(None), - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, + object_name: Annotated[Optional[str], Query()] = None, + user_id: Annotated[Optional[str], Query()] = None, + session_id: Annotated[Optional[str], Query()] = None, ) -> list[PicksSummaryResponse]: """Get all picks for a run with optional filtering.""" run = service.get_run(run_name) @@ -110,7 +111,7 @@ def get_pick_points( object_name: str, user_id: str, session_id: str, - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, ) -> PicksDetailResponse: """Get detailed picks with all points.""" pick = service.get_pick(run_name, object_name, user_id, session_id) @@ -151,7 +152,7 @@ def get_pick_points( def create_picks( run_name: str, request: CreatePicksRequest, - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, ) -> CreatePicksResponse: """Create a new empty picks collection.""" # Validate all name fields using copick rules @@ -165,7 +166,7 @@ def create_picks( raise HTTPException(status_code=400, detail=f"Invalid {field_name}: {error_msg}") try: - picks = service.create_picks( + service.create_picks( run_name, request.object_name, request.user_id, @@ -181,7 +182,7 @@ def create_picks( color=color, ) except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException(status_code=400, detail=str(e)) from e @router.put("/runs/{run_name}/picks/{object_name}/{user_id}/{session_id}", response_model=PicksDetailResponse) @@ -191,7 +192,7 @@ def update_picks( user_id: str, session_id: str, request: UpdatePicksRequest, - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, ) -> PicksDetailResponse: """Update picks with new points.""" # Check if picks are editable (session_id != "0") @@ -226,7 +227,7 @@ def update_picks( ], ) except ValueError as e: - raise HTTPException(status_code=404, detail=str(e)) + raise HTTPException(status_code=404, detail=str(e)) from e @router.delete("/runs/{run_name}/picks/{object_name}/{user_id}/{session_id}", status_code=204) @@ -235,7 +236,7 @@ def delete_picks( object_name: str, user_id: str, session_id: str, - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, ): """Delete a picks collection.""" # Check if picks are editable (session_id != "0") @@ -245,7 +246,7 @@ def delete_picks( try: service.delete_picks_collection(run_name, object_name, user_id, session_id) except ValueError as e: - raise HTTPException(status_code=404, detail=str(e)) + raise HTTPException(status_code=404, detail=str(e)) from e # --- Segmentations endpoints --- @@ -255,11 +256,11 @@ def delete_picks( def get_segmentations( run_name: str, request: Request, - name: Optional[str] = Query(None), - user_id: Optional[str] = Query(None), - session_id: Optional[str] = Query(None), - voxel_size: Optional[float] = Query(None), - service: CopickService = Depends(get_copick_service), + service: ServiceDependency, + name: Annotated[Optional[str], Query()] = None, + user_id: Annotated[Optional[str], Query()] = None, + session_id: Annotated[Optional[str], Query()] = None, + voxel_size: Annotated[Optional[float], Query()] = None, ) -> list[SegmentationSummaryResponse]: """Get all segmentations for a run with optional filtering.""" run = service.get_run(run_name) diff --git a/server/src/copick_web/app/routes/zarr_proxy.py b/server/src/copick_web/app/routes/zarr_proxy.py index a190a1f..dd9a3c7 100644 --- a/server/src/copick_web/app/routes/zarr_proxy.py +++ b/server/src/copick_web/app/routes/zarr_proxy.py @@ -1,8 +1,13 @@ -"""Zarr proxy routes for serving zarr data from fsspec stores.""" +"""HTTP proxy for objects exposed by copick-owned Zarr stores.""" import logging +import re +from dataclasses import dataclass +from typing import Annotated, Any -from fastapi import APIRouter, Depends, HTTPException, Response +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest +from zarr.core.buffer import default_buffer_prototype from ..services.copick_service import CopickService, get_copick_service @@ -10,93 +15,196 @@ router = APIRouter(prefix="/zarr", tags=["zarr"]) +_RANGE_PATTERN = re.compile(r"^(\d*)-(\d*)$") +_CACHE_CONTROL = "public, max-age=3600" + + +@dataclass(frozen=True) +class _SelectedRange: + """A validated HTTP byte range and its equivalent Zarr request.""" + + start: int + end: int + byte_request: RangeByteRequest | OffsetByteRequest | SuffixByteRequest + + @property + def length(self) -> int: + return self.end - self.start + 1 + + +class _RangeNotSatisfiable(ValueError): + """Raised when a Range header cannot select bytes from an object.""" + def _get_content_type(path: str) -> str: - """Determine content type based on file path.""" + """Determine content type based on the Zarr object path.""" if path.endswith((".zarray", ".zattrs", ".zgroup", "zarr.json")): return "application/json" return "application/octet-stream" -def _read_from_store(store, path: str) -> bytes: - """Read data from a zarr store.""" - logger.debug(f"Reading path '{path}' from store {type(store)}") +def _response_headers(content_length: int) -> dict[str, str]: + """Return headers shared by full, partial, and HEAD responses.""" + return { + "Accept-Ranges": "bytes", + "Cache-Control": _CACHE_CONTROL, + "Content-Length": str(content_length), + "Vary": "Range", + } + + +def _missing_path(path: str) -> HTTPException: + return HTTPException(status_code=404, detail=f"Path '{path}' not found in store") + + +async def _get_store_size(store: Any, path: str) -> int: + """Return object size without fetching its body when the store supports it.""" try: - data = store[path] - logger.debug(f"Got data of type {type(data)}, length {len(data) if hasattr(data, '__len__') else 'N/A'}") - if isinstance(data, (bytes, bytearray)): - return bytes(data) - # Handle memoryview or other buffer types - if hasattr(data, "tobytes"): - return data.tobytes() + return await store.getsize(path) + except FileNotFoundError as error: + logger.info("Path '%s' not found while obtaining its size", path) + raise _missing_path(path) from error + + +def _to_bytes(data: Any) -> bytes: + """Convert a Zarr buffer (or compatible buffer object) to response bytes.""" + if isinstance(data, bytes): + return data + if isinstance(data, (bytearray, memoryview)): return bytes(data) - except KeyError as e: - logger.warning(f"Path '{path}' not found in store: {e}") - raise HTTPException(status_code=404, detail=f"Path '{path}' not found in store") - except Exception as e: - logger.error(f"Error reading '{path}' from store: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error reading from store: {e}") + if hasattr(data, "to_bytes"): + return data.to_bytes() + if hasattr(data, "tobytes"): + return data.tobytes() + return bytes(data) + + +async def _read_from_store( + store: Any, + path: str, + byte_range: RangeByteRequest | OffsetByteRequest | SuffixByteRequest | None = None, +) -> bytes: + """Read an object, preserving the Zarr store's native range operation.""" + logger.debug("Reading path '%s' from %s with range %r", path, type(store), byte_range) + try: + data = await store.get( + path, + prototype=default_buffer_prototype(), + byte_range=byte_range, + ) + except FileNotFoundError as error: + raise _missing_path(path) from error + + if data is None: + raise _missing_path(path) + return _to_bytes(data) + + +def _parse_range(value: str, size: int) -> _SelectedRange: + """Parse one RFC 9110 byte range and construct the equivalent Zarr request.""" + unit, separator, range_value = value.partition("=") + if separator != "=" or unit.strip().lower() != "bytes" or "," in range_value: + raise _RangeNotSatisfiable + + match = _RANGE_PATTERN.fullmatch(range_value.strip()) + if match is None: + raise _RangeNotSatisfiable + start_value, end_value = match.groups() + if not start_value and not end_value: + raise _RangeNotSatisfiable + if size == 0: + raise _RangeNotSatisfiable -@router.get("/tomo/{run_name}/{voxel_size}/{tomo_type}/{path:path}") -def proxy_tomogram_zarr( + if not start_value: + suffix = int(end_value) + if suffix == 0: + raise _RangeNotSatisfiable + start = max(size - suffix, 0) + return _SelectedRange(start, size - 1, SuffixByteRequest(suffix)) + + start = int(start_value) + if start >= size: + raise _RangeNotSatisfiable + + if not end_value: + return _SelectedRange(start, size - 1, OffsetByteRequest(start)) + + requested_end = int(end_value) + if requested_end < start: + raise _RangeNotSatisfiable + end = min(requested_end, size - 1) + return _SelectedRange(start, end, RangeByteRequest(start, end + 1)) + + +async def _serve_zarr_object(request: Request, store: Any, path: str) -> Response: + """Serve a Zarr object with full, HEAD, and single-range behavior.""" + content_type = _get_content_type(path) + + if request.method == "HEAD": + size = await _get_store_size(store, path) + return Response(content=b"", media_type=content_type, headers=_response_headers(size)) + + range_header = request.headers.get("range") + if range_header is None: + data = await _read_from_store(store, path) + return Response(content=data, media_type=content_type, headers=_response_headers(len(data))) + + size = await _get_store_size(store, path) + try: + selected = _parse_range(range_header, size) + except _RangeNotSatisfiable: + headers = _response_headers(0) + headers["Content-Range"] = f"bytes */{size}" + return Response(status_code=416, content=b"", headers=headers) + + data = await _read_from_store(store, path, selected.byte_request) + if len(data) != selected.length: + raise RuntimeError( + f"Store returned {len(data)} bytes for range {selected.start}-{selected.end}; " + f"expected {selected.length}" + ) + + headers = _response_headers(len(data)) + headers["Content-Range"] = f"bytes {selected.start}-{selected.end}/{size}" + return Response(content=data, status_code=206, media_type=content_type, headers=headers) + + +@router.api_route("/tomo/{run_name}/{voxel_size}/{tomo_type}/{path:path}", methods=["GET", "HEAD"]) +async def proxy_tomogram_zarr( + request: Request, run_name: str, voxel_size: float, tomo_type: str, path: str, - service: CopickService = Depends(get_copick_service), + service: Annotated[CopickService, Depends(get_copick_service)], ) -> Response: - """Proxy zarr chunks for tomograms.""" - logger.info(f"Zarr proxy request: run={run_name}, vs={voxel_size}, type={tomo_type}, path={path}") - + """Proxy Zarr objects for a tomogram.""" store = service.get_tomogram_zarr_store(run_name, voxel_size, tomo_type) if store is None: - logger.warning(f"Tomogram not found: {run_name}/{voxel_size}/{tomo_type}") raise HTTPException( status_code=404, detail=f"Tomogram '{tomo_type}' not found for run '{run_name}' at voxel size {voxel_size}", ) + return await _serve_zarr_object(request, store, path) - logger.debug(f"Got store: {store}") - data = _read_from_store(store, path) - content_type = _get_content_type(path) - - logger.debug(f"Returning {len(data)} bytes with content-type {content_type}") - return Response( - content=data, - media_type=content_type, - headers={ - "Cache-Control": "public, max-age=3600", - }, - ) - - -@router.get("/seg/{run_name}/{seg_name}/{user_id}/{session_id}/{voxel_size}/{path:path}") -def proxy_segmentation_zarr( +@router.api_route("/seg/{run_name}/{seg_name}/{user_id}/{session_id}/{voxel_size}/{path:path}", methods=["GET", "HEAD"]) +async def proxy_segmentation_zarr( + request: Request, run_name: str, seg_name: str, user_id: str, session_id: str, voxel_size: float, path: str, - service: CopickService = Depends(get_copick_service), + service: Annotated[CopickService, Depends(get_copick_service)], ) -> Response: - """Proxy zarr chunks for segmentations.""" + """Proxy Zarr objects for a segmentation.""" store = service.get_segmentation_zarr_store(run_name, seg_name, user_id, session_id, voxel_size) if store is None: raise HTTPException( status_code=404, detail=f"Segmentation '{seg_name}' not found for run '{run_name}'", ) - - data = _read_from_store(store, path) - content_type = _get_content_type(path) - - return Response( - content=data, - media_type=content_type, - headers={ - "Cache-Control": "public, max-age=3600", - }, - ) + return await _serve_zarr_object(request, store, path) diff --git a/server/tests/test_zarr_proxy.py b/server/tests/test_zarr_proxy.py new file mode 100644 index 0000000..c03613e --- /dev/null +++ b/server/tests/test_zarr_proxy.py @@ -0,0 +1,297 @@ +"""Behavioral tests for the public Zarr HTTP proxy contract.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np +import pytest +from copick.util.ome import write_ome_zarr_3d +from fastapi import FastAPI +from fastapi.testclient import TestClient +from zarr.abc.store import OffsetByteRequest, RangeByteRequest, SuffixByteRequest +from zarr.storage import LocalStore + +from copick_web.app.main import app as production_app +from copick_web.app.routes.zarr_proxy import router +from copick_web.app.services.copick_service import get_copick_service + + +@dataclass +class RecordingStore: + objects: dict[str, bytes] + reads: list[tuple[str, Any]] = field(default_factory=list) + size_reads: list[str] = field(default_factory=list) + + async def getsize(self, key: str) -> int: + self.size_reads.append(key) + try: + return len(self.objects[key]) + except KeyError as error: + raise FileNotFoundError(key) from error + + async def get(self, key: str, prototype: Any, byte_range: Any = None): + self.reads.append((key, byte_range)) + value = self.objects.get(key) + if value is None: + return None + if isinstance(byte_range, RangeByteRequest): + value = value[byte_range.start : byte_range.end] + elif isinstance(byte_range, OffsetByteRequest): + value = value[byte_range.offset :] + elif isinstance(byte_range, SuffixByteRequest): + value = value[-byte_range.suffix :] + return prototype.buffer.from_bytes(value) + + +@dataclass +class FakeService: + store: Any + + def get_tomogram_zarr_store(self, *args: Any) -> Any: + return self.store + + def get_segmentation_zarr_store(self, *args: Any) -> Any: + return self.store + + +@pytest.fixture +def app_and_store() -> tuple[FastAPI, RecordingStore]: + store = RecordingStore( + { + "zarr.json": b'{"zarr_format":3}', + ".zarray": b'{"zarr_format":2}', + "0/0/0": b"0123456789", + "empty": b"", + } + ) + app = FastAPI() + app.include_router(router) + app.dependency_overrides[get_copick_service] = lambda: FakeService(store) + return app, store + + +def _client(app: FastAPI) -> TestClient: + return TestClient(app, raise_server_exceptions=False) + + +@pytest.mark.parametrize("path", ["zarr.json", ".zarray", "0/0/0"]) +def test_full_get_preserves_object_bytes(app_and_store: tuple[FastAPI, RecordingStore], path: str) -> None: + app, store = app_and_store + + response = _client(app).get(f"/zarr/tomo/run/10.0/denoised/{path}") + + assert response.status_code == 200 + assert response.content == store.objects[path] + assert response.headers["accept-ranges"] == "bytes" + assert response.headers["content-length"] == str(len(store.objects[path])) + assert response.headers["vary"] == "Range" + assert store.reads == [(path, None)] + assert store.size_reads == [] + + +def test_head_reports_size_without_fetching_the_body(app_and_store: tuple[FastAPI, RecordingStore]) -> None: + app, store = app_and_store + + response = _client(app).head("/zarr/tomo/run/10.0/denoised/0/0/0") + + assert response.status_code == 200 + assert response.content == b"" + assert response.headers["content-length"] == "10" + assert response.headers["accept-ranges"] == "bytes" + assert store.size_reads == ["0/0/0"] + assert store.reads == [] + + +@pytest.mark.parametrize( + ("range_header", "body", "content_range", "expected_request"), + [ + ("bytes=0-0", b"0", "bytes 0-0/10", RangeByteRequest(0, 1)), + ("bytes=9-9", b"9", "bytes 9-9/10", RangeByteRequest(9, 10)), + ("bytes=2-5", b"2345", "bytes 2-5/10", RangeByteRequest(2, 6)), + ("bytes=2-", b"23456789", "bytes 2-9/10", OffsetByteRequest(2)), + ("bytes=-3", b"789", "bytes 7-9/10", SuffixByteRequest(3)), + ("bytes=0-99", b"0123456789", "bytes 0-9/10", RangeByteRequest(0, 10)), + ("bytes=-99", b"0123456789", "bytes 0-9/10", SuffixByteRequest(99)), + ], +) +def test_single_ranges_are_served_with_bounded_store_reads( + app_and_store: tuple[FastAPI, RecordingStore], + range_header: str, + body: bytes, + content_range: str, + expected_request: Any, +) -> None: + app, store = app_and_store + + response = _client(app).get( + "/zarr/tomo/run/10.0/denoised/0/0/0", + headers={"Range": range_header}, + ) + + assert response.status_code == 206 + assert response.content == body + assert response.headers["content-range"] == content_range + assert response.headers["content-length"] == str(len(body)) + assert store.size_reads == ["0/0/0"] + assert store.reads == [("0/0/0", expected_request)] + assert store.reads[0][1] is not None + + +@pytest.mark.parametrize( + "range_header", + [ + "items=0-1", + "bytes=", + "bytes=-0", + "bytes=4-2", + "bytes=10-", + "bytes=0-1,3-4", + "bytes=abc-def", + ], +) +def test_invalid_and_unsatisfiable_ranges_do_not_read_the_object( + app_and_store: tuple[FastAPI, RecordingStore], range_header: str +) -> None: + app, store = app_and_store + + response = _client(app).get( + "/zarr/tomo/run/10.0/denoised/0/0/0", + headers={"Range": range_header}, + ) + + assert response.status_code == 416 + assert response.content == b"" + assert response.headers["content-range"] == "bytes */10" + assert response.headers["content-length"] == "0" + assert store.size_reads == ["0/0/0"] + assert store.reads == [] + + +def test_zero_length_object_supports_get_and_head_but_not_range( + app_and_store: tuple[FastAPI, RecordingStore], +) -> None: + app, store = app_and_store + client = _client(app) + + full = client.get("/zarr/tomo/run/10.0/denoised/empty") + head = client.head("/zarr/tomo/run/10.0/denoised/empty") + partial = client.get("/zarr/tomo/run/10.0/denoised/empty", headers={"Range": "bytes=0-"}) + + assert (full.status_code, full.content, full.headers["content-length"]) == (200, b"", "0") + assert (head.status_code, head.content, head.headers["content-length"]) == (200, b"", "0") + assert partial.status_code == 416 + assert partial.headers["content-range"] == "bytes */0" + assert store.reads == [("empty", None)] + + +@pytest.mark.parametrize("method", ["get", "head"]) +@pytest.mark.parametrize( + "url", + [ + "/zarr/tomo/run/10.0/denoised/missing", + "/zarr/seg/run/membrane/user/session/10.0/missing", + ], +) +def test_missing_objects_are_404(app_and_store: tuple[FastAPI, RecordingStore], method: str, url: str) -> None: + app, _ = app_and_store + + response = getattr(_client(app), method)(url) + + assert response.status_code == 404 + + +@pytest.mark.parametrize( + "url", + [ + "/zarr/tomo/run/10.0/denoised/0/0/0", + "/zarr/seg/run/membrane/user/session/10.0/0/0/0", + ], +) +def test_tomogram_and_segmentation_routes_have_range_parity( + app_and_store: tuple[FastAPI, RecordingStore], url: str +) -> None: + app, _ = app_and_store + + response = _client(app).get(url, headers={"Range": "bytes=3-6"}) + + assert response.status_code == 206 + assert response.content == b"3456" + assert response.headers["content-range"] == "bytes 3-6/10" + + +def test_missing_resources_are_404() -> None: + app = FastAPI() + app.include_router(router) + app.dependency_overrides[get_copick_service] = lambda: FakeService(None) + client = _client(app) + + assert client.get("/zarr/tomo/run/10.0/denoised/zarr.json").status_code == 404 + assert client.get("/zarr/seg/run/membrane/user/session/10.0/zarr.json").status_code == 404 + + +def test_backend_failures_are_not_reclassified_as_missing() -> None: + class FailingStore(RecordingStore): + async def get(self, key: str, prototype: Any, byte_range: Any = None): + raise PermissionError("backend credentials were rejected") + + app = FastAPI() + app.include_router(router) + app.dependency_overrides[get_copick_service] = lambda: FakeService(FailingStore({"zarr.json": b"{}"})) + + response = _client(app).get("/zarr/tomo/run/10.0/denoised/zarr.json") + + assert response.status_code == 500 + + +def test_canonical_copick_shard_is_served_from_a_real_zarr_store(tmp_path: Path) -> None: + """Exercise the public proxy against output written by copick 2.0.""" + zarr_path = tmp_path / "canonical.zarr" + values = np.arange(4 * 5 * 6, dtype=np.uint16).reshape(4, 5, 6) + write_ome_zarr_3d(str(zarr_path), {10.0: values}, chunk_size=(2, 3, 4)) + store = LocalStore(zarr_path, read_only=True) + + app = FastAPI() + app.include_router(router) + app.dependency_overrides[get_copick_service] = lambda: FakeService(store) + client = _client(app) + + root_metadata = client.get("/zarr/tomo/run/10.0/denoised/zarr.json") + array_metadata = client.get("/zarr/tomo/run/10.0/denoised/0/zarr.json") + shard_head = client.head("/zarr/tomo/run/10.0/denoised/0/0/0/0") + shard_tail = client.get( + "/zarr/tomo/run/10.0/denoised/0/0/0/0", + headers={"Range": "bytes=-64"}, + ) + + assert root_metadata.status_code == 200 + assert root_metadata.json()["attributes"]["ome"]["version"] == "0.5" + assert array_metadata.status_code == 200 + assert array_metadata.json()["codecs"][0]["name"] == "sharding_indexed" + assert shard_head.status_code == 200 + assert int(shard_head.headers["content-length"]) > 64 + assert shard_tail.status_code == 206 + assert len(shard_tail.content) == 64 + assert shard_tail.content == (zarr_path / "0/0/0/0").read_bytes()[-64:] + + +def test_browser_cors_exposes_range_response_headers() -> None: + store = RecordingStore({"0/0/0": b"0123456789"}) + production_app.dependency_overrides[get_copick_service] = lambda: FakeService(store) + try: + response = _client(production_app).get( + "/zarr/tomo/run/10.0/denoised/0/0/0", + headers={ + "Origin": "http://localhost:5173", + "Range": "bytes=2-5", + }, + ) + finally: + production_app.dependency_overrides.pop(get_copick_service, None) + + assert response.status_code == 206 + assert response.headers["access-control-allow-origin"] == "http://localhost:5173" + exposed = {header.strip().lower() for header in response.headers["access-control-expose-headers"].split(",")} + assert {"accept-ranges", "content-length", "content-range"} <= exposed