From d553926f915bb8f6eab258df7fb97001c34db792 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 7 Jul 2026 13:48:09 +0200 Subject: [PATCH 1/5] recipe: safetensors 0.8.0 Rust/PyO3 (maturin) recipe, tokenizers archetype minus the C++: the 0.8.0 sdist vendors the Rust core (safetensors/) next to the pyo3 bindings (bindings/python/), builds abi3-py310, and needs zero patches. Pure-Rust dep tree (memmap2, serde_json) - no libc++_shared host dep; the metal.rs Apple-silicon path is cfg-gated to macOS and compiles out on iOS/Android. Full 6-slice matrix green; android .so links only libpython/libdl/libc, 16KB-aligned. Unlocks model2vec (numpy-only static embeddings). Tests cover the 0.8.0 TensorSpec bytes API (constructor takes python-style dtype names, headers report format codes), the mmap safe_open path, and the numpy round-trip downstream consumers use. --- recipes/safetensors/meta.yaml | 8 +++ recipes/safetensors/tests/test_safetensors.py | 70 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 recipes/safetensors/meta.yaml create mode 100644 recipes/safetensors/tests/test_safetensors.py diff --git a/recipes/safetensors/meta.yaml b/recipes/safetensors/meta.yaml new file mode 100644 index 00000000..4046c785 --- /dev/null +++ b/recipes/safetensors/meta.yaml @@ -0,0 +1,8 @@ +package: + name: safetensors + version: "0.8.0" + +build: + number: 1 + script_env: + _PYTHON_SYSCONFIGDATA_NAME: '{sysconfigdata_name}' diff --git a/recipes/safetensors/tests/test_safetensors.py b/recipes/safetensors/tests/test_safetensors.py new file mode 100644 index 00000000..4a70e075 --- /dev/null +++ b/recipes/safetensors/tests/test_safetensors.py @@ -0,0 +1,70 @@ +import ctypes +import struct + + +def _spec(TensorSpec, data, shape): + # serialize() takes raw pointers (TensorSpec); a ctypes buffer keeps the + # memory alive for the duration of the call. Constructor takes + # python-style dtype names ("float32"); headers/output use safetensors + # format codes ("F32"). + buf = ctypes.create_string_buffer(data, len(data)) + spec = TensorSpec( + dtype="float32", shape=shape, data_ptr=ctypes.addressof(buf), data_len=len(data) + ) + return spec, buf + + +def test_serialize_deserialize_roundtrip(): + """safetensors is a PyO3 wrapper around the Rust core. Round-trip a + tensor through the bytes-level API — no numpy/torch needed.""" + from safetensors import TensorSpec, deserialize, serialize + + data = struct.pack("<2f", 1.0, 2.0) + spec, buf = _spec(TensorSpec, data, [2]) + blob = serialize({"emb": spec}) + out = dict(deserialize(bytes(blob))) + + assert out["emb"]["dtype"] == "F32" + assert list(out["emb"]["shape"]) == [2] + assert bytes(out["emb"]["data"]) == data + + +def test_serialize_file_roundtrip(tmp_path): + """serialize_file writes through the Rust file path; numpy-free (the + recipe-tester app only ships the recipe's own hard deps, and safetensors + has none — safe_open needs a framework module, so it lives in the + numpy-gated test below).""" + from safetensors import TensorSpec, deserialize, serialize_file + + data = struct.pack("<4f", 1.0, 2.0, 3.0, 4.0) + spec, buf = _spec(TensorSpec, data, [2, 2]) + path = str(tmp_path / "weights.safetensors") + serialize_file({"w": spec}, path) + + with open(path, "rb") as f: + out = dict(deserialize(f.read())) + assert list(out["w"]["shape"]) == [2, 2] + assert bytes(out["w"]["data"]) == data + + +def test_numpy_safe_open_roundtrip(tmp_path): + """The numpy integration + mmap safe_open path is what downstream + consumers (model2vec) use.""" + import pytest + + np = pytest.importorskip("numpy") + from safetensors import safe_open + from safetensors.numpy import load, save_file + + arr = np.arange(6, dtype=np.float32).reshape(2, 3) + path = str(tmp_path / "weights.safetensors") + save_file({"emb": arr}, path) + + with safe_open(path, framework="numpy") as f: + assert f.keys() == ["emb"] + assert np.array_equal(f.get_tensor("emb"), arr) + + with open(path, "rb") as fh: + import safetensors.numpy as st_np + + assert np.array_equal(st_np.load(fh.read())["emb"], arr) From 4352709afd29f510d484f2e7f555eca28f3593ac Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 7 Jul 2026 14:44:14 +0200 Subject: [PATCH 2/5] recipe-tester: support test-only deps via tests/requirements.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tester app installs flet + pytest + the recipe's Requires-Dist only, so a zero-runtime-dep recipe (safetensors) can't exercise integration paths on-device — its numpy test could only ever skip. stage_recipe.sh now expands an optional recipes//tests/requirements.txt (one PEP 508 spec per line, comments/blanks skipped) into the generated pyproject via a __TEST_DEPS__ token line in the template. Template expansion is a bash line loop with printf rather than sed: specs legitimately contain double quotes in markers, which have no safe portable sed-RHS escaping. Deps are emitted as TOML literal strings, so single quotes inside a spec are rejected with an error. Token matches are exact-line, so comments mentioning a token pass through verbatim. Empty-array expansion is guarded for macOS bash 3.2 under set -u. --- tests/recipe-tester/README.md | 21 ++++++++++ tests/recipe-tester/pyproject.toml.tpl | 5 ++- tests/recipe-tester/stage_recipe.sh | 58 +++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/tests/recipe-tester/README.md b/tests/recipe-tester/README.md index aac1d84a..b5ca96f6 100644 --- a/tests/recipe-tester/README.md +++ b/tests/recipe-tester/README.md @@ -11,6 +11,27 @@ This is the *runner*; the *tests* live in each recipe's `tests/` directory pinning the recipe under test. The same script is used by CI and local devs — one staging mechanism, one source of truth. +## Test-only dependencies + +The app installs `flet + pytest + `, so the only third-party +packages on the device are the recipe's own `Requires-Dist`. If a recipe's +tests need something the recipe itself doesn't require (e.g. `numpy` for +`safetensors`, whose numpy integration is an upstream extra), declare it in +an optional `recipes//tests/requirements.txt`: + +``` +# one PEP 508 spec per line; blanks and #-comments are skipped +numpy +``` + +`stage_recipe.sh` injects those into the generated `pyproject.toml`. Each +dep must be installable for the *mobile* target — pure-Python from PyPI, or +a recipe already published on `pypi.flet.dev` (or seeded into `dist/` via +the workflow's `prebuild_recipes` input) — the same constraint a real app +faces. Keep it minimal: prefer `pytest.importorskip` for genuinely optional +integrations, and use this file only when skipping would hide the coverage +that matters on-device. + ## Local quick-start You'll need: diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index 64aa9251..796f51cb 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -7,8 +7,11 @@ requires-python = ">=3.10" dependencies = [ "flet", "pytest", - # `stage_recipe.sh` rewrites the line below to pin the recipe under test (e.g. `"numpy==2.2.2"`). + # `stage_recipe.sh` rewrites the line below to pin the recipe under test (e.g. `"numpy==2.2.2"`), + # and replaces the token line after it with any test-only deps declared in + # the recipe's tests/requirements.txt (nothing emitted when the file is absent). "__RECIPE_DEP__", + __TEST_DEPS__ ] [dependency-groups] diff --git a/tests/recipe-tester/stage_recipe.sh b/tests/recipe-tester/stage_recipe.sh index 35ab476e..65d1b23d 100755 --- a/tests/recipe-tester/stage_recipe.sh +++ b/tests/recipe-tester/stage_recipe.sh @@ -45,21 +45,69 @@ else exit 1 fi -# 2. Substitute the __RECIPE_DEP__ token in the pyproject template and write -# a fresh pyproject.toml (which is gitignored). +# 2. Generate pyproject.toml from the template (gitignored): pin the recipe +# under test (__RECIPE_DEP__) and expand test-only deps (__TEST_DEPS__) +# from the recipe's optional tests/requirements.txt. DEP="$RECIPE" [ -n "$VERSION" ] && DEP="$RECIPE==$VERSION" -# Use a temp file + mv so the substitution is sed-portability-friendly -# (BSD sed and GNU sed differ on -i quoting). +# Test-only deps: packages the tests import that are NOT in the recipe's +# Requires-Dist (e.g. numpy for a zero-runtime-dep recipe like safetensors, +# whose numpy integration is extra-gated upstream). One PEP 508 spec per +# line; blanks and full-line comments are skipped. Each dep must resolve for +# the MOBILE target — pure-Python from PyPI, or a recipe published on +# pypi.flet.dev (or seeded into dist/) — the same constraint a real app faces. +REQS_FILE="$TEST_DIR/requirements.txt" +TEST_DEPS=() +if [ -f "$REQS_FILE" ]; then + while IFS= read -r line || [ -n "$line" ]; do + # ltrim/rtrim, then skip blanks and full-line comments + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [ -z "$line" ] && continue + [ "${line:0:1}" = "#" ] && continue + # Deps are emitted as TOML literal (single-quoted) strings so PEP 508 + # markers — which legitimately contain double quotes — pass through + # verbatim; a single quote inside the spec would end the TOML string. + if [[ "$line" == *"'"* ]]; then + echo "::error::single quotes are not supported in $REQS_FILE: $line" >&2 + exit 1 + fi + TEST_DEPS+=("$line") + done < "$REQS_FILE" +fi + +# Expand the template line-by-line with printf '%s' rather than sed: the +# replacement text (PEP 508 specs) may contain characters that are unsafe in +# a sed RHS, and BSD/GNU sed disagree on escaping rules. TPL="$SCRIPT_DIR/pyproject.toml.tpl" OUT="$SCRIPT_DIR/pyproject.toml" -sed "s|__RECIPE_DEP__|$DEP|" "$TPL" > "$OUT" +: > "$OUT" +while IFS= read -r tpl_line || [ -n "$tpl_line" ]; do + # Trimmed copy, so the token matches are exact-line (a comment merely + # *mentioning* a token must pass through verbatim, not expand). + trimmed="${tpl_line#"${tpl_line%%[![:space:]]*}"}" + trimmed="${trimmed%"${trimmed##*[![:space:]]}"}" + if [ "$trimmed" = '"__RECIPE_DEP__",' ]; then + printf '%s\n' "${tpl_line/__RECIPE_DEP__/$DEP}" >> "$OUT" + elif [ "$trimmed" = "__TEST_DEPS__" ]; then + # Replaced by zero or more dep lines. The ${arr[@]+...} guard keeps + # the empty-array expansion safe under `set -u` on macOS bash 3.2. + for dep in ${TEST_DEPS[@]+"${TEST_DEPS[@]}"}; do + printf " '%s',\n" "$dep" >> "$OUT" + done + else + printf '%s\n' "$tpl_line" >> "$OUT" + fi +done < "$TPL" echo "Staged recipe '$RECIPE' (dep: $DEP)" echo " recipe_tests/:" ls -1 "$TEST_DIR" | sed 's/^/ /' echo " pyproject.toml: generated (gitignored)" +if [ ${#TEST_DEPS[@]} -gt 0 ]; then + echo " test-only deps (tests/requirements.txt): ${TEST_DEPS[*]}" +fi echo "" echo "Next:" echo " cd $(realpath --relative-to="$PWD" "$SCRIPT_DIR" 2>/dev/null || echo "$SCRIPT_DIR")" From 034075d4ae4773cebd654a37c6466384ff5ed41f Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Tue, 7 Jul 2026 14:44:28 +0200 Subject: [PATCH 3/5] safetensors: run the numpy/safe_open test on-device via tests/requirements.txt --- recipes/safetensors/tests/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 recipes/safetensors/tests/requirements.txt diff --git a/recipes/safetensors/tests/requirements.txt b/recipes/safetensors/tests/requirements.txt new file mode 100644 index 00000000..37e357e8 --- /dev/null +++ b/recipes/safetensors/tests/requirements.txt @@ -0,0 +1,3 @@ +# safe_open + the numpy glue is the model2vec path; numpy is an upstream extra, +# not a Requires-Dist, so it must come in as a test-only dep. +numpy From 773ef38e16b94c626d4f6bb77bda347d35f89468 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Fri, 17 Jul 2026 14:41:28 +0200 Subject: [PATCH 4/5] recipe-tester: unpin flet version in pyproject.toml template --- tests/recipe-tester/pyproject.toml.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/recipe-tester/pyproject.toml.tpl b/tests/recipe-tester/pyproject.toml.tpl index 7fe4aecb..50d5af26 100644 --- a/tests/recipe-tester/pyproject.toml.tpl +++ b/tests/recipe-tester/pyproject.toml.tpl @@ -5,7 +5,7 @@ description = "Generic in-app pytest runner for mobile-forge recipe wheels." requires-python = ">=3.10" dependencies = [ - "flet >=0.86.0", + "flet", "pytest", # `stage_recipe.sh` rewrites the line below to pin the recipe under test (e.g. `"numpy==2.2.2"`), # and replaces the token line after it with any test-only deps declared in @@ -16,13 +16,13 @@ dependencies = [ [dependency-groups] dev = [ - "flet[all] >=0.86.0", + "flet[all]", ] [tool.flet] artifact = "recipe-tester" -# Flet >=0.86 compiles the app to .pyc and strips the source by default, but pytest +# Flet 0.86 compiles the app to .pyc and strips the source by default, but pytest # only collects .py test files — so the bundled recipe_tests would report "0 items" # (EXIT 5). Keep the app source (main.py + recipe_tests/*.py) as .py. This only # affects the app; bundled/package dependencies still get compiled. From fca3b1611c2da71fbd42e47ac90a9289ec6cc760 Mon Sep 17 00:00:00 2001 From: ndonkoHenri Date: Fri, 17 Jul 2026 14:43:49 +0200 Subject: [PATCH 5/5] safetensors: move numpy test dependency to meta.yaml for on-device testing --- recipes/safetensors/meta.yaml | 7 +++++++ recipes/safetensors/tests/requirements.txt | 3 --- recipes/safetensors/tests/test_safetensors.py | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 recipes/safetensors/tests/requirements.txt diff --git a/recipes/safetensors/meta.yaml b/recipes/safetensors/meta.yaml index 4046c785..262f2b60 100644 --- a/recipes/safetensors/meta.yaml +++ b/recipes/safetensors/meta.yaml @@ -6,3 +6,10 @@ build: number: 1 script_env: _PYTHON_SYSCONFIGDATA_NAME: '{sysconfigdata_name}' + +test: + # numpy is an upstream extra (safetensors[numpy]), not a Requires-Dist, but + # the numpy/safe_open path is what downstream consumers (model2vec) use — so + # install it for the on-device test. + requires: + - numpy diff --git a/recipes/safetensors/tests/requirements.txt b/recipes/safetensors/tests/requirements.txt deleted file mode 100644 index 37e357e8..00000000 --- a/recipes/safetensors/tests/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# safe_open + the numpy glue is the model2vec path; numpy is an upstream extra, -# not a Requires-Dist, so it must come in as a test-only dep. -numpy diff --git a/recipes/safetensors/tests/test_safetensors.py b/recipes/safetensors/tests/test_safetensors.py index 4a70e075..3f50d2d4 100644 --- a/recipes/safetensors/tests/test_safetensors.py +++ b/recipes/safetensors/tests/test_safetensors.py @@ -50,9 +50,8 @@ def test_serialize_file_roundtrip(tmp_path): def test_numpy_safe_open_roundtrip(tmp_path): """The numpy integration + mmap safe_open path is what downstream consumers (model2vec) use.""" - import pytest + import numpy as np - np = pytest.importorskip("numpy") from safetensors import safe_open from safetensors.numpy import load, save_file