From be70bdf4bd593d2998a335e096be57d01532481c Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 07:59:04 -0500 Subject: [PATCH 01/33] Switch dependency manager from poetry to uv --- poetry.lock | 553 ------------------------------------------ pyproject.toml | 77 +++--- uv.lock | 641 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 686 insertions(+), 585 deletions(-) delete mode 100644 poetry.lock create mode 100644 uv.lock diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index 538bc8be..00000000 --- a/poetry.lock +++ /dev/null @@ -1,553 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. - -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - -[[package]] -name = "astroid" -version = "2.15.6" -description = "An abstract syntax tree for Python with inference support." -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "astroid-2.15.6-py3-none-any.whl", hash = "sha256:389656ca57b6108f939cf5d2f9a2a825a3be50ba9d589670f393236e0a03b91c"}, - {file = "astroid-2.15.6.tar.gz", hash = "sha256:903f024859b7c7687d7a7f3a3f73b17301f8e42dfd9cc9df9d4418172d3e2dbd"}, -] - -[package.dependencies] -lazy-object-proxy = ">=1.4.0" -typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} -wrapt = [ - {version = ">=1.11,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.14,<2", markers = "python_version >= \"3.11\""}, -] - -[[package]] -name = "black" -version = "22.12.0" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.7" -files = [ - {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, - {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, - {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, - {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, - {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, - {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, - {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, - {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, - {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, - {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, - {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, - {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "click" -version = "8.1.5" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.5-py3-none-any.whl", hash = "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548"}, - {file = "click-8.1.5.tar.gz", hash = "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "dill" -version = "0.3.6" -description = "serialize all of python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, - {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] - -[[package]] -name = "exceptiongroup" -version = "1.1.2" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "isort" -version = "5.12.0" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, -] - -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "lazy-object-proxy" -version = "1.9.0" -description = "A fast and thorough lazy object proxy." -optional = false -python-versions = ">=3.7" -files = [ - {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, -] - -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.5" -files = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] - -[[package]] -name = "packaging" -version = "23.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] - -[[package]] -name = "pathlib" -version = "1.0.1" -description = "Object-oriented filesystem paths" -optional = false -python-versions = "*" -files = [ - {file = "pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147"}, - {file = "pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f"}, -] - -[[package]] -name = "pathspec" -version = "0.11.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, -] - -[[package]] -name = "pg8000" -version = "1.29.8" -description = "PostgreSQL interface library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pg8000-1.29.8-py3-none-any.whl", hash = "sha256:962e9d6687f76057bd6d9c9c0f67f503a503216bf60b3a4d71e4cb8c97f8326d"}, - {file = "pg8000-1.29.8.tar.gz", hash = "sha256:609cfbccea783e15f111cc0cb2f6d4e6b4c349a695c59505a29baba6fc79ffa9"}, -] - -[package.dependencies] -python-dateutil = ">=2.8.2" -scramp = ">=1.4.3" - -[[package]] -name = "platformdirs" -version = "3.8.1" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.7" -files = [ - {file = "platformdirs-3.8.1-py3-none-any.whl", hash = "sha256:cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c"}, - {file = "platformdirs-3.8.1.tar.gz", hash = "sha256:f87ca4fcff7d2b0f81c6a748a77973d7af0f4d526f98f308477c3c436c74d528"}, -] - -[package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.2.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pylint" -version = "2.17.4" -description = "python code static checker" -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "pylint-2.17.4-py3-none-any.whl", hash = "sha256:7a1145fb08c251bdb5cca11739722ce64a63db479283d10ce718b2460e54123c"}, - {file = "pylint-2.17.4.tar.gz", hash = "sha256:5dcf1d9e19f41f38e4e85d10f511e5b9c35e1aa74251bf95cdd8cb23584e2db1"}, -] - -[package.dependencies] -astroid = ">=2.15.4,<=2.17.0-dev0" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -dill = [ - {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\""}, -] -isort = ">=4.2.5,<6" -mccabe = ">=0.6,<0.8" -platformdirs = ">=2.2.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -tomlkit = ">=0.10.1" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} - -[package.extras] -spelling = ["pyenchant (>=3.2,<4.0)"] -testutils = ["gitpython (>3)"] - -[[package]] -name = "pytest" -version = "7.4.0" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] - -[[package]] -name = "scramp" -version = "1.4.4" -description = "An implementation of the SCRAM protocol." -optional = false -python-versions = ">=3.7" -files = [ - {file = "scramp-1.4.4-py3-none-any.whl", hash = "sha256:b142312df7c2977241d951318b7ee923d6b7a4f75ba0f05b621ece1ed616faa3"}, - {file = "scramp-1.4.4.tar.gz", hash = "sha256:b7022a140040f33cf863ab2657917ed05287a807b917950489b89b9f685d59bc"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "tomlkit" -version = "0.11.8" -description = "Style preserving TOML library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomlkit-0.11.8-py3-none-any.whl", hash = "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171"}, - {file = "tomlkit-0.11.8.tar.gz", hash = "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3"}, -] - -[[package]] -name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" -optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, -] - -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, -] - -[metadata] -lock-version = "2.0" -python-versions = ">=3.9" -content-hash = "29d95a36557ed6e054de245ce01f8cc49055e3b478d030a891aa3ee57b981245" diff --git a/pyproject.toml b/pyproject.toml index 13d2e42b..f954c4a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,42 +1,55 @@ -[tool.poetry] +[project] name = "hashstore" version = "1.1.0" description = "HashStore, an object storage system using content identifiers." -authors = ["Dou Mok ", "Matt Jones ", - "Matthew Brooke", "Jing Tao", "Jeanette Clark", "Ian M. Nesbitt"] +authors = [ + { name = "Dou Mok", email = "douming.mok@gmail.com" }, + { name = "Matt Jones", email = "gitcode@magisa.org" }, + { name = "Matthew Brooke" }, + { name = "Jing Tao" }, + { name = "Jeanette Clark" }, + { name = "Ian M. Nesbitt" }, +] +requires-python = ">=3.9" readme = "README.md" -keywords = ["filesystem", "object storage", "hashstore", "storage"] +keywords = [ + "filesystem", + "object storage", + "hashstore", + "storage", +] classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: System :: Filesystems" + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Filesystems", +] +dependencies = [ + "pathlib>=1.0.1", + "pyyaml>=6.0", ] -[tool.poetry.dependencies] -python = ">=3.9" -pathlib = ">=1.0.1" -pyyaml = ">=6.0" - -[tool.poetry_bumpversion.file."src/hashstore/__init__.py"] - -[tool.poetry.group.dev.dependencies] -pytest = ">=7.2.0" -black = ">=22.10.0" -pylint = ">=2.17.4" -pg8000 = ">=1.29.8" - -[tool.poetry.scripts] +[project.scripts] hashstore = "hashstore.hashstoreclient:main" +[dependency-groups] +dev = [ + "pytest>=7.2.0", + "black>=22.10.0", + "pylint>=2.17.4", + "pg8000>=1.29.8", +] + [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.poetry_bumpversion.file."src/hashstore/__init__.py"] diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..3bf3e461 --- /dev/null +++ b/uv.lock @@ -0,0 +1,641 @@ +version = 1 +revision = 3 +requires-python = ">=3.9" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version < '3.10'", +] + +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + +[[package]] +name = "astroid" +version = "3.3.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, +] + +[[package]] +name = "astroid" +version = "4.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/22/97df040e15d964e592d3a180598ace67e91b7c559d8298bdb3c949dc6e42/astroid-4.0.2.tar.gz", hash = "sha256:ac8fb7ca1c08eb9afec91ccc23edbd8ac73bb22cbdd7da1d488d9fb8d6579070", size = 405714, upload-time = "2025-11-09T21:21:18.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ac/a85b4bfb4cf53221513e27f33cc37ad158fce02ac291d18bee6b49ab477d/astroid-4.0.2-py3-none-any.whl", hash = "sha256:d7546c00a12efc32650b19a2bb66a153883185d3179ab0d4868086f807338b9b", size = 276354, upload-time = "2025-11-09T21:21:16.54Z" }, +] + +[[package]] +name = "black" +version = "25.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytokens" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/ad/33adf4708633d047950ff2dfdea2e215d84ac50ef95aff14a614e4b6e9b2/black-25.11.0.tar.gz", hash = "sha256:9a323ac32f5dc75ce7470501b887250be5005a01602e931a15e45593f70f6e08", size = 655669, upload-time = "2025-11-10T01:53:50.558Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/d2/6caccbc96f9311e8ec3378c296d4f4809429c43a6cd2394e3c390e86816d/black-25.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec311e22458eec32a807f029b2646f661e6859c3f61bc6d9ffb67958779f392e", size = 1743501, upload-time = "2025-11-10T01:59:06.202Z" }, + { url = "https://files.pythonhosted.org/packages/69/35/b986d57828b3f3dccbf922e2864223197ba32e74c5004264b1c62bc9f04d/black-25.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1032639c90208c15711334d681de2e24821af0575573db2810b0763bcd62e0f0", size = 1597308, upload-time = "2025-11-10T01:57:58.633Z" }, + { url = "https://files.pythonhosted.org/packages/39/8e/8b58ef4b37073f52b64a7b2dd8c9a96c84f45d6f47d878d0aa557e9a2d35/black-25.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0f7c461df55cf32929b002335883946a4893d759f2df343389c4396f3b6b37", size = 1656194, upload-time = "2025-11-10T01:57:10.909Z" }, + { url = "https://files.pythonhosted.org/packages/8d/30/9c2267a7955ecc545306534ab88923769a979ac20a27cf618d370091e5dd/black-25.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:f9786c24d8e9bd5f20dc7a7f0cdd742644656987f6ea6947629306f937726c03", size = 1347996, upload-time = "2025-11-10T01:57:22.391Z" }, + { url = "https://files.pythonhosted.org/packages/c4/62/d304786b75ab0c530b833a89ce7d997924579fb7484ecd9266394903e394/black-25.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:895571922a35434a9d8ca67ef926da6bc9ad464522a5fe0db99b394ef1c0675a", size = 1727891, upload-time = "2025-11-10T02:01:40.507Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/ffe8a006aa522c9e3f430e7b93568a7b2163f4b3f16e8feb6d8c3552761a/black-25.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb4f4b65d717062191bdec8e4a442539a8ea065e6af1c4f4d36f0cdb5f71e170", size = 1581875, upload-time = "2025-11-10T01:57:51.192Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c8/7c8bda3108d0bb57387ac41b4abb5c08782b26da9f9c4421ef6694dac01a/black-25.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d81a44cbc7e4f73a9d6ae449ec2317ad81512d1e7dce7d57f6333fd6259737bc", size = 1642716, upload-time = "2025-11-10T01:56:51.589Z" }, + { url = "https://files.pythonhosted.org/packages/34/b9/f17dea34eecb7cc2609a89627d480fb6caea7b86190708eaa7eb15ed25e7/black-25.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:7eebd4744dfe92ef1ee349dc532defbf012a88b087bb7ddd688ff59a447b080e", size = 1352904, upload-time = "2025-11-10T01:59:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/7f/12/5c35e600b515f35ffd737da7febdb2ab66bb8c24d88560d5e3ef3d28c3fd/black-25.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:80e7486ad3535636657aa180ad32a7d67d7c273a80e12f1b4bfa0823d54e8fac", size = 1772831, upload-time = "2025-11-10T02:03:47Z" }, + { url = "https://files.pythonhosted.org/packages/1a/75/b3896bec5a2bb9ed2f989a970ea40e7062f8936f95425879bbe162746fe5/black-25.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cced12b747c4c76bc09b4db057c319d8545307266f41aaee665540bc0e04e96", size = 1608520, upload-time = "2025-11-10T01:58:46.895Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b5/2bfc18330eddbcfb5aab8d2d720663cd410f51b2ed01375f5be3751595b0/black-25.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb2d54a39e0ef021d6c5eef442e10fd71fcb491be6413d083a320ee768329dd", size = 1682719, upload-time = "2025-11-10T01:56:55.24Z" }, + { url = "https://files.pythonhosted.org/packages/96/fb/f7dc2793a22cdf74a72114b5ed77fe3349a2e09ef34565857a2f917abdf2/black-25.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae263af2f496940438e5be1a0c1020e13b09154f3af4df0835ea7f9fe7bfa409", size = 1362684, upload-time = "2025-11-10T01:57:07.639Z" }, + { url = "https://files.pythonhosted.org/packages/ad/47/3378d6a2ddefe18553d1115e36aea98f4a90de53b6a3017ed861ba1bd3bc/black-25.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a1d40348b6621cc20d3d7530a5b8d67e9714906dfd7346338249ad9c6cedf2b", size = 1772446, upload-time = "2025-11-10T02:02:16.181Z" }, + { url = "https://files.pythonhosted.org/packages/ba/4b/0f00bfb3d1f7e05e25bfc7c363f54dc523bb6ba502f98f4ad3acf01ab2e4/black-25.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51c65d7d60bb25429ea2bf0731c32b2a2442eb4bd3b2afcb47830f0b13e58bfd", size = 1607983, upload-time = "2025-11-10T02:02:52.502Z" }, + { url = "https://files.pythonhosted.org/packages/99/fe/49b0768f8c9ae57eb74cc10a1f87b4c70453551d8ad498959721cc345cb7/black-25.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:936c4dd07669269f40b497440159a221ee435e3fddcf668e0c05244a9be71993", size = 1682481, upload-time = "2025-11-10T01:57:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/55/17/7e10ff1267bfa950cc16f0a411d457cdff79678fbb77a6c73b73a5317904/black-25.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:f42c0ea7f59994490f4dccd64e6b2dd49ac57c7c84f38b8faab50f8759db245c", size = 1363869, upload-time = "2025-11-10T01:58:24.608Z" }, + { url = "https://files.pythonhosted.org/packages/67/c0/cc865ce594d09e4cd4dfca5e11994ebb51604328489f3ca3ae7bb38a7db5/black-25.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35690a383f22dd3e468c85dc4b915217f87667ad9cce781d7b42678ce63c4170", size = 1771358, upload-time = "2025-11-10T02:03:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" }, + { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" }, + { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" }, + { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" }, + { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" }, + { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" }, +] + +[[package]] +name = "click" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "dill" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "hashstore" +version = "1.1.0" +source = { editable = "." } +dependencies = [ + { name = "pathlib" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "pg8000" }, + { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pytest", version = "9.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] + +[package.metadata] +requires-dist = [ + { name = "pathlib", specifier = ">=1.0.1" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=22.10.0" }, + { name = "pg8000", specifier = ">=1.29.8" }, + { name = "pylint", specifier = ">=2.17.4" }, + { name = "pytest", specifier = ">=7.2.0" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "isort" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" }, +] + +[[package]] +name = "isort" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathlib" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f", size = 49298, upload-time = "2014-09-03T15:41:57.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "pg8000" +version = "1.31.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "scramp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/9a/077ab21e700051e03d8c5232b6bcb9a1a4d4b6242c9a0226df2cfa306414/pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78", size = 118933, upload-time = "2025-09-14T09:16:49.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/07/5fd183858dff4d24840f07fc845f213cd371a19958558607ba22035dadd7/pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201", size = 57816, upload-time = "2025-09-14T09:16:47.798Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pylint" +version = "3.3.9" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "astroid", version = "3.3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "dill", marker = "python_full_version < '3.10'" }, + { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "mccabe", marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "tomlkit", marker = "python_full_version < '3.10'" }, + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/9d/81c84a312d1fa8133b0db0c76148542a98349298a01747ab122f9314b04e/pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a", size = 1525946, upload-time = "2025-10-05T18:41:43.786Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/a7/69460c4a6af7575449e615144aa2205b89408dc2969b87bc3df2f262ad0b/pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7", size = 523465, upload-time = "2025-10-05T18:41:41.766Z" }, +] + +[[package]] +name = "pylint" +version = "4.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "astroid", version = "4.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "dill", marker = "python_full_version >= '3.10'" }, + { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "mccabe", marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "tomlkit", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/92/d40f5d937517cc489ad848fc4414ecccc7592e4686b9071e09e64f5e378e/pylint-4.0.4-py3-none-any.whl", hash = "sha256:63e06a37d5922555ee2c20963eb42559918c20bd2b21244e4ef426e7c43b92e0", size = 536425, upload-time = "2025-11-30T13:29:02.53Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "packaging", marker = "python_full_version < '3.10'" }, + { name = "pluggy", marker = "python_full_version < '3.10'" }, + { name = "pygments", marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "packaging", marker = "python_full_version >= '3.10'" }, + { name = "pluggy", marker = "python_full_version >= '3.10'" }, + { name = "pygments", marker = "python_full_version >= '3.10'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/56/f013048ac4bc4c1d9be45afd4ab209ea62822fb1598f40687e6bf45dcea4/pytest-9.0.1.tar.gz", hash = "sha256:3e9c069ea73583e255c3b21cf46b8d3c56f6e3a1a8f6da94ccb0fcf57b9d73c8", size = 1564125, upload-time = "2025-11-12T13:05:09.333Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/8b/6300fb80f858cda1c51ffa17075df5d846757081d11ab4aa35cef9e6258b/pytest-9.0.1-py3-none-any.whl", hash = "sha256:67be0030d194df2dfa7b556f2e56fb3c3315bd5c8822c6951162b92b32ce7dad", size = 373668, upload-time = "2025-11-12T13:05:07.379Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytokens" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/8d/a762be14dae1c3bf280202ba3172020b2b0b4c537f94427435f19c413b72/pytokens-0.3.0.tar.gz", hash = "sha256:2f932b14ed08de5fcf0b391ace2642f858f1394c0857202959000b68ed7a458a", size = 17644, upload-time = "2025-11-05T13:36:35.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/25/d9db8be44e205a124f6c98bc0324b2bb149b7431c53877fc6d1038dddaf5/pytokens-0.3.0-py3-none-any.whl", hash = "sha256:95b2b5eaf832e469d141a378872480ede3f251a5a5041b8ec6e581d3ac71bbf3", size = 12195, upload-time = "2025-11-05T13:36:33.183Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/67fc8e68a75f738c9200422bf65693fb79a4cd0dc5b23310e5202e978090/pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", size = 184450, upload-time = "2025-09-25T21:33:00.618Z" }, + { url = "https://files.pythonhosted.org/packages/ae/92/861f152ce87c452b11b9d0977952259aa7df792d71c1053365cc7b09cc08/pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", size = 174319, upload-time = "2025-09-25T21:33:02.086Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cd/f0cfc8c74f8a030017a2b9c771b7f47e5dd702c3e28e5b2071374bda2948/pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", size = 737631, upload-time = "2025-09-25T21:33:03.25Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b2/18f2bd28cd2055a79a46c9b0895c0b3d987ce40ee471cecf58a1a0199805/pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", size = 836795, upload-time = "2025-09-25T21:33:05.014Z" }, + { url = "https://files.pythonhosted.org/packages/73/b9/793686b2d54b531203c160ef12bec60228a0109c79bae6c1277961026770/pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", size = 750767, upload-time = "2025-09-25T21:33:06.398Z" }, + { url = "https://files.pythonhosted.org/packages/a9/86/a137b39a611def2ed78b0e66ce2fe13ee701a07c07aebe55c340ed2a050e/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", size = 727982, upload-time = "2025-09-25T21:33:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/dd/62/71c27c94f457cf4418ef8ccc71735324c549f7e3ea9d34aba50874563561/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", size = 755677, upload-time = "2025-09-25T21:33:09.876Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/6f5e0d58bd924fb0d06c3a6bad00effbdae2de5adb5cda5648006ffbd8d3/pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", size = 142592, upload-time = "2025-09-25T21:33:10.983Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0c/25113e0b5e103d7f1490c0e947e303fe4a696c10b501dea7a9f49d4e876c/pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", size = 158777, upload-time = "2025-09-25T21:33:15.55Z" }, +] + +[[package]] +name = "scramp" +version = "1.4.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/77/6db18bab446c12cfbee22ca8f65d5b187966bd8f900aeb65db9e60d4be3d/scramp-1.4.6.tar.gz", hash = "sha256:fe055ebbebf4397b9cb323fcc4b299f219cd1b03fd673ca40c97db04ac7d107e", size = 16306, upload-time = "2025-07-05T14:44:03.977Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/bf/54b5d40bea1c1805175ead2d496c267f05eec87561687dd73ab76869d8d9/scramp-1.4.6-py3-none-any.whl", hash = "sha256:a0cf9d2b4624b69bac5432dd69fecfc55a542384fe73c3a23ed9b138cda484e1", size = 12812, upload-time = "2025-07-05T14:44:02.345Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tomli" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/ed/3f73f72945444548f33eba9a87fc7a6e969915e7b1acc8260b30e1f76a2f/tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549", size = 17392, upload-time = "2025-10-08T22:01:47.119Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/2e/299f62b401438d5fe1624119c723f5d877acc86a4c2492da405626665f12/tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45", size = 153236, upload-time = "2025-10-08T22:01:00.137Z" }, + { url = "https://files.pythonhosted.org/packages/86/7f/d8fffe6a7aefdb61bced88fcb5e280cfd71e08939da5894161bd71bea022/tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba", size = 148084, upload-time = "2025-10-08T22:01:01.63Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/24935fb6a2ee63e86d80e4d3b58b222dafaf438c416752c8b58537c8b89a/tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf", size = 234832, upload-time = "2025-10-08T22:01:02.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/da/75dfd804fc11e6612846758a23f13271b76d577e299592b4371a4ca4cd09/tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441", size = 242052, upload-time = "2025-10-08T22:01:03.836Z" }, + { url = "https://files.pythonhosted.org/packages/70/8c/f48ac899f7b3ca7eb13af73bacbc93aec37f9c954df3c08ad96991c8c373/tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845", size = 239555, upload-time = "2025-10-08T22:01:04.834Z" }, + { url = "https://files.pythonhosted.org/packages/ba/28/72f8afd73f1d0e7829bfc093f4cb98ce0a40ffc0cc997009ee1ed94ba705/tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c", size = 245128, upload-time = "2025-10-08T22:01:05.84Z" }, + { url = "https://files.pythonhosted.org/packages/b6/eb/a7679c8ac85208706d27436e8d421dfa39d4c914dcf5fa8083a9305f58d9/tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456", size = 96445, upload-time = "2025-10-08T22:01:06.896Z" }, + { url = "https://files.pythonhosted.org/packages/0a/fe/3d3420c4cb1ad9cb462fb52967080575f15898da97e21cb6f1361d505383/tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be", size = 107165, upload-time = "2025-10-08T22:01:08.107Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b7/40f36368fcabc518bb11c8f06379a0fd631985046c038aca08c6d6a43c6e/tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac", size = 154891, upload-time = "2025-10-08T22:01:09.082Z" }, + { url = "https://files.pythonhosted.org/packages/f9/3f/d9dd692199e3b3aab2e4e4dd948abd0f790d9ded8cd10cbaae276a898434/tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22", size = 148796, upload-time = "2025-10-08T22:01:10.266Z" }, + { url = "https://files.pythonhosted.org/packages/60/83/59bff4996c2cf9f9387a0f5a3394629c7efa5ef16142076a23a90f1955fa/tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f", size = 242121, upload-time = "2025-10-08T22:01:11.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/e5/7c5119ff39de8693d6baab6c0b6dcb556d192c165596e9fc231ea1052041/tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52", size = 250070, upload-time = "2025-10-08T22:01:12.498Z" }, + { url = "https://files.pythonhosted.org/packages/45/12/ad5126d3a278f27e6701abde51d342aa78d06e27ce2bb596a01f7709a5a2/tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8", size = 245859, upload-time = "2025-10-08T22:01:13.551Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a1/4d6865da6a71c603cfe6ad0e6556c73c76548557a8d658f9e3b142df245f/tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6", size = 250296, upload-time = "2025-10-08T22:01:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b7/a7a7042715d55c9ba6e8b196d65d2cb662578b4d8cd17d882d45322b0d78/tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876", size = 97124, upload-time = "2025-10-08T22:01:15.629Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/f22f100db15a68b520664eb3328fb0ae4e90530887928558112c8d1f4515/tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878", size = 107698, upload-time = "2025-10-08T22:01:16.51Z" }, + { url = "https://files.pythonhosted.org/packages/89/48/06ee6eabe4fdd9ecd48bf488f4ac783844fd777f547b8d1b61c11939974e/tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b", size = 154819, upload-time = "2025-10-08T22:01:17.964Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/88793757d54d8937015c75dcdfb673c65471945f6be98e6a0410fba167ed/tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae", size = 148766, upload-time = "2025-10-08T22:01:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/42/17/5e2c956f0144b812e7e107f94f1cc54af734eb17b5191c0bbfb72de5e93e/tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b", size = 240771, upload-time = "2025-10-08T22:01:20.106Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f4/0fbd014909748706c01d16824eadb0307115f9562a15cbb012cd9b3512c5/tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf", size = 248586, upload-time = "2025-10-08T22:01:21.164Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/fed85e114bde5e81ecf9bc5da0cc69f2914b38f4708c80ae67d0c10180c5/tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f", size = 244792, upload-time = "2025-10-08T22:01:22.417Z" }, + { url = "https://files.pythonhosted.org/packages/55/92/afed3d497f7c186dc71e6ee6d4fcb0acfa5f7d0a1a2878f8beae379ae0cc/tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05", size = 248909, upload-time = "2025-10-08T22:01:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/f8/84/ef50c51b5a9472e7265ce1ffc7f24cd4023d289e109f669bdb1553f6a7c2/tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606", size = 96946, upload-time = "2025-10-08T22:01:24.893Z" }, + { url = "https://files.pythonhosted.org/packages/b2/b7/718cd1da0884f281f95ccfa3a6cc572d30053cba64603f79d431d3c9b61b/tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999", size = 107705, upload-time = "2025-10-08T22:01:26.153Z" }, + { url = "https://files.pythonhosted.org/packages/19/94/aeafa14a52e16163008060506fcb6aa1949d13548d13752171a755c65611/tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e", size = 154244, upload-time = "2025-10-08T22:01:27.06Z" }, + { url = "https://files.pythonhosted.org/packages/db/e4/1e58409aa78eefa47ccd19779fc6f36787edbe7d4cd330eeeedb33a4515b/tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3", size = 148637, upload-time = "2025-10-08T22:01:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/26/b6/d1eccb62f665e44359226811064596dd6a366ea1f985839c566cd61525ae/tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc", size = 241925, upload-time = "2025-10-08T22:01:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/70/91/7cdab9a03e6d3d2bb11beae108da5bdc1c34bdeb06e21163482544ddcc90/tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0", size = 249045, upload-time = "2025-10-08T22:01:31.98Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/8c26874ed1f6e4f1fcfeb868db8a794cbe9f227299402db58cfcc858766c/tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879", size = 245835, upload-time = "2025-10-08T22:01:32.989Z" }, + { url = "https://files.pythonhosted.org/packages/fd/42/8e3c6a9a4b1a1360c1a2a39f0b972cef2cc9ebd56025168c4137192a9321/tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005", size = 253109, upload-time = "2025-10-08T22:01:34.052Z" }, + { url = "https://files.pythonhosted.org/packages/22/0c/b4da635000a71b5f80130937eeac12e686eefb376b8dee113b4a582bba42/tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463", size = 97930, upload-time = "2025-10-08T22:01:35.082Z" }, + { url = "https://files.pythonhosted.org/packages/b9/74/cb1abc870a418ae99cd5c9547d6bce30701a954e0e721821df483ef7223c/tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8", size = 107964, upload-time = "2025-10-08T22:01:36.057Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/5c46fff6432a712af9f792944f4fcd7067d8823157949f4e40c56b8b3c83/tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77", size = 163065, upload-time = "2025-10-08T22:01:37.27Z" }, + { url = "https://files.pythonhosted.org/packages/39/67/f85d9bd23182f45eca8939cd2bc7050e1f90c41f4a2ecbbd5963a1d1c486/tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf", size = 159088, upload-time = "2025-10-08T22:01:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/26/5a/4b546a0405b9cc0659b399f12b6adb750757baf04250b148d3c5059fc4eb/tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530", size = 268193, upload-time = "2025-10-08T22:01:39.712Z" }, + { url = "https://files.pythonhosted.org/packages/42/4f/2c12a72ae22cf7b59a7fe75b3465b7aba40ea9145d026ba41cb382075b0e/tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b", size = 275488, upload-time = "2025-10-08T22:01:40.773Z" }, + { url = "https://files.pythonhosted.org/packages/92/04/a038d65dbe160c3aa5a624e93ad98111090f6804027d474ba9c37c8ae186/tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67", size = 272669, upload-time = "2025-10-08T22:01:41.824Z" }, + { url = "https://files.pythonhosted.org/packages/be/2f/8b7c60a9d1612a7cbc39ffcca4f21a73bf368a80fc25bccf8253e2563267/tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f", size = 279709, upload-time = "2025-10-08T22:01:43.177Z" }, + { url = "https://files.pythonhosted.org/packages/7e/46/cc36c679f09f27ded940281c38607716c86cf8ba4a518d524e349c8b4874/tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0", size = 107563, upload-time = "2025-10-08T22:01:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/84/ff/426ca8683cf7b753614480484f6437f568fd2fda2edbdf57a2d3d8b27a0b/tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba", size = 119756, upload-time = "2025-10-08T22:01:45.234Z" }, + { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] From f19b491c02d25c91d4e28b847d3d14bbfe8bb655 Mon Sep 17 00:00:00 2001 From: Dou Mok Date: Thu, 4 Dec 2025 11:21:34 -0800 Subject: [PATCH 02/33] Update 'README.md' with instructions on how to install 'uv' for dependency management --- README.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7dd0bd32..0d2a997a 100644 --- a/README.md +++ b/README.md @@ -316,13 +316,23 @@ use_multiprocessing = os.getenv("USE_MULTIPROCESSING", "False") == "True" ## Development build -HashStore is a python package, and built using the [Python Poetry](https://python-poetry.org) -build tool. - -To install `hashstore` locally, create a virtual environment for python 3.9+, -install poetry, and then install or build the package with `poetry install` or `poetry build`, -respectively. Note, installing `hashstore` with poetry will also make the `hashstore` command -available through the command line terminal (see `HashStore Client` section below for details). +HashStore is a python package. We recommend installing it using `uv`. Instructions on how to install and set up `uv` can be found [here](https://gist.github.com/datadavev/3975f244e5db500ba0328ef771ca74dd). + +Friendly Notes: + - You may run into a `command not found: compdef` when adding code to your `.zshrc` file, this can be resolved by adjusting the code to be: + ```sh + # .zshrc + autoload -Uz compinit + compinit + eval "$(uv generate-shell-completion zsh)" + eval "$(uvx --generate-shell-completion zsh)" + ``` + - When downloading the script `uv-python-symlink`, an extension may be added to it, for example: `uv-python-symlink.txt`. It may also not have an executable status. You can execute the following to adjust it: + ```sh + $ mv uv-python-symlink uv-python-symlink.sh + chmod +x uv-python-symlink.sh + ``` + - After following the steps and navigating to the python project, `uv` may not have sufficient permissions to run. Follow the given prompts and execute `direnv allow` To run tests, navigate to the root directory and run `pytest`. The test suite contains tests that take a longer time to run (relating to the storage of large files) - to execute all tests, run From b91e8bb0f6640c368130fa5990ad6ca40653a716 Mon Sep 17 00:00:00 2001 From: Dou Mok Date: Thu, 4 Dec 2025 11:32:03 -0800 Subject: [PATCH 03/33] Add 'exceptiongroup' dependency to 'pyproject.toml' to resolve python 3.10 workflow issue --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f954c4a4..02e023b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ hashstore = "hashstore.hashstoreclient:main" [dependency-groups] dev = [ "pytest>=7.2.0", + "exceptiongroup>=1.1.0", "black>=22.10.0", "pylint>=2.17.4", "pg8000>=1.29.8", From 16a2598cd5ef794b74e81b65ff968f192caf4cb6 Mon Sep 17 00:00:00 2001 From: Dou Mok Date: Thu, 4 Dec 2025 11:32:33 -0800 Subject: [PATCH 04/33] Add python 3.11 to github workflow test --- .github/workflows/poetry-package-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/poetry-package-test.yml b/.github/workflows/poetry-package-test.yml index dc16d638..9d562d87 100644 --- a/.github/workflows/poetry-package-test.yml +++ b/.github/workflows/poetry-package-test.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From 243953dd16469cab81cd6e3bba3a298acc7dbb80 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:51:50 -0500 Subject: [PATCH 05/33] Add workflow for CI with uv, version bump, limit python upper version --- .github/workflows/uv-package-test.yml | 30 +++++++++++++++++++++++++++ pyproject.toml | 4 ++-- 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/uv-package-test.yml diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml new file mode 100644 index 00000000..8e7c27ba --- /dev/null +++ b/.github/workflows/uv-package-test.yml @@ -0,0 +1,30 @@ +name: Python CI with uv and pytest +on: + workflow_dispatch: + push: + branches: [ "main"] + pull_request: + branches: [ "main" ] +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] + steps: + - uses: actions/checkout@v4 + - name: Setup uv + uses: astral-sh/uv-setup-action@v1.2.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'uv' + - name: Create uv virtual environment + run: uv venv + - name: Install dependencies in uv environment + run: uv pip install -e '.[dev]' + - name: Run tests with pytest + run: uv run pytest diff --git a/pyproject.toml b/pyproject.toml index f954c4a4..95839ac9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hashstore" -version = "1.1.0" +version = "1.1.1" description = "HashStore, an object storage system using content identifiers." authors = [ { name = "Dou Mok", email = "douming.mok@gmail.com" }, @@ -10,7 +10,7 @@ authors = [ { name = "Jeanette Clark" }, { name = "Ian M. Nesbitt" }, ] -requires-python = ">=3.9" +requires-python = ">=3.9, <4.0" readme = "README.md" keywords = [ "filesystem", From 7ca4b33f4d52c0dc06f1e3e60980d97a94247978 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:03:59 -0500 Subject: [PATCH 06/33] Update uv CI workflow to use current --- .github/workflows/uv-package-test.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml index 8e7c27ba..8333412b 100644 --- a/.github/workflows/uv-package-test.yml +++ b/.github/workflows/uv-package-test.yml @@ -6,25 +6,23 @@ on: pull_request: branches: [ "main" ] jobs: - build: - + build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + - name: Setup uv - uses: astral-sh/uv-setup-action@v1.2.0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: astral-sh/setup-uv@v7 with: + version: '0.9.15' python-version: ${{ matrix.python-version }} - cache: 'uv' - - name: Create uv virtual environment - run: uv venv - - name: Install dependencies in uv environment - run: uv pip install -e '.[dev]' + + - name: Install the project + run: uv sync --locked --all-extras --dev + - name: Run tests with pytest - run: uv run pytest + run: uv run pytest tests From 1d3fdf68ee9dbc22fd13264fa381c139306657ed Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:05:21 -0500 Subject: [PATCH 07/33] uv CI withough lock --- .github/workflows/uv-package-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml index 8333412b..e9ebefe6 100644 --- a/.github/workflows/uv-package-test.yml +++ b/.github/workflows/uv-package-test.yml @@ -22,7 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install the project - run: uv sync --locked --all-extras --dev + run: uv sync --all-extras --dev - name: Run tests with pytest run: uv run pytest tests From 886de819637e685fef071f63f51e479880a15c15 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 23 Dec 2025 15:36:36 -0500 Subject: [PATCH 08/33] Update lockfile --- uv.lock | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 3bf3e461..bcdc3492 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.9" +requires-python = ">=3.9, <4.0" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version == '3.11.*'", @@ -149,7 +149,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -158,7 +158,7 @@ wheels = [ [[package]] name = "hashstore" -version = "1.1.0" +version = "1.1.1" source = { editable = "." } dependencies = [ { name = "pathlib" }, @@ -168,6 +168,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "black" }, + { name = "exceptiongroup" }, { name = "pg8000" }, { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -184,6 +185,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "black", specifier = ">=22.10.0" }, + { name = "exceptiongroup", specifier = ">=1.1.0" }, { name = "pg8000", specifier = ">=1.29.8" }, { name = "pylint", specifier = ">=2.17.4" }, { name = "pytest", specifier = ">=7.2.0" }, From 93eaf6c69fd2387ba89257b7e66cc570add807d5 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:03:08 -0500 Subject: [PATCH 09/33] Add support for folder store / retrieve. This is a WIP. --- pyproject.toml | 2 +- src/hashstore/filehashstore.py | 168 ++++++++++++++++++++++++++++++--- src/hashstore/hashstore.py | 66 ++++++++++++- 3 files changed, 220 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bddee275..a5d15884 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hashstore" -version = "1.1.1" +version = "1.2.0" description = "HashStore, an object storage system using content identifiers." authors = [ { name = "Dou Mok", email = "douming.mok@gmail.com" }, diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 74b9c600..88e42a23 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1,38 +1,40 @@ """Core module for FileHashStore""" import atexit +import fcntl +import hashlib +import inspect import io +import logging import multiprocessing +import os import shutil import threading -import hashlib -import os -import logging -import inspect -import fcntl -import yaml -from typing import List, Dict, Union, Optional, IO, Tuple, Set, Any +from contextlib import closing from dataclasses import dataclass from pathlib import Path -from contextlib import closing from tempfile import NamedTemporaryFile +from typing import IO, Any, Dict, List, Optional, Set, Tuple, Union + +import yaml + from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, - OrphanPidRefsFileFound, CidRefsFileNotFound, HashStoreRefsAlreadyExists, + IdentifierNotLocked, NonMatchingChecksum, NonMatchingObjSize, - PidRefsAlreadyExistsError, + OrphanPidRefsFileFound, PidNotFoundInCidRefsFile, + PidRefsAlreadyExistsError, PidRefsContentError, PidRefsDoesNotExist, PidRefsFileNotFound, RefsFileExistsButCidObjMissing, - UnsupportedAlgorithm, StoreObjectForPidAlreadyInProgress, - IdentifierNotLocked, + UnsupportedAlgorithm, ) @@ -246,7 +248,13 @@ def _write_properties(self, properties: Dict[str, Union[str, int]]) -> None: checked_properties = self._validate_properties(properties) # Collect configuration properties from validated & supplied dictionary - (_, store_depth, store_width, store_algorithm, store_metadata_namespace,) = [ + ( + _, + store_depth, + store_width, + store_algorithm, + store_metadata_namespace, + ) = [ checked_properties[property_name] for property_name in self.property_required_keys ] @@ -1022,8 +1030,140 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: logging.info(info_string) return hex_digest + def store_folder( + self, + pid: str, + root_path: str | Path, + child_path: Optional[str | Path] = None, + additional_algorithm: Optional[str] = None, + checksum: Optional[str] = None, + checksum_algorithm: Optional[str] = None, + expected_object_size: Optional[int] = None, + ) -> "ObjectMetadata": + """Store a folder (and subfolders) as container objects. + + Args: + pid (str): The context within which this folder is being stored + root_path (str): Path to the root of the folder. + child_path (str): Path to folder being stored relative to the root_path. If None, assumes root_path. + + Returns: + str: CID for the container + """ + # Get the relative path for the object / folder + root_path = Path(root_path) + if child_path is None: + child_path = root_path + else: + child_path = Path(child_path) + relative_path = child_path.relative_to(root_path) + path_pid = pid + container_name = "root" + if str(relative_path) != ".": + path_pid = f"{pid} {relative_path}" + container_name = str(relative_path) + + # Check if this container already exists + try: + # resolve pid, path to CID. This raises if not found + _entry = self._find_object(path_pid) + size = os.path.getsize( + self._build_hashstore_data_object_path(_entry["cid"]) + ) + return ObjectMetadata( + pid=path_pid, cid=_entry["cid"], obj_size=size, hex_digests={} + ) + except PidNotFoundInCidRefsFile: + pass + except PidRefsDoesNotExist: + pass + + manifest = [] + for item in child_path.iterdir(): + if item.is_dir(): + meta = self.store_folder( + pid, + root_path, + child_path=item.absolute(), + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=expected_object_size, + ) + manifest.append((0, meta.cid, item.name)) + elif item.is_file(): + item_pid = f"{pid} {item.relative_to(root_path)}" + meta = self.store_object(item_pid, str(item.absolute())) + manifest.append((1, meta.cid, item.name)) + manifest.sort(key=lambda x: (x[0], x[1])) + dest_stream = io.BytesIO() + dest_stream.name = container_name + for row in manifest: + dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) + dest_stream.seek(0) + # TODO: error handling + return self.store_object( + path_pid, + data=dest_stream, + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=expected_object_size, + ) + + def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + """Retrieve a folder (and subfolders) stored as container objects. + + Args: + pid (str): The context within which this folder is being retrieved + destination_path (str|Path): Path to the root of the folder to create. + child_path (str|Path): Path to folder being retrieved relative to the destination_path. If None, assumes destination_path. + Returns: + None + """ + # TODO: Error handling + # TODO: read access control considerations + destination_path = Path(destination_path) + if child_path is None: + child_path = Path("") + else: + child_path = Path(child_path) + path_pid = pid + if str(child_path) != ".": + path_pid = f"{pid} {child_path}" + + # Retrieve the container object + obj_stream = self.retrieve_object(path_pid) + with closing(obj_stream): + for line in obj_stream: + line = line.decode("utf-8").strip() + type_flag, cid, name = line.split(" ", 2) + if type_flag == "0": + # Directory + (destination_path / child_path / name).mkdir(parents=True, exist_ok=True) + self.retrieve_folder( + pid, + destination_path, + child_path=child_path / name, + ) + elif type_flag == "1": + # File + item_pid = f"{pid} {child_path / name}" + file_stream = self.retrieve_object(item_pid) + with closing(file_stream): + dest_file_path = destination_path / child_path / name + dest_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(dest_file_path, "wb") as dest_file: + shutil.copyfileobj(file_stream, dest_file) + # FileHashStore Core Methods + def _deserialize_container(self, cid) -> Dict[str, Any]: + pass + + def _serialize_container(self, container: Dict[str, Any]) -> str: + pass + def _find_object(self, pid: str) -> Dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. The `find_object` method validates the existence of an object based on the provided @@ -2768,7 +2908,7 @@ def _check_string(string: str, arg: str) -> None: :param str string: Value to check. :param str arg: Name of the argument to check. """ - if string is None or string.strip() == "" or any(ch.isspace() for ch in string): + if string is None or string.strip() == "": method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index 20a93fd8..c92d5257 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -1,8 +1,10 @@ """Hashstore Interface""" -from abc import ABC, abstractmethod import importlib.metadata import importlib.util +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional class HashStore(ABC): @@ -64,6 +66,68 @@ def store_object( """ raise NotImplementedError() + @abstractmethod + def store_folder( + self, + pid:str, + root_path:str|Path, + child_path:Optional[str|Path]=None, + additional_algorithm:Optional[str]=None, + checksum:Optional[str]=None, + checksum_algorithm:Optional[str]=None, + expected_object_size:Optional[int]=None, + ): + """Stores a folder and subfolders. + + The `store_folder` method stores a folder and its subfolders to HashStore. Each file within the folder + is processed and stored individually, following the same procedures as the `store_object` method. The + folder structure is preserved within HashStore, allowing for easy retrieval of the entire folder or + individual files as needed. + + The root of a folder is specified by the `root_path` argument and must be identified by a PID. + + This method performs a recursive, depth firth traversal of the folder structure, storing each file it encounters and + storing folders as a container object that lists the files and subfolders contained within it. + + Args: + pid (str): Identifier for the context of this folder hierarchy. + root_path (str | Path): The physical path to the root folder being stored. + child_path (Optional[str | Path], optional): Path to a subfolder of root_path. This is + normally None for the initial invocation of this method, and recursive calls will set the + child_path as needed. Defaults to None. + additional_algorithm (Optional[str], optional): See `store_object`. Defaults to None. + checksum (Optional[str], optional): See `store_object`. Defaults to None. + checksum_algorithm (Optional[str], optional): See `store_object`. Defaults to None. + expected_object_size (Optional[int], optional): See `store_object`. Defaults to None. + + Raises: + NotImplementedError: Must be implemented in subclass. + """ + raise NotImplementedError() + + @abstractmethod + def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + """Retrieves a folder and its subfolders from HashStore. + + The `retrieve_folder` method retrieves a folder and its subfolders from HashStore, reconstructing + the original folder structure at the specified target path. Each file within the folder is retrieved + individually, following the same procedures as the `retrieve_object` method. The folder structure + is preserved during retrieval, allowing for easy access to the entire folder or individual files as needed. + + The root of a folder is specified by the `pid` argument, which identifies the context of the folder hierarchy. The + optional child_path argument can be used to specify a subfolder within the root folder for retrieval. + + Output files and folders will be created under the `destination_path`. + + Args: + pid (str): Identifier for the context of this folder hierarchy. + destination_path (str | Path): The physical path where the retrieved folder will be reconstructed. + child_path (Optional[str|Path|], optional): Path to a subfolder of the root folder. This is + normally None for the initial invocation of this method, and recursive calls will set the + child_path as needed. Defaults to None. + """ + raise NotImplementedError() + @abstractmethod def tag_object(self, pid, cid): """Creates references that allow objects stored in HashStore to be discoverable. From 154a1cb29727fdef4320d4f0ea1c8a173cd61bfc Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:29:43 -0500 Subject: [PATCH 10/33] Added notes about folders in hashstore --- folder_operations.md | 114 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 folder_operations.md diff --git a/folder_operations.md b/folder_operations.md new file mode 100644 index 00000000..91eb1fdd --- /dev/null +++ b/folder_operations.md @@ -0,0 +1,114 @@ +# hashtree + +Describes storing directory trees in hashstore. + +## Assumptions + +- The root of a folder hierarchy is identified by a PID +- A folder hierarchy (including content) identified by a PID is immutable +- A mutation to a folder hierarchy results in a new folder hierarchy identified by a new PID +- Any subfolder may optionally be identified by a PID +- Any file contained within a folder hierarchy may be identified by a PID +- Permissions are associated with a PID and so apply to content of PID identified containers or files. +- A folder hierarchy may reference all or part of another identified folder hierarchy +- A folder is represented by a `container` in hashstore. + +## Containers + +Hashstore is augmented by adding an additional type of content that represents a `container`, the contents of which represent a single folder. A `container` has two types of entries: `file` that represents a single file and `folder` which represents a single subfolder. Each entry in a `container` has properties: `type`, `cid`, and `name`, where: + +`type` - Indicates if the entry is a folder (`0`) or file (`1`). + +`cid` - The content ID for the respective file or container. + +`name` - The name component of the path to the entry. i.e. The last path segment for a subfolder or the file name (without path) for a file. + +The CID for a container is computed from the serialized content on the container which includes the CID values for any subfolders. Hence, computing the CID for folders in a hierarchy requires a depth-first approach where the CIDs for leaves of a branch are computed before the branch. + +A container is serialized space delimited rows in a text file. Each row represents an entry in the container, with values `type`, `cid`, and `name` in that order. Since folder or file names *may* contain whitespace, the `name` entry consumes the remainder of the row. + +Since the CID for a container is dependent on its content, the content order is sorted by the `type` and `cid` values so hashing is consistent. Hence rows referencing subfolder containers will always appear before rows referencing files. + +For example, given the folder hierarchy: + +``` +PID_1 +├── A +│ ├── a1.txt +│ └── a2.txt +└── B + └── b1.csv +``` + +The following `container` entries are created (`cid` values are truncated): + +Container `ad5eb`: +``` +1 10fbd a1.txt +1 c880c a2.txt +``` + +Container `cc08d`: +``` +1 00e99 b1.csv +``` + +Container `dbc15`: +``` +0 ad5eb A +0 cc08d B +``` + +The hashstore entry for `PID_1` might be: +``` +$ cat refs/pids/53/b2/f2/58a2f3061a7bee4ba8b157aab217795c4692e2a2d8856e2fd97eb7fa3f +dbc1516e49e7437ea441f279570d32b1e2f149c44ab0a77682629215f4a5970b + +$ cat refs/cids/db/c1/51/6e49e7437ea441f279570d32b1e2f149c44ab0a77682629215f4a5970b +PID_1 +``` + +Each container is resolveable by the combination of PID and path. So for example, +the folder `B` within the context of `PID_1` can be resolved using the identifier `PID_1 B`. +Similarly, the file `A/a2.txt` can be resolved with the identifier `PID_1 A/a2.txt`. +Corresponding entries in hashstore `refs/pids` and `refs/cids` are created. + +## Operations + +### Get an object by path + +Given a PID and a path, retrieve the corresponding object (file or folder) from hashstore. + +Persistent identifiers for objects within a folder hierarchy are constructed by concatenating the PID with the path using a space as a delimiter. For example, to retrieve the object at path `data/file1.txt` within the folder hierarchy identified by PID `abc123`, the identifier would be `abc123 data/file1.txt`. + +``` +hashstore = HashStore(...) +path_pid = "" + " " + "" +object_stream = hashstore.get_object(path_pid) +``` + +### Store a new folder hierarchy + +To store a new folder hierarchy, recursively create `container` entries for each folder in the hierarchy, starting from the leaves and working up to the root. For each folder, create a `container` with entries for its subfolders and files, compute the CID for the container, and store it in hashstore. Finally, associate the root container's CID with the PID representing the entire folder hierarchy. + +This is achieved by the `hash_store.store_folder()` method. + +``` +hashstore = HashStore(...) +pid = "" +source_path = "" +hashstore.store_folder(pid, source_path) +``` + +### Retrieve folder hierarchy structure + +To retrieve the structure of a folder hierarchy identified by a PID, recursively resolve each `container` starting from the root PID. For each folder, read its `container` entries to identify subfolders and files, and continue resolving subfolders until the entire hierarchy is reconstructed. + +This is achieved by the `hash_store.retrieve_folder()` method. + +``` +hashstore = HashStore(...) +pid = "" +destination_path = "" +hashstore.retrieve_folder(pid, destination_path) +``` From e3056b7a8d4990c146b0184f2cacd08da6a85dd3 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:36:59 -0500 Subject: [PATCH 11/33] Adjust typehints for 3.9 --- folder_operations.md | 6 +++--- src/hashstore/filehashstore.py | 6 +++--- src/hashstore/hashstore.py | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/folder_operations.md b/folder_operations.md index 91eb1fdd..9ad9420c 100644 --- a/folder_operations.md +++ b/folder_operations.md @@ -84,14 +84,14 @@ Persistent identifiers for objects within a folder hierarchy are constructed by ``` hashstore = HashStore(...) path_pid = "" + " " + "" -object_stream = hashstore.get_object(path_pid) +object_stream = hashstore.retrieve_object(path_pid) ``` ### Store a new folder hierarchy To store a new folder hierarchy, recursively create `container` entries for each folder in the hierarchy, starting from the leaves and working up to the root. For each folder, create a `container` with entries for its subfolders and files, compute the CID for the container, and store it in hashstore. Finally, associate the root container's CID with the PID representing the entire folder hierarchy. -This is achieved by the `hash_store.store_folder()` method. +This is achieved by the `hashstore.store_folder()` method. ``` hashstore = HashStore(...) @@ -104,7 +104,7 @@ hashstore.store_folder(pid, source_path) To retrieve the structure of a folder hierarchy identified by a PID, recursively resolve each `container` starting from the root PID. For each folder, read its `container` entries to identify subfolders and files, and continue resolving subfolders until the entire hierarchy is reconstructed. -This is achieved by the `hash_store.retrieve_folder()` method. +This is achieved by the `hashstore.retrieve_folder()` method. ``` hashstore = HashStore(...) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 88e42a23..31af63a6 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1033,8 +1033,8 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: def store_folder( self, pid: str, - root_path: str | Path, - child_path: Optional[str | Path] = None, + root_path: Union[str, Path], + child_path: Optional[Union[str, Path]] = None, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, @@ -1111,7 +1111,7 @@ def store_folder( expected_object_size=expected_object_size, ) - def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path:Optional[Union[str, Path]]=None): """Retrieve a folder (and subfolders) stored as container objects. Args: diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index c92d5257..05b85f7b 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional +from typing import Optional, Union class HashStore(ABC): @@ -70,8 +70,8 @@ def store_object( def store_folder( self, pid:str, - root_path:str|Path, - child_path:Optional[str|Path]=None, + root_path:Union[str,Path], + child_path:Optional[Union[str,Path]]=None, additional_algorithm:Optional[str]=None, checksum:Optional[str]=None, checksum_algorithm:Optional[str]=None, @@ -106,7 +106,7 @@ def store_folder( raise NotImplementedError() @abstractmethod - def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + def retrieve_folder(self, pid:str, destination_path:Union[str,Path], child_path:Optional[Union[str,Path]]=None): """Retrieves a folder and its subfolders from HashStore. The `retrieve_folder` method retrieves a folder and its subfolders from HashStore, reconstructing From c5a5e7281bcd2e0bf1e863a2fed295355a583581 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:48:25 -0500 Subject: [PATCH 12/33] Adjust check_string to check for leading or trailing whitespace --- src/hashstore/filehashstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 31af63a6..470de60e 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -2908,7 +2908,7 @@ def _check_string(string: str, arg: str) -> None: :param str string: Value to check. :param str arg: Name of the argument to check. """ - if string is None or string.strip() == "": + if string is None or string.strip() == "" or string.strip() != string: method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" From b8ff3bf679c8e18729613629ff013830d6db46cc Mon Sep 17 00:00:00 2001 From: Dave Vieglais <605409+datadavev@users.noreply.github.com> Date: Tue, 20 Jan 2026 12:05:52 -0500 Subject: [PATCH 13/33] Rename header and update folder hierarchy example --- folder_operations.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/folder_operations.md b/folder_operations.md index 9ad9420c..3e286216 100644 --- a/folder_operations.md +++ b/folder_operations.md @@ -1,4 +1,4 @@ -# hashtree +# Folders in HashStore Describes storing directory trees in hashstore. @@ -32,11 +32,11 @@ Since the CID for a container is dependent on its content, the content order is For example, given the folder hierarchy: ``` -PID_1 -├── A +PID_1 <- dbc15 +├── A <- ad5eb │ ├── a1.txt │ └── a2.txt -└── B +└── B <- cc08d └── b1.csv ``` From 1b72b4dd5f074e66bcbdbb6ae4120d78ae7fef63 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 20 Feb 2026 10:44:52 -0500 Subject: [PATCH 14/33] KeyError instead of ValueError, add list_pids() --- src/hashstore/filehashstore.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 470de60e..e0fc6129 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -8,13 +8,14 @@ import logging import multiprocessing import os +import re import shutil import threading from contextlib import closing from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, Any, Dict, List, Optional, Set, Tuple, Union +from typing import IO, Any, Dict, Generator, List, Optional, Set, Tuple, Union import yaml @@ -755,8 +756,8 @@ def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[byt return metadata_stream else: err_msg = f"No metadata found for pid: {pid}" - self.fhs_logger.error(err_msg) - raise ValueError(err_msg) + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) def delete_object(self, pid: str) -> None: self.fhs_logger.debug("Request to delete object for id: %s", pid) @@ -1156,6 +1157,24 @@ def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path with open(dest_file_path, "wb") as dest_file: shutil.copyfileobj(file_stream, dest_file) + def list_pids(self, pattern: Optional[str]=None) -> Generator: + rpattern = None + if pattern is not None: + rpattern = re.compile(pattern) + ignore_names = [".DS_Store", ] + for cid_entry in self.cids.rglob('*'): + if cid_entry.is_file() and cid_entry.name not in ignore_names: + self.fhs_logger.debug(str(cid_entry)) + for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): + pid = entry.strip() + if len(pid) > 0: + if rpattern is not None: + if rpattern.fullmatch(pid): + yield pid + else: + yield pid + + # FileHashStore Core Methods def _deserialize_container(self, cid) -> Dict[str, Any]: @@ -2908,7 +2927,7 @@ def _check_string(string: str, arg: str) -> None: :param str string: Value to check. :param str arg: Name of the argument to check. """ - if string is None or string.strip() == "" or string.strip() != string: + if not string or string.strip() != string: method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" From 040868e30f8b7c1c1b83b34e25df90b32d86ee7e Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:32:54 -0400 Subject: [PATCH 15/33] some refactoring for folder support --- .vscode/settings.json | 3 +- folder_operations.md | 52 ++- hashstore_layout.drawio | 145 +++++++ pyproject.toml | 6 + src/hashstore/__init__.py | 2 +- src/hashstore/__main__.py | 312 ++++++++++++++ .../{hashstore.py => basehashstore.py} | 19 +- src/hashstore/filehashstore.py | 143 ++++++- src/hashstore/localstore.py | 403 ++++++++++++++++++ tests/test_hashstore.py | 2 +- 10 files changed, 1059 insertions(+), 28 deletions(-) create mode 100644 hashstore_layout.drawio create mode 100644 src/hashstore/__main__.py rename src/hashstore/{hashstore.py => basehashstore.py} (95%) create mode 100644 src/hashstore/localstore.py diff --git a/.vscode/settings.json b/.vscode/settings.json index b15ffaa4..cdd22c0f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,6 @@ "editor.formatOnSave": true, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" - } + }, + "python-envs.pythonProjects": [] } \ No newline at end of file diff --git a/folder_operations.md b/folder_operations.md index 9ad9420c..23412dae 100644 --- a/folder_operations.md +++ b/folder_operations.md @@ -1,6 +1,6 @@ # hashtree -Describes storing directory trees in hashstore. +Describes storing directory trees in hashstore (hs). ## Assumptions @@ -13,6 +13,16 @@ Describes storing directory trees in hashstore. - A folder hierarchy may reference all or part of another identified folder hierarchy - A folder is represented by a `container` in hashstore. +## Virtual hashstore + +When a folder is added to `hs`, it is necessary to calculate file and folder hashes and compare these with any existing content in the target `hs`. The efficiency of updating an existing folder entry in `hs` can be significantly improved by computing the hashes locally and determining what may need to be sent to the target `hs`. This is especially important for large folder structures that may have isolated changes. + +A virtual `hs` (`vhs`) is a local folder structure that is similar to a `hs` except that the content bytes are not stored (except for containers), only hashes of the content. Time stamps of the hash entries are compared with content time stamps to identify candidates for hash recalculation. If hash values have changed, then the files are tagged for upload to the target hs. + +A `vhs` is composed of CID and PID ref files, and container files for folder hashes. Even though content ids are calculated, the content files are not stored. + + + ## Containers Hashstore is augmented by adding an additional type of content that represents a `container`, the contents of which represent a single folder. A `container` has two types of entries: `file` that represents a single file and `folder` which represents a single subfolder. Each entry in a `container` has properties: `type`, `cid`, and `name`, where: @@ -112,3 +122,43 @@ pid = "" destination_path = "" hashstore.retrieve_folder(pid, destination_path) ``` + + +--- + +## `add` + +`add(PID:str, path:pathlib.Path)->None` + +Add an object or folder to `vhs`. + + +## `init` + +`init(path:pathlib.Path)->None` + +Initializes a `vhs` folder within the current folder. + + +## `status` + +`status()->VhsStatus` + +Reports the status of the entries in the `vhs` versus the current contents of +registered content. + + +## `update` + +`update(PID:str|None)` + +Recalculates CID values based on the current content of registered entries. + + +## `commit` + +`commit()` + +Makes entries in the `vhs` immutable preventing any further updates to existing +PIDs. Any further changes require new PID. + diff --git a/hashstore_layout.drawio b/hashstore_layout.drawio new file mode 100644 index 00000000..4b6c4c73 --- /dev/null +++ b/hashstore_layout.drawio @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml index a5d15884..8153e587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,14 +33,20 @@ classifiers = [ "Topic :: System :: Filesystems", ] dependencies = [ + "click>=8.3.1", "pathlib>=1.0.1", "pyyaml>=6.0", + "xattr-compat>=1.0.0", ] [project.scripts] hashstore = "hashstore.hashstoreclient:main" +hs = "hashstore.__main__:main" [dependency-groups] +cli = [ + "rich>=14.3.3", +] dev = [ "pytest>=7.2.0", "exceptiongroup>=1.1.0", diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index be656f5e..6d470289 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.hashstore import HashStore, HashStoreFactory +from hashstore.basehashstore import HashStore, HashStoreFactory __all__ = ("HashStore", "HashStoreFactory") __version__ = "1.1.0" diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py new file mode 100644 index 00000000..d4b09654 --- /dev/null +++ b/src/hashstore/__main__.py @@ -0,0 +1,312 @@ +import dataclasses +import datetime +import json +import logging +import os +import pathlib +import sys +import click +import rich +import rich.tree +import yaml +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +import hashstore +import hashstore.filehashstore_exceptions + +HASHSTORE_FOLDER_NAME = ".hashstore" +DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" + +def get_logger(): + return logging.getLogger("hs") + +def sizeof_fmt(num, suffix="B"): + for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + +def enumerate_dict(d): + for key, value in d.items(): + if isinstance(value, dict): + for subkey, subvalue in enumerate_dict(value): + yield f"{key}.{subkey}", subvalue + else: + yield key, value + + +def load_hashstore_properties(path:pathlib.Path)-> dict: + properties_file = path / "hashstore.yaml" + if not properties_file.exists(): + raise FileNotFoundError(f"Hashstore properties file not found: {properties_file}") + with open(properties_file, 'r') as f: + properties = yaml.load(f, Loader=Loader) + properties["store_path"] = str(path) + return properties + + +def locate_hashstore(path:pathlib.Path)->pathlib.Path: + # Iterate through the current directory and all its parents + for directory in [path] + list(path.parents): + # Use glob() to find files matching the pattern within the current directory + for file_path in directory.glob(HASHSTORE_FOLDER_NAME): + if file_path.is_dir(): + return file_path + return None + + +@click.group() +@click.option( + "--store", + type=click.Path(path_type=pathlib.Path, file_okay=False), + envvar="HASHSTORE_PATH", + default=None +) +@click.option("--config", default='~/.config/hashstore/defaults.yml', type=click.Path(), help="Path to configuration file") +@click.option("--log-level", default=None, help="Set the logging level (INFO)") +@click.pass_context +def main(ctx, store, config, log_level): + """Implements a high level file and folder operations CLI for hashstore. + """ + ctx.ensure_object(dict) + # Load defaults from config file + config = os.path.expanduser(config) + if os.path.exists(config): + with open(config, 'r') as f: + cfg = yaml.load(f, Loader=Loader) + ctx.default_map = cfg + if store is None: + store = locate_hashstore(pathlib.Path.cwd()) + if store is not None: + ctx.obj["hashstore_path"] = store + else: + store.expanduser() + ctx.obj["hashstore_path"] = store + ctx.obj["module_name"] = ctx.default_map.get("module_name", "hashstore.filehashstore") + ctx.obj["class_name"] = ctx.default_map.get("class_name", "FileHashStore") + # Set up logging + if log_level is None: + log_level = ctx.default_map.get('log_level', 'INFO') if 'cfg' in locals() else 'INFO' + logger = get_logger() + numeric_level = getattr(logging, log_level.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError(f"Invalid log level: {log_level}") + logging.basicConfig(level=numeric_level) + logger.setLevel(numeric_level) + logger.debug(f"Logging initialized at level: {log_level}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Configuration:") + for key, value in enumerate_dict(ctx.default_map): + logger.debug(f" {key}: {value}") + return 0 + + +@main.command() +@click.pass_context +def version(ctx): + """Show the version of Hashstore.""" + print(f"Hashstore version {hashstore.__version__}") + return 0 + + +@main.command() +@click.pass_context +@click.option("--depth", "-d", type=int, default=3, help="Hashstore hierarchy depth") +@click.option("--width", "-w", type=int, default=2, help="Hashstore hierarchy width") +@click.option("--algorithm", "-a", type=str, default="SHA-256", help="Hashstore algorithm") +@click.option("--metadata_namespace", type=str, default=None, help="Hashstore metadata namespace") +def create(ctx, depth, width, algorithm, metadata_namespace): + """Create a new hashstore at the specified root path.""" + logger = get_logger() + store = ctx.obj["hashstore_path"] + if not store.parent.exists(): + logger.error(f"Parent directory does not exist: {store.parent}") + return 1 + if store.exists(): + if (store / "hashstore.yaml").exists(): + logger.error(f"Hashstore already exists at: {store}") + return 1 + properties = { + "store_path": store, + "store_depth": depth, + "store_width": width, + "store_algorithm": algorithm, + "store_metadata_namespace": metadata_namespace, + } + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.info(f"Hashstore created at: {store}") + except Exception as e: + logger.error(f"Failed to create hashstore: {e}") + return 1 + return 0 + + +@main.command(name="add_folder") +@click.pass_context +@click.argument( + "root_path", + type=click.Path(path_type=pathlib.Path, file_okay=False, exists=True) +) +@click.argument("pid", type=str) +@click.option( + "-s", + "--sysmeta_path", + type=click.Path(path_type=pathlib.Path), + default=None, + help="Path to system metadata XML file" +) +@click.option( + "-p", + "--pattern", + type=str, + default=None, + help="Glob pattern for files to include (defaults to all)" +) +def add_object( + ctx, + root_path:pathlib.Path, + pid: str|None, + sysmeta_path:pathlib.Path|None, + pattern:str|None, +): + """Add a folder to the hashstore. + + PID is required and is used to reference the folder root. + + If object_path is a folder, a PID is generated for the folder contents added + recursively using the object relative paths as suffix to the PID. + + if the file doesn't exist: + add bytes + store pid + store sysmeta if provided + if the file exists: + if pid doesn't exist: + store pid by adding to list + if sysmeta provided: + store sysmeta + """ + logger = get_logger() + store = ctx.obj["hashstore_path"] + logger.info(f"Adding folder at path: {store} with PID: {pid} and sysmeta: {sysmeta_path}") + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + # Does the object already exist in the store? + try: + info = hash_store._find_object(pid=pid) + hash_store.tag_object(pid, info.get("cid")) + except hashstore.filehashstore_exceptions.PidRefsDoesNotExist: + info = None + # Object doesn't exist in store + root_path = root_path.absolute() + if info is None: + try: + info = hash_store.commit_folder(pid=pid, root_path=root_path, pattern=pattern) + print(json.dumps(dataclasses.asdict(info))) + except Exception as e: + logger.error(f"Failed to add object: {e}") + return 1 + if sysmeta_path is not None and sysmeta_path.is_file(): + hash_store.store_metadata(pid=pid, metadata=str(sysmeta_path), format_id=properties.get("store_metadata_namespace")) + return 1 + return 0 + + +@main.command("get") +@click.pass_context +@click.argument("pid", type=str) +@click.option("-s", "--stream", is_flag=True, help="Stream to stdout") +@click.option("-r", "--recursive", is_flag=True, help="Recurse into contents if PID is a folder.") +def get_object(ctx, pid, stream, recursive): + """Retrieve an object or folder from hashstore.""" + store = ctx.obj["hashstore_path"] + pass + + +@main.command("ls") +@click.pass_context +@click.option("-p", "--pattern", default=None, help="Optional regex pattern for PID matching.") +@click.option("-h", "--human-readable", is_flag=True, help="Display sizes in human readable format.") +@click.option("-r", "--reference", is_flag=True, help="Include path references.") +def list_pids(ctx, pattern, human_readable, reference): + """List PIDs in the hashstore.""" + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + total_objects = 0 + # iterate over the refs/cids folder, getting PIDs from each file. + print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") + for pid in hash_store.list_pids(pattern=pattern): + try: + pid_stat = hash_store.get_object_status(pid) + fsize = pid_stat.get("size", 0) + total_objects += 1 + t_modified = pid_stat.get("modtime", "-") + if t_modified != "-": + t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat(timespec="seconds") + t_access = pid_stat.get("accesstime", "-") + if t_access != "-": + t_access = datetime.datetime.fromtimestamp(t_access).isoformat(timespec="seconds") + if human_readable: + fsize = sizeof_fmt(fsize) + if fsize>0 or reference: + print(f"{fsize}\t{t_modified}\t{t_access}\t{pid}") + try: + meta = hash_store.retrieve_metadata(pid) + print(json.dumps(meta, indent=2)) + except KeyError: + pass + except KeyError: + logger.warning(f"PID status not available in hashstore: {pid}") + print(f"Total {total_objects}") + + +@main.command("tree") +@click.pass_context +@click.argument("pid", type=str) +def get_container_tree(ctx, pid)->None: + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + tree = { + pid:rich.tree.Tree(pid), + } + for entry in hash_store.folder_content(pid, depth_first=True): + print(f"{entry[0]} {entry[1]} {entry[3]} {entry[2]}") + parts = entry[3].rsplit("/", 1) + if entry[1] == 0: + tree[entry[3]] = tree[parts[0]].add(entry[3]) + else: + # file + tree[parts[0]].add(entry[3]) + rich.print(tree[pid]) + + +if __name__ == "__main__": + sys.exit(main(auto_envvar_prefix="HASHSTORE")) diff --git a/src/hashstore/hashstore.py b/src/hashstore/basehashstore.py similarity index 95% rename from src/hashstore/hashstore.py rename to src/hashstore/basehashstore.py index 05b85f7b..2e50d8cb 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Union +from typing import Optional, Union, Generator class HashStore(ABC): @@ -234,6 +234,23 @@ def get_hex_digest(self, pid, algorithm): """ raise NotImplementedError() + @abstractmethod + def list_pids(self, pattern:Optional[str]=None) -> Generator: + """Yields PIDs from the hashstore. + + :param str pattern: Optional regexp pattern to match. + """ + raise NotImplementedError() + + @abstractmethod + def get_object_status(self, pid) -> dict: + """Returns a dictionary of the object size, modtime, accesstime for the given pid. + + :param str pid: Object identifier + + :return: dict - Dictionary containing information about the object. + """ + raise NotImplementedError() class HashStoreFactory: """A factory class for creating `HashStore`-like objects. diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index e0fc6129..51e212ae 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -606,6 +606,30 @@ def tag_object(self, pid: str, cid: str) -> None: self.fhs_logger.error(err_msg) raise PidRefsAlreadyExistsError(err_msg) + def get_object_status(self, pid: str) -> dict: + logging.debug("Request to get object status for pid: %s", pid) + self._check_string(pid, "pid") + + object_status_dict = {} + try: + object_info_dict = self._find_object(pid) + object_cid = object_info_dict.get("cid") + if object_cid: + obj_path = self._get_hashstore_data_object_path(object_cid) + object_status_dict["size"] = os.path.getsize(obj_path) + object_status_dict["modtime"] = os.path.getmtime(obj_path) + object_status_dict["accesstime"] = os.path.getatime(obj_path) + else: + err_msg = f"No object found for pid: {pid}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + except KeyError as ke: + err_msg = f"No object found for pid: {pid}. Details: {ke}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + + return object_status_dict + def delete_if_invalid_object( self, object_metadata: "ObjectMetadata", @@ -1031,6 +1055,50 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: logging.info(info_string) return hex_digest + def _store_container( + self, + pid:str, + name:str, + manifest:list[tuple[int, str, str]], + additional_algorithm: Optional[str] = None, + checksum: Optional[str] = None, + checksum_algorithm: Optional[str] = None, + ) -> "ObjectMetadata": + self.fhs_logger.debug(f"_store_container {pid=}") + manifest.sort(key=lambda x: (x[0], x[1])) + dest_stream = io.BytesIO() + dest_stream.name = name + dest_stream.write(f"container {len(manifest)}\n".encode("utf-8")) + for row in manifest: + dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) + dest_stream.seek(0) + return self.store_object( + pid, + data=dest_stream, + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=None, + ) + + def _load_container(self, pid:str) -> list[tuple[int, str, str]]: + self.fhs_logger.debug(f"_load_container {pid=}") + obj_stream = self.retrieve_object(pid) + row = 0 + manifest = [] + with closing(obj_stream): + for line in obj_stream: + line = line.decode("utf-8").strip() + if row == 0: + if not line.startswith("container "): + msg = f"{pid} is not a container." + raise ValueError(msg) + else: + type_flag, cid, name = line.split(" ", 2) + manifest.append((int(type_flag), cid, name)) + row += 1 + return manifest + def store_folder( self, pid: str, @@ -1040,9 +1108,13 @@ def store_folder( checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, expected_object_size: Optional[int] = None, - ) -> "ObjectMetadata": + pattern: Optional[str] = None, + ) -> Optional["ObjectMetadata"]: """Store a folder (and subfolders) as container objects. + Traverses the folder hierarchy in a depth first manner so that + the leaf elements are computed for inclusion in parent hashes. + Args: pid (str): The context within which this folder is being stored root_path (str): Path to the root of the folder. @@ -1051,6 +1123,7 @@ def store_folder( Returns: str: CID for the container """ + raise NotImplementedError # Get the relative path for the object / folder root_path = Path(root_path) if child_path is None: @@ -1061,9 +1134,10 @@ def store_folder( path_pid = pid container_name = "root" if str(relative_path) != ".": - path_pid = f"{pid} {relative_path}" + path_pid = f"{pid}/{relative_path}" container_name = str(relative_path) + self.fhs_logger.debug("store_folder {path_pid=}") # Check if this container already exists try: # resolve pid, path to CID. This raises if not found @@ -1079,6 +1153,7 @@ def store_folder( except PidRefsDoesNotExist: pass + # Container doesn't exist manifest = [] for item in child_path.iterdir(): if item.is_dir(): @@ -1090,27 +1165,55 @@ def store_folder( checksum=checksum, checksum_algorithm=checksum_algorithm, expected_object_size=expected_object_size, + pattern=pattern, ) - manifest.append((0, meta.cid, item.name)) - elif item.is_file(): - item_pid = f"{pid} {item.relative_to(root_path)}" + if meta is not None: + manifest.append((0, meta.cid, item.name)) + elif pattern is None and item.is_file(): + # If no pattern then grab all files, otherwise defer to + # globbing match later. + item_pid = f"{pid}/{item.relative_to(root_path)}" + self.fhs_logger.debug("store_folder {item_pid=}") meta = self.store_object(item_pid, str(item.absolute())) manifest.append((1, meta.cid, item.name)) - manifest.sort(key=lambda x: (x[0], x[1])) - dest_stream = io.BytesIO() - dest_stream.name = container_name - for row in manifest: - dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) - dest_stream.seek(0) - # TODO: error handling - return self.store_object( + if pattern is not None: + # globbing pattern was specified. Grab the matching files here. + for item in child_path.glob(pattern): + if item.is_file(): + item_pid = f"{pid}/{item.relative_to(root_path)}" + meta = self.store_object(item_pid, str(item.absolute())) + manifest.append((1, meta.cid, item.name)) + if len(manifest) == 0: + return None + return self._store_container( path_pid, - data=dest_stream, + container_name, + manifest, additional_algorithm=additional_algorithm, checksum=checksum, - checksum_algorithm=checksum_algorithm, - expected_object_size=expected_object_size, - ) + checksum_algorithm=checksum_algorithm + ) + + def folder_content(self, pid:str, depth:int=0, depth_first:bool=True) -> Generator: + """Yield the content of a folder, breadth first, recursively. + (depth, type, CID, name) + """ + # Retrieve the container object + manifest = self._load_container(pid) + if depth_first: + for entry in manifest: + if entry[0] == 0: + yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") + yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) + else: + yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") + else: + for entry in manifest: + yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") + for entry in manifest: + if entry[0] == 0: + # folder, recurse + yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path:Optional[Union[str, Path]]=None): """Retrieve a folder (and subfolders) stored as container objects. @@ -1177,12 +1280,6 @@ def list_pids(self, pattern: Optional[str]=None) -> Generator: # FileHashStore Core Methods - def _deserialize_container(self, cid) -> Dict[str, Any]: - pass - - def _serialize_container(self, container: Dict[str, Any]) -> str: - pass - def _find_object(self, pid: str) -> Dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. The `find_object` method validates the existence of an object based on the provided diff --git a/src/hashstore/localstore.py b/src/hashstore/localstore.py new file mode 100644 index 00000000..3b8d3ec8 --- /dev/null +++ b/src/hashstore/localstore.py @@ -0,0 +1,403 @@ +"""Implements a virtual hashstore. + +A virtual hashstore is used for staging content that is to be added to a +hashstore. + +Staging associates PIDs with objects and folders, and computes their hashes. +When changes are made, a difference can be computed and used to efficiently +transmit changes to a hashtore. +""" +import collections.abc +import contextlib +import dataclasses +import hashlib +import io +import json +import os +import pathlib +import typing + +import xattr_compat as xattrs + +import hashstore.filehashstore +import hashstore.filehashstore_exceptions + +ORG_DATAONE_CID = "org.dataone.hashstore.cid" +ORG_DATAONE_CID_ALGORITHM = "sha265" +FOLDER_ENTRY_FOLDER = 0 +FOLDER_ENTRY_FILE = 1 + + +def depth_first_walk(obj_path:pathlib.Path) -> collections.abc.Generator[pathlib.Path]: + if obj_path.is_file(): + yield obj_path + return + if obj_path.is_dir(): + for item in obj_path.iterdir(): + if item.is_dir(): + yield from depth_first_walk(item) + elif item.is_file(): + yield item + yield obj_path + + +def compute_object_hash(obj_path: pathlib.Path, algorithm:str=ORG_DATAONE_CID_ALGORITHM) -> str: + """Computes and returns the hash of the specified object path. + + This has the side effect of setting the ORG_DATAONE_CID xattr to a JSON + struct that contains the current modification time, size, and hash value. + + The existing ORG_DATAONE_CID xattr is examined if present, and is returned + unless there is a mismatch of size or modification time. + + If obj_path is a folder, then the hash is computed as the hash of the list of + type, cid, and name values. Since it is necessary to verify the consistency + of the hashes for subfolders, a traversal to the leaves is necessary. Hence, + this method can be expensive with deeply nested subfolders since even traversing + """ + #Xattrs accepts os.PathLike + attrs = xattrs.Xattrs(obj_path) + obj_stat = obj_path.stat() + if obj_path.is_file(): + try: + # check for current existing cid value and return that if present + entry = json.loads(attrs.get(ORG_DATAONE_CID, {}).decode("utf-8")) + if entry["mtime"] == obj_stat.st_mtime and entry["size"] == obj_stat.st_size: + # nothing has changed, so use the existing value + return entry["cid"] + except KeyError: + # No or invalid xattr value, ignore and continue + pass + with obj_path.open('rb') as f: + # Use the builtin chunker, requires python >= 3.11 + digest = hashlib.file_digest(f, algorithm) + hexdigest = digest.hexdigest() + # Set the xattr + entry = json.dumps({ + "cid": hexdigest, + "mtime": obj_stat.st_mtime, + "size": obj_stat.st_size + }, ensure_ascii=False) + attrs[ORG_DATAONE_CID] = entry.encode("utf-8") + elif obj_path.is_dir(): + # Hash of a folder is the hash of the content. To check a folder, it + # is necessary to traverse to the leaves to verify / compute the + # hashes. + pass + else: + raise ValueError(f"Path is not a file or folder.") + return hexdigest + + +@dataclasses.dataclass +class FolderEntry: + """A single entry in a Folder instance. + + An entry represents a file or folder record within a folder object. + """ + kind: int # FOLDER_ENTRY_FOLDER | FOLDER_ENTRY_FILE + cid: str # The content id + name: str # name of folder or file + + def get_cid(self, algorithm:str=ORG_DATAONE_CID_ALGORITHM) -> str: + """Retrieves the contentId from the ORG_DATAONE_CID xattr or computes + the CID if the xattr value is out of date. + + A CID value is computed if the file size or modified timestamp do + not match the entries stored in the xattr structure or if the xattr + is not set or otherwise cannot be read. + """ + hasher = hashlib.new(self.algorithm) + pass + + +@dataclasses.dataclass +class Folder: + """An object that represents a single folder and its content. + """ + name: str # PID + "/" + path to the folder relative to root + entries: list[FolderEntry] = dataclasses.field(default_factory=list) + + def __iter__(self): + return self.entries.__iter__() + + def append(self, item: FolderEntry) -> None: + self.entries.append(item) + + def compute_cid(self, algorithm:str=ORG_DATAONE_CID_ALGORITHM) -> str: + """Computes the CID for this folder based on the content of the entries. + + The CID is computed as the hash of the list of type, cid, and name values. + The entries are sorted by type and name to ensure a consistent hash value. + """ + hasher = hashlib.new(algorithm) + self.entries.sort(key=lambda x: (x.kind, x.name)) + for entry in self.entries: + hasher.update(f"{entry.kind} {entry.cid} {entry.name}\n".encode("utf-8")) + return hasher.hexdigest() + + @classmethod + def deserialize(cls, stream: io.BytesIO, pid:str) -> "Folder": + manifest = Folder(name=pid) + with contextlib.closing(stream): + header = stream.readline().decode("utf-8").strip() + if not header.startswith("container "): + msg = f"{pid} is not a container." + raise ValueError(msg) + for line in stream: + line = line.decode("utf-8").strip() + type_flag, cid, name = line.split(" ", 2) + manifest.entries.append(FolderEntry(kind=int(type_flag), cid=cid, name=name)) + return manifest + + def serialize( + self, + ) -> io.BytesIO: + self.entries.sort(key=lambda x: (x.kind, x.name)) + dest_stream = io.BytesIO() + dest_stream.name = "tmp" + dest_stream.write(f"container {len(self.entries)}\n".encode("utf-8")) + for entry in self.entries: + dest_stream.write(f"{entry.kind} {entry.cid} {entry.name}\n".encode("utf-8")) + dest_stream.seek(0) + return dest_stream + + @property + def size(self) -> int: + return len(self.entries) + + +class VirtualHashStore(hashstore.filehashstore.FileHashStore): + + def __init__(self, properties): + super().__init__(properties) + + def _find_object(self, pid: str) -> dict[str, str]: + """Check if an object referenced by a pid exists and retrieve its content identifier. + The `find_object` method validates the existence of an object based on the provided + pid and returns the associated content identifier. + + :param str pid: Authority-based or persistent identifier of the object. + + :return: obj_info_dict: + - cid: content identifier + - cid_object_path: path to the object + - cid_refs_path: path to the cid refs file + - pid_refs_path: path to the pid refs file + - sysmeta_path: path to the sysmeta file + """ + self.fhs_logger.debug("Request to find object for for pid: %s", pid) + self._check_string(pid, "pid") + + pid_ref_abs_path = self._get_hashstore_pid_refs_path(pid) + if os.path.isfile(pid_ref_abs_path): + # Read the file to get the cid from the pid reference + pid_refs_cid = self._read_small_file_content(pid_ref_abs_path) + + # Confirm that the cid reference file exists + cid_ref_abs_path = self._get_hashstore_cid_refs_path(pid_refs_cid) + if os.path.isfile(cid_ref_abs_path): + # Check that the pid is actually found in the cid reference file + if self._is_string_in_refs_file(pid, cid_ref_abs_path): + # Object must also exist in order to return the cid retrieved + if self._exists("objects", pid_refs_cid): + cid_object_path = self._get_hashstore_data_object_path( + pid_refs_cid + ) + else: + cid_object_path = None + sysmeta_doc_name = self._computehash(pid + self.sysmeta_ns) + metadata_directory = self._computehash(pid) + metadata_rel_path = pathlib.Path(*self._shard(metadata_directory)) + sysmeta_full_path = ( + self._get_store_path("metadata") + / metadata_rel_path + / sysmeta_doc_name + ) + obj_info_dict = { + "cid": pid_refs_cid, + "cid_object_path": cid_object_path, + "cid_refs_path": cid_ref_abs_path, + "pid_refs_path": pid_ref_abs_path, + "sysmeta_path": ( + sysmeta_full_path + if os.path.isfile(sysmeta_full_path) + else "Does not exist." + ), + } + return obj_info_dict + else: + # If not, it is an orphan pid refs file + err_msg = ( + f"Pid reference file exists with cid: {pid_refs_cid} for pid: {pid} but " + f"is missing from cid refs file: {cid_ref_abs_path}" + ) + self.fhs_logger.error(err_msg) + raise hashstore.filehashstore_exceptions.PidNotFoundInCidRefsFile(err_msg) + else: + err_msg = ( + f"Pid reference file exists with cid: {pid_refs_cid} but cid reference file " + + f"not found: {cid_ref_abs_path} for pid: {pid}" + ) + self.fhs_logger.error(err_msg) + raise hashstore.filehashstore_exceptions.OrphanPidRefsFileFound(err_msg) + else: + err_msg = ( + f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" + ) + self.fhs_logger.error(err_msg) + raise hashstore.filehashstore_exceptions.PidRefsDoesNotExist(err_msg) + + + def commit_folder( + self, + pid: str, + root_path: typing.Union[str, pathlib.Path], + child_path: typing.Optional[typing.Union[str, pathlib.Path]] = None, + additional_algorithm: typing.Optional[str] = None, + checksum: typing.Optional[str] = None, + checksum_algorithm: typing.Optional[str] = None, + expected_object_size: typing.Optional[int] = None, + pattern: typing.Optional[str] = None, + ) -> typing.Optional[hashstore.filehashstore.ObjectMetadata]: + """Store a folder (and subfolders) as container objects. + + Traverses the folder hierarchy in a depth first manner so that + the leaf elements are computed for inclusion in parent hashes. + + Args: + pid (str): The context within which this folder is being stored + root_path (str): Path to the root of the folder. + child_path (str): Path to folder being stored relative to the root_path. If None, assumes root_path. + + Returns: + str: CID for the container + """ + # Get the relative path for the object / folder + root_path = pathlib.Path(root_path) + if child_path is None: + child_path = root_path + else: + child_path = pathlib.Path(child_path) + relative_path = child_path.relative_to(root_path) + path_pid = pid + container_name = "root" + if str(relative_path) != ".": + path_pid = f"{pid}/{relative_path}" + container_name = str(relative_path) + + # Check if this container already exists + try: + # resolve pid, path to CID. This raises if not found + _entry = self._find_object(path_pid) + size = os.path.getsize( + self._build_hashstore_data_object_path(_entry["cid"]) + ) + return hashstore.filehashstore.ObjectMetadata( + pid=path_pid, cid=_entry["cid"], obj_size=size, hex_digests={} + ) + except hashstore.filehashstore.PidNotFoundInCidRefsFile: + pass + except hashstore.filehashstore.PidRefsDoesNotExist: + pass + + # Container doesn't exist + manifest = Folder(name=path_pid) + for item in child_path.iterdir(): + if item.is_dir(): + meta = self.commit_folder( + pid, + root_path, + child_path=item.absolute(), + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=expected_object_size, + pattern=pattern, + ) + if meta is not None: + manifest.append(FolderEntry(kind=0, cid=meta.cid, name=item.name)) + elif pattern is None and item.is_file(): + # If no pattern then grab all files, otherwise defer to + # globbing match later. + item_pid = f"{pid}/{item.relative_to(root_path)}" + self.fhs_logger.debug("store_folder {item_pid=}") + with item.open("r") as item_stream: + _cid = self._computehash(item_stream) + self._store_hashstore_refs_files(item_pid, _cid) + manifest.append(FolderEntry(kind=1, cid=_cid, name=item.name)) + if pattern is not None: + # globbing pattern was specified. Grab the matching files here. + for item in child_path.glob(pattern): + if item.is_file(): + item_pid = f"{pid}/{item.relative_to(root_path)}" + with item.open("r") as item_stream: + _cid = self._computehash(item_stream) + self._store_hashstore_refs_files(item_pid, _cid) + manifest.append(FolderEntry(kind=1, cid=_cid, name=item.name)) + if manifest.size == 0: + return None + + return self.store_object( + path_pid, + data=manifest.serialize(), + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=None, + ) + + + def folder_content(self, pid:str, depth:int=0, depth_first:bool=True) -> typing.Generator: + """Yield the content of a folder, breadth first, recursively. + (depth, type, CID, name) + """ + # Retrieve the container object + with contextlib.closing(self.retrieve_object(pid)) as obj_stream: + manifest = Folder.deserialize(obj_stream, pid) + if depth_first: + for entry in manifest: + if entry.kind == 0: + yield (depth, entry.kind, entry.cid, f"{pid}/{entry.name}") + yield from self.folder_content(f"{pid}/{entry.name}", depth=depth+1, depth_first=depth_first) + else: + yield (depth, entry.kind, entry.cid, f"{pid}/{entry.name}") + else: + for entry in manifest: + yield (depth, entry.kind, entry.cid, f"{pid}/{entry.name}") + for entry in manifest: + if entry.kind == 0: + # folder, recurse + yield from self.folder_content(f"{pid}/{entry.name}", depth=depth+1, depth_first=depth_first) + + def get_object_status(self, pid: str) -> dict: + self.fhs_logger.debug("Request to get object status for pid: %s", pid) + self._check_string(pid, "pid") + + object_status_dict = {} + try: + object_info_dict = self._find_object(pid) + object_cid = object_info_dict.get("cid") + if object_cid: + try: + obj_path = self._get_hashstore_data_object_path(object_cid) + object_status_dict["size"] = os.path.getsize(obj_path) + object_status_dict["modtime"] = os.path.getmtime(obj_path) + object_status_dict["accesstime"] = os.path.getatime(obj_path) + except FileNotFoundError as e: + msg = f"No object found for CID {object_cid}" + self.fhs_logger.warning(msg) + object_status_dict["size"] = 0 + object_status_dict["modtime"] = "-" + object_status_dict["accesstime"] = "-" + else: + err_msg = f"No object found for pid: {pid}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + except KeyError as ke: + err_msg = f"No object found for pid: {pid}. Details: {ke}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + + return object_status_dict + diff --git a/tests/test_hashstore.py b/tests/test_hashstore.py index 02d83e3b..dbc24a2d 100644 --- a/tests/test_hashstore.py +++ b/tests/test_hashstore.py @@ -2,7 +2,7 @@ import os import pytest -from hashstore.hashstore import HashStoreFactory +from hashstore.basehashstore import HashStoreFactory from hashstore.filehashstore import FileHashStore From 74f1870e09a7d67739ecb4958060ae5fe210d4ef Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:38:27 -0400 Subject: [PATCH 16/33] Dependency updates require 3.10 minimum python version --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8153e587..00fc7ed7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ { name = "Jeanette Clark" }, { name = "Ian M. Nesbitt" }, ] -requires-python = ">=3.9, <4.0" +requires-python = ">=3.10, <4.0" readme = "README.md" keywords = [ "filesystem", @@ -35,6 +35,7 @@ classifiers = [ dependencies = [ "click>=8.3.1", "pathlib>=1.0.1", + "pyarrow>=24.0.0", "pyyaml>=6.0", "xattr-compat>=1.0.0", ] From fda4af6702f15346b54ac054de851b618c31f6ea Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:39:56 -0400 Subject: [PATCH 17/33] Adjust folder related method signatures --- src/hashstore/basehashstore.py | 104 ++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 2e50d8cb..6620c845 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,9 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Union, Generator +from typing import Generator, Optional, Union + +import hashstore.folderentry class HashStore(ABC): @@ -69,36 +71,41 @@ def store_object( @abstractmethod def store_folder( self, - pid:str, - root_path:Union[str,Path], - child_path:Optional[Union[str,Path]]=None, - additional_algorithm:Optional[str]=None, - checksum:Optional[str]=None, - checksum_algorithm:Optional[str]=None, - expected_object_size:Optional[int]=None, + pid: str, + path: str, + entries: hashstore.folderentry.FolderEntries, + additional_algorithm: Optional[str] = None, + checksum: Optional[str] = None, + checksum_algorithm: Optional[str] = None, + verify_entry_cids: bool = True, ): - """Stores a folder and subfolders. - - The `store_folder` method stores a folder and its subfolders to HashStore. Each file within the folder - is processed and stored individually, following the same procedures as the `store_object` method. The - folder structure is preserved within HashStore, allowing for easy retrieval of the entire folder or - individual files as needed. - - The root of a folder is specified by the `root_path` argument and must be identified by a PID. - - This method performs a recursive, depth firth traversal of the folder structure, storing each file it encounters and - storing folders as a container object that lists the files and subfolders contained within it. + """Store a folder object. + + A Folder is a list of entries that appear in a folder. Each entry + may be a file or a Folder. This method is used instead of store_object + because Folders have special requirements to ensure deterministic serialization. + + The Folder is tagged with an identifier that is "{PID} {path}", that is, the + PID followed by a single space, then the path. If the path portion is an empty + string, ".", or "/" then the Folder is the root Folder. + + Note that since the hash of a Folder is computed from hashes of its content, + a Folder hierarchy must be stored starting with the leaves. This method + will raise a ValueError if the hash of an entry does not already exist in + the hashstore. Hence the general pattern for storing a folder hierarchy is + to do a depth first traversal of the hierarchy, storing the files (ensuring + their hashes are available) and computing the hash for the containing folder + for use in the parent folder reference to the child. Args: - pid (str): Identifier for the context of this folder hierarchy. - root_path (str | Path): The physical path to the root folder being stored. - child_path (Optional[str | Path], optional): Path to a subfolder of root_path. This is - normally None for the initial invocation of this method, and recursive calls will set the - child_path as needed. Defaults to None. - additional_algorithm (Optional[str], optional): See `store_object`. Defaults to None. - checksum (Optional[str], optional): See `store_object`. Defaults to None. - checksum_algorithm (Optional[str], optional): See `store_object`. Defaults to None. - expected_object_size (Optional[int], optional): See `store_object`. Defaults to None. + pid (str): The context within which this folder is being stored + path (str): Path to this folder relative to the root. + entries (list[FolderEntry]): A list of FolderEntry objects. + verify_entry_cids: If True then FolderEntry CID values are + verified to to ensure they exist in the hashstore. + + Returns: + ObjectMetadata: The computed ObjectMetadata for this entry. Raises: NotImplementedError: Must be implemented in subclass. @@ -106,25 +113,27 @@ def store_folder( raise NotImplementedError() @abstractmethod - def retrieve_folder(self, pid:str, destination_path:Union[str,Path], child_path:Optional[Union[str,Path]]=None): - """Retrieves a folder and its subfolders from HashStore. - - The `retrieve_folder` method retrieves a folder and its subfolders from HashStore, reconstructing - the original folder structure at the specified target path. Each file within the folder is retrieved - individually, following the same procedures as the `retrieve_object` method. The folder structure - is preserved during retrieval, allowing for easy access to the entire folder or individual files as needed. - - The root of a folder is specified by the `pid` argument, which identifies the context of the folder hierarchy. The - optional child_path argument can be used to specify a subfolder within the root folder for retrieval. - - Output files and folders will be created under the `destination_path`. + def retrieve_folder( + self, + pid: str, + path: str, + ) -> hashstore.folderentry.FolderEntries: + """Retrieve a FolderEntries instance from the hashstore. + + We first check to see if a CID is available for the combination of + "{PID} {path}", and if so, return that entry. Otherwise, we iterate + over path segments to find the correspoding FolderEntry, if any. + This iterative approach is necesary if since entire trees are not + stored when a new version of a folder hierarchy is stored. Hence, it + may be necessary to jump back to a branch that is recorded in an + earlier version but not recorded in the current version since it + was unchanged between versions. Args: - pid (str): Identifier for the context of this folder hierarchy. - destination_path (str | Path): The physical path where the retrieved folder will be reconstructed. - child_path (Optional[str|Path|], optional): Path to a subfolder of the root folder. This is - normally None for the initial invocation of this method, and recursive calls will set the - child_path as needed. Defaults to None. + pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved + path (str): Path within the context to the desired entry + Returns: + FolderEntries """ raise NotImplementedError() @@ -235,9 +244,9 @@ def get_hex_digest(self, pid, algorithm): raise NotImplementedError() @abstractmethod - def list_pids(self, pattern:Optional[str]=None) -> Generator: + def list_pids(self, pattern: Optional[str] = None) -> Generator: """Yields PIDs from the hashstore. - + :param str pattern: Optional regexp pattern to match. """ raise NotImplementedError() @@ -252,6 +261,7 @@ def get_object_status(self, pid) -> dict: """ raise NotImplementedError() + class HashStoreFactory: """A factory class for creating `HashStore`-like objects. From 880c355567d5fdc1252c9b697cadddf2e9116816 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:40:22 -0400 Subject: [PATCH 18/33] Dependency updates --- uv.lock | 329 ++++++++++++++++++++++---------------------------------- 1 file changed, 126 insertions(+), 203 deletions(-) diff --git a/uv.lock b/uv.lock index bcdc3492..2151d931 100644 --- a/uv.lock +++ b/uv.lock @@ -1,11 +1,10 @@ version = 1 revision = 3 -requires-python = ">=3.9, <4.0" +requires-python = ">=3.10, <4.0" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version < '3.11'", ] [[package]] @@ -17,32 +16,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, ] -[[package]] -name = "astroid" -version = "3.3.11" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, -] - [[package]] name = "astroid" version = "4.0.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/22/97df040e15d964e592d3a180598ace67e91b7c559d8298bdb3c949dc6e42/astroid-4.0.2.tar.gz", hash = "sha256:ac8fb7ca1c08eb9afec91ccc23edbd8ac73bb22cbdd7da1d488d9fb8d6579070", size = 405714, upload-time = "2025-11-09T21:21:18.373Z" } wheels = [ @@ -54,13 +33,11 @@ name = "black" version = "25.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "click" }, { name = "mypy-extensions" }, { name = "packaging" }, { name = "pathspec" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "platformdirs" }, { name = "pytokens" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, @@ -87,39 +64,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" }, { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" }, { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" }, - { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" }, - { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" }, - { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" }, - { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" }, { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" }, ] -[[package]] -name = "click" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, -] - [[package]] name = "click" version = "8.3.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } wheels = [ @@ -158,31 +111,39 @@ wheels = [ [[package]] name = "hashstore" -version = "1.1.1" +version = "1.2.0" source = { editable = "." } dependencies = [ + { name = "click" }, { name = "pathlib" }, + { name = "pyarrow" }, { name = "pyyaml" }, + { name = "xattr-compat" }, ] [package.dev-dependencies] +cli = [ + { name = "rich" }, +] dev = [ { name = "black" }, { name = "exceptiongroup" }, { name = "pg8000" }, - { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "9.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pylint" }, + { name = "pytest" }, ] [package.metadata] requires-dist = [ + { name = "click", specifier = ">=8.3.1" }, { name = "pathlib", specifier = ">=1.0.1" }, + { name = "pyarrow", specifier = ">=24.0.0" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "xattr-compat", specifier = ">=1.0.0" }, ] [package.metadata.requires-dev] +cli = [{ name = "rich", specifier = ">=14.3.3" }] dev = [ { name = "black", specifier = ">=22.10.0" }, { name = "exceptiongroup", specifier = ">=1.1.0" }, @@ -191,39 +152,10 @@ dev = [ { name = "pytest", specifier = ">=7.2.0" }, ] -[[package]] -name = "importlib-metadata" -version = "8.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, @@ -231,31 +163,23 @@ wheels = [ [[package]] name = "isort" -version = "6.1.0" +version = "7.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, ] [[package]] -name = "isort" -version = "7.0.0" +name = "markdown-it-py" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", +dependencies = [ + { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] [[package]] @@ -267,6 +191,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -316,27 +249,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/07/5fd183858dff4d24840f07fc845f213cd371a19958558607ba22035dadd7/pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201", size = 57816, upload-time = "2025-09-14T09:16:47.798Z" }, ] -[[package]] -name = "platformdirs" -version = "4.4.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, -] - [[package]] name = "platformdirs" version = "4.5.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, @@ -351,6 +267,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" }, + { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" }, + { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" }, + { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" }, + { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" }, + { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ba/464cc70761c2a525d97ebd84e21c31ebd47f3ef4bdcee117009f51c46f24/pyarrow-24.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:c42ab9439498270139cc63e18847a02afe5c8b3ed9c931266533cfe378bd3591", size = 27251730, upload-time = "2026-04-21T10:46:30.913Z" }, + { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, + { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, + { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -360,91 +333,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] -[[package]] -name = "pylint" -version = "3.3.9" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "astroid", version = "3.3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "dill", marker = "python_full_version < '3.10'" }, - { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "mccabe", marker = "python_full_version < '3.10'" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, - { name = "tomlkit", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/9d/81c84a312d1fa8133b0db0c76148542a98349298a01747ab122f9314b04e/pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a", size = 1525946, upload-time = "2025-10-05T18:41:43.786Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/a7/69460c4a6af7575449e615144aa2205b89408dc2969b87bc3df2f262ad0b/pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7", size = 523465, upload-time = "2025-10-05T18:41:41.766Z" }, -] - [[package]] name = "pylint" version = "4.0.4" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "astroid", version = "4.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "dill", marker = "python_full_version >= '3.10'" }, - { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mccabe", marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, - { name = "tomlkit", marker = "python_full_version >= '3.10'" }, + { name = "astroid" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "dill" }, + { name = "isort" }, + { name = "mccabe" }, + { name = "platformdirs" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomlkit" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a6/92/d40f5d937517cc489ad848fc4414ecccc7592e4686b9071e09e64f5e378e/pylint-4.0.4-py3-none-any.whl", hash = "sha256:63e06a37d5922555ee2c20963eb42559918c20bd2b21244e4ef426e7c43b92e0", size = 536425, upload-time = "2025-11-30T13:29:02.53Z" }, ] -[[package]] -name = "pytest" -version = "8.4.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, -] - [[package]] name = "pytest" version = "9.0.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/56/f013048ac4bc4c1d9be45afd4ab209ea62822fb1598f40687e6bf45dcea4/pytest-9.0.1.tar.gz", hash = "sha256:3e9c069ea73583e255c3b21cf46b8d3c56f6e3a1a8f6da94ccb0fcf57b9d73c8", size = 1564125, upload-time = "2025-11-12T13:05:09.333Z" } wheels = [ @@ -534,15 +453,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, - { url = "https://files.pythonhosted.org/packages/9f/62/67fc8e68a75f738c9200422bf65693fb79a4cd0dc5b23310e5202e978090/pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", size = 184450, upload-time = "2025-09-25T21:33:00.618Z" }, - { url = "https://files.pythonhosted.org/packages/ae/92/861f152ce87c452b11b9d0977952259aa7df792d71c1053365cc7b09cc08/pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", size = 174319, upload-time = "2025-09-25T21:33:02.086Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cd/f0cfc8c74f8a030017a2b9c771b7f47e5dd702c3e28e5b2071374bda2948/pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", size = 737631, upload-time = "2025-09-25T21:33:03.25Z" }, - { url = "https://files.pythonhosted.org/packages/ef/b2/18f2bd28cd2055a79a46c9b0895c0b3d987ce40ee471cecf58a1a0199805/pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", size = 836795, upload-time = "2025-09-25T21:33:05.014Z" }, - { url = "https://files.pythonhosted.org/packages/73/b9/793686b2d54b531203c160ef12bec60228a0109c79bae6c1277961026770/pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", size = 750767, upload-time = "2025-09-25T21:33:06.398Z" }, - { url = "https://files.pythonhosted.org/packages/a9/86/a137b39a611def2ed78b0e66ce2fe13ee701a07c07aebe55c340ed2a050e/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", size = 727982, upload-time = "2025-09-25T21:33:08.708Z" }, - { url = "https://files.pythonhosted.org/packages/dd/62/71c27c94f457cf4418ef8ccc71735324c549f7e3ea9d34aba50874563561/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", size = 755677, upload-time = "2025-09-25T21:33:09.876Z" }, - { url = "https://files.pythonhosted.org/packages/29/3d/6f5e0d58bd924fb0d06c3a6bad00effbdae2de5adb5cda5648006ffbd8d3/pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", size = 142592, upload-time = "2025-09-25T21:33:10.983Z" }, - { url = "https://files.pythonhosted.org/packages/f0/0c/25113e0b5e103d7f1490c0e947e303fe4a696c10b501dea7a9f49d4e876c/pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", size = 158777, upload-time = "2025-09-25T21:33:15.55Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] [[package]] @@ -634,10 +557,10 @@ wheels = [ ] [[package]] -name = "zipp" -version = "3.23.0" +name = "xattr-compat" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/82/5ffb78d69936e7148d6ee907dd44581b6478dbfc313cda32c122b264f505/xattr-compat-1.0.0.tar.gz", hash = "sha256:5ef02e523585276d78a4ebbc05f2d08f0ddc200116c847826b1a18d8aa71f5bc", size = 5300, upload-time = "2021-03-03T21:20:30.288Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, + { url = "https://files.pythonhosted.org/packages/06/51/4708370113789e0ab876c9bcb895a66fa116bb9e90e8940f6f6c932db5c2/xattr_compat-1.0.0-py3-none-any.whl", hash = "sha256:5fafcd360b274b362a46c88871473c26236a858071fbbe787a05f8df062899fa", size = 7130, upload-time = "2021-03-03T21:20:29.09Z" }, ] From e84f4de0eff551be880f12a0cc42e434bce49c84 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:43:17 -0400 Subject: [PATCH 19/33] Simplify folder methods, build recursion is responsibility of caller; relax pid check to match hashstore requirements; make find_object a public method --- src/hashstore/filehashstore.py | 347 +++++++++++++++------------------ 1 file changed, 153 insertions(+), 194 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 51e212ae..331c3807 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -19,6 +19,7 @@ import yaml +import hashstore.folderentry from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, @@ -780,7 +781,7 @@ def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[byt return metadata_stream else: err_msg = f"No metadata found for pid: {pid}" - self.fhs_logger.warning(err_msg) + self.fhs_logger.warning(err_msg) raise KeyError(err_msg) def delete_object(self, pid: str) -> None: @@ -1055,219 +1056,172 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: logging.info(info_string) return hex_digest - def _store_container( - self, - pid:str, - name:str, - manifest:list[tuple[int, str, str]], - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, - ) -> "ObjectMetadata": - self.fhs_logger.debug(f"_store_container {pid=}") - manifest.sort(key=lambda x: (x[0], x[1])) - dest_stream = io.BytesIO() - dest_stream.name = name - dest_stream.write(f"container {len(manifest)}\n".encode("utf-8")) - for row in manifest: - dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) - dest_stream.seek(0) - return self.store_object( - pid, - data=dest_stream, - additional_algorithm=additional_algorithm, - checksum=checksum, - checksum_algorithm=checksum_algorithm, - expected_object_size=None, - ) - - def _load_container(self, pid:str) -> list[tuple[int, str, str]]: - self.fhs_logger.debug(f"_load_container {pid=}") - obj_stream = self.retrieve_object(pid) - row = 0 - manifest = [] - with closing(obj_stream): - for line in obj_stream: - line = line.decode("utf-8").strip() - if row == 0: - if not line.startswith("container "): - msg = f"{pid} is not a container." - raise ValueError(msg) - else: - type_flag, cid, name = line.split(" ", 2) - manifest.append((int(type_flag), cid, name)) - row += 1 - return manifest - def store_folder( self, pid: str, - root_path: Union[str, Path], - child_path: Optional[Union[str, Path]] = None, + path: str, + entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, - expected_object_size: Optional[int] = None, - pattern: Optional[str] = None, + verify_entry_cids: bool = True, ) -> Optional["ObjectMetadata"]: - """Store a folder (and subfolders) as container objects. + """Store a folder object. - Traverses the folder hierarchy in a depth first manner so that - the leaf elements are computed for inclusion in parent hashes. + A Folder is a list of entries that appear in a folder. Each entry + may be a file or a Folder. This method is used instead of store_object + because Folders have special requirements to ensure deterministic serialization. + + The Folder is tagged with an identifier that is "{PID} {path}", that is, the + PID followed by a single space, then the path. If the path portion is an empty + string, ".", or "/" then the Folder is the root Folder. + + Note that since the hash of a Folder is computed from hashes of its content, + a Folder hierarchy must be stored starting with the leaves. This method + will raise a ValueError if the hash of an entry does not already exist in + the hashstore. Hence the general pattern for storing a folder hierarchy is + to do a depth first traversal of the hierarchy, storing the files (ensuring + their hashes are available) and computing the hash for the containing folder + for use in the parent folder reference to the child. Args: pid (str): The context within which this folder is being stored - root_path (str): Path to the root of the folder. - child_path (str): Path to folder being stored relative to the root_path. If None, assumes root_path. - + path (str): Path to this folder relative to the root. + entries (list[FolderEntry]): A list of FolderEntry objects. + verify_entry_cids: If True then FolderEntry CID values are + verified to to ensure they exist in the hashstore. + Returns: - str: CID for the container + ObjectMetadata: The computed ObjectMetadata for this entry. """ - raise NotImplementedError - # Get the relative path for the object / folder - root_path = Path(root_path) - if child_path is None: - child_path = root_path - else: - child_path = Path(child_path) - relative_path = child_path.relative_to(root_path) - path_pid = pid - container_name = "root" - if str(relative_path) != ".": - path_pid = f"{pid}/{relative_path}" - container_name = str(relative_path) - - self.fhs_logger.debug("store_folder {path_pid=}") - # Check if this container already exists - try: - # resolve pid, path to CID. This raises if not found - _entry = self._find_object(path_pid) - size = os.path.getsize( - self._build_hashstore_data_object_path(_entry["cid"]) - ) - return ObjectMetadata( - pid=path_pid, cid=_entry["cid"], obj_size=size, hex_digests={} - ) - except PidNotFoundInCidRefsFile: - pass - except PidRefsDoesNotExist: - pass + if path in ("", ".", "/"): + path = "" + folder_pid = f"{pid} {path}" if path != "" else pid + self._check_string(folder_pid, "PID") + if verify_entry_cids: + # check that each entry CID is present in the hashstore. + for entry in entries: + if not self._exists("objects", entry.cid): + raise ValueError( + f"object {entry.name} cid {entry.cid} does not exist." + ) + # Sort the entries by cid + entries.sort(key=lambda entry: entry.cid) - # Container doesn't exist - manifest = [] - for item in child_path.iterdir(): - if item.is_dir(): - meta = self.store_folder( - pid, - root_path, - child_path=item.absolute(), - additional_algorithm=additional_algorithm, - checksum=checksum, - checksum_algorithm=checksum_algorithm, - expected_object_size=expected_object_size, - pattern=pattern, - ) - if meta is not None: - manifest.append((0, meta.cid, item.name)) - elif pattern is None and item.is_file(): - # If no pattern then grab all files, otherwise defer to - # globbing match later. - item_pid = f"{pid}/{item.relative_to(root_path)}" - self.fhs_logger.debug("store_folder {item_pid=}") - meta = self.store_object(item_pid, str(item.absolute())) - manifest.append((1, meta.cid, item.name)) - if pattern is not None: - # globbing pattern was specified. Grab the matching files here. - for item in child_path.glob(pattern): - if item.is_file(): - item_pid = f"{pid}/{item.relative_to(root_path)}" - meta = self.store_object(item_pid, str(item.absolute())) - manifest.append((1, meta.cid, item.name)) - if len(manifest) == 0: - return None - return self._store_container( - path_pid, - container_name, - manifest, - additional_algorithm=additional_algorithm, - checksum=checksum, - checksum_algorithm=checksum_algorithm - ) - - def folder_content(self, pid:str, depth:int=0, depth_first:bool=True) -> Generator: - """Yield the content of a folder, breadth first, recursively. - (depth, type, CID, name) - """ - # Retrieve the container object - manifest = self._load_container(pid) - if depth_first: - for entry in manifest: - if entry[0] == 0: - yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") - yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) - else: - yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") - else: - for entry in manifest: - yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") - for entry in manifest: - if entry[0] == 0: - # folder, recurse - yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) + hash_algorithms = { + self.algorithm: hashlib.new(self.algorithm), + } + if additional_algorithm is not None: + hash_algorithms[additional_algorithm] = hashlib.new(additional_algorithm) + + # Compute the CID for the folder by iterating over the + # list of CIDs and updating the hash being computed. This + # is about as fast as serializing the cids to a big string + # but much more memory efficient. + for entry in entries: + v = entry.cid.encode("utf-8") + for fhasher in hash_algorithms.values(): + fhasher.update(v) + hex_digests = { + name: hasher.hexdigest() for name, hasher in hash_algorithms.items() + } + folder_cid = hex_digests[self.algorithm] + + # Compute the physical path for the computed CID + cid_path = self._build_hashstore_data_object_path(folder_cid) + # ensure the folder path exists + self._create_path(Path(os.path.dirname(cid_path))) + + # Store the entries to disk. Records are serialized as parquet + # which is an indeterminate file format. Hence the need for + # computing the hashes seperately to the actual bytes on disk. + # The parquet format is very efficient especially as the number + # of folder entries increases. + obj_size = entries.to_parquet(cid_path, pid=folder_pid) + self.tag_object(folder_pid, folder_cid) + return ObjectMetadata( + pid=folder_pid, + cid=folder_cid, + hex_digests=hex_digests, + obj_size=obj_size, + ) - def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path:Optional[Union[str, Path]]=None): - """Retrieve a folder (and subfolders) stored as container objects. + def retrieve_folder( + self, + pid: str, + path: str, + ) -> hashstore.folderentry.FolderEntries: + """Retrieve a FolderEntries instance from the hashstore. + + We first check to see if a CID is available for the combination of + "{PID} {path}", and if so, return that entry. Otherwise, we iterate + over path segments to find the correspoding FolderEntry, if any. + This iterative approach is necesary if since entire trees are not + stored when a new version of a folder hierarchy is stored. Hence, it + may be necessary to jump back to a branch that is recorded in an + earlier version but not recorded in the current version since it + was unchanged between versions. Args: - pid (str): The context within which this folder is being retrieved - destination_path (str|Path): Path to the root of the folder to create. - child_path (str|Path): Path to folder being retrieved relative to the destination_path. If None, assumes destination_path. + pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved + path (str): Path within the context to the desired entry Returns: - None + FolderEntries """ - # TODO: Error handling - # TODO: read access control considerations - destination_path = Path(destination_path) - if child_path is None: - child_path = Path("") - else: - child_path = Path(child_path) - path_pid = pid - if str(child_path) != ".": - path_pid = f"{pid} {child_path}" - - # Retrieve the container object - obj_stream = self.retrieve_object(path_pid) - with closing(obj_stream): - for line in obj_stream: - line = line.decode("utf-8").strip() - type_flag, cid, name = line.split(" ", 2) - if type_flag == "0": - # Directory - (destination_path / child_path / name).mkdir(parents=True, exist_ok=True) - self.retrieve_folder( - pid, - destination_path, - child_path=child_path / name, - ) - elif type_flag == "1": - # File - item_pid = f"{pid} {child_path / name}" - file_stream = self.retrieve_object(item_pid) - with closing(file_stream): - dest_file_path = destination_path / child_path / name - dest_file_path.parent.mkdir(parents=True, exist_ok=True) - with open(dest_file_path, "wb") as dest_file: - shutil.copyfileobj(file_stream, dest_file) - - def list_pids(self, pattern: Optional[str]=None) -> Generator: + if path in ("", ".", "/"): + path = "" + folder_pid = f"{pid} {path}" if path != "" else pid + self._check_string(folder_pid, "PID") + # try direct reference to CID using folder_pid + try: + object_info_dict = self._find_object(folder_pid) + folder_cid = object_info_dict.get("cid") + if folder_cid is None: + raise PidRefsDoesNotExist("Entry has no cid?") + cid_path = object_info_dict.get("cid_object_path") + # self._build_hashstore_data_object_path(folder_cid) + return hashstore.folderentry.FolderEntries.from_parquet(cid_path) + except PidRefsDoesNotExist: + pass + + # otherwise, get the root, split the path, and start iterating. + # This will raise PidRefsDoesNotExist if the root PID isn't there + object_info_dict = self._find_object(pid) + folder_cid = object_info_dict.get("cid") + if folder_cid is None: + # Should never reach this... + raise PidRefsDoesNotExist("Entry has no cid?") + cid_path = object_info_dict.get("cid_object_path") + # self._build_hashstore_data_object_path(folder_cid) + current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) + path_segments = path.split("/") + # iterate over segments, saving last + for name in path_segments: + entry = current_folder.entry_by_name(name) + if entry is None: + raise KeyError(f"PID {pid} {path} not found.") + if entry.type == hashstore.folderentry.FTYPE_FILE: + # it's a file! + raise ValueError(f"Path {path} is a file.") + object_info_dict = self._find_object(pid) + folder_cid = object_info_dict.get("cid") + if folder_cid is None: + # Should never reach this... + raise PidRefsDoesNotExist("Entry has no cid?") + cid_path = object_info_dict.get("cid_object_path") + # cid_path = self._build_hashstore_data_object_path(folder_cid) + current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) + return current_folder + + def list_pids(self, pattern: Optional[str] = None) -> Generator: rpattern = None if pattern is not None: rpattern = re.compile(pattern) - ignore_names = [".DS_Store", ] - for cid_entry in self.cids.rglob('*'): + ignore_names = [ + ".DS_Store", + ] + for cid_entry in self.cids.rglob("*"): if cid_entry.is_file() and cid_entry.name not in ignore_names: - self.fhs_logger.debug(str(cid_entry)) for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): pid = entry.strip() if len(pid) > 0: @@ -1276,7 +1230,6 @@ def list_pids(self, pattern: Optional[str]=None) -> Generator: yield pid else: yield pid - # FileHashStore Core Methods @@ -3018,8 +2971,14 @@ def _check_integer(file_size: int) -> None: @staticmethod def _check_string(string: str, arg: str) -> None: - """Check whether a string is None or empty - or if it contains an illegal character; - throws an exception if so. + """Raises ValueError if string is empty or has leading or trailing whitespace. + + Note: This checks string for valid use as a PID or CID in hashstore. It is + the responsibility of the calling application to ensure the string is a + valid PID or CID in the context of the application. For example, hashstore + allows whitespace in PIDs though Metacat / DataONE does not. This allows + for storing a path segment after a Metacat PID which is needed for referencing + entries within a folder identified by a PID. :param str string: Value to check. :param str arg: Name of the argument to check. From 067955b97645a2b2756b1183a667b0a91662c82d Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:43:51 -0400 Subject: [PATCH 20/33] Add structure for folder entries --- src/hashstore/folderentry.py | 118 +++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/hashstore/folderentry.py diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py new file mode 100644 index 00000000..6c5dfa30 --- /dev/null +++ b/src/hashstore/folderentry.py @@ -0,0 +1,118 @@ +"""Implements FolderEntry class.""" + +import collections.abc +import dataclasses +import json +import logging +import os + +import pyarrow +import pyarrow.parquet + +FTYPE_FOLDER = 0 +"""FolderEntry is for a FolderEntry""" +FTYPE_FILE = 1 +"""FolderEntry is for a file.""" +# TODO: Ratify this key +PARQUET_METADATA_KEY = b"https://ns.dataone.org/types/FolderEntries" +"""Key in parquet file metadata pointing to dict of properties.""" +PARQUET_READ_BATCH_SIZE = 10000 +"""Number of entries to read at a time from FolderEntries parquet file.""" + + +def get_logger(): + return logging.getLogger("FolderEntry") + + +@dataclasses.dataclass +class FolderEntry: + """Represents a file or folder entry in a folder manifest.""" + + name: str + """The name portion of the path (not full path) for the file or folder.""" + cid: str + """The content hash (CID) for the entry.""" + type: int # '1' for file, '0' for directory + """The type of manifest entry: '1' for file, '0' for directory.""" + size: int = 0 + """Size of the file in bytes or number of entries for directories.""" + formatid: str | None = None + """Optional format identifier for files.""" + + def __post_init__(self): + if self.type not in (FTYPE_FILE, FTYPE_FOLDER): + raise ValueError(f"Invalid type: {self.type}") + + def __repr__(self) -> str: + # Representation of a FolderEntry + return json.dumps( + { + "cid": self.cid, + "type": self.type, + "name": self.name, + "size": self.size, + "formatid": self.formatid, + }, + ensure_ascii=False, + ) + + +class FolderEntries(list[FolderEntry]): + def entry_by_name(self, name) -> FolderEntry | None: + """Find the entry with name that matches.""" + for entry in self: + if entry.name == name: + return entry + return None + + def to_parquet( + self, pq_path: str, pid: str | None = None, writer_args: dict = {} + ) -> int: + """Writes the list of folder entries to a parquet file. + + See also: https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html + + #TODO: There are quite a few options for tweaking the written parquet file, + # e.g. with respect to column sorting, a UI may prefer sorting by type or formatid + + args: + pq_path: path to destination parquet file + pid: Optional PID+path used to create this folder. + writer_args: optional dict of arguments for the parquet writer. + """ + # Add some metadata to the parquet file to help identify it as a list of FolderEntries + pq_metadata = { + "type": "FolderEntries", + "version": "1.0", + "pid": pid, + } + metadata_bytes = json.dumps(pq_metadata).encode("utf-8") + table = pyarrow.Table.from_pylist([dataclasses.asdict(entry) for entry in self]) + table = table.replace_schema_metadata({PARQUET_METADATA_KEY: metadata_bytes}) + pyarrow.parquet.write_table(table, pq_path, **writer_args) + return os.path.getsize(pq_path) + + @classmethod + def from_parquet(cls, pq_path) -> "FolderEntries": + """Create an instance of FolderEntries from a parquet source.""" + pq_metadata = pyarrow.parquet.read_metadata(pq_path) + try: + metadata = json.loads(pq_metadata.metadata[PARQUET_METADATA_KEY].decode()) + _ = metadata["version"] + except KeyError: + raise ValueError(f"File {pq_path} is not a FolderEntry list.") + + pq_file = pyarrow.parquet.ParquetFile(pq_path) + entries = cls() + for batch in pq_file.iter_batches(batch_size=PARQUET_READ_BATCH_SIZE): + for row in batch.to_pylist(): + entries.append( + FolderEntry( + name=row["name"], + cid=row["cid"], + type=row["type"], + size=row["size"], + formatid=row["formatid"], + ) + ) + return entries From 45ef848ac2bd8b71a92c8112fea0427ffddac9f2 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:47:24 -0400 Subject: [PATCH 21/33] Make find_object less noisy for common and expected cases --- src/hashstore/filehashstore.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 331c3807..d4aa6c6c 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -613,7 +613,7 @@ def get_object_status(self, pid: str) -> dict: object_status_dict = {} try: - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) object_cid = object_info_dict.get("cid") if object_cid: obj_path = self._get_hashstore_data_object_path(object_cid) @@ -742,7 +742,7 @@ def retrieve_object(self, pid: str) -> IO[bytes]: self.fhs_logger.debug("Request to retrieve object for pid: %s", pid) self._check_string(pid, "pid") - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) object_cid = object_info_dict.get("cid") entity = "objects" @@ -801,7 +801,7 @@ def delete_object(self, pid: str) -> None: self._synchronize_object_locked_pids(pid) try: - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) cid = object_info_dict.get("cid") # Proceed with next steps - cid has been retrieved without any issues @@ -1044,7 +1044,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: entity = "objects" algorithm = self._clean_algorithm(algorithm) - object_cid = self._find_object(pid).get("cid") + object_cid = self.find_object(pid).get("cid") if not self._exists(entity, object_cid): err_msg = f"No object found for pid: {pid}" self.fhs_logger.error(err_msg) @@ -1174,7 +1174,7 @@ def retrieve_folder( self._check_string(folder_pid, "PID") # try direct reference to CID using folder_pid try: - object_info_dict = self._find_object(folder_pid) + object_info_dict = self.find_object(folder_pid) folder_cid = object_info_dict.get("cid") if folder_cid is None: raise PidRefsDoesNotExist("Entry has no cid?") @@ -1186,7 +1186,7 @@ def retrieve_folder( # otherwise, get the root, split the path, and start iterating. # This will raise PidRefsDoesNotExist if the root PID isn't there - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... @@ -1203,7 +1203,7 @@ def retrieve_folder( if entry.type == hashstore.folderentry.FTYPE_FILE: # it's a file! raise ValueError(f"Path {path} is a file.") - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... @@ -1233,7 +1233,7 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: # FileHashStore Core Methods - def _find_object(self, pid: str) -> Dict[str, str]: + def find_object(self, pid: str) -> Dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. The `find_object` method validates the existence of an object based on the provided pid and returns the associated content identifier. @@ -1269,7 +1269,7 @@ def _find_object(self, pid: str) -> Dict[str, str]: self.fhs_logger.error(err_msg) raise RefsFileExistsButCidObjMissing(err_msg) else: - sysmeta_doc_name = self._computehash(pid + self.sysmeta_ns) + sysmeta_doc_name = self._computehash(f"{pid}{self.sysmeta_ns}") metadata_directory = self._computehash(pid) metadata_rel_path = Path(*self._shard(metadata_directory)) sysmeta_full_path = ( @@ -1310,7 +1310,8 @@ def _find_object(self, pid: str) -> Dict[str, str]: err_msg = ( f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" ) - self.fhs_logger.error(err_msg) + # This can be an expected case, so not really an error. + self.fhs_logger.debug(err_msg) raise PidRefsDoesNotExist(err_msg) def _store_and_validate_data( @@ -1782,7 +1783,7 @@ def _untag_object(self, pid: str, cid: str) -> None: # which will throw custom exceptions if there is an issue with the reference files, # which help us determine the path to proceed with. try: - obj_info_dict = self._find_object(pid) + obj_info_dict = self.find_object(pid) cid_to_check = obj_info_dict["cid"] self._validate_and_check_cid_lock(pid, cid, cid_to_check) From 97ffe890c4f9666f5eede0c9f80ea746c9e5d2ae Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:52:52 -0400 Subject: [PATCH 22/33] Make find_object part of base hashstore, tweak hints --- src/hashstore/basehashstore.py | 21 +++++++++++++++++++++ src/hashstore/filehashstore.py | 8 ++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 6620c845..1b337545 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -261,6 +261,27 @@ def get_object_status(self, pid) -> dict: """ raise NotImplementedError() + @abstractmethod + def find_object(self, pid: str) -> dict[str, str]: + """Check if an object referenced by a pid exists and retrieve its content identifier. + + The `find_object` method validates the existence of an object based on the provided + pid and returns the associated content identifier and information about how to + retrieve various accoutrements. Note that the returned dict will contain values + relevant to the type of store, but will always contain a `cid` key if the object + is present. + + :param str pid: Authority-based or persistent identifier of the object. + + :return: obj_info_dict: + - cid: content identifier + - cid_object_path: path to the object + - cid_refs_path: path to the cid refs file + - pid_refs_path: path to the pid refs file + - sysmeta_path: path to the sysmeta file + """ + raise NotImplementedError() + class HashStoreFactory: """A factory class for creating `HashStore`-like objects. diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index d4aa6c6c..b815412d 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1233,10 +1233,14 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: # FileHashStore Core Methods - def find_object(self, pid: str) -> Dict[str, str]: + def find_object(self, pid: str) -> dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. + The `find_object` method validates the existence of an object based on the provided - pid and returns the associated content identifier. + pid and returns the associated content identifier and information about how to + retrieve various accoutrements. Note that the returned dict will contain values + relevant to the type of store, but will always contain a `cid` key if the object + is present. :param str pid: Authority-based or persistent identifier of the object. From d324b9f4e516707014b3f38a1ce698cbf7648c69 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:55:46 -0400 Subject: [PATCH 23/33] WIP: adjust cli for revised hashstore folder support --- src/hashstore/__main__.py | 202 +++++++++++++++++++++++++------------- 1 file changed, 135 insertions(+), 67 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index d4b09654..4ffb1da3 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -5,10 +5,12 @@ import os import pathlib import sys + import click import rich import rich.tree import yaml + try: from yaml import CLoader as Loader except ImportError: @@ -20,9 +22,11 @@ HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" + def get_logger(): return logging.getLogger("hs") + def sizeof_fmt(num, suffix="B"): for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): if abs(num) < 1024.0: @@ -30,26 +34,29 @@ def sizeof_fmt(num, suffix="B"): num /= 1024.0 return f"{num:.1f}Yi{suffix}" + def enumerate_dict(d): for key, value in d.items(): if isinstance(value, dict): for subkey, subvalue in enumerate_dict(value): yield f"{key}.{subkey}", subvalue - else: + else: yield key, value -def load_hashstore_properties(path:pathlib.Path)-> dict: +def load_hashstore_properties(path: pathlib.Path) -> dict: properties_file = path / "hashstore.yaml" if not properties_file.exists(): - raise FileNotFoundError(f"Hashstore properties file not found: {properties_file}") - with open(properties_file, 'r') as f: + raise FileNotFoundError( + f"Hashstore properties file not found: {properties_file}" + ) + with open(properties_file, "r") as f: properties = yaml.load(f, Loader=Loader) properties["store_path"] = str(path) return properties -def locate_hashstore(path:pathlib.Path)->pathlib.Path: +def locate_hashstore(path: pathlib.Path) -> pathlib.Path | None: # Iterate through the current directory and all its parents for directory in [path] + list(path.parents): # Use glob() to find files matching the pattern within the current directory @@ -61,36 +68,44 @@ def locate_hashstore(path:pathlib.Path)->pathlib.Path: @click.group() @click.option( - "--store", - type=click.Path(path_type=pathlib.Path, file_okay=False), - envvar="HASHSTORE_PATH", - default=None + "--store", + type=click.Path(path_type=pathlib.Path, file_okay=False), + envvar="HASHSTORE_PATH", + default=None, +) +@click.option( + "--config", + default="~/.config/hashstore/defaults.yml", + type=click.Path(), + help="Path to configuration file", ) -@click.option("--config", default='~/.config/hashstore/defaults.yml', type=click.Path(), help="Path to configuration file") @click.option("--log-level", default=None, help="Set the logging level (INFO)") @click.pass_context def main(ctx, store, config, log_level): - """Implements a high level file and folder operations CLI for hashstore. - """ + """Implements a high level file and folder operations CLI for hashstore.""" ctx.ensure_object(dict) # Load defaults from config file config = os.path.expanduser(config) if os.path.exists(config): - with open(config, 'r') as f: + with open(config, "r") as f: cfg = yaml.load(f, Loader=Loader) ctx.default_map = cfg if store is None: store = locate_hashstore(pathlib.Path.cwd()) if store is not None: - ctx.obj["hashstore_path"] = store + ctx.obj["hashstore_path"] = store else: store.expanduser() ctx.obj["hashstore_path"] = store - ctx.obj["module_name"] = ctx.default_map.get("module_name", "hashstore.filehashstore") + ctx.obj["module_name"] = ctx.default_map.get( + "module_name", "hashstore.filehashstore" + ) ctx.obj["class_name"] = ctx.default_map.get("class_name", "FileHashStore") # Set up logging if log_level is None: - log_level = ctx.default_map.get('log_level', 'INFO') if 'cfg' in locals() else 'INFO' + log_level = ( + ctx.default_map.get("log_level", "INFO") if "cfg" in locals() else "INFO" + ) logger = get_logger() numeric_level = getattr(logging, log_level.upper(), None) if not isinstance(numeric_level, int): @@ -117,8 +132,12 @@ def version(ctx): @click.pass_context @click.option("--depth", "-d", type=int, default=3, help="Hashstore hierarchy depth") @click.option("--width", "-w", type=int, default=2, help="Hashstore hierarchy width") -@click.option("--algorithm", "-a", type=str, default="SHA-256", help="Hashstore algorithm") -@click.option("--metadata_namespace", type=str, default=None, help="Hashstore metadata namespace") +@click.option( + "--algorithm", "-a", type=str, default="SHA-256", help="Hashstore algorithm" +) +@click.option( + "--metadata_namespace", type=str, default=None, help="Hashstore metadata namespace" +) def create(ctx, depth, width, algorithm, metadata_namespace): """Create a new hashstore at the specified root path.""" logger = get_logger() @@ -139,7 +158,9 @@ def create(ctx, depth, width, algorithm, metadata_namespace): } hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.info(f"Hashstore created at: {store}") except Exception as e: logger.error(f"Failed to create hashstore: {e}") @@ -150,38 +171,37 @@ def create(ctx, depth, width, algorithm, metadata_namespace): @main.command(name="add_folder") @click.pass_context @click.argument( - "root_path", - type=click.Path(path_type=pathlib.Path, file_okay=False, exists=True) + "root_path", type=click.Path(path_type=pathlib.Path, file_okay=False, exists=True) ) @click.argument("pid", type=str) @click.option( - "-s", - "--sysmeta_path", - type=click.Path(path_type=pathlib.Path), - default=None, - help="Path to system metadata XML file" + "-s", + "--sysmeta_path", + type=click.Path(path_type=pathlib.Path), + default=None, + help="Path to system metadata XML file", ) @click.option( "-p", "--pattern", type=str, default=None, - help="Glob pattern for files to include (defaults to all)" + help="Glob pattern for files to include (defaults to all)", ) def add_object( - ctx, - root_path:pathlib.Path, - pid: str|None, - sysmeta_path:pathlib.Path|None, - pattern:str|None, + ctx, + root_path: pathlib.Path, + pid: str | None, + sysmeta_path: pathlib.Path | None, + pattern: str | None, ): """Add a folder to the hashstore. PID is required and is used to reference the folder root. - - If object_path is a folder, a PID is generated for the folder contents added + + If object_path is a folder, a PID is generated for the folder contents added recursively using the object relative paths as suffix to the PID. - + if the file doesn't exist: add bytes store pid @@ -194,11 +214,15 @@ def add_object( """ logger = get_logger() store = ctx.obj["hashstore_path"] - logger.info(f"Adding folder at path: {store} with PID: {pid} and sysmeta: {sysmeta_path}") + logger.info( + f"Adding folder at path: {store} with PID: {pid} and sysmeta: {sysmeta_path}" + ) properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.debug(f"Hashstore opened at: {store}") except Exception as e: logger.error(f"Failed to open hashstore: {e}") @@ -213,13 +237,19 @@ def add_object( root_path = root_path.absolute() if info is None: try: - info = hash_store.commit_folder(pid=pid, root_path=root_path, pattern=pattern) + info = hash_store.commit_folder( + pid=pid, root_path=root_path, pattern=pattern + ) print(json.dumps(dataclasses.asdict(info))) except Exception as e: logger.error(f"Failed to add object: {e}") return 1 if sysmeta_path is not None and sysmeta_path.is_file(): - hash_store.store_metadata(pid=pid, metadata=str(sysmeta_path), format_id=properties.get("store_metadata_namespace")) + hash_store.store_metadata( + pid=pid, + metadata=str(sysmeta_path), + format_id=properties.get("store_metadata_namespace"), + ) return 1 return 0 @@ -228,7 +258,9 @@ def add_object( @click.pass_context @click.argument("pid", type=str) @click.option("-s", "--stream", is_flag=True, help="Stream to stdout") -@click.option("-r", "--recursive", is_flag=True, help="Recurse into contents if PID is a folder.") +@click.option( + "-r", "--recursive", is_flag=True, help="Recurse into contents if PID is a folder." +) def get_object(ctx, pid, stream, recursive): """Retrieve an object or folder from hashstore.""" store = ctx.obj["hashstore_path"] @@ -237,17 +269,32 @@ def get_object(ctx, pid, stream, recursive): @main.command("ls") @click.pass_context -@click.option("-p", "--pattern", default=None, help="Optional regex pattern for PID matching.") -@click.option("-h", "--human-readable", is_flag=True, help="Display sizes in human readable format.") +@click.option( + "-p", "--pattern", default=None, help="Optional regex pattern for PID matching." +) +@click.option( + "-h", + "--human-readable", + is_flag=True, + help="Display sizes in human readable format.", +) +@click.option( + "-m", + "--show-metadata", + is_flag=True, + help="Show metadata for entry if available.", +) @click.option("-r", "--reference", is_flag=True, help="Include path references.") -def list_pids(ctx, pattern, human_readable, reference): +def list_pids(ctx, pattern, human_readable, show_metadata, reference): """List PIDs in the hashstore.""" logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.debug(f"Hashstore opened at: {store}") except Exception as e: logger.error(f"Failed to open hashstore: {e}") @@ -256,56 +303,77 @@ def list_pids(ctx, pattern, human_readable, reference): # iterate over the refs/cids folder, getting PIDs from each file. print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") for pid in hash_store.list_pids(pattern=pattern): - try: + try: pid_stat = hash_store.get_object_status(pid) fsize = pid_stat.get("size", 0) total_objects += 1 t_modified = pid_stat.get("modtime", "-") if t_modified != "-": - t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat(timespec="seconds") + t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat( + timespec="seconds" + ) t_access = pid_stat.get("accesstime", "-") if t_access != "-": - t_access = datetime.datetime.fromtimestamp(t_access).isoformat(timespec="seconds") + t_access = datetime.datetime.fromtimestamp(t_access).isoformat( + timespec="seconds" + ) if human_readable: fsize = sizeof_fmt(fsize) - if fsize>0 or reference: + if fsize > 0 or reference: print(f"{fsize}\t{t_modified}\t{t_access}\t{pid}") - try: - meta = hash_store.retrieve_metadata(pid) - print(json.dumps(meta, indent=2)) - except KeyError: - pass + if show_metadata: + try: + meta = hash_store.retrieve_metadata(pid).read().decode() + print(meta) + # print(json.dumps(meta, indent=2)) + except KeyError: + pass except KeyError: logger.warning(f"PID status not available in hashstore: {pid}") print(f"Total {total_objects}") +def iterate_folder(hs, tree, pid, path=""): + current_folder = hs.retrieve_folder(pid, path=path) + for entry in current_folder: + branch = tree.add(entry.name) + if entry.type == 0: + _path = f"{path}/{entry.name}" if path != "" else entry.name + iterate_folder(hs, branch, pid, path=_path) + + @main.command("tree") @click.pass_context @click.argument("pid", type=str) -def get_container_tree(ctx, pid)->None: +def get_container_tree(ctx, pid) -> None: logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.debug(f"Hashstore opened at: {store}") except Exception as e: logger.error(f"Failed to open hashstore: {e}") return 1 - tree = { - pid:rich.tree.Tree(pid), - } - for entry in hash_store.folder_content(pid, depth_first=True): - print(f"{entry[0]} {entry[1]} {entry[3]} {entry[2]}") - parts = entry[3].rsplit("/", 1) - if entry[1] == 0: - tree[entry[3]] = tree[parts[0]].add(entry[3]) - else: - # file - tree[parts[0]].add(entry[3]) - rich.print(tree[pid]) + parts = pid.split(" ", 1) + path = "" + if len(parts) > 1: + path = parts[1].strip() + tree = rich.tree.Tree(pid) + iterate_folder(hash_store, tree, parts[0], path=path) + # parts = [] + # for entry in iterate_folder(hash_store, pid): + # print(f"{entry.type} {entry.name} {entry.size} {entry.cid}") + # # parts = entry[3].rsplit("/", 1) + # if entry.type == 0: + # tree[entry.name] = tree[parts[0]].add(entry[3]) + # # else: + # # # file + # # tree[parts[0]].add(entry[3]) + rich.print(tree) if __name__ == "__main__": From 0970b1e31ebca306abeba46630050824067c7e33 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 8 May 2026 15:08:43 -0400 Subject: [PATCH 24/33] Added option to capture object creation events to an index file --- src/hashstore/filehashstore.py | 21 ++++++++++++-- src/hashstore/pidlogger.py | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 src/hashstore/pidlogger.py diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index b815412d..69ac8632 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -20,7 +20,7 @@ import yaml import hashstore.folderentry -from hashstore import HashStore +from hashstore import HashStore, pidlogger from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -134,6 +134,8 @@ def __init__(self, properties=None): self._create_path(self.refs / "tmp") self._create_path(self.refs / "pids") self._create_path(self.refs / "cids") + # pidlog is used to create an index of cid - pid + self.pidlog = logging.getLogger("pid_logger") # Variables to orchestrate parallelization # Check to see whether a multiprocessing or threading sync lock should be used @@ -578,6 +580,7 @@ def store_object( cid = object_metadata.cid self.tag_object(pid, cid) self.fhs_logger.info("Successfully stored object for pid: %s", pid) + self.pidlog.info(cid, extra={"pid": pid}) finally: # Release pid self._release_object_locked_pids(pid) @@ -1068,6 +1071,8 @@ def store_folder( ) -> Optional["ObjectMetadata"]: """Store a folder object. + #TODO: important: Make thread and multi processing safe. + A Folder is a list of entries that appear in a folder. Each entry may be a file or a Folder. This method is used instead of store_object because Folders have special requirements to ensure deterministic serialization. @@ -1139,6 +1144,7 @@ def store_folder( # of folder entries increases. obj_size = entries.to_parquet(cid_path, pid=folder_pid) self.tag_object(folder_pid, folder_cid) + self.pidlog.info(folder_cid, extra={"pid": folder_pid}) return ObjectMetadata( pid=folder_pid, cid=folder_cid, @@ -1214,22 +1220,31 @@ def retrieve_folder( return current_folder def list_pids(self, pattern: Optional[str] = None) -> Generator: + """Yield create_timestamp, CID, PID. + + Iterates over all CID entries and yields the create timestamp, + CID value, and PID value for all entries or those PIDs that match + the optionally provided regexp pattern. + """ rpattern = None if pattern is not None: rpattern = re.compile(pattern) ignore_names = [ ".DS_Store", ] + cids_path = str(self.cids) for cid_entry in self.cids.rglob("*"): if cid_entry.is_file() and cid_entry.name not in ignore_names: + cid_value = str(cid_entry).replace(cids_path, "").replace("/", "") + ctime = cid_entry.stat().st_ctime for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): pid = entry.strip() if len(pid) > 0: if rpattern is not None: if rpattern.fullmatch(pid): - yield pid + yield ctime, cid_value, pid else: - yield pid + yield ctime, cid_value, pid # FileHashStore Core Methods diff --git a/src/hashstore/pidlogger.py b/src/hashstore/pidlogger.py new file mode 100644 index 00000000..1091e9c0 --- /dev/null +++ b/src/hashstore/pidlogger.py @@ -0,0 +1,51 @@ +"""Implements a multi-process safe logger for generating a cid-pid index. + +This implementation writes an NDJSON file with each line containing a json array +with elements: + [0] timestamp + [1] cid + [2] pid + +The logging module is used because it is multi-thread and -process safe. + +The resulting ndjson file can be loaded into duckdb for example with: + + CREATE TABLE pids AS SELECT + to_timestamp(json[1]::DOUBLE) AS ctime, + json[2]->> '$' AS cid, + json[3]->>'$' AS pid + FROM read_json('pid_index.ndjson'); + +or create a parquet representation: + + duckdb -c "COPY (SELECT to_timestamp(json[1]::DOUBLE) AS ctime, + json[2]->> '\\$' AS cid, json[3]->>'\\$' AS pid FROM + read_json('pid_index.ndjson')) TO 'pid_index.parquet' (FORMAT parquet)" +""" + +import json +import logging + + +class PidIndexFormatter(logging.Formatter): + def format(self, record) -> str: + pid_record = ( + record.created, + record.getMessage().strip(), + record.pid if hasattr(record, "pid") else None, + ) + return json.dumps(pid_record) + + +def getPidLogger(log_file_name: str): + """Return a logger for the pid index. + + PID records are added to the index like: + logger.info(CID, extra={"pid":PID}) + """ + logger = logging.getLogger("pid_logger") + logger.setLevel(logging.INFO) + handler = logging.FileHandler(log_file_name) + handler.setFormatter(PidIndexFormatter()) + logger.addHandler(handler) + return logger From f7b24bec1528b3dca860bf979b3b0030f4d455e6 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 8 May 2026 16:44:14 -0400 Subject: [PATCH 25/33] Added folder inspection methods --- src/hashstore/__main__.py | 98 ++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 22 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index 4ffb1da3..e791ba70 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -285,7 +285,8 @@ def get_object(ctx, pid, stream, recursive): help="Show metadata for entry if available.", ) @click.option("-r", "--reference", is_flag=True, help="Include path references.") -def list_pids(ctx, pattern, human_readable, show_metadata, reference): +@click.option("-l", "--list-only", is_flag=True, help="Just list the cid, pid values.") +def list_pids(ctx, pattern, human_readable, show_metadata, reference, list_only): """List PIDs in the hashstore.""" logger = get_logger() store = ctx.obj["hashstore_path"] @@ -301,8 +302,12 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference): return 1 total_objects = 0 # iterate over the refs/cids folder, getting PIDs from each file. - print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") - for pid in hash_store.list_pids(pattern=pattern): + if not list_only: + print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") + for ctime, cid, pid in hash_store.list_pids(pattern=pattern): + if list_only: + print(json.dumps((ctime, cid, pid))) + continue try: pid_stat = hash_store.get_object_status(pid) fsize = pid_stat.get("size", 0) @@ -330,22 +335,80 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference): pass except KeyError: logger.warning(f"PID status not available in hashstore: {pid}") - print(f"Total {total_objects}") + if not list_only: + print(f"Total {total_objects}") -def iterate_folder(hs, tree, pid, path=""): - current_folder = hs.retrieve_folder(pid, path=path) - for entry in current_folder: - branch = tree.add(entry.name) - if entry.type == 0: - _path = f"{path}/{entry.name}" if path != "" else entry.name - iterate_folder(hs, branch, pid, path=_path) +@main.command("finfo") +@click.pass_context +@click.argument("pid", type=str) +def get_folder_info(ctx, pid) -> None: + """Compute basic stats for a folder and sub-folders.""" + + def iterate_folder(hs, stats, pid, path="", depth: int = 1): + if depth > stats["max_depth"]: + stats["max_depth"] = depth + current_folder = hs.retrieve_folder(pid, path=path) + for entry in current_folder: + if entry.type == 0: + _path = f"{path}/{entry.name}" if path != "" else entry.name + stats["total_folders"] = stats["total_folders"] + 1 + iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) + else: + stats["total_bytes"] = stats["total_bytes"] + entry.size + stats["total_files"] += 1 + + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + + info = { + "total_files": 0, + "total_bytes": 0, + "total_folders": 0, + "max_depth": 0, + } + parts = pid.split(" ", 1) + path = "" + if len(parts) > 1: + path = parts[1].strip() + + iterate_folder(hash_store, info, parts[0], path=path, depth=0) + print(json.dumps(info, indent=2)) @main.command("tree") @click.pass_context @click.argument("pid", type=str) -def get_container_tree(ctx, pid) -> None: +@click.option("-n", "--no-files", is_flag=True, help="Show folders but not content.") +def get_folder_tree(ctx, pid: str, no_files: bool) -> None: + """Generate a tree representation of the folder and sub-folders.""" + + def iterate_folder(hs, tree, pid, path="", with_files: bool = True): + current_folder = hs.retrieve_folder(pid, path=path) + n = 0 + s = 0 + for entry in current_folder: + if entry.type == 0 or with_files: + branch = tree.add(entry.name) + if entry.type == 0: + _path = f"{path}/{entry.name}" if path != "" else entry.name + iterate_folder(hs, branch, pid, path=_path, with_files=with_files) + else: + n += 1 + s += entry.size + if not with_files: + branch = tree.add(f"{n:,} files, {s:,} bytes") + logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) @@ -363,16 +426,7 @@ def get_container_tree(ctx, pid) -> None: if len(parts) > 1: path = parts[1].strip() tree = rich.tree.Tree(pid) - iterate_folder(hash_store, tree, parts[0], path=path) - # parts = [] - # for entry in iterate_folder(hash_store, pid): - # print(f"{entry.type} {entry.name} {entry.size} {entry.cid}") - # # parts = entry[3].rsplit("/", 1) - # if entry.type == 0: - # tree[entry.name] = tree[parts[0]].add(entry[3]) - # # else: - # # # file - # # tree[parts[0]].add(entry[3]) + iterate_folder(hash_store, tree, parts[0], path=path, with_files=not no_files) rich.print(tree) From 9b129311beeab6bb88a6e243b418d7b833d81a7e Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 10:20:03 -0400 Subject: [PATCH 26/33] Enable folder creation without folderEntry cids --- src/hashstore/filehashstore.py | 19 +++++++++++++++---- src/hashstore/folderentry.py | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 69ac8632..714bcbda 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1077,6 +1077,10 @@ def store_folder( may be a file or a Folder. This method is used instead of store_object because Folders have special requirements to ensure deterministic serialization. + If FolderEntries have cid = '' | None, then the cid is looked up using + + pid + " " + path + "/" + entry.name + The Folder is tagged with an identifier that is "{PID} {path}", that is, the PID followed by a single space, then the path. If the path portion is an empty string, ".", or "/" then the Folder is the root Folder. @@ -1106,10 +1110,17 @@ def store_folder( if verify_entry_cids: # check that each entry CID is present in the hashstore. for entry in entries: - if not self._exists("objects", entry.cid): - raise ValueError( - f"object {entry.name} cid {entry.cid} does not exist." - ) + if entry.cid is None or entry.cid == "": + # no cid provided, so look it up + _entry_pid = f"{folder_pid}/{entry.name}" + _meta = self.find_object(_entry_pid) + entry.cid = _meta["cid"] + else: + # verify provided cid is legit + if not self._exists("objects", entry.cid): + raise ValueError( + f"object {entry.name} cid {entry.cid} does not exist." + ) # Sort the entries by cid entries.sort(key=lambda entry: entry.cid) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 6c5dfa30..bae3dda1 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -56,6 +56,18 @@ def __repr__(self) -> str: ensure_ascii=False, ) + @classmethod + def parquet_schema(cls): + return pyarrow.schema( + ( + ("cid", pyarrow.string()), + ("type", pyarrow.bool_()), + ("name", pyarrow.string()), + ("size", pyarrow.int64()), + ("formatid", pyarrow.string()), + ) + ) + class FolderEntries(list[FolderEntry]): def entry_by_name(self, name) -> FolderEntry | None: @@ -86,10 +98,11 @@ def to_parquet( "version": "1.0", "pid": pid, } + pq_schema = FolderEntry.parquet_schema() metadata_bytes = json.dumps(pq_metadata).encode("utf-8") table = pyarrow.Table.from_pylist([dataclasses.asdict(entry) for entry in self]) table = table.replace_schema_metadata({PARQUET_METADATA_KEY: metadata_bytes}) - pyarrow.parquet.write_table(table, pq_path, **writer_args) + pyarrow.parquet.write_table(table, pq_path, schema=pq_schema, **writer_args) return os.path.getsize(pq_path) @classmethod From 9cf4f214ec923c04a2ff3ffc93ac10cb6419b610 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 11:09:28 -0400 Subject: [PATCH 27/33] Change type to bool for efficiency --- src/hashstore/filehashstore.py | 4 ++-- src/hashstore/folderentry.py | 23 +++++++++-------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 714bcbda..8f414018 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1121,7 +1121,7 @@ def store_folder( raise ValueError( f"object {entry.name} cid {entry.cid} does not exist." ) - # Sort the entries by cid + # Sort the entries by cid for consistent hashing entries.sort(key=lambda entry: entry.cid) hash_algorithms = { @@ -1217,7 +1217,7 @@ def retrieve_folder( entry = current_folder.entry_by_name(name) if entry is None: raise KeyError(f"PID {pid} {path} not found.") - if entry.type == hashstore.folderentry.FTYPE_FILE: + if entry.is_file: # it's a file! raise ValueError(f"Path {path} is a file.") object_info_dict = self.find_object(pid) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index bae3dda1..3797f169 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -9,10 +9,6 @@ import pyarrow import pyarrow.parquet -FTYPE_FOLDER = 0 -"""FolderEntry is for a FolderEntry""" -FTYPE_FILE = 1 -"""FolderEntry is for a file.""" # TODO: Ratify this key PARQUET_METADATA_KEY = b"https://ns.dataone.org/types/FolderEntries" """Key in parquet file metadata pointing to dict of properties.""" @@ -32,23 +28,19 @@ class FolderEntry: """The name portion of the path (not full path) for the file or folder.""" cid: str """The content hash (CID) for the entry.""" - type: int # '1' for file, '0' for directory + is_file: bool # True for file, False for Folder """The type of manifest entry: '1' for file, '0' for directory.""" size: int = 0 """Size of the file in bytes or number of entries for directories.""" formatid: str | None = None """Optional format identifier for files.""" - def __post_init__(self): - if self.type not in (FTYPE_FILE, FTYPE_FOLDER): - raise ValueError(f"Invalid type: {self.type}") - def __repr__(self) -> str: # Representation of a FolderEntry return json.dumps( { "cid": self.cid, - "type": self.type, + "is_file": self.is_file, "name": self.name, "size": self.size, "formatid": self.formatid, @@ -61,7 +53,7 @@ def parquet_schema(cls): return pyarrow.schema( ( ("cid", pyarrow.string()), - ("type", pyarrow.bool_()), + ("is_file", pyarrow.bool_()), ("name", pyarrow.string()), ("size", pyarrow.int64()), ("formatid", pyarrow.string()), @@ -100,9 +92,12 @@ def to_parquet( } pq_schema = FolderEntry.parquet_schema() metadata_bytes = json.dumps(pq_metadata).encode("utf-8") - table = pyarrow.Table.from_pylist([dataclasses.asdict(entry) for entry in self]) + table = pyarrow.Table.from_pylist( + [dataclasses.asdict(entry) for entry in self], + schema=pq_schema, + ) table = table.replace_schema_metadata({PARQUET_METADATA_KEY: metadata_bytes}) - pyarrow.parquet.write_table(table, pq_path, schema=pq_schema, **writer_args) + pyarrow.parquet.write_table(table, pq_path, **writer_args) return os.path.getsize(pq_path) @classmethod @@ -123,7 +118,7 @@ def from_parquet(cls, pq_path) -> "FolderEntries": FolderEntry( name=row["name"], cid=row["cid"], - type=row["type"], + is_file=row["is_file"], size=row["size"], formatid=row["formatid"], ) From 67e2668ee679eba610439c29d888dc0a4c3850bc Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 16:55:11 -0400 Subject: [PATCH 28/33] Use alternate path delimiter --- src/hashstore/__main__.py | 23 +++++++++++++------- src/hashstore/filehashstore.py | 39 ++++++++++++++++++++++++++++------ src/hashstore/folderentry.py | 2 +- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index e791ba70..abad4d6e 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -18,6 +18,7 @@ import hashstore import hashstore.filehashstore_exceptions +from hashstore.folderentry import PATH_DELIMITER HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" @@ -350,13 +351,15 @@ def iterate_folder(hs, stats, pid, path="", depth: int = 1): stats["max_depth"] = depth current_folder = hs.retrieve_folder(pid, path=path) for entry in current_folder: - if entry.type == 0: - _path = f"{path}/{entry.name}" if path != "" else entry.name - stats["total_folders"] = stats["total_folders"] + 1 - iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) - else: + if entry.is_file == 0: stats["total_bytes"] = stats["total_bytes"] + entry.size stats["total_files"] += 1 + else: + _path = ( + f"{path}{PATH_DELIMITER}{entry.name}" if path != "" else entry.name + ) + stats["total_folders"] = stats["total_folders"] + 1 + iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) logger = get_logger() store = ctx.obj["hashstore_path"] @@ -398,10 +401,14 @@ def iterate_folder(hs, tree, pid, path="", with_files: bool = True): n = 0 s = 0 for entry in current_folder: - if entry.type == 0 or with_files: + if not entry.is_file or with_files: branch = tree.add(entry.name) - if entry.type == 0: - _path = f"{path}/{entry.name}" if path != "" else entry.name + if not entry.is_file: + _path = ( + f"{path}{PATH_DELIMITER}{entry.name}" + if path != "" + else entry.name + ) iterate_folder(hs, branch, pid, path=_path, with_files=with_files) else: n += 1 diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 8f414018..50d992d4 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1103,7 +1103,8 @@ def store_folder( Returns: ObjectMetadata: The computed ObjectMetadata for this entry. """ - if path in ("", ".", "/"): + delim = hashstore.folderentry.PATH_DELIMITER + if path in ("", ".", delim): path = "" folder_pid = f"{pid} {path}" if path != "" else pid self._check_string(folder_pid, "PID") @@ -1112,7 +1113,7 @@ def store_folder( for entry in entries: if entry.cid is None or entry.cid == "": # no cid provided, so look it up - _entry_pid = f"{folder_pid}/{entry.name}" + _entry_pid = f"{folder_pid}{delim}{entry.name}" _meta = self.find_object(_entry_pid) entry.cid = _meta["cid"] else: @@ -1163,6 +1164,17 @@ def store_folder( obj_size=obj_size, ) + def _make_pidpath(self, pid, path_segments): + if len(path_segments) < 1: + return pid + return f"{pid} {hashstore.folderentry.PATH_DELIMITER.join(path_segments)}" + + def _split_pidpath(self, pidpath: str) -> tuple[str, list[str]]: + parts = pidpath.split(" ") + if len(parts) == 1: + return (parts[0], []) + return (parts[0], parts[1].split(hashstore.folderentry.PATH_DELIMITER)) + def retrieve_folder( self, pid: str, @@ -1181,11 +1193,12 @@ def retrieve_folder( Args: pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved - path (str): Path within the context to the desired entry + path (str): Path within the context to the desired entry, using folderentry.PATH_DELIMITER Returns: FolderEntries """ - if path in ("", ".", "/"): + delim = hashstore.folderentry.PATH_DELIMITER + if path in ("", ".", delim): path = "" folder_pid = f"{pid} {path}" if path != "" else pid self._check_string(folder_pid, "PID") @@ -1211,8 +1224,10 @@ def retrieve_folder( cid_path = object_info_dict.get("cid_object_path") # self._build_hashstore_data_object_path(folder_cid) current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - path_segments = path.split("/") # iterate over segments, saving last + current_pid = pid + path_segments = path.split(delim) + segment_index = 0 for name in path_segments: entry = current_folder.entry_by_name(name) if entry is None: @@ -1220,7 +1235,19 @@ def retrieve_folder( if entry.is_file: # it's a file! raise ValueError(f"Path {path} is a file.") - object_info_dict = self.find_object(pid) + segment_index += 1 + try: + # Does name form a registered PID? + object_info_dict = self.find_object(name) + # if so, then return follow + return self.retrieve_folder( + name, delim.join(path_segments[segment_index:]) + ) + except PidRefsDoesNotExist: + # continue but trye with pid + path + pass + folder_pid = self._make_pidpath(current_pid, path_segments[:segment_index]) + object_info_dict = self.find_object(name) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 3797f169..38e206b7 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -1,6 +1,5 @@ """Implements FolderEntry class.""" -import collections.abc import dataclasses import json import logging @@ -14,6 +13,7 @@ """Key in parquet file metadata pointing to dict of properties.""" PARQUET_READ_BATCH_SIZE = 10000 """Number of entries to read at a time from FolderEntries parquet file.""" +PATH_DELIMITER = "→" def get_logger(): From b1403b3f61832997089199daedffef5cc6eb3292 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 22:28:31 -0400 Subject: [PATCH 29/33] switch path delim, path as list instead of str --- src/hashstore/filehashstore.py | 92 ++++++++++++---------------------- src/hashstore/folderentry.py | 20 ++++++-- 2 files changed, 46 insertions(+), 66 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 50d992d4..27e6e85a 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1061,8 +1061,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: def store_folder( self, - pid: str, - path: str, + pathpid: list[str], entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, @@ -1079,11 +1078,12 @@ def store_folder( If FolderEntries have cid = '' | None, then the cid is looked up using - pid + " " + path + "/" + entry.name + pid + DELIM + path + DELIM + entry.name - The Folder is tagged with an identifier that is "{PID} {path}", that is, the - PID followed by a single space, then the path. If the path portion is an empty - string, ".", or "/" then the Folder is the root Folder. + The Folder is tagged with an identifier that is "{PID}{DELIM}{path}", that is, the + PID followed by a single delimiter, then the path. If the path portion is an empty + list or a list of length 1 with the first element ".", or DELIM then the Folder is + the root Folder within the context of PID. Note that since the hash of a Folder is computed from hashes of its content, a Folder hierarchy must be stored starting with the leaves. This method @@ -1095,7 +1095,7 @@ def store_folder( Args: pid (str): The context within which this folder is being stored - path (str): Path to this folder relative to the root. + path (list[str]): Path to this folder relative to the root. entries (list[FolderEntry]): A list of FolderEntry objects. verify_entry_cids: If True then FolderEntry CID values are verified to to ensure they exist in the hashstore. @@ -1103,17 +1103,23 @@ def store_folder( Returns: ObjectMetadata: The computed ObjectMetadata for this entry. """ - delim = hashstore.folderentry.PATH_DELIMITER - if path in ("", ".", delim): - path = "" - folder_pid = f"{pid} {path}" if path != "" else pid + # delim = hashstore.folderentry.PATH_DELIMITER + # if path in ("", ".", delim): + # path = "" + # folder_pid = f"{pid} {path}" if path != "" else pid + folder_pid = hashstore.folderentry.join_pathpid(pathpid) self._check_string(folder_pid, "PID") if verify_entry_cids: # check that each entry CID is present in the hashstore. for entry in entries: if entry.cid is None or entry.cid == "": # no cid provided, so look it up - _entry_pid = f"{folder_pid}{delim}{entry.name}" + _entry_pid = hashstore.folderentry.join_pathpid( + pathpid + + [ + entry.name, + ] + ) _meta = self.find_object(_entry_pid) entry.cid = _meta["cid"] else: @@ -1164,21 +1170,9 @@ def store_folder( obj_size=obj_size, ) - def _make_pidpath(self, pid, path_segments): - if len(path_segments) < 1: - return pid - return f"{pid} {hashstore.folderentry.PATH_DELIMITER.join(path_segments)}" - - def _split_pidpath(self, pidpath: str) -> tuple[str, list[str]]: - parts = pidpath.split(" ") - if len(parts) == 1: - return (parts[0], []) - return (parts[0], parts[1].split(hashstore.folderentry.PATH_DELIMITER)) - def retrieve_folder( self, - pid: str, - path: str, + pathpid: list[str], ) -> hashstore.folderentry.FolderEntries: """Retrieve a FolderEntries instance from the hashstore. @@ -1197,12 +1191,10 @@ def retrieve_folder( Returns: FolderEntries """ - delim = hashstore.folderentry.PATH_DELIMITER - if path in ("", ".", delim): - path = "" - folder_pid = f"{pid} {path}" if path != "" else pid + folder_pid = hashstore.folderentry.join_pathpid(pathpid) self._check_string(folder_pid, "PID") - # try direct reference to CID using folder_pid + # try direct reference to CID using folder_pid. This works if + # there is no branching to other PID contexts (typical case) try: object_info_dict = self.find_object(folder_pid) folder_cid = object_info_dict.get("cid") @@ -1214,46 +1206,24 @@ def retrieve_folder( except PidRefsDoesNotExist: pass - # otherwise, get the root, split the path, and start iterating. + # otherwise, iterate over the path, following a branch if needed. + # get the root, split the path, and start iterating. # This will raise PidRefsDoesNotExist if the root PID isn't there - object_info_dict = self.find_object(pid) + object_info_dict = self.find_object(pathpid[0]) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... raise PidRefsDoesNotExist("Entry has no cid?") cid_path = object_info_dict.get("cid_object_path") # self._build_hashstore_data_object_path(folder_cid) + # Get the root folder, then find the next path element in the folder current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - # iterate over segments, saving last - current_pid = pid - path_segments = path.split(delim) - segment_index = 0 - for name in path_segments: - entry = current_folder.entry_by_name(name) + for idx in range(1, len(pathpid)): + entry = current_folder.entry_by_name(pathpid[idx]) if entry is None: - raise KeyError(f"PID {pid} {path} not found.") - if entry.is_file: - # it's a file! - raise ValueError(f"Path {path} is a file.") - segment_index += 1 - try: - # Does name form a registered PID? - object_info_dict = self.find_object(name) - # if so, then return follow - return self.retrieve_folder( - name, delim.join(path_segments[segment_index:]) - ) - except PidRefsDoesNotExist: - # continue but trye with pid + path - pass - folder_pid = self._make_pidpath(current_pid, path_segments[:segment_index]) - object_info_dict = self.find_object(name) - folder_cid = object_info_dict.get("cid") - if folder_cid is None: - # Should never reach this... - raise PidRefsDoesNotExist("Entry has no cid?") - cid_path = object_info_dict.get("cid_object_path") - # cid_path = self._build_hashstore_data_object_path(folder_cid) + raise KeyError(f"PID {pathpid} not found.") + # given the entry, we have the cid. Use that to get the next folder + cid_path = self._get_hashstore_data_object_path(entry.cid) current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) return current_folder diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 38e206b7..92eda30a 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -13,13 +13,25 @@ """Key in parquet file metadata pointing to dict of properties.""" PARQUET_READ_BATCH_SIZE = 10000 """Number of entries to read at a time from FolderEntries parquet file.""" -PATH_DELIMITER = "→" +PATH_DELIMITER = "⫽" def get_logger(): return logging.getLogger("FolderEntry") +def split_pathpid(pathpid: str) -> list[str]: + pathpid = pathpid.strip(PATH_DELIMITER) + parts = pathpid.split(PATH_DELIMITER) + return parts + + +def join_pathpid(path: list[str]) -> str: + # remove "", strings with only white space + cleaned = [s.strip() for s in path] + return PATH_DELIMITER.join(list(filter(str.strip, cleaned))) + + @dataclasses.dataclass class FolderEntry: """Represents a file or folder entry in a folder manifest.""" @@ -69,9 +81,7 @@ def entry_by_name(self, name) -> FolderEntry | None: return entry return None - def to_parquet( - self, pq_path: str, pid: str | None = None, writer_args: dict = {} - ) -> int: + def to_parquet(self, pq_path: str, pid: str, writer_args: dict = {}) -> int: """Writes the list of folder entries to a parquet file. See also: https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html @@ -81,7 +91,7 @@ def to_parquet( args: pq_path: path to destination parquet file - pid: Optional PID+path used to create this folder. + pid: PID+path used to create this folder. writer_args: optional dict of arguments for the parquet writer. """ # Add some metadata to the parquet file to help identify it as a list of FolderEntries From 8e852e3320fedc035e79d80cb5202e6f69f64a3a Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 12 May 2026 07:55:42 -0400 Subject: [PATCH 30/33] Refactor to use path segments in api --- src/hashstore/__main__.py | 50 ++++++-------- src/hashstore/basehashstore.py | 43 ++++++++++-- src/hashstore/filehashstore.py | 121 ++++++++++++++++++++------------- src/hashstore/folderentry.py | 6 +- 4 files changed, 136 insertions(+), 84 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index abad4d6e..e6a4d49f 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -18,7 +18,7 @@ import hashstore import hashstore.filehashstore_exceptions -from hashstore.folderentry import PATH_DELIMITER +import hashstore.folderentry HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" @@ -345,23 +345,25 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference, list_only) @click.argument("pid", type=str) def get_folder_info(ctx, pid) -> None: """Compute basic stats for a folder and sub-folders.""" + logger = get_logger() - def iterate_folder(hs, stats, pid, path="", depth: int = 1): + def iterate_folder(hs, stats, path, depth: int = 1): + logger.debug("Current path=%s", path) if depth > stats["max_depth"]: stats["max_depth"] = depth - current_folder = hs.retrieve_folder(pid, path=path) + current_folder = hs.retrieve_folder(path) for entry in current_folder: - if entry.is_file == 0: + logger.debug(str(entry)) + if entry.is_file: stats["total_bytes"] = stats["total_bytes"] + entry.size stats["total_files"] += 1 else: - _path = ( - f"{path}{PATH_DELIMITER}{entry.name}" if path != "" else entry.name - ) stats["total_folders"] = stats["total_folders"] + 1 - iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) + _path = path + [ + entry.name, + ] + iterate_folder(hs, stats, _path, depth=depth + 1) - logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() @@ -380,12 +382,9 @@ def iterate_folder(hs, stats, pid, path="", depth: int = 1): "total_folders": 0, "max_depth": 0, } - parts = pid.split(" ", 1) - path = "" - if len(parts) > 1: - path = parts[1].strip() + path = hashstore.folderentry.split_pidpath(pid) - iterate_folder(hash_store, info, parts[0], path=path, depth=0) + iterate_folder(hash_store, info, path, depth=0) print(json.dumps(info, indent=2)) @@ -395,28 +394,24 @@ def iterate_folder(hs, stats, pid, path="", depth: int = 1): @click.option("-n", "--no-files", is_flag=True, help="Show folders but not content.") def get_folder_tree(ctx, pid: str, no_files: bool) -> None: """Generate a tree representation of the folder and sub-folders.""" + logger = get_logger() - def iterate_folder(hs, tree, pid, path="", with_files: bool = True): - current_folder = hs.retrieve_folder(pid, path=path) + def iterate_folder(hs, tree, path, with_files: bool = True): + current_folder = hs.retrieve_folder(path) n = 0 s = 0 for entry in current_folder: if not entry.is_file or with_files: branch = tree.add(entry.name) if not entry.is_file: - _path = ( - f"{path}{PATH_DELIMITER}{entry.name}" - if path != "" - else entry.name - ) - iterate_folder(hs, branch, pid, path=_path, with_files=with_files) + _path = path + [entry.name] + iterate_folder(hs, branch, _path, with_files=with_files) else: n += 1 s += entry.size if not with_files: branch = tree.add(f"{n:,} files, {s:,} bytes") - logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() @@ -428,12 +423,9 @@ def iterate_folder(hs, tree, pid, path="", with_files: bool = True): except Exception as e: logger.error(f"Failed to open hashstore: {e}") return 1 - parts = pid.split(" ", 1) - path = "" - if len(parts) > 1: - path = parts[1].strip() - tree = rich.tree.Tree(pid) - iterate_folder(hash_store, tree, parts[0], path=path, with_files=not no_files) + path = hashstore.folderentry.split_pidpath(pid) + tree = rich.tree.Tree(path[0]) + iterate_folder(hash_store, tree, path, with_files=not no_files) rich.print(tree) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 1b337545..93c50b34 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Generator, Optional, Union +from typing import IO, Generator, Optional, Union import hashstore.folderentry @@ -68,11 +68,33 @@ def store_object( """ raise NotImplementedError() + @abstractmethod + def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: + """Return object info dict given a path. + + A path may reference another path: + c_1 -> sub_1 -> c_0 -> sub_2 -> x + In such cases, the full path is not stored as a cidref, instead + we have: + path name + c_1 sub_1 + c_1, sub_1 c_0 <- change of context + c_0 sub_2 + c_0, sub_2 x + c_0, sub_2, x + + Hence it is necessary to walk the path to find the next context, + switch to that context, then continue looking for the target. + + An alternative strategy is to load the CID from each folder along + the path, but that is more IO and iterations to find the target. + """ + raise NotADirectoryError() + @abstractmethod def store_folder( self, - pid: str, - path: str, + pidpath: list[str], entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, @@ -115,8 +137,7 @@ def store_folder( @abstractmethod def retrieve_folder( self, - pid: str, - path: str, + pidpath: list[str], ) -> hashstore.folderentry.FolderEntries: """Retrieve a FolderEntries instance from the hashstore. @@ -182,6 +203,18 @@ def retrieve_object(self, pid): """ raise NotImplementedError() + @abstractmethod + def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: + """Retrieve an object from disk using a persistent identifier (pid). The `retrieve_object` + method opens and returns a buffered object stream ready for reading if the object + associated with the provided `pid` exists on disk. + + :param str pid: Authority-based identifier. + + :return: io.BufferedReader - Buffered stream of the data object. + """ + raise NotImplementedError() + @abstractmethod def retrieve_metadata(self, pid, format_id): """Retrieve the metadata object from disk using a persistent identifier (pid) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 27e6e85a..f0f21367 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1061,7 +1061,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: def store_folder( self, - pathpid: list[str], + pidpath: list[str], entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, @@ -1107,15 +1107,15 @@ def store_folder( # if path in ("", ".", delim): # path = "" # folder_pid = f"{pid} {path}" if path != "" else pid - folder_pid = hashstore.folderentry.join_pathpid(pathpid) + folder_pid = hashstore.folderentry.join_pidpath(pidpath) self._check_string(folder_pid, "PID") if verify_entry_cids: # check that each entry CID is present in the hashstore. for entry in entries: if entry.cid is None or entry.cid == "": # no cid provided, so look it up - _entry_pid = hashstore.folderentry.join_pathpid( - pathpid + _entry_pid = hashstore.folderentry.join_pidpath( + pidpath + [ entry.name, ] @@ -1170,62 +1170,89 @@ def store_folder( obj_size=obj_size, ) + def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: + """Return object info dict given a path. + + A path may reference another path: + c_1 -> sub_1 -> c_0 -> sub_2 -> x + In such cases, the full path is not stored as a cidref, instead + we have: + path name + c_1 sub_1 + c_1, sub_1 c_0 <- change of context + c_0 sub_2 + c_0, sub_2 x + c_0, sub_2, x + + Hence it is necessary to walk the path to find the next context, + switch to that context, then continue looking for the target. + + An alternative strategy is to load the CID from each folder along + the path, but that is more IO and iterations to find the target. + """ + pid = hashstore.folderentry.join_pidpath(pidpath) + self._check_string(pid, "PID") + # first try the literal path + try: + return self.find_object(pid) + except PidRefsDoesNotExist as e: + # End of the line + if len(pidpath) == 1: + raise e + pass + # Path changes context at some point (or doesn't exist) + context_switch = 1 + for cpos in range(1, len(pidpath)): + # walk the path to find where context switches + try: + current_pid = hashstore.folderentry.join_pidpath(pidpath[:cpos]) + _ = self.find_object(current_pid) + except PidRefsDoesNotExist: + # context switch, start over with segment from current context onwards + context_switch = cpos + break + return self.resolve_pidpath(pidpath[context_switch - 1 :]) + + def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: + object_info_dict = self.resolve_pidpath(pidpath) + object_cid = object_info_dict.get("cid") + entity = "objects" + if object_cid: + self.fhs_logger.debug( + "Metadata exists for pid: %s, retrieving object.", pidpath + ) + obj_stream = self._open(entity, object_cid) + else: + err_msg = f"No object found for pid: {pidpath}" + self.fhs_logger.error(err_msg) + raise ValueError(err_msg) + self.fhs_logger.info("Retrieved object for pid: %s", pidpath) + return obj_stream + def retrieve_folder( self, - pathpid: list[str], + pidpath: list[str], ) -> hashstore.folderentry.FolderEntries: """Retrieve a FolderEntries instance from the hashstore. - We first check to see if a CID is available for the combination of - "{PID} {path}", and if so, return that entry. Otherwise, we iterate - over path segments to find the correspoding FolderEntry, if any. - This iterative approach is necesary if since entire trees are not - stored when a new version of a folder hierarchy is stored. Hence, it - may be necessary to jump back to a branch that is recorded in an - earlier version but not recorded in the current version since it - was unchanged between versions. + Given a sequence of path segments to a FolderEntries object, + return the object. Args: - pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved - path (str): Path within the context to the desired entry, using folderentry.PATH_DELIMITER + pidpath (list[str]): Path segments to the folder Returns: FolderEntries + Raises: + PidRefsDoesNotExist """ - folder_pid = hashstore.folderentry.join_pathpid(pathpid) - self._check_string(folder_pid, "PID") - # try direct reference to CID using folder_pid. This works if - # there is no branching to other PID contexts (typical case) - try: - object_info_dict = self.find_object(folder_pid) - folder_cid = object_info_dict.get("cid") - if folder_cid is None: - raise PidRefsDoesNotExist("Entry has no cid?") - cid_path = object_info_dict.get("cid_object_path") - # self._build_hashstore_data_object_path(folder_cid) - return hashstore.folderentry.FolderEntries.from_parquet(cid_path) - except PidRefsDoesNotExist: - pass - - # otherwise, iterate over the path, following a branch if needed. - # get the root, split the path, and start iterating. - # This will raise PidRefsDoesNotExist if the root PID isn't there - object_info_dict = self.find_object(pathpid[0]) + # this will raise if pidpath isn't found + object_info_dict = self.resolve_pidpath(pidpath) + # have info, load the folder object from the cid path folder_cid = object_info_dict.get("cid") if folder_cid is None: - # Should never reach this... - raise PidRefsDoesNotExist("Entry has no cid?") + raise PidRefsDoesNotExist(f"Entry {pidpath} has no cid?") cid_path = object_info_dict.get("cid_object_path") - # self._build_hashstore_data_object_path(folder_cid) - # Get the root folder, then find the next path element in the folder - current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - for idx in range(1, len(pathpid)): - entry = current_folder.entry_by_name(pathpid[idx]) - if entry is None: - raise KeyError(f"PID {pathpid} not found.") - # given the entry, we have the cid. Use that to get the next folder - cid_path = self._get_hashstore_data_object_path(entry.cid) - current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - return current_folder + return hashstore.folderentry.FolderEntries.from_parquet(cid_path) def list_pids(self, pattern: Optional[str] = None) -> Generator: """Yield create_timestamp, CID, PID. diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 92eda30a..58470fcb 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -20,13 +20,13 @@ def get_logger(): return logging.getLogger("FolderEntry") -def split_pathpid(pathpid: str) -> list[str]: - pathpid = pathpid.strip(PATH_DELIMITER) +def split_pidpath(pidpath: str) -> list[str]: + pathpid = pidpath.strip(PATH_DELIMITER) parts = pathpid.split(PATH_DELIMITER) return parts -def join_pathpid(path: list[str]) -> str: +def join_pidpath(path: list[str]) -> str: # remove "", strings with only white space cleaned = [s.strip() for s in path] return PATH_DELIMITER.join(list(filter(str.strip, cleaned))) From 8c2a67346497fe542457e0f0132d2c44c4c4fe6d Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 12 May 2026 08:43:18 -0400 Subject: [PATCH 31/33] Fix recursion step, allow delimtier to be specified --- src/hashstore/__main__.py | 6 +++--- src/hashstore/filehashstore.py | 4 +++- src/hashstore/folderentry.py | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index e6a4d49f..2e8c7bd7 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -382,7 +382,7 @@ def iterate_folder(hs, stats, path, depth: int = 1): "total_folders": 0, "max_depth": 0, } - path = hashstore.folderentry.split_pidpath(pid) + path = hashstore.folderentry.split_pidpath(pid, delimiter="|") iterate_folder(hash_store, info, path, depth=0) print(json.dumps(info, indent=2)) @@ -423,8 +423,8 @@ def iterate_folder(hs, tree, path, with_files: bool = True): except Exception as e: logger.error(f"Failed to open hashstore: {e}") return 1 - path = hashstore.folderentry.split_pidpath(pid) - tree = rich.tree.Tree(path[0]) + path = hashstore.folderentry.split_pidpath(pid, delimiter="|") + tree = rich.tree.Tree(pid) iterate_folder(hash_store, tree, path, with_files=not no_files) rich.print(tree) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index f0f21367..a1a2f725 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1190,6 +1190,7 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: An alternative strategy is to load the CID from each folder along the path, but that is more IO and iterations to find the target. """ + self.fhs_logger.debug("Resolve: %s", pidpath) pid = hashstore.folderentry.join_pidpath(pidpath) self._check_string(pid, "PID") # first try the literal path @@ -1202,8 +1203,9 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: pass # Path changes context at some point (or doesn't exist) context_switch = 1 - for cpos in range(1, len(pidpath)): + for cpos in range(1, len(pidpath) + 1): # walk the path to find where context switches + self.fhs_logger.debug("At: %s", pidpath[:cpos]) try: current_pid = hashstore.folderentry.join_pidpath(pidpath[:cpos]) _ = self.find_object(current_pid) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 58470fcb..511b45e0 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -20,16 +20,16 @@ def get_logger(): return logging.getLogger("FolderEntry") -def split_pidpath(pidpath: str) -> list[str]: - pathpid = pidpath.strip(PATH_DELIMITER) - parts = pathpid.split(PATH_DELIMITER) +def split_pidpath(pidpath: str, delimiter: str = PATH_DELIMITER) -> list[str]: + pathpid = pidpath.strip(delimiter) + parts = pathpid.split(delimiter) return parts -def join_pidpath(path: list[str]) -> str: +def join_pidpath(path: list[str], delimiter: str = PATH_DELIMITER) -> str: # remove "", strings with only white space cleaned = [s.strip() for s in path] - return PATH_DELIMITER.join(list(filter(str.strip, cleaned))) + return delimiter.join(list(filter(str.strip, cleaned))) @dataclasses.dataclass From 07e32a707c6f9f45c91ba5bfa19ca0d541365653 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 14 May 2026 11:08:34 -0400 Subject: [PATCH 32/33] Initial store - WIP --- src/hashstore/__init__.py | 4 +- src/hashstore/basehashstore.py | 6 +++ src/hashstore/filehashstore.py | 78 +++++++++++++++++++++++++--------- src/hashstore/folderentry.py | 4 +- src/hashstore/pidlogger.py | 28 ++++++------ 5 files changed, 83 insertions(+), 37 deletions(-) diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index 6d470289..6e58b02b 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.basehashstore import HashStore, HashStoreFactory +from hashstore.basehashstore import HashStore, HashStoreFactory, PidObserver -__all__ = ("HashStore", "HashStoreFactory") +__all__ = ("HashStore", "HashStoreFactory", "PidObserver") __version__ = "1.1.0" diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 93c50b34..14c40fc0 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -9,6 +9,12 @@ import hashstore.folderentry +class PidObserver(ABC): + @abstractmethod + def update(self, cid: str, pid: str | None = None): + pass + + class HashStore(ABC): """HashStore is a content-addressable file management system that utilizes an object's content identifier (hex digest/checksum) to address files.""" diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index a1a2f725..6461bcb2 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -20,7 +20,7 @@ import yaml import hashstore.folderentry -from hashstore import HashStore, pidlogger +from hashstore import HashStore, PidObserver from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -85,6 +85,7 @@ class FileHashStore(HashStore): def __init__(self, properties=None): self.fhs_logger = logging.getLogger(__name__) + self.pid_watchers = [] # Now check properties if properties: # Validate properties against existing configuration if present @@ -134,8 +135,6 @@ def __init__(self, properties=None): self._create_path(self.refs / "tmp") self._create_path(self.refs / "pids") self._create_path(self.refs / "cids") - # pidlog is used to create an index of cid - pid - self.pidlog = logging.getLogger("pid_logger") # Variables to orchestrate parallelization # Check to see whether a multiprocessing or threading sync lock should be used @@ -518,6 +517,13 @@ def lookup_algo(algo_to_translate): # Public API / HashStore Interface Methods + def add_watcher(self, watcher: PidObserver) -> None: + self.pid_watchers.append(watcher) + + def notify(self, cid, pid): + for watcher in self.pid_watchers: + watcher.update(cid, pid) + def store_object( self, pid: Optional[str] = None, @@ -580,7 +586,7 @@ def store_object( cid = object_metadata.cid self.tag_object(pid, cid) self.fhs_logger.info("Successfully stored object for pid: %s", pid) - self.pidlog.info(cid, extra={"pid": pid}) + self.notify(cid, pid) finally: # Release pid self._release_object_locked_pids(pid) @@ -1162,7 +1168,7 @@ def store_folder( # of folder entries increases. obj_size = entries.to_parquet(cid_path, pid=folder_pid) self.tag_object(folder_pid, folder_cid) - self.pidlog.info(folder_cid, extra={"pid": folder_pid}) + self.notify(folder_cid, folder_pid) return ObjectMetadata( pid=folder_pid, cid=folder_cid, @@ -1173,6 +1179,10 @@ def store_folder( def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: """Return object info dict given a path. + #TODO: + # - write test cases for this. + # - consider adding a recursion trap + A path may reference another path: c_1 -> sub_1 -> c_0 -> sub_2 -> x In such cases, the full path is not stored as a cidref, instead @@ -1197,23 +1207,46 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: try: return self.find_object(pid) except PidRefsDoesNotExist as e: - # End of the line - if len(pidpath) == 1: + # End of the line ? + if len(pidpath) < 2: raise e - pass - # Path changes context at some point (or doesn't exist) - context_switch = 1 - for cpos in range(1, len(pidpath) + 1): - # walk the path to find where context switches - self.fhs_logger.debug("At: %s", pidpath[:cpos]) - try: - current_pid = hashstore.folderentry.join_pidpath(pidpath[:cpos]) - _ = self.find_object(current_pid) - except PidRefsDoesNotExist: - # context switch, start over with segment from current context onwards - context_switch = cpos - break - return self.resolve_pidpath(pidpath[context_switch - 1 :]) + # continue + # Does the root context exist? + try: + object_info_dict = self.find_object(pidpath[0]) + cid_object_path = object_info_dict.get("cid_object_path") + current_folder = hashstore.folderentry.FolderEntries.from_parquet( + cid_object_path + ) + except PidRefsDoesNotExist as e: + # nope + raise e + object_info_dict = { + "cid": None, + "cid_object_path": None, + "cid_refs_path": None, + "pid_refs_path": None, + "sysmeta_path": "Does not exist.", + } + entry = None + # walk the path, following cids referened by folder + for idx in range(1, len(pidpath)): + _name = pidpath[idx] + entry = current_folder.entry_by_name(_name) + if entry is None: + raise PidRefsDoesNotExist(f"PID not found: {pid}") + object_info_dict["cid"] = entry.cid + object_info_dict["cid_object_path"] = self._get_hashstore_data_object_path( + entry.cid + ) + object_info_dict["cid_ref_path"] = self._get_hashstore_cid_refs_path( + entry.cid + ) + if idx < len(pidpath): + current_folder = hashstore.folderentry.FolderEntries.from_parquet( + object_info_dict["cid_object_path"] + ) + return object_info_dict def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: object_info_dict = self.resolve_pidpath(pidpath) @@ -1262,6 +1295,9 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: Iterates over all CID entries and yields the create timestamp, CID value, and PID value for all entries or those PIDs that match the optionally provided regexp pattern. + + Note that this can be really slow when there's a large number of + ref files. """ rpattern = None if pattern is not None: diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 511b45e0..a4e4d15e 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -41,9 +41,9 @@ class FolderEntry: cid: str """The content hash (CID) for the entry.""" is_file: bool # True for file, False for Folder - """The type of manifest entry: '1' for file, '0' for directory.""" + """The type of manifest entry: False for folder, True for file.""" size: int = 0 - """Size of the file in bytes or number of entries for directories.""" + """Size of the file in bytes or number of entries for folders.""" formatid: str | None = None """Optional format identifier for files.""" diff --git a/src/hashstore/pidlogger.py b/src/hashstore/pidlogger.py index 1091e9c0..8901f7a7 100644 --- a/src/hashstore/pidlogger.py +++ b/src/hashstore/pidlogger.py @@ -26,6 +26,8 @@ import json import logging +from hashstore.basehashstore import PidObserver + class PidIndexFormatter(logging.Formatter): def format(self, record) -> str: @@ -37,15 +39,17 @@ def format(self, record) -> str: return json.dumps(pid_record) -def getPidLogger(log_file_name: str): - """Return a logger for the pid index. - - PID records are added to the index like: - logger.info(CID, extra={"pid":PID}) - """ - logger = logging.getLogger("pid_logger") - logger.setLevel(logging.INFO) - handler = logging.FileHandler(log_file_name) - handler.setFormatter(PidIndexFormatter()) - logger.addHandler(handler) - return logger +class PidLogObserver(PidObserver): + def __init__(self, log_file_name: str | None = None): + self.logger = logging.getLogger("pid_logger") + self.logger.handlers.clear() + self.logger.propagate = False + if log_file_name is None: + return + self.logger.setLevel(logging.INFO) + handler = logging.FileHandler(log_file_name) + handler.setFormatter(PidIndexFormatter()) + self.logger.addHandler(handler) + + def update(self, cid: str, pid: str | None = None): + self.logger.info(cid, extra={"pid": pid}) From fa1eb13a04b07f574a9422b4d3b73fa0ee8103aa Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 14 May 2026 11:58:47 -0400 Subject: [PATCH 33/33] Limit code changes to folder support --- src/hashstore/__init__.py | 4 +-- src/hashstore/basehashstore.py | 6 ---- src/hashstore/filehashstore.py | 44 ++++++++++----------------- src/hashstore/pidlogger.py | 55 ---------------------------------- 4 files changed, 17 insertions(+), 92 deletions(-) delete mode 100644 src/hashstore/pidlogger.py diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index 6e58b02b..6d470289 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.basehashstore import HashStore, HashStoreFactory, PidObserver +from hashstore.basehashstore import HashStore, HashStoreFactory -__all__ = ("HashStore", "HashStoreFactory", "PidObserver") +__all__ = ("HashStore", "HashStoreFactory") __version__ = "1.1.0" diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 14c40fc0..93c50b34 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -9,12 +9,6 @@ import hashstore.folderentry -class PidObserver(ABC): - @abstractmethod - def update(self, cid: str, pid: str | None = None): - pass - - class HashStore(ABC): """HashStore is a content-addressable file management system that utilizes an object's content identifier (hex digest/checksum) to address files.""" diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 6461bcb2..cfadf451 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -20,7 +20,7 @@ import yaml import hashstore.folderentry -from hashstore import HashStore, PidObserver +from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -85,7 +85,6 @@ class FileHashStore(HashStore): def __init__(self, properties=None): self.fhs_logger = logging.getLogger(__name__) - self.pid_watchers = [] # Now check properties if properties: # Validate properties against existing configuration if present @@ -517,13 +516,6 @@ def lookup_algo(algo_to_translate): # Public API / HashStore Interface Methods - def add_watcher(self, watcher: PidObserver) -> None: - self.pid_watchers.append(watcher) - - def notify(self, cid, pid): - for watcher in self.pid_watchers: - watcher.update(cid, pid) - def store_object( self, pid: Optional[str] = None, @@ -586,7 +578,6 @@ def store_object( cid = object_metadata.cid self.tag_object(pid, cid) self.fhs_logger.info("Successfully stored object for pid: %s", pid) - self.notify(cid, pid) finally: # Release pid self._release_object_locked_pids(pid) @@ -1168,7 +1159,6 @@ def store_folder( # of folder entries increases. obj_size = entries.to_parquet(cid_path, pid=folder_pid) self.tag_object(folder_pid, folder_cid) - self.notify(folder_cid, folder_pid) return ObjectMetadata( pid=folder_pid, cid=folder_cid, @@ -1179,14 +1169,14 @@ def store_folder( def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: """Return object info dict given a path. - #TODO: - # - write test cases for this. - # - consider adding a recursion trap - A path may reference another path: c_1 -> sub_1 -> c_0 -> sub_2 -> x + + Similarly, a path rooted in on PID may reference a path rooted in another PID. + In such cases, the full path is not stored as a cidref, instead - we have: + we have in the example above: + path name c_1 sub_1 c_1, sub_1 c_0 <- change of context @@ -1194,11 +1184,8 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: c_0, sub_2 x c_0, sub_2, x - Hence it is necessary to walk the path to find the next context, - switch to that context, then continue looking for the target. - - An alternative strategy is to load the CID from each folder along - the path, but that is more IO and iterations to find the target. + Hence it is necessary to walk the path following folder entry CIDs + to locate the target if it is not directly resolvable by the full path. """ self.fhs_logger.debug("Resolve: %s", pidpath) pid = hashstore.folderentry.join_pidpath(pidpath) @@ -1210,8 +1197,14 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: # End of the line ? if len(pidpath) < 2: raise e - # continue # Does the root context exist? + object_info_dict = { + "cid": None, + "cid_object_path": None, + "cid_refs_path": None, + "pid_refs_path": None, + "sysmeta_path": "Does not exist.", + } try: object_info_dict = self.find_object(pidpath[0]) cid_object_path = object_info_dict.get("cid_object_path") @@ -1221,13 +1214,6 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: except PidRefsDoesNotExist as e: # nope raise e - object_info_dict = { - "cid": None, - "cid_object_path": None, - "cid_refs_path": None, - "pid_refs_path": None, - "sysmeta_path": "Does not exist.", - } entry = None # walk the path, following cids referened by folder for idx in range(1, len(pidpath)): diff --git a/src/hashstore/pidlogger.py b/src/hashstore/pidlogger.py deleted file mode 100644 index 8901f7a7..00000000 --- a/src/hashstore/pidlogger.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Implements a multi-process safe logger for generating a cid-pid index. - -This implementation writes an NDJSON file with each line containing a json array -with elements: - [0] timestamp - [1] cid - [2] pid - -The logging module is used because it is multi-thread and -process safe. - -The resulting ndjson file can be loaded into duckdb for example with: - - CREATE TABLE pids AS SELECT - to_timestamp(json[1]::DOUBLE) AS ctime, - json[2]->> '$' AS cid, - json[3]->>'$' AS pid - FROM read_json('pid_index.ndjson'); - -or create a parquet representation: - - duckdb -c "COPY (SELECT to_timestamp(json[1]::DOUBLE) AS ctime, - json[2]->> '\\$' AS cid, json[3]->>'\\$' AS pid FROM - read_json('pid_index.ndjson')) TO 'pid_index.parquet' (FORMAT parquet)" -""" - -import json -import logging - -from hashstore.basehashstore import PidObserver - - -class PidIndexFormatter(logging.Formatter): - def format(self, record) -> str: - pid_record = ( - record.created, - record.getMessage().strip(), - record.pid if hasattr(record, "pid") else None, - ) - return json.dumps(pid_record) - - -class PidLogObserver(PidObserver): - def __init__(self, log_file_name: str | None = None): - self.logger = logging.getLogger("pid_logger") - self.logger.handlers.clear() - self.logger.propagate = False - if log_file_name is None: - return - self.logger.setLevel(logging.INFO) - handler = logging.FileHandler(log_file_name) - handler.setFormatter(PidIndexFormatter()) - self.logger.addHandler(handler) - - def update(self, cid: str, pid: str | None = None): - self.logger.info(cid, extra={"pid": pid})