From 00c1c64b92e02ea5b574ed955438e9dfcf32f218 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:08:05 -0400 Subject: [PATCH 1/5] chore: replace hatch environments with uv + ci scripts --- .github/workflows/codspeed.yml | 13 ++- .github/workflows/docs.yml | 16 +-- .github/workflows/gpu_test.yml | 24 +---- .github/workflows/hypothesis.yaml | 23 +---- .github/workflows/lint.yml | 5 + .github/workflows/test.yml | 94 +++++++----------- changes/4096.misc.md | 1 + ci/run-coverage.sh | 16 +++ ci/run-hypothesis.sh | 12 +++ ci/sync-upstream.sh | 18 ++++ docs/contributing.md | 40 +++++--- pyproject.toml | 158 ++++-------------------------- uv.lock | 2 +- 13 files changed, 160 insertions(+), 262 deletions(-) create mode 100644 changes/4096.misc.md create mode 100644 ci/run-coverage.sh create mode 100644 ci/run-hypothesis.sh create mode 100644 ci/sync-upstream.sh diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 115b9d0bf7..3574d4e7d6 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -27,12 +27,17 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - version: '1.16.5' + enable-cache: true + python-version: '3.12' + # Pre-build the locked env so the measured `run` below is just pytest, not a + # cold uv sync — keeps the walltime sample clean. + - name: Sync locked benchmark env + run: uv sync --locked --no-default-groups --group test -p 3.12 - name: Run the benchmarks uses: CodSpeedHQ/action@c145068895e045cc725ee76fcd2307624b65c3af # v4.17.5 with: mode: walltime - run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed + run: uv run --no-sync -p 3.12 pytest tests/benchmarks --codspeed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ab745beec1..dc55d44d8b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,13 +23,17 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - run: uv sync --group docs - # --strict turns warnings into errors, so a docs code block that fails to execute - # at build time (e.g. a non-exec python fence disrupting a later exec="true" block) - # fails CI instead of merging as a silent warning. - - run: uv run mkdocs build --strict + with: + python-version: '3.12' + # Mirror the former hatch `docs` env exactly: remote extra + docs group, + # from the universal lock. --strict turns warnings into errors, so a docs + # code block that fails to execute at build time (e.g. a non-exec python + # fence disrupting a later exec="true" block) fails CI instead of merging + # as a silent warning. + - run: uv sync --locked --no-default-groups --extra remote --group docs + - run: uv run --no-sync mkdocs build --strict env: DISABLE_MKDOCS_2_WARNING: "true" NO_MKDOCS_2_WARNING: "true" - - run: uv run python ci/check_unlinked_types.py + - run: uv run --no-sync python ci/check_unlinked_types.py continue-on-error: true diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index f27307efff..9129549a43 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -12,8 +12,6 @@ on: env: LD_LIBRARY_PATH: /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64 - # Use the uv from astral-sh/setup-uv instead of hatch's bundled (pyapp) uv. - HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv permissions: contents: read @@ -56,28 +54,16 @@ jobs: echo $PATH echo $LD_LIBRARY_PATH nvcc -V - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc with: - version: '1.16.5' - - name: Set Up Hatch Env - env: - HATCH_ENV: gputest.py${{ matrix.python-version }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + enable-cache: true + python-version: ${{ matrix.python-version }} - name: Run Tests - env: - HATCH_ENV: gputest.py${{ matrix.python-version }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + uv sync --locked --no-default-groups --group test --extra gpu --extra optional + uv pip install pytest-examples + bash ci/run-coverage.sh -m gpu - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index d15b5b1405..2050a8edbf 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -18,8 +18,6 @@ concurrency: env: FORCE_COLOR: 3 - # Use the uv from astral-sh/setup-uv instead of hatch's bundled (pyapp) uv. - HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv jobs: @@ -51,23 +49,11 @@ jobs: else echo "HYPOTHESIS_PROFILE=ci" >> $GITHUB_ENV fi - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc with: - version: '1.16.5' - - name: Set Up Hatch Env - env: - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + enable-cache: true + python-version: ${{ matrix.python-version }} # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - name: Restore cached hypothesis directory id: restore-hypothesis-cache @@ -81,11 +67,10 @@ jobs: - name: Run slow Hypothesis tests if: success() id: status - env: - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" - hatch env run --env "$HATCH_ENV" run-hypothesis + uv sync --locked --no-default-groups --group test --group remote-tests --extra remote --extra optional --extra cli --extra cast-value-rs + bash ci/run-hypothesis.sh # explicitly save the cache so it gets updated, also do this even if it fails. - name: Save cached hypothesis directory diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 32e521377d..50a53a6945 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,9 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: enable-cache: true + # Fail fast if uv.lock is stale vs pyproject.toml. Every test job installs + # with `uv sync --locked`, so a drifted lock would otherwise only surface + # as a confusing failure there. + - name: Check uv.lock is up to date + run: uv lock --check - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51ce958f90..1f6551c7cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,11 +13,6 @@ on: permissions: contents: read -env: - # Use the uv from astral-sh/setup-uv; without an explicit path hatch - # bootstraps its own (pyapp) uv, which fails on non-3.12 runners. - HATCH_ENV_TYPE_VIRTUAL_UV_PATH: uv - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -60,27 +55,26 @@ jobs: with: fetch-depth: 0 # grab all branches and tags persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 - - name: Set Up Hatch Env - env: - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + # One universal uv.lock serves every os/python cell of this matrix; + # `uv sync --locked` fails if it is stale vs pyproject.toml — a + # cross-platform drift gate. setup-uv's python-version sets UV_PYTHON, so uv + # selects the right interpreter. The shared coverage runner lives in + # ci/run-coverage.sh so CI and local dev invoke an identical command. - name: Run Tests env: HYPOTHESIS_PROFILE: ci - HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + EXTRAS="" + if [ "${{ matrix.dependency-set }}" = "optional" ]; then + EXTRAS="--extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests" + fi + uv sync --locked --no-default-groups --group test $EXTRAS + bash ci/run-coverage.sh - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 @@ -109,26 +103,22 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 - - name: Set Up Hatch Env - env: - HATCH_ENV: ${{ matrix.dependency-set }} - run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + # Both intentionally unlocked: min_deps derives floors via uv's --resolution + # lowest-direct (replacing the hand-pinned tool.hatch.envs.min_deps); + # upstream overlays nightly + git mains (ci/sync-upstream.sh). - name: Run Tests - env: - HATCH_ENV: ${{ matrix.dependency-set }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + if [ "${{ matrix.dependency-set }}" = "min_deps" ]; then + uv sync --resolution lowest-direct --no-default-groups --group test --group remote-tests --extra remote --extra optional + else + bash ci/sync-upstream.sh + fi + bash ci/run-coverage.sh - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: @@ -144,21 +134,16 @@ jobs: with: fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 - - name: Set Up Hatch Env - run: | - hatch run doctest:pip list - - name: Run Tests + with: + enable-cache: true + python-version: '3.13' + - name: Run doctests run: | - hatch run doctest:test + uv sync --locked --no-default-groups --extra remote --group remote-tests + uv pip list + uv run --no-sync --with pytest-examples pytest tests/test_docs.py -v benchmarks: name: Benchmark smoke test @@ -168,18 +153,15 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - cache: 'pip' - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - - name: Install Hatch - run: python -m pip install hatch==1.16.5 + with: + enable-cache: true + python-version: '3.13' - name: Run Benchmarks run: | - hatch env run --env "test.py3.13-minimal" run-benchmark + uv sync --locked --no-default-groups --group test + uv run --no-sync pytest --benchmark-enable tests/benchmarks test-complete: name: Test complete diff --git a/changes/4096.misc.md b/changes/4096.misc.md new file mode 100644 index 0000000000..baea81738c --- /dev/null +++ b/changes/4096.misc.md @@ -0,0 +1 @@ +Replaced hatch with uv for development environments and CI task management. Contributors now create the development environment with `uv sync --locked` and run tests with `uv run pytest` (or the shared runners in `ci/`, e.g. `bash ci/run-coverage.sh`); see the contributing guide for details. The `tool.hatch.envs.*` environments and their run scripts were removed from `pyproject.toml`, and a committed `uv.lock` now pins the full transitive dependency closure for reproducible, cross-platform CI. hatchling remains the build backend, so package builds and version derivation are unchanged. diff --git a/ci/run-coverage.sh b/ci/run-coverage.sh new file mode 100644 index 0000000000..814b77be64 --- /dev/null +++ b/ci/run-coverage.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Coverage-instrumented pytest run + XML report — the former hatch `run-coverage` +# script, shared by the test / upstream / min_deps / gpu CI jobs and runnable +# locally. Assumes the target environment is already synced (e.g. +# `uv sync --locked --group test`). Extra args are forwarded to pytest, e.g. +# bash ci/run-coverage.sh -m gpu +set -euo pipefail + +# In CI, log the resolved environment so version issues are debuggable. +# In GitHub Actions, the CI environment variable is always set to true. +if [ -n "${CI:-}" ]; then uv pip list; fi + +uv run --no-sync coverage run --source=src -m pytest \ + --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy "$@" +uv run --no-sync coverage xml diff --git a/ci/run-hypothesis.sh b/ci/run-hypothesis.sh new file mode 100644 index 0000000000..5009c38f27 --- /dev/null +++ b/ci/run-hypothesis.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Slow Hypothesis run — the former hatch `run-hypothesis` script. Assumes the +# target environment is already synced. Extra args are forwarded to pytest. +set -euo pipefail + +# In CI, log the resolved environment so version issues are debuggable. +# In GitHub Actions, the CI environment variable is always set to true. +if [ -n "${CI:-}" ]; then uv pip list; fi + +uv run --no-sync coverage run --source=src -m pytest -nauto \ + --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* "$@" +uv run --no-sync coverage xml diff --git a/ci/sync-upstream.sh b/ci/sync-upstream.sh new file mode 100644 index 0000000000..3bc6ae4b94 --- /dev/null +++ b/ci/sync-upstream.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Bleeding-edge environment — the former hatch `tool.hatch.envs.upstream`. +# Latest test deps + nightly numpy + git mains of the core stack. Intentionally +# unlocked (no `--locked`): this job exists to catch upstream breakage early. +set -euo pipefail + +uv sync --no-default-groups --group test --group remote-tests --extra remote +uv pip install --prerelease=allow \ + --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + numpy \ + "packaging @ git+https://github.com/pypa/packaging" \ + "numcodecs @ git+https://github.com/zarr-developers/numcodecs" \ + "s3fs @ git+https://github.com/fsspec/s3fs" \ + "universal_pathlib @ git+https://github.com/fsspec/universal_pathlib" \ + "typing_extensions @ git+https://github.com/python/typing_extensions" \ + "donfig @ git+https://github.com/pytroll/donfig" \ + "obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore" diff --git a/docs/contributing.md b/docs/contributing.md index 750f7c7a65..fcf724e9e7 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -80,17 +80,18 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git ### Creating a development environment -To work with the Zarr source code, it is recommended to use [hatch](https://hatch.pypa.io/latest/index.html) to create and manage development environments. Hatch will automatically install all Zarr dependencies using the same versions as are used by the core developers and continuous integration services. Assuming you have a Python 3 interpreter already installed, and you have cloned the Zarr source code and your current working directory is the root of the repository, you can do something like the following: +To work with the Zarr source code, we recommend [uv](https://docs.astral.sh/uv/) to create and manage development environments. uv installs all Zarr dependencies pinned to the same versions used by continuous integration, via the committed `uv.lock`. Assuming you have [installed uv](https://docs.astral.sh/uv/getting-started/installation/), cloned the Zarr source code, and your current working directory is the root of the repository, run: ```bash -pip install hatch -hatch env show # list all available environments +uv sync --locked # create .venv/ with Zarr and all development dependencies ``` -To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.: +This creates a virtual environment in `.venv/`. Prefix commands with `uv run` to run them inside it, or activate it directly with `source .venv/bin/activate`. To use a specific Python version, pass `-p`, e.g. `uv sync --locked -p 3.13`. + +To verify that your development environment is working, you can run the unit tests: ```bash -hatch env run --env test.py3.12-optional run +uv run pytest ``` ### Creating a branch @@ -125,10 +126,17 @@ Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite -Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: +Zarr includes a suite of unit tests. The simplest way to run them is within your development environment (see [creating a development environment](#creating-a-development-environment) above): + +```bash +uv run pytest +``` + +To reproduce a CI test environment exactly — the optional integration dependencies, with coverage — sync that environment and run the shared coverage script: ```bash -hatch env run --env test.py3.12-optional run +uv sync --locked --no-default-groups --group test --extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests +bash ci/run-coverage.sh ``` All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. @@ -187,18 +195,18 @@ If you would like to skip the failing checks and push the code for further discu > **Note:** Test coverage for Zarr-Python 3 is currently not at 100%. This is a known issue and help is welcome to bring test coverage back to 100%. See issue #2613 for more details. -Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: +Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. From your development environment, running: ```bash -hatch env run --env test.py3.12-optional run-coverage +bash ci/run-coverage.sh ``` -will automatically run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. +will run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. -You can also generate an HTML coverage report by running: +You can then generate an HTML coverage report from the collected data by running: ```bash -hatch env run --env test.py3.12-optional run-coverage-html +uv run --no-sync coverage html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. @@ -212,15 +220,15 @@ Zarr uses mkdocs for documentation, hosted on readthedocs.org. Documentation is The documentation can be built locally by running: ```bash -hatch --env docs run build +uv run --group docs --extra remote mkdocs build ``` -The resulting built documentation will be available in the `docs/_build/html` folder. +The resulting built documentation will be available in the `site/` folder. -Hatch can also be used to serve continuously updating version of the documentation during development at [http://0.0.0.0:8000/](http://0.0.0.0:8000/). This can be done by running: +You can also serve a continuously updating version of the documentation during development at [http://0.0.0.0:8000/](http://0.0.0.0:8000/) by running: ```bash -hatch --env docs run serve +uv run --group docs --extra remote mkdocs serve --watch src ``` #### Adding executable code blocks in the documentation diff --git a/pyproject.toml b/pyproject.toml index 02e66c67e8..2ec3cd8aec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,9 @@ gpu = [ ] cast-value-rs = ["cast-value-rs"] cli = ["typer"] -optional = ["universal-pathlib"] +# Floor is the minimum we support and test (exercised by the `min_deps` CI job +# via `uv --resolution lowest-direct`); was the old `min_deps` hatch pin. +optional = ["universal-pathlib>=0.2.0"] [project.scripts] zarr = "zarr._cli.cli:app" @@ -83,15 +85,15 @@ Discussions = "https://github.com/zarr-developers/zarr-python/discussions" documentation = "https://zarr.readthedocs.io/" homepage = "https://github.com/zarr-developers/zarr-python" -# Dev *tooling* is pinned to exact versions for reproducible CI: the hatch envs -# (see `tool.hatch.envs.*`) and bare `uv run` resolve these groups fresh from -# PyPI and do NOT consult uv.lock, so an unrelated tooling release can break CI -# without any change on our side (e.g. the pytest 9.1.0 `duplicate -# parametrization` regression). Runtime/integration deps (fsspec, obstore, s3fs, -# botocore, numcodecs, universal-pathlib) are intentionally left floating so the -# `optional` test matrix keeps exercising their latest releases; their floor and -# bleeding edge are covered by the `min_deps` and `upstream` hatch envs. Bump the -# pins deliberately, e.g. via dependabot or `uv lock --upgrade`. +# Dev *tooling* is pinned to exact versions for reproducible CI, and the full +# transitive closure is locked in `uv.lock` (CI installs with `uv sync --locked`). +# Runtime/integration deps (fsspec, obstore, s3fs, botocore, numcodecs, +# universal-pathlib) instead carry floor (`>=`) constraints, so the `optional` +# test matrix keeps exercising their latest releases; their floor is covered by +# the `min_deps` CI job (`uv sync --resolution lowest-direct`, which derives the +# floors from these `>=` constraints) and their bleeding edge by the `upstream` +# job (`scripts/sync-upstream.sh`). Bump the pins deliberately, e.g. via +# dependabot or `uv lock --upgrade`. [dependency-groups] test = [ "coverage==7.14.1", @@ -170,137 +172,10 @@ version.raw-options = { git_describe_command = "git describe --dirty --tags --lo [tool.hatch.build] hooks.vcs.version-file = "src/zarr/_version.py" -[tool.hatch.envs.dev] -dependency-groups = ["dev"] - -[tool.hatch.envs.test] -dependency-groups = ["test"] - -[tool.hatch.envs.test.env-vars] - -[[tool.hatch.envs.test.matrix]] -python = ["3.12", "3.13", "3.14"] -deps = ["minimal", "optional"] - -[tool.hatch.envs.test.overrides] -matrix.deps.features = [ - {value = "remote", if = ["optional"]}, - {value = "optional", if = ["optional"]}, - {value = "cli", if = ["optional"]}, - {value = "cast-value-rs", if = ["optional"]}, -] -matrix.deps.dependency-groups = [ - {value = "remote-tests", if = ["optional"]}, -] - -[tool.hatch.envs.test.scripts] -run-coverage = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy {args:}", - "coverage xml", -] -run-coverage-html = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks {args:}", - "coverage html", -] -run = "pytest --ignore tests/benchmarks" -run-verbose = "run-coverage --verbose" -run-hypothesis = [ - "coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* {args:}", - "coverage xml", -] -run-benchmark = "pytest --benchmark-enable tests/benchmarks" -serve-coverage-html = "python -m http.server -d htmlcov 8000" -list-env = "pip list" - -[tool.hatch.envs.gputest] -template = "test" -extra-dependencies = [ - "universal_pathlib", - # Needed so tests/test_docs.py is collectable under `pytest -m gpu`; otherwise its - # module-level importorskip("pytest_examples") skips the whole module and the gpu - # docs example is never executed on GPU hardware. - "pytest-examples", -] -features = ["gpu"] - -[[tool.hatch.envs.gputest.matrix]] -python = ["3.12", "3.13"] - -[tool.hatch.envs.gputest.scripts] -run-coverage = [ - "coverage run --source=src -m pytest -m gpu --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks {args:}", - "coverage xml", -] -run = "pytest -m gpu --ignore tests/benchmarks" - -[tool.hatch.envs.upstream] -template = 'test' -python = "3.14" -extra-dependencies = [ - 'packaging @ git+https://github.com/pypa/packaging', - 'numpy', # from scientific-python-nightly-wheels - 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', - 's3fs @ git+https://github.com/fsspec/s3fs', - 'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib', - 'typing_extensions @ git+https://github.com/python/typing_extensions', - 'donfig @ git+https://github.com/pytroll/donfig', - 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', -] - -[tool.hatch.envs.upstream.env-vars] -PIP_INDEX_URL = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/" -PIP_EXTRA_INDEX_URL = "https://pypi.org/simple/" -PIP_PRE = "1" - -[tool.hatch.envs.min_deps] -description = """Test environment for minimum supported dependencies - -See Spec 0000 for details and drop schedule: https://scientific-python.org/specs/spec-0000/ -""" -template = "test" -python = "3.12" -features = ["remote"] -dependency-groups = ["remote-tests"] -extra-dependencies = [ - 'packaging==22.*', - 'numpy==2.0.*', - 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs - 'fsspec==2023.10.0', - 's3fs==2023.10.0', - 'universal_pathlib==0.2.0', - 'typing_extensions==4.14.*', - 'donfig==0.8.*', - 'obstore==0.5.*', -] - -[tool.hatch.envs.default] -installer = "uv" - -[tool.hatch.envs.docs] -features = ['remote'] -dependency-groups = ['docs'] - -[tool.hatch.envs.docs.env-vars] -DISABLE_MKDOCS_2_WARNING = "true" -NO_MKDOCS_2_WARNING = "true" - -[tool.hatch.envs.docs.scripts] -serve = "mkdocs serve --watch src" -build = "mkdocs build" -check = "mkdocs build --strict" -readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" - -[tool.hatch.envs.doctest] -description = "Test environment for validating executable code blocks in documentation" -features = ['remote'] -dependency-groups = ['remote-tests'] -extra-dependencies = [ - "pytest-examples", -] - -[tool.hatch.envs.doctest.scripts] -test = "pytest tests/test_docs.py -v" -list-env = "pip list" +# NOTE: `tool.hatch.envs.*` used to define the test/docs/min_deps/upstream +# environments and their run scripts. Those are gone: CI now drives `uv` directly +# (`uv sync --locked ...`) with the shared runners in `scripts/`, and the dep sets +# (extras/groups) live in `.github/workflows/`. See docs/contributing.md. [tool.ruff] line-length = 100 @@ -471,6 +346,7 @@ ignore = [ "PC170", # use PyGrep hooks - no *.rst files to check "PC180", # for JavaScript - not interested "PC902", # pre-commit.ci custom autofix message - not using autofix + "PY007", # task running is uv + ci/*.sh shell scripts, not nox/tox/hatch envs ] [tool.numpydoc_validation] diff --git a/uv.lock b/uv.lock index 799ea6e45a..e66e131b20 100644 --- a/uv.lock +++ b/uv.lock @@ -4084,7 +4084,7 @@ requires-dist = [ { name = "packaging", specifier = ">=22.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, - { name = "universal-pathlib", marker = "extra == 'optional'" }, + { name = "universal-pathlib", marker = "extra == 'optional'", specifier = ">=0.2.0" }, ] provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] From 98dcde719834bcd6fc010ffc4b07be5ae02fc2e4 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:19:17 -0400 Subject: [PATCH 2/5] chore: use just --- .github/workflows/docs.yml | 14 +-- .github/workflows/gpu_test.yml | 5 +- .github/workflows/hypothesis.yaml | 3 +- .github/workflows/test.yml | 42 +++------ Justfile | 140 ++++++++++++++++++++++++++++++ changes/4096.misc.md | 2 +- ci/run-coverage.sh | 16 ---- ci/run-hypothesis.sh | 12 --- ci/sync-upstream.sh | 18 ---- docs/contributing.md | 27 ++++-- pyproject.toml | 10 +-- 11 files changed, 183 insertions(+), 106 deletions(-) create mode 100644 Justfile delete mode 100644 ci/run-coverage.sh delete mode 100644 ci/run-hypothesis.sh delete mode 100644 ci/sync-upstream.sh diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dc55d44d8b..97fea8ec3f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,15 +25,9 @@ jobs: - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: python-version: '3.12' - # Mirror the former hatch `docs` env exactly: remote extra + docs group, - # from the universal lock. --strict turns warnings into errors, so a docs - # code block that fails to execute at build time (e.g. a non-exec python - # fence disrupting a later exec="true" block) fails CI instead of merging - # as a silent warning. - - run: uv sync --locked --no-default-groups --extra remote --group docs - - run: uv run --no-sync mkdocs build --strict - env: - DISABLE_MKDOCS_2_WARNING: "true" - NO_MKDOCS_2_WARNING: "true" + # `just docs-build` runs `mkdocs build --strict` (warnings are errors, so a + # docs code block that fails to execute at build time fails CI) against the + # locked docs env. check_unlinked_types runs in that same synced env. + - run: uvx --from rust-just just docs-build - run: uv run --no-sync python ci/check_unlinked_types.py continue-on-error: true diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 9129549a43..d4306680c9 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -60,10 +60,7 @@ jobs: enable-cache: true python-version: ${{ matrix.python-version }} - name: Run Tests - run: | - uv sync --locked --no-default-groups --group test --extra gpu --extra optional - uv pip install pytest-examples - bash ci/run-coverage.sh -m gpu + run: uvx --from rust-just just gpu - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index 2050a8edbf..38ad64d78b 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -69,8 +69,7 @@ jobs: id: status run: | echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" - uv sync --locked --no-default-groups --group test --group remote-tests --extra remote --extra optional --extra cli --extra cast-value-rs - bash ci/run-hypothesis.sh + uvx --from rust-just just hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. - name: Save cached hypothesis directory diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f6551c7cb..a37d1a35cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,21 +60,16 @@ jobs: with: enable-cache: true python-version: ${{ matrix.python-version }} - # One universal uv.lock serves every os/python cell of this matrix; - # `uv sync --locked` fails if it is stale vs pyproject.toml — a - # cross-platform drift gate. setup-uv's python-version sets UV_PYTHON, so uv - # selects the right interpreter. The shared coverage runner lives in - # ci/run-coverage.sh so CI and local dev invoke an identical command. + # The `just` recipe (see Justfile) runs `uv sync --locked`, so one universal + # uv.lock serves every os/python cell — and it fails if the lock is stale vs + # pyproject.toml (a cross-platform drift gate). setup-uv's python-version + # sets UV_PYTHON so uv picks the right interpreter. CI and local dev invoke + # the identical recipe; `uvx --from rust-just` provides `just` without a + # separate install. - name: Run Tests env: HYPOTHESIS_PROFILE: ci - run: | - EXTRAS="" - if [ "${{ matrix.dependency-set }}" = "optional" ]; then - EXTRAS="--extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests" - fi - uv sync --locked --no-default-groups --group test $EXTRAS - bash ci/run-coverage.sh + run: uvx --from rust-just just test-${{ matrix.dependency-set }} - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 @@ -108,17 +103,11 @@ jobs: with: enable-cache: true python-version: ${{ matrix.python-version }} - # Both intentionally unlocked: min_deps derives floors via uv's --resolution - # lowest-direct (replacing the hand-pinned tool.hatch.envs.min_deps); - # upstream overlays nightly + git mains (ci/sync-upstream.sh). + # min_deps and upstream are intentionally unlocked (floors via --resolution + # lowest-direct; nightly + git mains overlay). The recipe names match the + # matrix values — see the Justfile. - name: Run Tests - run: | - if [ "${{ matrix.dependency-set }}" = "min_deps" ]; then - uv sync --resolution lowest-direct --no-default-groups --group test --group remote-tests --extra remote --extra optional - else - bash ci/sync-upstream.sh - fi - bash ci/run-coverage.sh + run: uvx --from rust-just just ${{ matrix.dependency-set }} - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: @@ -140,10 +129,7 @@ jobs: enable-cache: true python-version: '3.13' - name: Run doctests - run: | - uv sync --locked --no-default-groups --extra remote --group remote-tests - uv pip list - uv run --no-sync --with pytest-examples pytest tests/test_docs.py -v + run: uvx --from rust-just just doctest benchmarks: name: Benchmark smoke test @@ -159,9 +145,7 @@ jobs: enable-cache: true python-version: '3.13' - name: Run Benchmarks - run: | - uv sync --locked --no-default-groups --group test - uv run --no-sync pytest --benchmark-enable tests/benchmarks + run: uvx --from rust-just just benchmark test-complete: name: Test complete diff --git a/Justfile b/Justfile new file mode 100644 index 0000000000..c8843bbed1 --- /dev/null +++ b/Justfile @@ -0,0 +1,140 @@ +# zarr-python developer task runner (https://github.com/casey/just). +# +# `just` is a thin verb-runner over uv; uv owns the environments. Each recipe is +# the single source of truth for a dev/CI task — CI calls the same recipes (via +# `uvx --from rust-just just `), so local and CI behavior cannot drift. +# +# Install just: uv tool install rust-just (or `brew install just`, `cargo install just`) +# List recipes: just (or `just --list`) +# +# The matrix lives in GitHub Actions; pass the Python version via UV_PYTHON +# (setup-uv sets it from `python-version`). Locally, override per call, e.g. +# UV_PYTHON=3.13 just test-optional + +# Extras + groups that make up the "optional" (full integration) test environment. +optional_deps := "--extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests" + +[private] +default: + @just --list + +[doc("Run the unit tests with the minimal dependency set")] +test-minimal *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the unit tests with the full (optional) integration dependency set")] +test-optional *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Generate an HTML coverage report (optional deps); open htmlcov/index.html")] +coverage-html *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks {{args}} + uv run --no-sync coverage html + +[doc("Run the slow Hypothesis property tests")] +hypothesis *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test {{optional_deps}} + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest -nauto --run-slow-hypothesis \ + tests/test_properties.py tests/test_store/test_stateful* {{args}} + uv run --no-sync coverage xml + +[doc("Validate executable code blocks in the docs (tests/test_docs.py)")] +doctest *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group remote-tests + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync --with pytest-examples pytest tests/test_docs.py -v {{args}} + +[doc("Run the benchmark suite (minimal deps)")] +benchmark *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test + uv run --no-sync pytest --benchmark-enable tests/benchmarks {{args}} + +[doc("Run the tests against the lowest supported direct dependency versions")] +min_deps *args: + #!/usr/bin/env bash + set -euo pipefail + # uv derives the floors from the `>=` constraints in pyproject.toml. + uv sync --resolution lowest-direct --no-default-groups \ + --group test --group remote-tests --extra remote --extra optional + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the tests against bleeding-edge (nightly + git main) dependencies")] +upstream *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --no-default-groups --group test --group remote-tests --extra remote + uv pip install --prerelease=allow \ + --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + numpy \ + "packaging @ git+https://github.com/pypa/packaging" \ + "numcodecs @ git+https://github.com/zarr-developers/numcodecs" \ + "s3fs @ git+https://github.com/fsspec/s3fs" \ + "universal_pathlib @ git+https://github.com/fsspec/universal_pathlib" \ + "typing_extensions @ git+https://github.com/python/typing_extensions" \ + "donfig @ git+https://github.com/pytroll/donfig" \ + "obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore" + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Run the GPU tests (requires CUDA + a GPU); `pytest -m gpu`")] +gpu *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --group test --extra gpu --extra optional + uv pip install pytest-examples + if [ -n "${CI:-}" ]; then uv pip list; fi + uv run --no-sync coverage run --source=src -m pytest -m gpu --ignore tests/benchmarks \ + --junitxml=junit.xml -o junit_family=legacy {{args}} + uv run --no-sync coverage xml + +[doc("Build the documentation (strict: warnings are errors)")] +docs-build *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group docs + DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ + uv run --no-sync mkdocs build --strict {{args}} + +[doc("Serve the documentation locally with live reload at http://0.0.0.0:8000/")] +docs-serve *args: + #!/usr/bin/env bash + set -euo pipefail + uv sync --locked --no-default-groups --extra remote --group docs + DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true \ + uv run --no-sync mkdocs serve --watch src {{args}} + +[doc("Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...)")] +lint *args: + prek run --all-files {{args}} + +[doc("Check that uv.lock is in sync with pyproject.toml")] +lock-check: + uv lock --check diff --git a/changes/4096.misc.md b/changes/4096.misc.md index baea81738c..f2c30699a7 100644 --- a/changes/4096.misc.md +++ b/changes/4096.misc.md @@ -1 +1 @@ -Replaced hatch with uv for development environments and CI task management. Contributors now create the development environment with `uv sync --locked` and run tests with `uv run pytest` (or the shared runners in `ci/`, e.g. `bash ci/run-coverage.sh`); see the contributing guide for details. The `tool.hatch.envs.*` environments and their run scripts were removed from `pyproject.toml`, and a committed `uv.lock` now pins the full transitive dependency closure for reproducible, cross-platform CI. hatchling remains the build backend, so package builds and version derivation are unchanged. +Replaced hatch with uv and [just](https://github.com/casey/just) for development environments and CI task management. Contributors create the development environment with `uv sync --locked` and run tasks via `just` recipes defined in the `Justfile` (e.g. `just test-optional`, `just docs-build`; run `just` to list them). The recipes are thin wrappers over uv and are shared by CI, so local and CI behavior cannot drift. The `tool.hatch.envs.*` environments were removed and a committed `uv.lock` now pins the full transitive dependency closure for reproducible, cross-platform CI. hatchling remains the build backend, so package builds and version derivation are unchanged. diff --git a/ci/run-coverage.sh b/ci/run-coverage.sh deleted file mode 100644 index 814b77be64..0000000000 --- a/ci/run-coverage.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# Coverage-instrumented pytest run + XML report — the former hatch `run-coverage` -# script, shared by the test / upstream / min_deps / gpu CI jobs and runnable -# locally. Assumes the target environment is already synced (e.g. -# `uv sync --locked --group test`). Extra args are forwarded to pytest, e.g. -# bash ci/run-coverage.sh -m gpu -set -euo pipefail - -# In CI, log the resolved environment so version issues are debuggable. -# In GitHub Actions, the CI environment variable is always set to true. -if [ -n "${CI:-}" ]; then uv pip list; fi - -uv run --no-sync coverage run --source=src -m pytest \ - --ignore tests/benchmarks \ - --junitxml=junit.xml -o junit_family=legacy "$@" -uv run --no-sync coverage xml diff --git a/ci/run-hypothesis.sh b/ci/run-hypothesis.sh deleted file mode 100644 index 5009c38f27..0000000000 --- a/ci/run-hypothesis.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# Slow Hypothesis run — the former hatch `run-hypothesis` script. Assumes the -# target environment is already synced. Extra args are forwarded to pytest. -set -euo pipefail - -# In CI, log the resolved environment so version issues are debuggable. -# In GitHub Actions, the CI environment variable is always set to true. -if [ -n "${CI:-}" ]; then uv pip list; fi - -uv run --no-sync coverage run --source=src -m pytest -nauto \ - --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* "$@" -uv run --no-sync coverage xml diff --git a/ci/sync-upstream.sh b/ci/sync-upstream.sh deleted file mode 100644 index 3bc6ae4b94..0000000000 --- a/ci/sync-upstream.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Bleeding-edge environment — the former hatch `tool.hatch.envs.upstream`. -# Latest test deps + nightly numpy + git mains of the core stack. Intentionally -# unlocked (no `--locked`): this job exists to catch upstream breakage early. -set -euo pipefail - -uv sync --no-default-groups --group test --group remote-tests --extra remote -uv pip install --prerelease=allow \ - --index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ - --extra-index-url https://pypi.org/simple/ \ - numpy \ - "packaging @ git+https://github.com/pypa/packaging" \ - "numcodecs @ git+https://github.com/zarr-developers/numcodecs" \ - "s3fs @ git+https://github.com/fsspec/s3fs" \ - "universal_pathlib @ git+https://github.com/fsspec/universal_pathlib" \ - "typing_extensions @ git+https://github.com/python/typing_extensions" \ - "donfig @ git+https://github.com/pytroll/donfig" \ - "obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore" diff --git a/docs/contributing.md b/docs/contributing.md index fcf724e9e7..56bfe119a5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -88,6 +88,13 @@ uv sync --locked # create .venv/ with Zarr and all development dependencies This creates a virtual environment in `.venv/`. Prefix commands with `uv run` to run them inside it, or activate it directly with `source .venv/bin/activate`. To use a specific Python version, pass `-p`, e.g. `uv sync --locked -p 3.13`. +Common developer tasks (running the matrix test environments, building docs, etc.) are defined as [`just`](https://github.com/casey/just) recipes in the `Justfile`. Install `just` and list the available recipes with: + +```bash +uv tool install rust-just # or: brew install just +just # list all recipes +``` + To verify that your development environment is working, you can run the unit tests: ```bash @@ -132,13 +139,15 @@ Zarr includes a suite of unit tests. The simplest way to run them is within your uv run pytest ``` -To reproduce a CI test environment exactly — the optional integration dependencies, with coverage — sync that environment and run the shared coverage script: +To reproduce a CI test environment exactly — the same dependency set CI uses, with coverage — run the matching `just` recipe. These sync the locked environment and run the test suite the way CI does: ```bash -uv sync --locked --no-default-groups --group test --extra remote --extra optional --extra cli --extra cast-value-rs --group remote-tests -bash ci/run-coverage.sh +just test-minimal # the minimal dependency set +just test-optional # the full optional/integration dependency set ``` +Pass extra pytest arguments through, e.g. `just test-optional -k test_attributes`, and select a Python version with `UV_PYTHON`, e.g. `UV_PYTHON=3.13 just test-optional`. + All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. > **Note:** Previous versions of Zarr-Python made extensive use of doctests. These tests were not maintained during the 3.0 refactor but may be brought back in the future. See issue #2614 for more details. @@ -195,18 +204,18 @@ If you would like to skip the failing checks and push the code for further discu > **Note:** Test coverage for Zarr-Python 3 is currently not at 100%. This is a known issue and help is welcome to bring test coverage back to 100%. See issue #2613 for more details. -Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. From your development environment, running: +Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: ```bash -bash ci/run-coverage.sh +just test-optional ``` will run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. -You can then generate an HTML coverage report from the collected data by running: +You can also generate a browsable HTML coverage report by running: ```bash -uv run --no-sync coverage html +just coverage-html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. @@ -220,7 +229,7 @@ Zarr uses mkdocs for documentation, hosted on readthedocs.org. Documentation is The documentation can be built locally by running: ```bash -uv run --group docs --extra remote mkdocs build +just docs-build ``` The resulting built documentation will be available in the `site/` folder. @@ -228,7 +237,7 @@ The resulting built documentation will be available in the `site/` folder. You can also serve a continuously updating version of the documentation during development at [http://0.0.0.0:8000/](http://0.0.0.0:8000/) by running: ```bash -uv run --group docs --extra remote mkdocs serve --watch src +just docs-serve ``` #### Adding executable code blocks in the documentation diff --git a/pyproject.toml b/pyproject.toml index 2ec3cd8aec..dbf4961dcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ homepage = "https://github.com/zarr-developers/zarr-python" # test matrix keeps exercising their latest releases; their floor is covered by # the `min_deps` CI job (`uv sync --resolution lowest-direct`, which derives the # floors from these `>=` constraints) and their bleeding edge by the `upstream` -# job (`scripts/sync-upstream.sh`). Bump the pins deliberately, e.g. via +# job (the `just upstream` recipe). Bump the pins deliberately, e.g. via # dependabot or `uv lock --upgrade`. [dependency-groups] test = [ @@ -173,9 +173,9 @@ version.raw-options = { git_describe_command = "git describe --dirty --tags --lo hooks.vcs.version-file = "src/zarr/_version.py" # NOTE: `tool.hatch.envs.*` used to define the test/docs/min_deps/upstream -# environments and their run scripts. Those are gone: CI now drives `uv` directly -# (`uv sync --locked ...`) with the shared runners in `scripts/`, and the dep sets -# (extras/groups) live in `.github/workflows/`. See docs/contributing.md. +# environments and their run scripts. Those are gone: tasks are now `just` +# recipes (see the Justfile) that wrap `uv sync --locked ...`; CI runs the same +# recipes via `uvx --from rust-just just `. See docs/contributing.md. [tool.ruff] line-length = 100 @@ -346,7 +346,7 @@ ignore = [ "PC170", # use PyGrep hooks - no *.rst files to check "PC180", # for JavaScript - not interested "PC902", # pre-commit.ci custom autofix message - not using autofix - "PY007", # task running is uv + ci/*.sh shell scripts, not nox/tox/hatch envs + "PY007", # task runner is a Justfile (just); not in PY007's nox/tox/hatch allowlist ] [tool.numpydoc_validation] From 693f694e50b9ae4cd7c6b0d93238b9ee9f8f6865 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:25:09 -0400 Subject: [PATCH 3/5] chore: use just for python helpers --- .github/workflows/check_changelogs.yml | 4 ++-- .github/workflows/docs.yml | 2 +- Justfile | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index 10a21d21c3..e35d99fb16 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -25,7 +25,7 @@ jobs: uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - name: Check zarr-python changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py + run: uvx --from rust-just just check-changelogs - name: Check zarr-metadata changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-metadata/changes + run: uvx --from rust-just just check-changelogs packages/zarr-metadata/changes diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 97fea8ec3f..48357d259b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,5 +29,5 @@ jobs: # docs code block that fails to execute at build time fails CI) against the # locked docs env. check_unlinked_types runs in that same synced env. - run: uvx --from rust-just just docs-build - - run: uv run --no-sync python ci/check_unlinked_types.py + - run: uvx --from rust-just just check-doc-links continue-on-error: true diff --git a/Justfile b/Justfile index c8843bbed1..9c38908d29 100644 --- a/Justfile +++ b/Justfile @@ -138,3 +138,11 @@ lint *args: [doc("Check that uv.lock is in sync with pyproject.toml")] lock-check: uv lock --check + +[doc("Check changelog entry filenames (pass a directory to check, default: changes/)")] +check-changelogs *dir: + uv run --no-sync python ci/check_changelog_entries.py {{dir}} + +[doc("Report unlinked types in the built docs (run `just docs-build` first)")] +check-doc-links *args: + uv run --no-sync python ci/check_unlinked_types.py {{args}} From 318202881172d1d89dbf8c543a673eb508c4fd09 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:40:11 -0400 Subject: [PATCH 4/5] fix: min deps --- .github/workflows/test.yml | 6 +++--- .gitignore | 1 + Justfile | 12 +++++++++--- ci/min-deps-constraints.txt | 28 ++++++++++++++++++++++++++++ docs/contributing.md | 15 +++++++++++++++ pyproject.toml | 16 +++++++++------- tests/test_examples.py | 5 ++++- uv.lock | 2 +- 8 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 ci/min-deps-constraints.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a37d1a35cf..a0d095ec9a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -103,9 +103,9 @@ jobs: with: enable-cache: true python-version: ${{ matrix.python-version }} - # min_deps and upstream are intentionally unlocked (floors via --resolution - # lowest-direct; nightly + git mains overlay). The recipe names match the - # matrix values — see the Justfile. + # min_deps and upstream are intentionally unlocked (floor constraints from + # ci/min-deps-constraints.txt; nightly + git mains overlay). The recipe names + # match the matrix values — see the Justfile. - name: Run Tests run: uvx --from rust-just just ${{ matrix.dependency-set }} - name: Upload coverage diff --git a/.gitignore b/.gitignore index 3284865d6c..2b61e1efda 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .Python env/ .venv/ +.venv-min-deps/ build/ develop-eggs/ dist/ diff --git a/Justfile b/Justfile index 9c38908d29..ea08508999 100644 --- a/Justfile +++ b/Justfile @@ -75,9 +75,15 @@ benchmark *args: min_deps *args: #!/usr/bin/env bash set -euo pipefail - # uv derives the floors from the `>=` constraints in pyproject.toml. - uv sync --resolution lowest-direct --no-default-groups \ - --group test --group remote-tests --extra remote --extra optional + # Build the env imperatively with `uv pip` (does not read/write uv.lock) so + # the floors in ci/min-deps-constraints.txt apply only here. Runtime deps are + # pinned to their floors; everything else (e.g. flask/werkzeug) resolves to + # its latest compatible release. Isolated venv so the dev .venv is untouched. + export UV_PROJECT_ENVIRONMENT="${UV_PROJECT_ENVIRONMENT:-.venv-min-deps}" + export VIRTUAL_ENV="$UV_PROJECT_ENVIRONMENT" + uv venv --clear "$UV_PROJECT_ENVIRONMENT" + uv pip install --editable '.[remote,optional]' --group test --group remote-tests \ + --constraint ci/min-deps-constraints.txt if [ -n "${CI:-}" ]; then uv pip list; fi uv run --no-sync coverage run --source=src -m pytest --ignore tests/benchmarks \ --junitxml=junit.xml -o junit_family=legacy {{args}} diff --git a/ci/min-deps-constraints.txt b/ci/min-deps-constraints.txt new file mode 100644 index 0000000000..5639edfd1e --- /dev/null +++ b/ci/min-deps-constraints.txt @@ -0,0 +1,28 @@ +# Lowest supported dependency versions (the SPEC 0 floor), exercised by the +# `min_deps` CI job (`just min_deps`). These pin the *runtime* dependencies to +# their minimums; transitive deps (e.g. flask, werkzeug) are left to resolve to +# their latest compatible release. Keep each pin in sync with the matching `>=` +# lower bound in pyproject.toml. SPEC 0 drop schedule: +# https://scientific-python.org/specs/spec-0000/ +# +# Why a hand-pinned list, and not uv's `--resolution lowest-direct` (which would +# derive the floors automatically from the `>=` bounds)? +# * It tests the EXACT floor we advertise. lowest-direct resolves to the lowest +# *mutually consistent* set, which can sit above a declared floor and thus +# silently skip a broken minimum (it resolved packaging 23.0, not 22.0). +# * lowest-direct drives under-constrained *transitive* deps to their absolute +# floor too: moto only says `flask!=2.2.0,!=2.2.1`, so flask floored to 2.0.3 +# (which imports `url_quote`, removed in werkzeug 2.3+) -> ImportError. +# +# Applied with `uv pip install --constraint`, which does not touch uv.lock. The +# same pins in a [dependency-groups] entry would leak into the single universal +# lock and drag the locked envs down too. +packaging==22.* +numpy==2.0.* +numcodecs==0.14.1 +fsspec==2023.10.0 +s3fs==2023.10.0 +universal_pathlib==0.2.0 +typing_extensions==4.14.* +donfig==0.8.* +obstore==0.5.* diff --git a/docs/contributing.md b/docs/contributing.md index 56bfe119a5..922e74a0d6 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -152,6 +152,21 @@ All tests are automatically run via GitHub Actions for every pull request and mu > **Note:** Previous versions of Zarr-Python made extensive use of doctests. These tests were not maintained during the 3.0 refactor but may be brought back in the future. See issue #2614 for more details. +#### Minimum supported dependencies + +Zarr follows [SPEC 0](https://scientific-python.org/specs/spec-0000/) for the minimum versions of its dependencies. CI exercises those minimums in a dedicated `min_deps` job, which you can reproduce locally with: + +```bash +just min_deps +``` + +The supported floor for each runtime dependency is declared twice, and the two must be kept in sync: + +- the `>=` lower bound in `pyproject.toml` (`[project.dependencies]` and `[project.optional-dependencies]`) — what users actually get; and +- an exact pin in `ci/min-deps-constraints.txt` — what the `min_deps` job installs. + +When you raise a floor, update **both**. The `min_deps` job builds an isolated environment with `uv pip install --constraint ci/min-deps-constraints.txt`, which pins the runtime dependencies to their floors while letting transitive packages (e.g. flask, werkzeug) resolve to their latest compatible release. We pin the floors by hand rather than deriving them with uv's `--resolution lowest-direct` on purpose: hand-pinning tests the *exact* version we advertise (auto-derivation resolves to the lowest mutually consistent set, which can sit above the declared floor and hide a broken minimum), and it avoids dragging the committed `uv.lock` down. See the comments in `ci/min-deps-constraints.txt` for details. + ### Code standards - using prek All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. diff --git a/pyproject.toml b/pyproject.toml index dbf4961dcc..04af90b6c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,10 +30,14 @@ maintainers = [ { name = "Deepak Cherian" } ] requires-python = ">=3.12" +# The `>=` lower bounds below (and in [project.optional-dependencies]) are the +# minimum versions we support. The `min_deps` CI job tests them via the matching +# pins in `ci/min-deps-constraints.txt` — keep the two in sync when you bump a +# floor (per the SPEC 0 drop schedule: https://scientific-python.org/specs/spec-0000/). dependencies = [ 'packaging>=22.0', 'numpy>=2', - 'numcodecs>=0.14', + 'numcodecs>=0.14.1', 'google-crc32c>=1.5', 'typing_extensions>=4.14', 'donfig>=0.8', @@ -70,9 +74,7 @@ gpu = [ ] cast-value-rs = ["cast-value-rs"] cli = ["typer"] -# Floor is the minimum we support and test (exercised by the `min_deps` CI job -# via `uv --resolution lowest-direct`); was the old `min_deps` hatch pin. -optional = ["universal-pathlib>=0.2.0"] +optional = ["universal-pathlib"] [project.scripts] zarr = "zarr._cli.cli:app" @@ -90,9 +92,9 @@ homepage = "https://github.com/zarr-developers/zarr-python" # Runtime/integration deps (fsspec, obstore, s3fs, botocore, numcodecs, # universal-pathlib) instead carry floor (`>=`) constraints, so the `optional` # test matrix keeps exercising their latest releases; their floor is covered by -# the `min_deps` CI job (`uv sync --resolution lowest-direct`, which derives the -# floors from these `>=` constraints) and their bleeding edge by the `upstream` -# job (the `just upstream` recipe). Bump the pins deliberately, e.g. via +# the `min_deps` CI job (hand-pinned in `ci/min-deps-constraints.txt`) and their +# bleeding edge by the `upstream` job (the `just upstream` recipe). Bump the +# floors/pins deliberately, e.g. via # dependabot or `uv lock --upgrade`. [dependency-groups] test = [ diff --git a/tests/test_examples.py b/tests/test_examples.py index 9f8085e8c2..45d7d07160 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -58,7 +58,10 @@ def resave_script(source_path: Path, dest_path: Path) -> None: local Zarr project directory in the PEP-723 header. """ source_text = source_path.read_text() - dest_text = set_dep(source_text, f"zarr @ file:///{ZARR_PROJECT_PATH}") + # Use as_uri() to build a well-formed file URL. Naive f"file:///{abs_path}" + # yields four slashes (the path already starts with "/"), which strict + # packaging versions (our floor, 22.0) reject as an invalid URL. + dest_text = set_dep(source_text, f"zarr @ {ZARR_PROJECT_PATH.as_uri()}") dest_path.write_text(dest_text) diff --git a/uv.lock b/uv.lock index e66e131b20..799ea6e45a 100644 --- a/uv.lock +++ b/uv.lock @@ -4084,7 +4084,7 @@ requires-dist = [ { name = "packaging", specifier = ">=22.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, - { name = "universal-pathlib", marker = "extra == 'optional'", specifier = ">=0.2.0" }, + { name = "universal-pathlib", marker = "extra == 'optional'" }, ] provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] From 11fe7be8f907bdae09f4137c55c239c51dd0b888 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 24 Jun 2026 21:28:14 -0400 Subject: [PATCH 5/5] chore: pin min supported versions at the patch level --- ci/min-deps-constraints.txt | 22 +++++++++++++--------- pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/ci/min-deps-constraints.txt b/ci/min-deps-constraints.txt index 5639edfd1e..3234e04c47 100644 --- a/ci/min-deps-constraints.txt +++ b/ci/min-deps-constraints.txt @@ -1,10 +1,13 @@ # Lowest supported dependency versions (the SPEC 0 floor), exercised by the -# `min_deps` CI job (`just min_deps`). These pin the *runtime* dependencies to -# their minimums; transitive deps (e.g. flask, werkzeug) are left to resolve to -# their latest compatible release. Keep each pin in sync with the matching `>=` -# lower bound in pyproject.toml. SPEC 0 drop schedule: +# `min_deps` CI job (`just min_deps`). Each pin is the EXACT lowest version that +# satisfies the matching `>=` bound in pyproject.toml — so the job verifies the +# precise floor we advertise (not merely "some version in the floor's minor +# series"). Keep the two in sync. SPEC 0 drop schedule: # https://scientific-python.org/specs/spec-0000/ # +# These pin the *runtime* dependencies; transitive deps (e.g. flask, werkzeug) +# are left to resolve to their latest compatible release. +# # Why a hand-pinned list, and not uv's `--resolution lowest-direct` (which would # derive the floors automatically from the `>=` bounds)? # * It tests the EXACT floor we advertise. lowest-direct resolves to the lowest @@ -17,12 +20,13 @@ # Applied with `uv pip install --constraint`, which does not touch uv.lock. The # same pins in a [dependency-groups] entry would leak into the single universal # lock and drag the locked envs down too. -packaging==22.* -numpy==2.0.* +packaging==22.0 +numpy==2.0.0 numcodecs==0.14.1 +google-crc32c==1.5.0 +typing_extensions==4.14.0 +donfig==0.8.0 fsspec==2023.10.0 s3fs==2023.10.0 +obstore==0.5.1 universal_pathlib==0.2.0 -typing_extensions==4.14.* -donfig==0.8.* -obstore==0.5.* diff --git a/pyproject.toml b/pyproject.toml index 04af90b6c6..b093bdb498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ gpu = [ ] cast-value-rs = ["cast-value-rs"] cli = ["typer"] -optional = ["universal-pathlib"] +optional = ["universal-pathlib>=0.2.0"] [project.scripts] zarr = "zarr._cli.cli:app" diff --git a/uv.lock b/uv.lock index 799ea6e45a..6c230efa66 100644 --- a/uv.lock +++ b/uv.lock @@ -4078,13 +4078,13 @@ requires-dist = [ { name = "donfig", specifier = ">=0.8" }, { name = "fsspec", marker = "extra == 'remote'", specifier = ">=2023.10.0" }, { name = "google-crc32c", specifier = ">=1.5" }, - { name = "numcodecs", specifier = ">=0.14" }, + { name = "numcodecs", specifier = ">=0.14.1" }, { name = "numpy", specifier = ">=2" }, { name = "obstore", marker = "extra == 'remote'", specifier = ">=0.5.1" }, { name = "packaging", specifier = ">=22.0" }, { name = "typer", marker = "extra == 'cli'" }, { name = "typing-extensions", specifier = ">=4.14" }, - { name = "universal-pathlib", marker = "extra == 'optional'" }, + { name = "universal-pathlib", marker = "extra == 'optional'", specifier = ">=0.2.0" }, ] provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"]