Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 41 additions & 10 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,40 +14,71 @@ on:

jobs:
checks:
runs-on: ubuntu-latest
name: checks (${{ matrix.platform }}, ${{ matrix.python-version }})
# Keep fork PRs and standalone forks outside the Namespace workspace.
runs-on: >-
${{ github.repository == 'Hebbian-Robotics/hflow'
&& (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
&& fromJSON(format('["{0}-with-cache","nscloud-cache-tag-{1}-{2}","nscloud-cache-allow-commit-from-main"]',
matrix.namespace-runner, matrix.cache-tag-prefix, matrix.python-version))
|| matrix.github-runner }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
# Test the supported floor and current runtime. Static analysis runs once below.
python-version: ["3.11", "3.14"]
include:
- platform: Linux
python-version: "3.11"
namespace-runner: nscloud-ubuntu-24.04-amd64-4x16
github-runner: ubuntu-latest
cache-tag-prefix: hflow-ci-uv
- platform: Linux
python-version: "3.14"
namespace-runner: nscloud-ubuntu-24.04-amd64-4x16
github-runner: ubuntu-latest
cache-tag-prefix: hflow-ci-uv
- platform: macOS
python-version: "3.14"
namespace-runner: nscloud-macos-sequoia-arm64-6x14
github-runner: macos-15
cache-tag-prefix: hflow-ci-macos-arm64-uv
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install ffmpeg (the test suite pins to the system binary)
- name: Install FFmpeg on Linux
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg
- name: Install FFmpeg on macOS
if: runner.os == 'macOS'
run: brew install ffmpeg
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
python-version: ${{ matrix.python-version }}
enable-cache: ${{ runner.environment == 'github-hosted' }}
- name: Attach the uv cache volume
if: runner.environment == 'self-hosted'
uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1.7.0
with:
cache: uv
- name: Sync environment
# Not --all-extras: the mediapipe extra is a model, deliberately not
# exercised here (see pyproject.toml).
run: uv sync --locked
- name: Lint
if: matrix.python-version == '3.14'
if: matrix.platform == 'Linux' && matrix.python-version == '3.14'
run: uv run ruff check
- name: Format check
if: matrix.python-version == '3.14'
if: matrix.platform == 'Linux' && matrix.python-version == '3.14'
run: uv run ruff format --check
- name: Typecheck
if: matrix.python-version == '3.14'
if: matrix.platform == 'Linux' && matrix.python-version == '3.14'
run: uv run ty check
- name: Tests
# Public ubuntu-latest runners have four CPUs; keep media-heavy tests within that limit.
run: uv run pytest -q -n 4
- name: Install minimum supported NumPy
if: matrix.python-version == '3.14'
if: matrix.platform == 'Linux' && matrix.python-version == '3.14'
run: uv pip install "numpy==2.3.5"
- name: Test minimum NumPy compatibility
if: matrix.python-version == '3.14'
if: matrix.platform == 'Linux' && matrix.python-version == '3.14'
run: >-
uv run --no-sync pytest -q -n 4
tests/test_resample_derive.py tests/test_episode_to_arrow.py tests/test_video_import.py
11 changes: 10 additions & 1 deletion .github/workflows/pinned-assets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ permissions:

jobs:
availability:
runs-on: ubuntu-latest
runs-on: >-
${{ github.repository == 'Hebbian-Robotics/hflow'
&& fromJSON('["nscloud-ubuntu-24.04-amd64-2x4-with-cache","nscloud-cache-tag-hflow-ci-uv-3.11","nscloud-cache-allow-commit-from-main"]')
|| 'ubuntu-latest' }}
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -19,6 +22,12 @@ jobs:
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
python-version: "3.11"
enable-cache: ${{ runner.environment == 'github-hosted' }}
- name: Attach the uv cache volume
if: runner.environment == 'self-hosted'
uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1.7.0
with:
cache: uv
- name: Check both pinned Linux FFmpeg assets are available
env:
HFLOW_NETWORK_TESTS: "1"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr-limit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ permissions:

jobs:
enforce_open_pr_cap:
runs-on: ubuntu-latest
runs-on: ${{ github.repository == 'Hebbian-Robotics/hflow' && 'nscloud-ubuntu-24.04-amd64-2x4' || 'ubuntu-latest' }}
if: github.event.pull_request.user.type != 'Bot'
steps:
- name: Close pull requests over the per-author cap
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ permissions:

jobs:
build:
runs-on: ubuntu-latest
runs-on: ${{ github.repository == 'Hebbian-Robotics/hflow' && 'nscloud-ubuntu-24.04-amd64-4x16' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand Down Expand Up @@ -64,6 +64,7 @@ jobs:

publish:
needs: build
# PyPA supports the publishing action on GitHub-hosted Ubuntu.
runs-on: ubuntu-latest
environment:
name: pypi
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/welcome.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ permissions:

jobs:
welcome:
runs-on: ubuntu-latest
runs-on: ${{ github.repository == 'Hebbian-Robotics/hflow' && 'nscloud-ubuntu-24.04-amd64-2x4' || 'ubuntu-latest' }}
if: (github.event.pull_request.user.type || github.event.issue.user.type) != 'Bot'
steps:
- name: Welcome first-time contributors
Expand Down
23 changes: 14 additions & 9 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,14 @@ macOS, install the Xcode Command Line Tools with `xcode-select --install`.

## Platform support

Linux and macOS both work for native development. CI runs on Linux only, so
run the quality checks yourself before opening a pull request on macOS.
Native overlay builds require CPython on Linux, so the packaging tests that
compile one are skipped on macOS.
Linux and macOS both work for native development. CI covers Linux x86-64 and
Apple Silicon. Native overlay builds require CPython on Linux, so the packaging
tests that compile one are skipped on macOS.

Upstream builds and checks use Namespace runners. CI keeps a separate uv Cache
Volume per platform and Python version, and only runs on `main` can update those
volumes. CI for pull requests from forks uses GitHub-hosted runners on the
same platforms. The PyPI publishing job uses GitHub-hosted Ubuntu.

Native Windows is not supported: pipeline and storage operations use `fcntl`
for file locking (`src/hflow/storage.py`), which does not exist on Windows. Work inside WSL2 with an Ubuntu
Expand Down Expand Up @@ -116,8 +120,9 @@ uv run --locked --project examples/path_to_example \
pytest -q examples/path_to_example/tests
```

CI runs the suite on Python 3.11 and 3.14. When changing compatibility-sensitive
code, exercise both ends locally:
CI runs the suite on Python 3.11 and 3.14 on Linux, and on Python 3.14 on macOS.
Static analysis runs once, on Linux with Python 3.14. When changing
compatibility-sensitive code, exercise both Python versions locally:

```bash
uv run --python 3.11 --locked pytest -q
Expand All @@ -127,9 +132,9 @@ uv run --python 3.14 --locked pytest -q
CI uses four pytest workers for the default suite. To match its test execution
locally, run `uv run pytest -q -n 4`.

After the normal suite, CI runs targeted resampling, Arrow conversion, and video
import tests with the minimum supported NumPy on Python 3.14. To reproduce that
compatibility check after syncing a Python 3.14 environment:
After the normal suite on Linux with Python 3.14, CI runs targeted resampling,
Arrow conversion, and video import tests with the minimum supported NumPy.
To reproduce that compatibility check after syncing a Python 3.14 environment:

```bash
uv pip install "numpy==2.3.5"
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
</a>
</p>

<p align="center">
<a href="https://namespace.so/github-actions?utm_source=oss&amp;utm_campaign=hflow">
<img src="https://namespace.so/oss/badge.svg" alt="CI powered by Namespace, HFlow's CI sponsor">
</a>
</p>

Hebbian Robotics (YC S26) is building HFlow, an open source SDK for scalable
multimodal data pipelines in robotics and physical AI. It makes data tooling and
practices typically developed inside large robotics teams accessible to teams of
Expand Down