Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
version: 2

updates:
# Refresh the versions pinned in uv.lock, which CI tests against. `widen` edits pyproject.toml only
# when a release falls above an upper cap: it raises the cap and leaves the lower bound untouched.
- package-ecosystem: "uv"
directory: "/"
target-branch: "develop"
schedule:
interval: "weekly"
versioning-strategy: "widen"
groups:
uv-dependencies:
# Minor and patch releases of everything, in one weekly PR. Major releases get a PR each:
# for dash, plotly and pandas they raise the upper cap, which needs manual checks first.
patterns: ["*"]
update-types: ["minor", "patch"]
cooldown:
default-days: 7 # Keep in sync with `exclude-newer` in pyproject.toml

# Keep the third-party GitHub Actions used by the workflows up to date.
- package-ecosystem: "github-actions"
directory: "/"
target-branch: "develop"
schedule:
interval: "weekly"
groups:
actions-dependencies:
patterns:
- "*"
cooldown:
default-days: 7
95 changes: 52 additions & 43 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,39 +7,50 @@ on:
- develop
pull_request:
types: [opened, synchronize, reopened]
workflow_dispatch:

concurrency:
# Cancel older runs for the same PR/branch when new commits arrive.
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
code_quality:
name: Code Quality
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Git checkout
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it best pratice to target a commit hash instead of a tag?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it's the practice GitHub recommends in its security hardening guide for Actions: pinning to a full-length commit SHA is the only way to reference an action immutably.

A tag like v4 is a movable pointer. Whoever controls the action's repo (or an attacker who compromises it) can repoint the tag to different code, and our workflows would run it on the next trigger with no change on our side. A SHA, on the contrary can't be moved.

with:
fetch-depth: 0 # required for setuptools-scm to read git tags
- name: Set up Python 3.12
uses: actions/setup-python@v6
persist-credentials: false
# Oldest supported Python: mypy checks against the installed libraries' stubs, which follow the
# interpreter, so code that only works on newer Pythons is caught here.
- name: Set up uv and Python 3.11
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml # it would be better on a lock file
python-version: "3.11"
activate-environment: true
enable-cache: true
cache-dependency-glob: uv.lock
# Locked environment: `dev` + `test` groups (`default-groups`), the latter because mypy needs the
# optional dependencies that ship types. `--locked` also fails when uv.lock is out of date.
- name: Install dev dependencies
# mypy needs `test` optional dependencies that ship types
run: |
python -m pip install --upgrade pip
python -m pip install . --group dev --group test
run: uv sync --locked
- name: Check linting
run: make lint
- name: Check formatting
run: make format
- name: Check type annotations
run: make typecheck
- name: Audit workflows
env:
GH_TOKEN: ${{ github.token }} # enables zizmor's online audits (e.g. known-vulnerable actions)
run: make workflows

test:
name: Test (Python ${{ matrix.python-version }})
Expand All @@ -55,30 +66,31 @@ jobs:
- python-version: "3.12"
coverage: true
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 0 # required for setuptools-scm to read git tags
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
persist-credentials: false
- name: Set up uv and Python ${{ matrix.python-version }}
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: pyproject.toml
# Full suite, NLP included. CPU-only torch first: the PyPI Linux wheel drags in several GB of
# CUDA libraries the tests never use. The `nlp` extra's `torch>=` bound is then already
# satisfied and pip keeps it.
activate-environment: true
enable-cache: true
cache-dependency-glob: uv.lock
# Full suite, NLP included, pinned to uv.lock through a constraints file. Not `uv sync`: uv.lock
# holds the PyPI torch wheel, which drags in several GB of CUDA libraries the tests never use.
# `--torch-backend cpu` installs the same torch version from the CPU-only index instead.
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
python -m pip install '.[nlp]' --group test
uv export --locked --no-hashes --no-emit-project --extra nlp --group test -o constraints.txt
uv pip install --torch-backend cpu -c constraints.txt '.[nlp]' --group test
# NLP test modules skip (not fail) when their imports are missing, so fail loudly here instead.
- name: Check NLP stack is importable
run: python -c "import torch, transformers, sentence_transformers, captum; print(torch.__version__)"
# Models the NLP tests download from the Hugging Face Hub: reuse them instead of hitting the Hub
# from every job. The key follows the tests, so a newly referenced model gets saved too.
- name: Cache Hugging Face models
uses: actions/cache@v6
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/huggingface
key: hf-models-${{ hashFiles('tests/**/*.py') }}
Expand All @@ -91,7 +103,7 @@ jobs:
run: make coverage-report
- name: Upload coverage to Codecov
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v6
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
Expand All @@ -100,36 +112,33 @@ jobs:
# Pure-Python package: building it once is enough.
- name: Build python package
if: ${{ matrix.coverage }}
run: |
python -m pip install --upgrade build
python -m build
run: uv build

pandas_compat:
name: Pandas Compat (${{ matrix.pandas }})
lowest_deps:
name: Lowest Dependencies
needs: code_quality
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
# Lower bound + last 2.x. Latest pandas 3.x is already what the `test` jobs resolve to.
pandas: ["2.2.2", "2.3.*"]
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 0 # required for setuptools-scm to read git tags
- name: Set up Python 3.12
uses: actions/setup-python@v6
persist-credentials: false
# Oldest supported Python: the oldest dependency releases may ship no wheels for newer ones.
- name: Set up uv and Python 3.11
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml
# Classical-install jobs: deliberately without the `nlp` extra.
python-version: "3.11"
activate-environment: true
enable-cache: true
cache-dependency-glob: pyproject.toml
# Each direct dependency at the lower bound pyproject.toml declares (transitive ones at their
# newest), so a PR relying on a newer API fails here until the bound is raised. Classical install:
# the extras the `test` group pulls are listed explicitly so they count as direct too.
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install . --group test "pandas==${{ matrix.pandas }}"
python -c "import pandas; print(pandas.__version__)"
uv pip install --resolution lowest-direct '.[xgboost,catboost,lightgbm,lime,report]' --group test
python -c "import pandas, plotly, sklearn; print(pandas.__version__, plotly.__version__, sklearn.__version__)"
- name: Check NLP stack is absent
run: python -c "import importlib.util as u; assert u.find_spec('torch') is None, 'torch installed'"
- name: Test
Expand Down
29 changes: 17 additions & 12 deletions .github/workflows/publish-release-candidate.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,28 @@ concurrency:
cancel-in-progress: false


permissions:
contents: read

jobs:
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 15

steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
persist-credentials: false
- name: Set up uv
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
python-version: "3.13"
- name: Build package
run: |
python -m pip install --upgrade build
python -m build
run: uv build
- name: Store the distribution packages
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: python-package-distributions
path: dist/
Expand All @@ -47,12 +49,12 @@ jobs:

steps:
- name: Download all the dists
uses: actions/download-artifact@v4
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: python-package-distributions
path: dist/
- name: Publish distribution to TestPyPI
uses: pypa/gh-action-pypi-publish@release/v1
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
with:
repository-url: https://test.pypi.org/legacy/

Expand All @@ -64,12 +66,12 @@ jobs:
permissions: {}

steps:
- uses: actions/download-artifact@v4
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: python-package-distributions
path: dist/
- name: Setup Python
uses: actions/setup-python@v6
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.13"
- name: Resolve package version from tag
Expand All @@ -86,8 +88,9 @@ jobs:
run: |
python3 -m venv .venv-smoke
. .venv-smoke/bin/activate
# Only shapash comes from TestPyPI: anyone can upload there, so dependencies must come from PyPI.
for attempt in 1 2 3 4 5; do
if pip install --pre -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "shapash==${RELEASE_VERSION}"; then
if pip install --no-deps -i https://test.pypi.org/simple/ "shapash==${RELEASE_VERSION}"; then
break
fi
if [ "${attempt}" -eq 5 ]; then
Expand All @@ -97,4 +100,6 @@ jobs:
echo "Install not available yet on TestPyPI, retrying in 30s (attempt ${attempt}/5)..."
sleep 30
done
# Already installed, so pip keeps it and installs its dependencies from PyPI.
pip install "shapash==${RELEASE_VERSION}"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this works?
You target pypi.org here where REALESE_VERSION does not exists.
How can it fetch its dependencies?
Am I misreading something?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When shapash==X is already installed (here the release candidate from test.pypi), pip's resolver takes the installed copy ("Requirement already satisfied"). It doesn't need that version to exist on PyPI. It reads the installed package's metadata and fetches only its dependencies from PyPI.

python -c "import shapash; print(shapash.__version__)"
24 changes: 13 additions & 11 deletions .github/workflows/publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,28 @@ concurrency:
group: pypi-publish
cancel-in-progress: false

permissions:
contents: read

jobs:
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 15

steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
persist-credentials: false
- name: Set up uv
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
with:
python-version: "3.13"
- name: Build package
run: |
python -m pip install --upgrade build
python -m build
run: uv build
- name: Store the distribution packages
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: python-package-distributions
path: dist/
Expand All @@ -45,12 +47,12 @@ jobs:

steps:
- name: Download all the dists
uses: actions/download-artifact@v4
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: python-package-distributions
path: dist/
- name: Publish distribution to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

smoke-test-pypi:
name: Smoke test install from PyPI
Expand All @@ -60,12 +62,12 @@ jobs:
permissions: {}

steps:
- uses: actions/download-artifact@v4
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: python-package-distributions
path: dist/
- name: Set up Python
uses: actions/setup-python@v6
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.13"
- name: Resolve package version from tag
Expand Down
14 changes: 11 additions & 3 deletions .github/workflows/sonarqube.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,24 @@ on:
types: [opened, synchronize, reopened]

name: SonarQube
permissions:
contents: read

jobs:
sonarqube:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
# Skip PRs from forks and from Dependabot: neither gets the repository secrets.
if: >-
github.event_name != 'pull_request' ||
(github.event.pull_request.head.repo.full_name == github.repository &&
github.event.pull_request.user.login != 'dependabot[bot]')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
fetch-depth: 0
persist-credentials: false
- name: SonarQube Scanner
uses: SonarSource/sonarqube-scan-action@v8.2.2 # see the latest version at https://github.com/marketplace/actions/official-sonarqube-scan
uses: SonarSource/sonarqube-scan-action@ba9859eae8dd6bd29e412f25ddbbef3d032000f4 # v8.2.2
env:
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
Loading
Loading