Skip to content

chore: set up dynamic versioning hatch plugin #471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mckornfield wants to merge 2 commits into
base: main
Choose a base branch
from
+510 −833
Open

chore: set up dynamic versioning hatch plugin #471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6ac2720
chore: set up dynamic versioning hatch plugin
mckornfield Jun 23, 2026
03d6e49
Merge branch 'main' into dynamic-versioning/mck
mckornfield Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions .github/actions/build-nemo-platform-wheel/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
name: Build nemo-platform wheel
description: >
Set up the build toolchain (uv, plus node/pnpm when building nemo-platform
so the hatch hook can compile Studio assets), stamp the SDK version, and
so the hatch hook can compile Studio assets), resolve the SDK version, and
run `uv build --wheel --package <pkg>`. The build itself — including
Studio asset compilation and wheel content force-includes — lives in the
package's hatch_build.py; this action only handles environment setup,
the pre-build version stamp, and capturing the produced wheel.
the pre-build version override, and capturing the produced wheel.

Both ci.yaml's wheel-test job and release-bundle.yaml's build-sdks matrix
call this action so the test wheel and the published wheel come out of
Expand Down Expand Up @@ -78,14 +78,12 @@ runs:
version-file: ${{ inputs.source-root }}/pyproject.toml
cache-dependency-glob: ${{ inputs.source-root }}/uv.lock

# Stamp early, before pnpm/node setup, so bad cadence/label/timestamp
# Resolve early, before pnpm/node setup, so bad cadence/label/timestamp
# inputs fail fast instead of after spending ~20s on tool setup. The
# version is written into packages/nmp_common/.../version.py and the
# generated _version.py before `uv build` runs (hatchling reads
# version.py via its regex source but does not write it). The
# --print-version flag emits just the resolved version on stdout;
# the human banner goes to stderr.
- name: Stamp SDK version
# resolved version is passed to uv-dynamic-versioning through its supported
# bypass environment variable. The --print-version flag emits just the
# resolved version on stdout; the human banner goes to stderr.
- name: Resolve SDK version
id: stamp
shell: bash
env:
Expand Down Expand Up @@ -176,6 +174,7 @@ runs:
SOURCE_ROOT: ${{ inputs.source-root }}
OUT_DIR: ${{ inputs.out-dir }}
PACKAGE: ${{ inputs.package }}
UV_DYNAMIC_VERSIONING_BYPASS: ${{ steps.stamp.outputs.wheel-version }}
run: |
set -euo pipefail
if [[ "${OUT_DIR}" != /* ]]; then
Expand Down
82 changes: 35 additions & 47 deletions .github/scripts/stamp_sdk_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Stamp the selected SDK version before building a release wheel."""
"""Resolve the package version to inject into dynamic-versioned wheel builds."""

import argparse
import ast
import re
import subprocess
import sys
from pathlib import Path
from typing import Literal

Cadence = Literal["nightly", "rc", "release"]
SEMVER_CORE_PATTERN = r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)"
RELEASE_CORE_TAG_PATTERN = re.compile(rf"^({SEMVER_CORE_PATTERN})(?:-rc\d+)?$")


class StampError(Exception):
"""Raised when the SDK version cannot be stamped safely."""
"""Raised when the SDK version cannot be resolved safely."""


def safe_sdk_id(sdk_id: str) -> str:
Expand All @@ -24,52 +25,40 @@ def safe_sdk_id(sdk_id: str) -> str:
return sdk_id


def read_assignment(path: Path, name: str) -> str:
if not path.is_file():
raise StampError(f"version file is missing: {path}")
def _semver_key(version: str) -> tuple[int, int, int]:
major, minor, patch = version.split(".")
return int(major), int(minor), int(patch)

prefix = f"{name} = "
values: list[object] = []
for line in path.read_text(encoding="utf-8").splitlines():
if line.startswith(prefix):
try:
values.append(ast.literal_eval(line.removeprefix(prefix).strip()))
except (SyntaxError, ValueError) as error:
raise StampError(f"{name} in {path} must be a string literal") from error

if len(values) != 1:
raise StampError(f"expected exactly one {name} assignment in {path}, found {len(values)}")
if not isinstance(values[0], str) or not values[0]:
raise StampError(f"{name} in {path} must be a non-empty string")
return values[0]


def replace_assignment(path: Path, name: str, value: str) -> None:
if not path.is_file():
raise StampError(f"version file is missing: {path}")

prefix = f"{name} = "
lines = path.read_text(encoding="utf-8").splitlines(keepends=True)
matches = [index for index, line in enumerate(lines) if line.startswith(prefix)]
if len(matches) != 1:
raise StampError(f"expected exactly one {name} assignment in {path}, found {len(matches)}")
def latest_release_core(source_root: Path) -> str | None:
result = subprocess.run(
["git", "-C", str(source_root), "tag", "--merged", "HEAD", "--list"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
return None

index = matches[0]
newline = "\n" if lines[index].endswith("\n") else ""
lines[index] = f'{name} = "{value}"{newline}'
path.write_text("".join(lines), encoding="utf-8")
release_cores = []
for tag in result.stdout.splitlines():
match = RELEASE_CORE_TAG_PATTERN.fullmatch(tag)
if match:
release_cores.append(match.group(1))
if not release_cores:
return None
return max(release_cores, key=_semver_key)


def resolve_sdk_version(
cadence: Cadence,
release_label: str,
nightly_timestamp: str,
shared_sdk_version_path: Path,
source_root: Path,
) -> str:
if cadence == "nightly":
base_version = read_assignment(shared_sdk_version_path, "platform_sdk_version")
if not re.fullmatch(SEMVER_CORE_PATTERN, base_version):
raise StampError(f"nightly base SDK version must be SemVer core MAJOR.MINOR.PATCH: {base_version}")
base_version = latest_release_core(source_root) or "0.0.0"
if not re.fullmatch(r"\d{14}", nightly_timestamp):
Comment on lines +33 to 62

@coderabbitai coderabbitai Bot Jun 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

Fail when release tags cannot be resolved.

Lines 41-62 silently turn git failures or missing fetched tags into 0.0.0.dev..., which can publish a wrongly ordered nightly. Make tag resolution failure explicit.

Proposed fix 
-def latest_release_core(source_root: Path) -> str | None:
+def latest_release_core(source_root: Path) -> str:
     result = subprocess.run(
         ["git", "-C", str(source_root), "tag", "--merged", "HEAD", "--list"],
         check=False,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         text=True,
     )
     if result.returncode != 0:
-        return None
+        detail = result.stderr.strip() or f"git exited with {result.returncode}"
+        raise StampError(f"failed to inspect merged release tags: {detail}")
 
     release_cores = []
     for tag in result.stdout.splitlines():
         match = RELEASE_CORE_TAG_PATTERN.fullmatch(tag)
         if match:
             release_cores.append(match.group(1))
     if not release_cores:
-        return None
+        raise StampError("no merged SemVer release tags found; fetch tags before resolving nightly version")
     return max(release_cores, key=_semver_key)
@@
     if cadence == "nightly":
-        base_version = latest_release_core(source_root) or "0.0.0"
+        base_version = latest_release_core(source_root)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
def latest_release_core(source_root: Path) -> str | None:
result = subprocess.run(
["git", "-C", str(source_root), "tag", "--merged", "HEAD", "--list"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
return None
index = matches[0]
newline = "\n" if lines[index].endswith("\n") else ""
lines[index] = f'{name} = "{value}"{newline}'
path.write_text("".join(lines), encoding="utf-8")
release_cores = []
for tag in result.stdout.splitlines():
match = RELEASE_CORE_TAG_PATTERN.fullmatch(tag)
if match:
release_cores.append(match.group(1))
if not release_cores:
return None
return max(release_cores, key=_semver_key)
def resolve_sdk_version(
cadence: Cadence,
release_label: str,
nightly_timestamp: str,
shared_sdk_version_path: Path,
source_root: Path,
) -> str:
if cadence == "nightly":
base_version = read_assignment(shared_sdk_version_path, "platform_sdk_version")
if not re.fullmatch(SEMVER_CORE_PATTERN, base_version):
raise StampError(f"nightly base SDK version must be SemVer core MAJOR.MINOR.PATCH: {base_version}")
base_version = latest_release_core(source_root) or "0.0.0"
if not re.fullmatch(r"\d{14}", nightly_timestamp):
def latest_release_core(source_root: Path) -> str:
result = subprocess.run(
["git", "-C", str(source_root), "tag", "--merged", "HEAD", "--list"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or f"git exited with {result.returncode}"
raise StampError(f"failed to inspect merged release tags: {detail}")
release_cores = []
for tag in result.stdout.splitlines():
match = RELEASE_CORE_TAG_PATTERN.fullmatch(tag)
if match:
release_cores.append(match.group(1))
if not release_cores:
raise StampError("no merged SemVer release tags found; fetch tags before resolving nightly version")
return max(release_cores, key=_semver_key)
def resolve_sdk_version(
cadence: Cadence,
release_label: str,
nightly_timestamp: str,
source_root: Path,
) -> str:
if cadence == "nightly":
base_version = latest_release_core(source_root)
if not re.fullmatch(r"\d{14}", nightly_timestamp):
🧰 Tools
🪛 ast-grep (0.44.0)

[error] 33-39: Command coming from incoming request
Context: subprocess.run(
["git", "-C", str(source_root), "tag", "--merged", "HEAD", "--list"],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
Note: [CWE-78] Improper Neutralization of Special Elements used in an OS Command ('OS Command Injection').

(subprocess-from-request)

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In @.github/scripts/stamp_sdk_version.py around lines 33 - 62, The nightly
version fallback in latest_release_core and resolve_sdk_version is too silent:
git tag lookup failures or missing release tags should not degrade to
0.0.0.dev.... Update latest_release_core to raise or propagate a clear error
when subprocess.run fails or no release cores are found, and have
resolve_sdk_version fail fast instead of fabricating a version when release tags
cannot be resolved. Use the existing latest_release_core, resolve_sdk_version,
and RELEASE_CORE_TAG_PATTERN symbols to locate the logic.

raise StampError("nightly timestamp must be YYYYMMDDHHMMSS")
return f"{base_version}.dev{nightly_timestamp}"
Comment on lines 60 to 64

@coderabbitai coderabbitai Bot Jun 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Verify PEP 440 ordering. Expect both comparisons to print False with the current scheme.
python - <<'PY'
from packaging.version import Version

print(Version("1.2.0.dev20260512010101") > Version("1.2.0"))
print(Version("2.1.0.dev20260512010101") > Version("2.1.0rc0"))
PY

Repository: NVIDIA-NeMo/nemo-platform

Length of output: 175

Make nightly versions sort after the latest release

{base_version}.devYYYYMMDDHHMMSS sorts before both base_version and rc releases, so nightlies can look older than published artifacts. Bump the target core or use a scheme that orders after the latest release.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In @.github/scripts/stamp_sdk_version.py around lines 60 - 64, The nightly
version in stamp_sdk_version.py currently uses base_version.devYYYYMMDDHHMMSS,
which can sort before the latest release and rc builds. Update the nightly
branch in the version-building logic (the cadence == "nightly" path in the
stamping function) so it produces a version that orders after the current
release line, either by bumping the target core version before appending the
nightly suffix or by switching to a version scheme that consistently sorts above
published and rc artifacts.

Expand All @@ -96,13 +85,12 @@ def stamp_sdk_version(
nightly_timestamp: str,
) -> str:
safe_sdk_id(sdk_id)
shared_sdk_version_path = source_root / "packages/nmp_common/src/nmp/common/version.py"
generated_sdk_version_path = source_root / "sdk/python/nemo-platform/src/nemo_platform/_version.py"
sdk_version = resolve_sdk_version(cadence, release_label, nightly_timestamp, shared_sdk_version_path)

replace_assignment(shared_sdk_version_path, "platform_sdk_version", sdk_version)
replace_assignment(generated_sdk_version_path, "__version__", sdk_version)
return sdk_version
return resolve_sdk_version(
cadence=cadence,
release_label=release_label,
nightly_timestamp=nightly_timestamp,
source_root=source_root,
)


def parse_args(argv: list[str]) -> argparse.Namespace:
Expand Down Expand Up @@ -138,10 +126,10 @@ def main(argv: list[str]) -> int:

if args.print_version:
# Machine-readable mode: human banner to stderr, just the version on stdout.
print(f"Stamped sdk:{args.sdk_id} version {sdk_version}.", file=sys.stderr)
print(f"Resolved sdk:{args.sdk_id} version {sdk_version}.", file=sys.stderr)
print(sdk_version)
else:
print(f"Stamped sdk:{args.sdk_id} version {sdk_version}.")
print(f"Resolved sdk:{args.sdk_id} version {sdk_version}.")
return 0


Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/release-bundle.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,8 @@ jobs:
repository: ${{ needs.plan-release.outputs.source_repo }}
ref: ${{ needs.plan-release.outputs.source_sha }}
path: source
fetch-depth: 0

@matthewgrossman matthewgrossman Jun 25, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this just speed things up?

@mckornfield mckornfield Jun 25, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

opposite, it disables a shallow clone lol

fetch-tags: true

# Toolchain setup + version stamp + `uv build` are factored into the
# composite action, which ci.yaml's wheel-test job also calls so the
Expand Down
39 changes: 25 additions & 14 deletions RELEASING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@

This document describes the end-to-end process for cutting and shipping a new version of NeMo Platform.

> **Who can release?** Any team member can open the version bump PR and trigger the workflow. The `release-stable` GitHub Actions environment requires approval from a member of the `nmp_devops` team before the workflow proceeds.
> **Who can release?** Any team member can trigger the workflow. The `release-stable` GitHub Actions environment requires approval from a member of the `nmp_devops` team before the workflow proceeds.

---

## Overview

> The examples in this document use `0.1.2` as the version being released and `0.1.3` as the next development version.
> The examples in this document use `0.1.2` as the version being released.

```
merge version bump PR (0.1.3) → trigger release-stable.yaml at parent commit (0.1.2) → nmp_devops approval → Platform-Deploy publishes to PyPI
trigger release-stable.yaml with source_sha + version → nmp_devops approval → workflow tags source_sha → Platform-Deploy publishes to PyPI
```

Artifacts published on a stable release:
Expand All @@ -20,31 +20,40 @@ Artifacts published on a stable release:

Nightly builds go to `pypi.nvidia.com` (NVIDIA's internal/public PyPI mirror), **not** public PyPI.

## Versioning model

Release and nightly wheel versions are resolved at build time. The release workflow runs `.github/scripts/stamp_sdk_version.py`, then passes the resolved version to Hatch through `UV_DYNAMIC_VERSIONING_BYPASS`.

Dynamic versioning is intentionally limited to packages that need release/nightly wheel metadata:
- `packages/nemo_platform` (`nemo-platform`)
- `packages/nemo_platform_plugin` (`nemo-platform-plugin`)
- `sdk/python/nemo-platform` (`nemo-platform-sdk`, consumed by the released wrappers and SDK tooling)

All other first-party workspace packages use static stub versions, normally `0.0.0`, because they are implementation packages rather than independently released artifacts. Do not add `nmp-dynamic-versioning` to another package unless that package is added to the release catalog or otherwise needs published wheel metadata.

`packages/nmp_build_tools` centralizes the Hatch version source and its defaults, but that package itself is also an internal stub-version package. The OpenAPI specs are schema inputs for SDK generation and intentionally keep a fixed `info.version: 0.0.0`; package release versions should not be copied into the specs.

---

## Step 1 — Merge the version bump PR first
## Step 1 — Choose the source SHA and release version

Before triggering the release, merge a PR that bumps `version.py` to the *next* version (e.g. `0.1.2` → `0.1.3`). You will then release from the commit *just before* that merge — the last commit where `version.py` still reads `0.1.2`. This means:
- The bump is already on `main` before you touch the release workflow — no race window.
- Nightlies built after the bump PR merges will immediately produce `0.1.3.dev...` strings.
Pick the full 40-character commit SHA on `main` that should be released, plus the SemVer core version to publish, for example `0.1.2`. There is no version bump PR to merge first: the stable workflow creates the release tag at `source_sha`, and the wheel build receives the package version from the workflow input.

The bump PR must also regenerate the OpenAPI spec and SDKs, since they embed the version:
If the API surface changed since the last SDK update, regenerate the OpenAPI spec and SDKs before releasing:

```bash
make update-sdk
```

This runs `make refresh-openapi` (regenerates `openapi/openapi.yaml` and plugin specs) and then syncs the Python and web SDKs via Stainless. Requires `STAINLESS_API_KEY` to be set — see `sdk/README.md` for setup instructions.
This runs `make refresh-openapi` (regenerates `openapi/openapi.yaml` and plugin specs) and then syncs the Python and web SDKs via Stainless. Requires `STAINLESS_API_KEY` to be set — see `sdk/README.md` for setup instructions. The generated OpenAPI specs should keep `info.version: 0.0.0`.

To find the right SHA:

```bash
git log --oneline main | head -5
# The second line is the commit just before the bump PR
# Pick the commit to release and copy its full 40-character SHA.
```

Or find it in the GitHub commit history — it's the parent of the bump PR's merge commit.

---

## Step 2 — Trigger the stable release workflow
Expand All @@ -54,9 +63,11 @@ Navigate to the [`release-stable.yaml` workflow](https://github.com/NVIDIA-NeMo/
| Input | Required | Description |
|---|---|---|
| `source_sha` | Yes | The full 40-character commit SHA to release from (must be on `main`). |
| `version` | Yes | SemVer core version string to release, e.g. `0.1.2`. Must match the value in `version.py` at `source_sha` (not the current `main`, which has already been bumped to `0.1.3`). |
| `version` | Yes | SemVer core version string to release, e.g. `0.1.2`. This becomes the stable git tag and wheel version. |
| `release_date` | No | `YYYY-MM-DD`. Provide only on the first run for a given version; leave blank on reruns. |
| `release_scope` | No | `sdks` (default) — releases both `nemo-platform` and `nemo-platform-plugin`. Use `custom` + `sdk_ids` to release a subset. |
| `release_scope` | No | `all` (default) releases every catalog SDK and container. Use `sdks`, `containers`, or `custom` for narrower releases. |
| `sdk_ids` | No | Comma-separated SDK IDs for `release_scope: custom`; must exist in `release/assets.yaml`. |
| `container_ids` | No | Comma-separated container IDs for `release_scope: custom`; must exist in `release/assets.yaml`. |

The workflow runs from the **`main` branch** by default. The `source_sha` must be reachable from that branch.

Expand Down
2 changes: 1 addition & 1 deletion openapi/ga/individual/platform.openapi.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion openapi/ga/openapi.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion openapi/openapi.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion packages/data_designer_nemo/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
[project]
name = "data-designer-nemo"
version = "0.1.0"
description = "NeMo-platform integration helpers for the Data Designer library."
readme = "README.md"
requires-python = ">=3.11,<3.15"
Expand All @@ -12,6 +11,7 @@ dependencies = [
"duckdb",
"pydantic>=2",
]
version = "0.0.0"

[project.optional-dependencies]
test = [
Expand All @@ -26,10 +26,12 @@ build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/data_designer_nemo"]


[tool.uv.sources]
nemo-platform = { workspace = true }

[tool.pytest.ini_options]
testpaths = ["tests"]
asyncio_mode = "auto"
pythonpath = ["src"]

3 changes: 2 additions & 1 deletion packages/filesets/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
[project]
name = "filesets"
version = "0.0.0"
description = "Extended FilesetsResource with fsspec filesystem support"
readme = "README.md"
authors = [{ name = "NVIDIA", email = "nemo@nvidia.com" }]
Expand All @@ -9,6 +8,7 @@ dependencies = [
"anyio>=4.0.0",
"fsspec>=2023.1.0",
]
version = "0.0.0"

[dependency-groups]
dev = [
Expand All @@ -24,6 +24,7 @@ build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/filesets"]


[tool.vendor-package]
package = "filesets"
package_root = "packages/filesets"
Expand Down
3 changes: 2 additions & 1 deletion packages/models/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
[project]
name = "models"
version = "0.0.0"
description = "Extended ModelsResource with high-level helper methods"
readme = "README.md"
authors = [{ name = "NVIDIA", email = "nemo@nvidia.com" }]
Expand All @@ -9,6 +8,7 @@ dependencies = [
"nemo-platform-sdk",
"openai",
]
version = "0.0.0"

[build-system]
requires = ["hatchling"]
Expand All @@ -17,6 +17,7 @@ build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/models"]


[dependency-groups]
dev = []

Expand Down
Loading
Loading