diff --git a/.github/workflows/back-merge.yml b/.github/workflows/back-merge.yml new file mode 100644 index 0000000000..4a3392a85f --- /dev/null +++ b/.github/workflows/back-merge.yml @@ -0,0 +1,157 @@ +# Opens an automated PR back-merging `main` into `dev` after every release, so +# the prerelease cycle reopens from the freshly published stable version. +# +# It cannot trigger on `push` to main: the release commit Lerna creates is +# `[no ci] Release: x.y.z`, which makes GitHub skip push-triggered workflows. +# Instead it runs via `workflow_run` once the CD workflow finishes on `main`. +# +# Deterministic conflicts (version fields, CHANGELOGs) are resolved by +# scripts/back-merge-resolve.mjs; generated files and the lockfile are +# regenerated. Any leftover (non-deterministic) conflict is auto-resolved to the +# dev side and the PR is opened as a draft flagged for manual review, so a human +# confirms no `main` hotfix was dropped before merging. +name: Back-merge main to dev + +on: + workflow_run: + workflows: ["CD"] + types: [completed] + +permissions: + contents: write + pull-requests: write + +concurrency: + group: back-merge-main-to-dev + cancel-in-progress: false + +jobs: + back-merge: + name: Back-merge + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_branch == 'main' + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + with: + token: ${{ secrets.VTEX_GITHUB_BOT_TOKEN }} + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10.28.0 + + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: 24.13.0 + cache: "pnpm" + + - name: Configure CI Git User + run: | + git config user.name vtexgithubbot + git config user.email vtexgithubbot@github.com + + - name: Prepare sync branch + id: prep + run: | + git fetch origin main dev --tags + VERSION=$(node -p "require('./lerna.json').version") + BRANCH="chore/sync-dev-with-main-$VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "branch=$BRANCH" >> "$GITHUB_OUTPUT" + if git merge-base --is-ancestor origin/main origin/dev; then + echo "main is already contained in dev; nothing to back-merge." + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + git checkout -B "$BRANCH" origin/dev + fi + + - name: Merge main and resolve + if: steps.prep.outputs.skip == 'false' + id: resolve + run: | + # Let the merge report conflicts without aborting the step. + git merge --no-ff origin/main \ + -m "chore: back-merge main into dev (v${{ steps.prep.outputs.version }})" || true + + # Deterministic conflicts: version fields + CHANGELOGs. + node ./scripts/back-merge-resolve.mjs + + # Whatever remains is either regenerated (lockfile / generated code) + # or needs a human. Keep the dev side to clear markers, and flag the + # non-regenerable ones for review. + DRAFT=false + FLAGGED="" + for f in $(git diff --name-only --diff-filter=U); do + case "$f" in + pnpm-lock.yaml|*/@generated/*|*/__generated__/*) ;; + *) DRAFT=true; FLAGGED="${FLAGGED}- \`${f}\`"$'\n' ;; + esac + git checkout --ours -- "$f" + git add -- "$f" + done + echo "draft=$DRAFT" >> "$GITHUB_OUTPUT" + { + echo "flagged<> "$GITHUB_OUTPUT" + + - name: Regenerate lockfile and codegen + if: steps.prep.outputs.skip == 'false' + run: | + pnpm i + pnpm --filter @faststore/api build + pnpm --filter @faststore/api generate + pnpm --filter @faststore/core generate + git add -A + + - name: Commit and push + if: steps.prep.outputs.skip == 'false' + run: | + if [ -f .git/MERGE_HEAD ]; then + git commit --no-edit + elif ! git diff --cached --quiet; then + git commit -m "chore: regenerate lockfile and codegen after back-merge" + fi + git push --force-with-lease origin "${{ steps.prep.outputs.branch }}" + + - name: Open pull request + if: steps.prep.outputs.skip == 'false' + env: + GH_TOKEN: ${{ secrets.VTEX_GITHUB_BOT_TOKEN }} + VERSION: ${{ steps.prep.outputs.version }} + BRANCH: ${{ steps.prep.outputs.branch }} + DRAFT: ${{ steps.resolve.outputs.draft }} + FLAGGED: ${{ steps.resolve.outputs.flagged }} + run: | + EXISTING=$(gh pr list --head "$BRANCH" --base dev --state open --json number --jq '.[0].number' || true) + if [ -n "$EXISTING" ]; then + echo "PR already open for $BRANCH: #$EXISTING" + exit 0 + fi + + # Build the body via printf + env vars (never shell-interpolate the + # flagged file list — it contains backticks). + BODY_FILE="$RUNNER_TEMP/pr-body.md" + { + printf 'Automated back-merge of `main` (v%s) into `dev`, to reopen the prerelease cycle from the freshly published stable version.\n\n' "$VERSION" + printf 'Deterministic conflicts (version fields and CHANGELOGs) were resolved automatically; the lockfile and generated files were regenerated.\n' + if [ "$DRAFT" = "true" ]; then + printf '\n> :warning: **Needs manual review.** Non-deterministic conflicts were auto-resolved by keeping the `dev` side in the files below. Verify no `main` hotfix was dropped before merging:\n\n%s\n' "$FLAGGED" + fi + } > "$BODY_FILE" + + DRAFT_FLAG="" + if [ "$DRAFT" = "true" ]; then DRAFT_FLAG="--draft"; fi + + gh pr create \ + --base dev \ + --head "$BRANCH" \ + --title "chore: update dev with main (v$VERSION)" \ + --body-file "$BODY_FILE" $DRAFT_FLAG diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d62e7c1952..b9487d42ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,3 +92,24 @@ jobs: run: | cd starter yarn build + + # Informational check (never blocks): flags FastStore packages whose latest + # published version lacks an npm provenance/attestation — the symptom of a + # manual publish instead of the CD OIDC Trusted Publisher (what broke + # @faststore/diagnostics in the v4 launch). See scripts/check-provenance.mjs. + provenance: + name: npm provenance (informational) + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 24.13.0 + + - name: Check provenance of published packages + continue-on-error: true + run: node ./scripts/check-provenance.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0af475cd8c..d394888838 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,6 +67,9 @@ jobs: - name: Verify publish manifests run: pnpm release:verify + - name: Diagnose changed packages + run: node ./scripts/diagnose-changed.mjs + - name: Publish (main) if: github.ref_name == 'main' run: pnpm release diff --git a/package.json b/package.json index ba16a83195..c02185ed25 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,9 @@ "test:coverage": "turbo run test:coverage", "ci:dkcicd": "pnpm build && pnpm lint && pnpm size && pnpm test:coverage && mkdir -p coverage && pnpm dlx lcov-result-merger 'packages/*/coverage/lcov.info' coverage/lcov.info --prepend-source-files", "size": "turbo run size", - "release": "lerna version --conventional-commits --conventional-graduate --yes && pnpm -r publish --access public --no-git-checks", + "release": "lerna version --conventional-commits --conventional-graduate --force-conventional-graduate --yes && pnpm -r publish --access public --no-git-checks", "release:dev": "lerna version --conventional-commits --conventional-prerelease --preid=dev --force-publish --yes && pnpm -r publish --access public --tag dev --no-git-checks", - "release:v3": "lerna version --conventional-commits --yes && pnpm -r publish --access public --tag v3-latest --no-git-checks", + "release:v3": "lerna version --conventional-commits --force-publish --yes && pnpm -r publish --access public --tag v3-latest --no-git-checks", "release:verify": "node ./scripts/verify-publish-manifests.mjs", "clean": "turbo run clean && rm -rf node_modules", "check:spec-kit": "node ./scripts/check-spec-kit.mjs", diff --git a/scripts/back-merge-resolve.mjs b/scripts/back-merge-resolve.mjs new file mode 100644 index 0000000000..cc08baaee4 --- /dev/null +++ b/scripts/back-merge-resolve.mjs @@ -0,0 +1,176 @@ +#!/usr/bin/env node +// Resolves the *deterministic* merge conflicts produced when back-merging +// `main` into `dev` (after a release). It is meant to run while a `git merge` +// is in progress with conflicts, and it only touches conflicts whose +// resolution is unambiguous: +// +// 1. Version fields — `lerna.json` and every `packages/*/package.json`. In +// fixed mode the only conflicting hunk is `version`; we take the stable +// value from `main` (theirs) while keeping every other dev-side change in +// the file (new deps, scripts, etc.). +// 2. CHANGELOGs — `CHANGELOG.md` and `packages/*/CHANGELOG.md`. Both sides +// only ever *prepend* entries, so we union the conflicting blocks (ours +// first, then any theirs line not already present), never deleting an +// entry. This preserves `[no ci] Release: X.Y.Z` lines from both sides. +// +// Anything else (source files, generated files, lockfile) is intentionally +// left untouched — the workflow regenerates generated files / the lockfile and +// flags any remaining conflict for manual review. The script prints a summary +// and always exits 0; the caller decides what to do with leftover conflicts. + +import { execFileSync } from 'node:child_process' +import { readFileSync, writeFileSync } from 'node:fs' + +function git(args, opts = {}) { + return execFileSync('git', args, { encoding: 'utf8', ...opts }) +} + +function unmergedFiles() { + const out = git(['diff', '--name-only', '--diff-filter=U']) + return out + .split('\n') + .map((s) => s.trim()) + .filter(Boolean) +} + +function isVersionFile(file) { + return file === 'lerna.json' || /^packages\/[^/]+\/package\.json$/.test(file) +} + +function isChangelog(file) { + return ( + file === 'CHANGELOG.md' || /^packages\/[^/]+\/CHANGELOG\.md$/.test(file) + ) +} + +// Read a specific merge stage of a conflicted file. Stage 2 = ours (dev), +// stage 3 = theirs (main). Returns null when the stage is absent. +function readStage(stage, file) { + try { + return git(['show', `:${stage}:${file}`]) + } catch { + return null + } +} + +// Take the dev-side manifest as the base (keeps dev-only edits) and only +// overwrite `version` with the stable value coming from main. +function resolveVersionFile(file) { + const ours = readStage(2, file) + const theirs = readStage(3, file) + if (ours == null || theirs == null) return false + + const oursJson = JSON.parse(ours) + const theirsJson = JSON.parse(theirs) + if (typeof theirsJson.version !== 'string') return false + + oursJson.version = theirsJson.version + + // Preserve trailing newline style used across the repo manifests. + writeFileSync(file, `${JSON.stringify(oursJson, null, 2)}\n`) + git(['add', '--', file]) + return theirsJson.version +} + +// Union the two sides of every conflict block, ours first, appending theirs +// lines that are not already present (dedupe by trimmed, non-empty content). +function unionResolveContent(content) { + const lines = content.split('\n') + const out = [] + let i = 0 + + while (i < lines.length) { + if (lines[i].startsWith('<<<<<<<')) { + i++ // skip the "<<<<<<< ours" marker + const ours = [] + while ( + i < lines.length && + !lines[i].startsWith('|||||||') && + !lines[i].startsWith('=======') + ) { + ours.push(lines[i]) + i++ + } + // Skip an optional diff3 base section (||||||| ... =======). + if (i < lines.length && lines[i].startsWith('|||||||')) { + i++ + while (i < lines.length && !lines[i].startsWith('=======')) i++ + } + i++ // skip the "=======" marker + const theirs = [] + while (i < lines.length && !lines[i].startsWith('>>>>>>>')) { + theirs.push(lines[i]) + i++ + } + i++ // skip the ">>>>>>>" marker + + const merged = [...ours] + const seen = new Set( + ours.map((l) => l.trim()).filter((l) => l.length > 0) + ) + for (const line of theirs) { + const key = line.trim() + if (key.length > 0 && seen.has(key)) continue + if (key.length > 0) seen.add(key) + merged.push(line) + } + out.push(...merged) + } else { + out.push(lines[i]) + i++ + } + } + + return out.join('\n') +} + +function resolveChangelog(file) { + // The working-tree copy still holds the conflict markers written by git. + let raw + try { + raw = readFileSync(file, 'utf8') + } catch { + return false + } + const resolved = unionResolveContent(raw) + writeFileSync(file, resolved) + git(['add', '--', file]) + return true +} + +function main() { + const conflicts = unmergedFiles() + if (conflicts.length === 0) { + console.log('No conflicts to resolve.') + return + } + + const resolvedVersions = [] + const resolvedChangelogs = [] + const skipped = [] + + for (const file of conflicts) { + if (isVersionFile(file)) { + const version = resolveVersionFile(file) + if (version) resolvedVersions.push(`${file} -> ${version}`) + else skipped.push(file) + } else if (isChangelog(file)) { + if (resolveChangelog(file)) resolvedChangelogs.push(file) + else skipped.push(file) + } else { + skipped.push(file) + } + } + + console.log('Back-merge deterministic resolution summary:') + console.log(` version files resolved: ${resolvedVersions.length}`) + for (const v of resolvedVersions) console.log(` ${v}`) + console.log(` CHANGELOGs unioned: ${resolvedChangelogs.length}`) + for (const c of resolvedChangelogs) console.log(` ${c}`) + if (skipped.length > 0) { + console.log(` left for the workflow: ${skipped.length}`) + for (const s of skipped) console.log(` ${s}`) + } +} + +main() diff --git a/scripts/check-provenance.mjs b/scripts/check-provenance.mjs new file mode 100644 index 0000000000..fd2b8b2175 --- /dev/null +++ b/scripts/check-provenance.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node +// Informational (non-blocking): flags FastStore packages whose latest published +// version has no npm provenance/attestation. A missing attestation usually means +// the last publish was done manually (personal account / token) instead of via +// the CD OIDC Trusted Publisher — the exact gap that broke @faststore/diagnostics +// during the v4 launch (E404 then E422). +// +// There is no clean public API to read the Trusted Publisher configuration, so +// this infers it from the presence of a provenance attestation on the latest +// published version. It never fails the build; it only surfaces a warning so a +// maintainer can configure the Trusted Publisher before the next CI release. + +import { execFileSync } from 'node:child_process' +import { readFileSync, readdirSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const __dirname = fileURLToPath(new URL('.', import.meta.url)) +const PACKAGES_DIR = resolve(__dirname, '..', 'packages') + +function publicPackageNames() { + return readdirSync(PACKAGES_DIR, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join(PACKAGES_DIR, entry.name, 'package.json')) + .map((path) => { + try { + return JSON.parse(readFileSync(path, 'utf8')) + } catch { + return null + } + }) + .filter((pkg) => pkg && !pkg.private && pkg.name) + .map((pkg) => pkg.name) +} + +function npmView(spec, field) { + try { + const args = ['view', spec] + if (field) args.push(field) + args.push('--json') + return execFileSync('npm', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }).trim() + } catch { + return '' + } +} + +const flagged = [] +const ok = [] +const skipped = [] + +for (const name of publicPackageNames()) { + const latestRaw = npmView(name, 'dist-tags.latest') + if (!latestRaw) { + skipped.push(`${name} (no latest dist-tag / not published)`) + continue + } + const latest = JSON.parse(latestRaw) + const attestations = npmView(`${name}@${latest}`, 'dist.attestations') + + if (attestations && attestations !== 'null' && attestations !== '{}') { + ok.push(`${name}@${latest}`) + } else { + flagged.push(`${name}@${latest}`) + } +} + +console.log('npm provenance check (latest dist-tag):') +for (const p of ok) console.log(` ok: ${p}`) +for (const p of skipped) console.log(` skipped: ${p}`) +for (const p of flagged) console.log(` NO prov: ${p}`) + +if (flagged.length > 0) { + const list = flagged.join(', ') + console.log( + `::warning title=Missing npm provenance::${list} — the latest published ` + + 'version has no provenance/attestation, which usually means a manual ' + + 'publish rather than the CD OIDC Trusted Publisher. Configure the ' + + 'Trusted Publisher for these packages before the next CI release to ' + + 'avoid E404/E422 failures.' + ) +} + +// Always succeed: informational check only. +process.exit(0) diff --git a/scripts/diagnose-changed.mjs b/scripts/diagnose-changed.mjs new file mode 100644 index 0000000000..0802b11f40 --- /dev/null +++ b/scripts/diagnose-changed.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +// Diagnostic (non-blocking): reports whether Lerna would find any changed +// package to version on the current branch. It makes the "No changed packages +// to version" situation obvious in the CI log — with a clear reason — instead +// of a silent no-op that people misread as a broken release. +// +// This does NOT gate the release. The release scripts force-publish (dev/v3) or +// force-conventional-graduate (main), so packages are still published even when +// Lerna reports no natural changes; this step only explains what is happening. + +import { execFileSync } from 'node:child_process' + +function run(cmd, args) { + try { + return { code: 0, out: execFileSync(cmd, args, { encoding: 'utf8' }) } + } catch (err) { + return { + code: err.status ?? 1, + out: `${err.stdout ?? ''}${err.stderr ?? ''}`, + } + } +} + +// `lerna changed` lists packages with changes since the last release tag and +// exits non-zero when there are none. +const { code, out } = run('pnpm', ['exec', 'lerna', 'changed', '--long']) +if (out.trim()) process.stdout.write(`${out.trimEnd()}\n`) + +const nothingChanged = code !== 0 || /No changed packages/i.test(out) + +if (nothingChanged) { + const reason = + 'No changed packages to version. The commit likely only touched files ' + + 'outside packages/ or matched lerna.json > ignoreChanges ' + + '(**/*.md, **/*.test.*, **/*.snap, **/__fixtures__/**). ' + + 'The release still force-publishes all packages, so the version moves only ' + + 'by the prerelease/graduate counter — confirm this is intended.' + console.log(`::warning title=No changed packages to version::${reason}`) +} else { + console.log( + 'Changed packages detected — the release will version real changes.' + ) +} + +// Always succeed: this is an informational diagnostic, not a gate. +process.exit(0) diff --git a/scripts/verify-publish-manifests.mjs b/scripts/verify-publish-manifests.mjs index f24ca48834..7cec1c5df1 100755 --- a/scripts/verify-publish-manifests.mjs +++ b/scripts/verify-publish-manifests.mjs @@ -1,16 +1,22 @@ #!/usr/bin/env node -// Verifies that the package manifests we are about to publish do not contain -// unresolved `workspace:` or `catalog:` protocols. These protocols are pnpm -// workspace primitives that MUST be rewritten to concrete versions before a -// package is uploaded to the npm registry. Publishing them as-is breaks -// installation for any consumer outside of this monorepo. +// Verifies that the package manifests we are about to publish are safe to +// upload to the npm registry. Two guards run per package: +// +// 1. No unresolved `workspace:` / `catalog:` protocols. These are pnpm +// workspace primitives that MUST be rewritten to concrete versions before +// publish; shipping them breaks installation for consumers outside this +// monorepo. +// 2. A valid `repository.url` that matches this repo. npm provenance (OIDC +// Trusted Publisher) compares the manifest repo against the CI repo, so a +// missing/empty/mismatched `repository.url` fails publish with `E422` +// (the exact failure that hit `@faststore/diagnostics` in the v4 launch). // // Strategy: pack every non-private package in `packages/*` with `pnpm pack` // (which is the same code path `pnpm publish` uses to produce the tarball) // and inspect the `package/package.json` entry inside each tarball. // -// Exits with status 1 if any unresolved protocol is found, so it can be wired -// into CI as a guard before the actual publish step. +// Exits with status 1 if any guard fails, so it can be wired into CI as a +// guard before the actual publish step. import { execFileSync } from 'node:child_process' import { @@ -21,7 +27,7 @@ import { rmSync, } from 'node:fs' import { tmpdir } from 'node:os' -import { join, resolve } from 'node:path' +import { basename, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { createGunzip } from 'node:zlib' @@ -41,6 +47,25 @@ function readJson(path) { return JSON.parse(readFileSync(path, 'utf8')) } +// Normalize a git/repository URL for comparison: drop the `git+` prefix, a +// trailing `.git`, and any trailing slash. Accepts a string or the object form +// (`{ type, url, directory }`). +function normalizeRepoUrl(repository) { + const raw = + typeof repository === 'string' ? repository : (repository?.url ?? '') + return raw + .trim() + .replace(/^git\+/, '') + .replace(/\.git$/, '') + .replace(/\/$/, '') +} + +// The repository URL every published package must point at. Read from the root +// manifest so this script stays repo-agnostic (works on forks/mirrors too). +const EXPECTED_REPO_URL = normalizeRepoUrl( + readJson(join(ROOT, 'package.json')).repository +) + function listPublicPackages() { return readdirSync(PACKAGES_DIR, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) @@ -125,6 +150,34 @@ function findUnresolvedProtocols(manifest) { return offenders } +// Returns an array of human-readable problems with the manifest's `repository` +// field (empty array means it is valid). +function findRepositoryProblems(manifest, expectedDir) { + if (!EXPECTED_REPO_URL) return [] // no root repo url to compare against + + const problems = [] + const repository = manifest.repository + const url = normalizeRepoUrl(repository) + + if (!url) { + problems.push('repository.url is missing or empty') + } else if (url !== EXPECTED_REPO_URL) { + problems.push(`repository.url = "${url}" (expected "${EXPECTED_REPO_URL}")`) + } + + // `repository.directory` is optional, but when present it must point at the + // package's own folder so provenance/tooling can locate the source. + if (repository && typeof repository === 'object' && repository.directory) { + if (repository.directory !== expectedDir) { + problems.push( + `repository.directory = "${repository.directory}" (expected "${expectedDir}")` + ) + } + } + + return problems +} + async function main() { const packages = listPublicPackages() if (packages.length === 0) { @@ -160,12 +213,17 @@ async function main() { ) const manifest = JSON.parse(manifestRaw) const offenders = findUnresolvedProtocols(manifest) + const expectedDir = `packages/${basename(pkgDir)}` + const repoProblems = findRepositoryProblems(manifest, expectedDir) - if (offenders.length > 0) { + if (offenders.length > 0 || repoProblems.length > 0) { console.log('FAIL') for (const { field, name, version } of offenders) { console.error(` ${field}.${name} = "${version}"`) } + for (const problem of repoProblems) { + console.error(` ${problem}`) + } failed = true } else { console.log('OK') @@ -177,9 +235,13 @@ async function main() { if (failed) { console.error( - '\nUnresolved workspace:/catalog: protocols would be published.\n' + - 'Make sure release scripts use `pnpm -r publish` (not `lerna publish` /\n' + - '`npm publish`) so the protocols are rewritten before upload.' + '\nOne or more manifests are not safe to publish:\n' + + ' • Unresolved workspace:/catalog: protocols — make sure release\n' + + ' scripts use `pnpm -r publish` (not `lerna publish` / `npm publish`)\n' + + ' so the protocols are rewritten before upload.\n' + + ` • repository.url must match "${EXPECTED_REPO_URL}" (and\n` + + ' repository.directory must point at the package) — a missing or\n' + + ' mismatched value fails npm provenance with E422.' ) process.exit(1) }