diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml index 4429c578..7eeffe75 100644 --- a/.github/actions/bootstrap/action.yml +++ b/.github/actions/bootstrap/action.yml @@ -1,6 +1,16 @@ name: Bootstrap description: Set up Node + pnpm, install dependencies, and restore the turbo cache. +inputs: + turbo-cache: + description: >- + Restore and save .turbo/cache. Only a job that actually runs turbo may set this. + Jobs share one cache key per commit, so a job that touches the cache without + producing task outputs would save an empty one and lock the real producer out + of saving (actions/cache refuses to overwrite its own primary key). + required: false + default: 'true' + runs: using: composite steps: @@ -30,11 +40,14 @@ runs: shell: bash run: pnpm install --frozen-lockfile - # Same key per commit -> the build job's cache is an exact hit for every downstream job. - # turbo 2.x writes to .turbo/cache at the repo root, not node_modules/.cache/turbo. + # Keyed on the HEAD commit, not `github.sha`: on `pull_request` that is the ephemeral + # merge sha, which repeats whenever the base hasn't moved - actions/cache then hits its + # own primary key, logs "not saving cache", and discards every task hash the run just + # computed. turbo 2.x writes to .turbo/cache at the repo root, not node_modules/.cache. - name: Cache turbo + if: inputs.turbo-cache == 'true' uses: actions/cache@v4 with: path: .turbo/cache - key: turbo-${{ runner.os }}-${{ github.sha }} + key: turbo-${{ runner.os }}-${{ github.event.pull_request.head.sha || github.sha }} restore-keys: turbo-${{ runner.os }}- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad3075fc..e704100f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,11 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + # No turbo here - taking the shared per-commit cache key would lock the verify + # job out of saving its own task outputs. - uses: ./.github/actions/bootstrap + with: + turbo-cache: 'false' - name: Commitlint run: pnpm commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} @@ -59,14 +63,15 @@ jobs: - uses: ./.github/actions/bootstrap - name: Prepare migration database run: pnpm db:setup:test + # Through turbo, not `pnpm --filter`, so the artifact lands in the turbo cache + # and the `build` inside Verify below is a hit instead of a second full tsc. - name: Build migration runner - run: pnpm --filter @openora/core build + run: pnpm exec turbo run build --filter=@openora/core - name: Run migrations run: pnpm db:migrate - # Fans out types, lint, format, boundaries, module shape, deprecations, unit - # + tool tests through turbo (parallel + cached). `build` runs implicitly as - # the `check:types` dependency. + # Fans out lint, format, boundaries, module shape, deprecations, unit + tool + # tests through turbo (parallel + cached). `build` runs implicitly as the + # test dependency and is what typechecks (every package builds with tsc). + # `pnpm verify` ends with check:drift - no separate step needed. - name: Verify run: pnpm verify - - name: Drift check - run: pnpm check:drift diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 184bc4c3..34af7494 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -40,19 +40,18 @@ jobs: steps: - uses: actions/checkout@v4 + # Restores the turbo cache the `ci` job just populated for this same commit, so the + # build below is a cache hit rather than a second full compile of every package. + - uses: ./.github/actions/bootstrap + + # `setup-node` again purely for the npm registry auth the publish step needs - + # bootstrap installs node but does not write an authenticated .npmrc. - uses: actions/setup-node@v4 with: node-version: 26 registry-url: https://registry.npmjs.org scope: '@openora' - - name: Enable corepack (pnpm) - run: | - npm install -g corepack@latest - corepack enable - corepack prepare pnpm@11.8.0 --activate - - - run: pnpm install --frozen-lockfile # Build only the publishable packages - the apps are private and never published. - run: pnpm exec turbo run build --filter='!./apps/*' diff --git a/.rulesync/commands/verify.md b/.rulesync/commands/verify.md index 5f271ce2..2af33f1a 100644 --- a/.rulesync/commands/verify.md +++ b/.rulesync/commands/verify.md @@ -9,7 +9,9 @@ Parse $ARGUMENTS for an optional `--filter ` flag. If filter provided, run: `pnpm verify --filter ` Otherwise run: `pnpm verify` -This executes `turbo run check:types lint test:unit` (or scoped to the filter). +This executes `turbo run build check:lint check:format check:boundaries check:shape check:hygiene +check:deprecations test:unit test:integration test:tools`, then `pnpm check:drift` (or scoped to the +filter). `build` is what typechecks - see `docs/standards/enforcement.md`. After completion: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 49de1ca4..bb4fcfb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,28 +27,28 @@ Backoffice login: `admin@oss.dev` / `password123` (see `pnpm db:seed --help` for Scripts are grouped by prefix: `check:*` reports, `fix:*` rewrites, `gen:*` emits, `db:*` touches Postgres, `test:*` runs suites. Every `check:*` and `test:*` task goes through turbo, so repeat runs hit the cache. -| Command | What it does | -| ------------------------- | ---------------------------------------------------------------------------- | -| `pnpm dev` | turbo dev across docs, mcp | -| `pnpm verify` | the full gate - every `check:*` plus `test:unit` + `test:tools`, in parallel | -| `pnpm regen` | tsconfig paths + drizzle generate + catalog | -| `pnpm check:types` | `tsc --noEmit` across the workspace | -| `pnpm check:lint` | oxlint (incl. the `oss-boundaries/*` plugin) | -| `pnpm check:format` | oxfmt in check mode | -| `pnpm check:boundaries` | dependency-cruiser whole-graph boundary + cycle gate | -| `pnpm check:shape` | module layout conformance | -| `pnpm check:deprecations` | fails on any use of a `@deprecated` symbol | -| `pnpm check:drift` | regenerates the catalog and fails if the committed output is stale | -| `pnpm fix:lint` | oxlint `--fix` | -| `pnpm fix:format` | oxfmt write + final-newline pass | -| `pnpm test:unit` | vitest, no external services | -| `pnpm test:integration` | service/router tests against real Postgres | -| `pnpm test:tools` | `node --test` over `tools/__tests__` | -| `pnpm test:scaffold` | scaffolds a throwaway module and verifies it, then cleans up | -| `pnpm gen:agents` | regenerate the per-tool agent files from `.rulesync/` via rulesync | -| `pnpm db:migrate` | apply every module's migrations | -| `pnpm db:seed` | demo data (idempotent) | -| `pnpm db:setup:test` | provision the integration-test database | +| Command | What it does | +| ------------------------- | ------------------------------------------------------------------- | +| `pnpm dev` | turbo dev across docs, mcp | +| `pnpm verify` | the full gate - build, every `check:*`, tests, then the drift check | +| `pnpm regen` | tsconfig paths + drizzle generate + catalog | +| `pnpm check:types` | `tsc --noEmit` across the workspace | +| `pnpm check:lint` | oxlint (incl. the `oss-boundaries/*` plugin) | +| `pnpm check:format` | oxfmt in check mode | +| `pnpm check:boundaries` | dependency-cruiser whole-graph boundary + cycle gate | +| `pnpm check:shape` | module layout conformance | +| `pnpm check:deprecations` | fails on any use of a `@deprecated` symbol | +| `pnpm check:drift` | regenerates the catalog and fails if the committed output is stale | +| `pnpm fix:lint` | oxlint `--fix` | +| `pnpm fix:format` | oxfmt write + final-newline pass | +| `pnpm test:unit` | vitest, no external services | +| `pnpm test:integration` | service/router tests against real Postgres | +| `pnpm test:tools` | `node --test` over `tools/__tests__` | +| `pnpm test:scaffold` | scaffolds a throwaway module and verifies it, then cleans up | +| `pnpm gen:agents` | regenerate the per-tool agent files from `.rulesync/` via rulesync | +| `pnpm db:migrate` | apply every module's migrations | +| `pnpm db:seed` | demo data (idempotent) | +| `pnpm db:setup:test` | provision the integration-test database | `check:deprecations` excludes `packages/core/src/pam/identity/adapters/identity-reader.service.ts`: depretec mis-resolves Drizzle's `SQL.as` overloads and flags the non-deprecated `as(alias: string)` one. Drop the exclusion once depretec resolves overloads correctly. diff --git a/docs/standards/enforcement.md b/docs/standards/enforcement.md index d1bfcc37..92aa3048 100644 --- a/docs/standards/enforcement.md +++ b/docs/standards/enforcement.md @@ -7,5 +7,6 @@ Detail for the enforcement line in `conventions`. Read this when a gate fails or - Module structure + naming are lint-enforced (`oss-module-shape/*` oxlint JS plugin, `tools/lint/oxlint-module-shape-plugin.mjs`): files sit in a canonical layer dir, `service/` files end `.service.ts`, `__tests__/` files end `.test.ts`, an infra-backed test ends `.int.test.ts`, filenames kebab-case, no inline `pgEnum` value arrays. - oxlint config is split: the published `@openora/core/oxlint/oxlintrc.json` holds the universal, stack-agnostic rules (base rules, `typescript/no-explicit-any`, `typescript/no-non-null-assertion`, `typescript/consistent-type-definitions`, `import/no-cycle`, `import/no-duplicates`) - the single source of truth a consumer extends via `"extends": ["./node_modules/@openora/core/oxlint/oxlintrc.json"]`. The root `.oxlintrc.json` here `extends` that shared config and adds only OSS-internal rules (`oss-boundaries/*`, `oss-module-shape/*`, `unicorn/filename-case`) that need the local `jsPlugins`. - Generated migrations are byte-sensitive: drizzle hashes each file to decide what is already applied, so formatting hooks skip `**/drizzle/migrations/*.sql` and nothing may hand-edit them. -- Pre-commit runs `pnpm check:boundaries` + `pnpm check:types`; CI runs `pnpm verify` + the no-drift check. +- Pre-commit runs `pnpm check:boundaries` + `pnpm check:types`; CI runs `pnpm verify`, which ends with the no-drift check. +- `pnpm verify` runs `build` rather than `check:types`: every package builds with `tsc` over the tsconfig its `check:types` uses, so the build IS the typecheck and running both compiles the workspace twice. `pnpm check:types` stays for ad-hoc and pre-commit use. - Agent rules mirror this standard - generated from `.rulesync/` via `pnpm gen:agents`. diff --git a/package.json b/package.json index 58ff379f..0b441767 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "build": "turbo run build", "dev": "turbo run dev", "clean": "turbo run clean && rm -rf node_modules", - "verify": "turbo run check:types check:lint check:format check:boundaries check:shape check:hygiene check:deprecations test:unit test:integration test:tools && pnpm check:drift", + "verify": "turbo run build check:lint check:format check:boundaries check:shape check:hygiene check:deprecations test:unit test:integration test:tools && pnpm check:drift", "check:types": "turbo run check:types", "check:lint": "oxlint .", "check:format": "oxfmt --check", diff --git a/packages/core/src/testing/real-infra.ts b/packages/core/src/testing/real-infra.ts index ac9251ac..434f972d 100644 --- a/packages/core/src/testing/real-infra.ts +++ b/packages/core/src/testing/real-infra.ts @@ -10,7 +10,10 @@ const ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@localhost:5432/postgres'; const REDIS_URL = process.env['TEST_REDIS_URL'] ?? 'redis://localhost:6379'; -const REDIS_LOGICAL_DATABASE_COUNT = 16; +// Databases 0-7 belong to this tier; `@openora/testing` claims 8-15 (see its redis.ts). +// The split is what lets both integration suites run concurrently without flushing +// each other's keys - do not widen it without moving the other tier too. +const REDIS_LOGICAL_DATABASE_COUNT = 8; const REDIS_DATABASE = Number(process.env['VITEST_POOL_ID'] ?? 1) % REDIS_LOGICAL_DATABASE_COUNT; function withRedisDatabase(baseUrl: string, database: number): string { diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index d24e8820..702b922a 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -8,5 +8,10 @@ export default defineConfig({ environment: 'node', include: ['src/**/*.{test,spec}.ts'], exclude: ['dist/**', 'node_modules/**', 'src/**/*.int.{test,spec}.ts'], + // Deliberately left isolated. `isolate: false` cuts this tier from ~53s to ~16s by + // sharing one module graph across files, but a shared graph defeats `vi.mock` - a + // file that already imported the real module wins, and auth.test.ts/migrate.test.ts + // fail depending on how files group onto workers. It also buys no CI wall-clock: + // this tier runs concurrently underneath the much longer integration tier. }, }); diff --git a/packages/core/vitest.integration.config.ts b/packages/core/vitest.integration.config.ts index 9927826f..17b2d191 100644 --- a/packages/core/vitest.integration.config.ts +++ b/packages/core/vitest.integration.config.ts @@ -10,5 +10,9 @@ export default defineConfig({ exclude: ['dist/**', 'node_modules/**'], testTimeout: 30_000, hookTimeout: 30_000, + // This tier partitions Redis by `VITEST_POOL_ID % 8` (real-infra.ts), leaving 8-15 + // to `@openora/testing` so both integration suites can run concurrently. More than + // 8 workers would wrap that modulo and let two workers flush each other's keys. + maxWorkers: 8, }, }); diff --git a/packages/testing/src/redis.ts b/packages/testing/src/redis.ts index 62d2bde5..03705bb9 100644 --- a/packages/testing/src/redis.ts +++ b/packages/testing/src/redis.ts @@ -5,8 +5,8 @@ const INFRA_HINT = 'integration tests need redis - run `docker compose up -d`'; /** * Logical databases handed to `bootTestApp`, allocated downward from 15. The core - * unit harness allocates upward from `VITEST_POOL_ID % 16`, so the two tiers stay - * clear of each other when both run against the same Redis. + * harness is pinned to 0-7 (`VITEST_POOL_ID % 8`), so the two tiers stay clear of + * each other and both integration suites can run concurrently against one Redis. */ const HIGHEST_DATABASE = 15; const LOWEST_DATABASE = 8; diff --git a/turbo.json b/turbo.json index 176112ea..bb8ea7c0 100644 --- a/turbo.json +++ b/turbo.json @@ -3,6 +3,9 @@ "ui": "tui", "globalDependencies": ["packages/config/**", ".oxlintrc.json", ".oxfmtrc.json"], "tasks": { + // Every package builds with `tsc` over the same tsconfig its `check:types` uses, so a + // successful build IS the typecheck - `verify` runs this instead of `check:types` to + // avoid compiling the workspace twice. `check:types` stays for ad-hoc/pre-commit use. "build": { "dependsOn": ["^build"], "outputs": ["dist/**", ".next/**", "!.next/cache/**"] @@ -22,19 +25,6 @@ "dependsOn": ["^build", "build"], "outputs": [] }, - "@openora/testing#test:integration": { - "dependsOn": ["^build", "build", "@openora/core#test:integration"], - "cache": false, - "passThroughEnv": [ - "TEST_DATABASE_URL", - "DATABASE_URL", - "DATABASE_ADMIN_URL", - "TEST_ADMIN_DATABASE_URL", - "TEST_REDIS_URL", - "BETTER_AUTH_SECRET", - "BETTER_AUTH_URL" - ] - }, "test:integration": { "dependsOn": ["^build", "build"], "cache": false, @@ -42,6 +32,8 @@ "TEST_DATABASE_URL", "DATABASE_URL", "DATABASE_ADMIN_URL", + "TEST_ADMIN_DATABASE_URL", + "TEST_REDIS_URL", "BETTER_AUTH_SECRET", "BETTER_AUTH_URL" ]