test(serenity): API-level e2e against the Semrush vendor mocks

package.json

@@ -93,13 +93,13 @@
     "@adobe/spacecat-shared-http-utils": "1.31.0",
     "@adobe/spacecat-shared-ims-client": "1.12.7",
     "@adobe/spacecat-shared-launchdarkly-client": "1.3.0",
-    "@adobe/spacecat-shared-project-engine-client": "1.2.0",
+    "@adobe/spacecat-shared-project-engine-client": "1.3.0",
     "@adobe/spacecat-shared-rum-api-client": "2.44.0",
     "@adobe/spacecat-shared-scrape-client": "2.6.3",
     "@adobe/spacecat-shared-slack-client": "1.6.7",
     "@adobe/spacecat-shared-tier-client": "1.5.1",
     "@adobe/spacecat-shared-tokowaka-client": "1.19.0",
-    "@adobe/spacecat-shared-user-manager-client": "1.1.0",
+    "@adobe/spacecat-shared-user-manager-client": "1.3.0",
     "@adobe/spacecat-shared-utils": "1.123.0",
     "@adobe/spacecat-shared-vault-secrets": "1.3.5",
     "@aws-sdk/client-s3": "3.1045.0",

src/controllers/serenity.js

@@ -180,10 +180,31 @@ function mapError(e, log) {
  * missing OR if the caller authenticated by some other mechanism. The
  * upstream gateway only understands IMS user tokens; we refuse to forward
  * anything else.
+ *
+ * SECURITY MODEL — this proxy is NOT the auth boundary; Semrush is. The bearer
+ * we forward is validated AGAIN by the real Semrush gateway on every upstream
+ * call (it rejects an invalid/expired/forged token with 401/403, which the
+ * transport surfaces as a SerenityTransportError). This local check is only a
+ * fail-fast + shape guard so we do not forward a token Semrush will obviously
+ * reject; it never substitutes for the upstream's own validation.
+ *
+ * Test-only escape hatch: when `SERENITY_ALLOW_NON_IMS_AUTH === 'true'` the
+ * IMS-type check is skipped so an authenticated NON-IMS caller (e.g. the
+ * locally-signed JWT the integration-test harness mints) can reach the
+ * handlers. This is sound because (a) production auth is unaffected — Semrush
+ * still validates the forwarded token end to end — and (b) the integration
+ * tests run against the Semrush vendor MOCKS, which intentionally do not
+ * validate the bearer, so the token's value never matters there, only that an
+ * authenticated identity is present. Mirrors `SERENITY_ALLOW_WORKSPACE_DELETE`
+ * in rest-transport.js: an explicit opt-in flag that NO deployed environment
+ * sets (it is never written to Vault `dx_mysticat/<env>/api-service`); it is
+ * for local + automated E2E only. The Authorization-header requirement still
+ * holds — a bearer must be present to forward upstream.
  */
 function requireImsBearer(ctx) {
   const authInfo = ctx?.attributes?.authInfo;
-  if (authInfo?.getType && authInfo.getType() !== 'ims') {
+  const allowNonIms = ctx?.env?.SERENITY_ALLOW_NON_IMS_AUTH === 'true';
+  if (!allowNonIms && authInfo?.getType && authInfo.getType() !== 'ims') {
     throw new ErrorWithStatusCode(
       'Serenity proxy requires IMS authentication',
       401,
@@ -216,13 +237,22 @@ export function brandPointerReloader(ctx, brandUuid) {
   };
 }
 
+// Logged at most once per process: makes an accidental SERENITY_ALLOW_NON_IMS_AUTH
+// enablement in a deployed environment visible in the logs (the flag bypasses the
+// IMS-type gate — it must only ever be set for local/automated E2E).
+let warnedNonImsAuth = false;
+
 function SerenityController(context, log, env) {
   if (!isNonEmptyObject(context)) {
     throw new Error('Context required');
   }
   if (!log) {
     throw new Error('Log required');
   }
+  if (!warnedNonImsAuth && (context?.env || env)?.SERENITY_ALLOW_NON_IMS_AUTH === 'true') {
+    warnedNonImsAuth = true;
+    log.warn('[serenity] SERENITY_ALLOW_NON_IMS_AUTH is enabled — the IMS-type auth gate is bypassed. This is test-only and must never be set in a deployed environment.');
+  }
 
   /**
    * Verifies the caller has access to the addressed org AND the brand

src/support/serenity/rest-transport.js

@@ -67,53 +67,91 @@ export function redactUpstreamMessage(e) {
 }
 
 /**
- * Resolves and validates the upstream base URL. The URL is required and must
- * arrive via `env.SEMRUSH_PROJECTS_BASE_URL` — sourced from Vault
- * (`dx_mysticat/<env>/api-service`) and injected through AWS Secrets Manager.
- * No source default: the upstream host is operational config that must be
- * settable per-environment without a code change.
+ * Validates a raw base-URL value and returns its canonical `protocol//host`
+ * origin — never the raw value. A misconfigured value like
+ * `https://host/path-prefix` or `https://user:pass@host` would otherwise
+ * silently bleed path/userinfo into every outbound request (and the userinfo
+ * form would leak credentials in each fetch's `Authorization`-adjacent
+ * metadata). Always returning the parsed origin closes both classes of
+ * injection. The https requirement is kept for both resolvers.
  *
- * Returns the canonical `protocol//host` origin — never the raw value. A
- * misconfigured value like `https://host/path-prefix` or
- * `https://user:pass@host` would otherwise silently bleed path/userinfo into
- * every outbound request (and the userinfo form would leak credentials in
- * each fetch's `Authorization`-adjacent metadata). Always returning the
- * parsed origin closes both classes of injection.
+ * `varName` names the env var in every error message so a misconfiguration
+ * points the operator at the exact key to fix.
  *
  * Failure mapping (controller `mapError`):
  *   - Missing/invalid/non-https → throws ErrorWithStatusCode(503,
  *     'configurationError'): operational failure, not a runtime bug.
+ *
+ * @param {string | undefined} raw - The raw env value.
+ * @param {string} varName - The env var name, used in error messages.
+ * @returns {string} The canonical `protocol//host` origin.
  */
-function baseUrl(env) {
-  const raw = typeof env?.SEMRUSH_PROJECTS_BASE_URL === 'string'
-    ? env.SEMRUSH_PROJECTS_BASE_URL.trim()
-    : env?.SEMRUSH_PROJECTS_BASE_URL;
-  if (!hasText(raw)) {
+function normalizeBaseUrl(raw, varName) {
+  // Default a non-string (incl. undefined) to '' so the value is always a
+  // string for hasText/replace below — hasText is not a TS type guard, so the
+  // narrowing has to be structural (see this dir's CLAUDE.md).
+  const trimmed = typeof raw === 'string' ? raw.trim() : '';
+  if (!hasText(trimmed)) {
     throw new ErrorWithStatusCode(
-      'SEMRUSH_PROJECTS_BASE_URL is not set. Configure it via Vault '
+      `${varName} is not set. Configure it via Vault `
       + '(dx_mysticat/<env>/api-service) or .env for local dev.',
       503,
     );
   }
-  const candidate = raw.replace(/\/$/, '');
+  const candidate = trimmed.replace(/\/$/, '');
   let parsed;
   try {
     parsed = new URL(candidate);
   } catch {
     throw new ErrorWithStatusCode(
-      `SEMRUSH_PROJECTS_BASE_URL is not a valid URL: ${candidate}`,
+      `${varName} is not a valid URL: ${candidate}`,
       503,
     );
   }
   if (parsed.protocol !== 'https:') {
     throw new ErrorWithStatusCode(
-      `SEMRUSH_PROJECTS_BASE_URL must use https (got ${parsed.protocol})`,
+      `${varName} must use https (got ${parsed.protocol})`,
       503,
     );
   }
   return `${parsed.protocol}//${parsed.host}`;
 }
 
+/**
+ * Resolves the Project Engine gateway origin. Required; arrives via
+ * `env.SEMRUSH_PROJECTS_BASE_URL` — sourced from Vault
+ * (`dx_mysticat/<env>/api-service`) and injected through AWS Secrets Manager.
+ * No source default: the upstream host is operational config that must be
+ * settable per-environment without a code change.
+ *
+ * @param {object} env
+ * @returns {string} canonical `protocol//host` origin
+ */
+function baseUrl(env) {
+  return normalizeBaseUrl(env?.SEMRUSH_PROJECTS_BASE_URL, 'SEMRUSH_PROJECTS_BASE_URL');
+}
+
+/**
+ * Resolves the User Manager gateway origin. Reads `env.SEMRUSH_USERS_BASE_URL`,
+ * **falling back to `SEMRUSH_PROJECTS_BASE_URL` when unset** — Project Engine
+ * and User Manager are distinct Semrush services that share a single host in
+ * every deployed environment today, so the fallback keeps prod/stage/dev Vault
+ * config working untouched. Decoupling lets local + E2E setups point the User
+ * Manager calls at a SEPARATE (mock) host without a path-routing reverse proxy
+ * (LLMO / api-service#2656). The error message names whichever var was the
+ * effective source so a misconfiguration is unambiguous.
+ *
+ * @param {object} env
+ * @returns {string} canonical `protocol//host` origin
+ */
+function usersBaseUrl(env) {
+  const explicit = hasText(env?.SEMRUSH_USERS_BASE_URL);
+  return normalizeBaseUrl(
+    explicit ? env.SEMRUSH_USERS_BASE_URL : env?.SEMRUSH_PROJECTS_BASE_URL,
+    explicit ? 'SEMRUSH_USERS_BASE_URL' : 'SEMRUSH_PROJECTS_BASE_URL',
+  );
+}
+
 /**
  * Wraps global fetch with the transport's 15s ceiling and the
  * `Accept: application/json` header sent on every call — neither of which the
@@ -183,11 +221,17 @@ function unwrap(method, result) {
  * separate user-manager gateway.
  *
  * @param {object} args
- * @param {object} args.env - Environment (reads SEMRUSH_PROJECTS_BASE_URL override).
+ * @param {object} args.env - Environment (reads SEMRUSH_PROJECTS_BASE_URL and,
+ *   for the User Manager gateway, SEMRUSH_USERS_BASE_URL — falling back to the
+ *   projects host when the latter is unset).
  * @param {string} args.imsToken - IMS user bearer token (without 'Bearer ' prefix).
  */
 export function createSerenityTransport({ env, imsToken }) {
   const root = baseUrl(env);
+  // User Manager has its OWN origin (SEMRUSH_USERS_BASE_URL), falling back to
+  // `root` when unset — see usersBaseUrl(). Lets the sub-workspace lifecycle
+  // calls hit a separate (mock) host independently of Project Engine.
+  const usersRoot = usersBaseUrl(env);
 
   // Fail-closed guard for the destructive workspace delete. Deleting a
   // sub-workspace must be IMPOSSIBLE in every deployed environment
@@ -225,8 +269,10 @@ export function createSerenityTransport({ env, imsToken }) {
 
   // Typed User Manager client over the sub-workspace lifecycle gateway. Same
   // shape as the project client; appends its own '/enterprise/users/api' prefix.
+  // Uses `usersRoot` (SEMRUSH_USERS_BASE_URL, or `root` by fallback) so the
+  // lifecycle gateway can be a separate host from Project Engine.
   const users = createSerenityUserManagerApiClient({
-    baseUrl: root,
+    baseUrl: usersRoot,
     authToken,
     maxRetries: 0,
     fetch: createTimeoutFetch(DEFAULT_TIMEOUT_MS),

test/controllers/serenity.test.js

@@ -258,6 +258,28 @@ describe('SerenityController', () => {
       expect(response.status).to.equal(401);
     });
 
+    // Test-only escape hatch (SERENITY_ALLOW_NON_IMS_AUTH). The integration-test
+    // harness mints a non-IMS (JWT) token; with the flag set, the IMS-type gate
+    // is skipped so the handler runs (the Semrush mock ignores the forwarded
+    // bearer). The Authorization header is still required (asserted below).
+    it('lets a non-IMS caller through when SERENITY_ALLOW_NON_IMS_AUTH is set (reaches the handler, not 401)', async () => {
+      handlers.handleListPrompts.resolves({ items: [], total: 0 });
+      const controller = SerenityController({ env: {} }, fakeLog(), {});
+      const ctx = fakeContext({ authType: 'jwt', env: { SERENITY_ALLOW_NON_IMS_AUTH: 'true' } });
+      const response = await controller.listPrompts(ctx);
+      expect(response.status).to.equal(200);
+      expect(handlers.handleListPrompts).to.have.been.calledOnce;
+    });
+
+    it('still 401s a non-IMS caller with the flag set but NO Authorization header', async () => {
+      const controller = SerenityController({ env: {} }, fakeLog(), {});
+      const ctx = fakeContext({
+        authType: 'jwt', bearer: null, env: { SERENITY_ALLOW_NON_IMS_AUTH: 'true' },
+      });
+      const response = await controller.listPrompts(ctx);
+      expect(response.status).to.equal(401);
+    });
+
     it('400s when :brandId is not a UUID (the new guard)', async () => {
       const controller = SerenityController({ env: {} }, fakeLog(), {});
       const ctx = fakeContext({ brandId: 'adobe-brand-name' });

test/it/README.md

@@ -38,9 +38,9 @@ test/it/
 │
 └── postgres/                     # PostgreSQL backend
     ├── harness.js                # Mocha root hooks (beforeAll/afterAll)
-    ├── setup.js                  # Docker Compose startup + PostgREST polling
+    ├── setup.js                  # Container stack startup/poll + Semrush mock reset/version-pinning
     ├── seed.js                   # TRUNCATE + re-seed via PostgREST HTTP API
-    ├── docker-compose.yml        # PostgreSQL + PostgREST containers
+    ├── docker-compose.yml        # PostgreSQL + PostgREST + MinIO + Semrush vendor mocks
     ├── .mocharc.postgres.yml     # Mocha config
     ├── seed-data/                # Seed data in snake_case
     │   ├── organizations.js
@@ -187,6 +187,37 @@ Each `describe` block calls `before(() => resetData())` which truncates all data
 
 **PostgreSQL seed** (`postgres/seed.js`): POSTs rows directly to PostgREST (snake_case). Also seeds entities like `async_jobs`.
 
+## Serenity E2E: Semrush vendor mocks
+
+The `/serenity/*` routes proxy to two external Semrush APIs (Project Engine + User Manager). To exercise them end to end — without a real IMS account or a live vendor — the harness boots **two stateful [Counterfact](https://counterfact.dev) mock containers**, published as **public GHCR images** from the same `spacecat-shared` packages that provide the typed clients:
+
+| Service (compose) | Image | Serves prefix | Host port |
+|---|---|---|---|
+| `project-engine-mock` | `ghcr.io/adobe/spacecat-shared-project-engine-client-mock` | `/enterprise/projects/api` | 8443 |
+| `user-manager-mock` | `ghcr.io/adobe/spacecat-shared-user-manager-client-mock` | `/enterprise/users/api` | 8444 |
+
+The images are public, so **no registry credentials are needed** for them (only the data-service image still needs ECR). Each serves self-signed HTTPS (Caddy → Counterfact on `:4010` internally).
+
+### How it's wired here
+
+- **Image tag = the installed client version.** `setup.js` (`installedVersion()`) reads the installed `@adobe/spacecat-shared-{project-engine,user-manager}-client` version and exports `SERENITY_PE_MOCK_TAG` / `SERENITY_UM_MOCK_TAG`, which `docker-compose.yml` interpolates. The mock is built from the same package as the client, so this guarantees the test runs against the contract version we ship. There is **no fallback** — the compose `${...:?}` form fails hard if the tag is unset, and a client bumped to a version whose mock image isn't published yet fails the pull (forcing the mock to be released in lockstep).
+- **Transport target (`env.js`):** `SEMRUSH_PROJECTS_BASE_URL` → PE mock, `SEMRUSH_USERS_BASE_URL` → UM mock (the User-Manager origin split landed in api-service#2656; it falls back to the projects host when unset, so production needs no new config). `NODE_TLS_REJECT_UNAUTHORIZED=0` trusts the self-signed certs (IT process only). **None of these need Vault / deployed-env config.**
+- **Auth (`SERENITY_ALLOW_NON_IMS_AUTH=true`, test-only):** the serenity controller normally forwards only IMS-typed tokens. This flag lets the harness's non-IMS JWT through. It's sound only against the mocks (which ignore the forwarded bearer); in production the real Semrush gateway validates the token end to end, so this never weakens deployed auth. The flag is never set in any deployed environment.
+- **Statefulness / isolation:** the mocks are stateful within a run. Auth-exempt control routes — `POST /<prefix>/__reset`, `POST /<prefix>/__seed`, `GET /<prefix>/__dump` — manage that; `setup.js` exposes `resetSemrushMocks()` for tests that mutate mock state (used by the upcoming activate/deactivate + market-create flows). Readiness is polled via `waitForSemrushMocks()`.
+- **Seed alignment:** the mock fixtures use fixed workspace/project UUIDs, so the api-service seed (org `semrush_workspace_id` = the mock parent workspace, brand `semrush_workspace_id` = the mock child) is aligned to them. See the upstream `mock/seeds.js`.
+- **Test factory:** `shared/tests/serenity.js`, wired in `postgres/serenity.test.js`.
+
+### Where the mocks are developed (and how to learn more)
+
+The mocks live in **`adobe/spacecat-shared`**, inside each client package (`packages/spacecat-shared-{project-engine,user-manager}-client/`). Authoritative docs are on that repo's `main`:
+
+- `docs/mock-usage.md` — control routes (`__reset`/`__seed`/`__dump`/`__quota`), auth behaviour, seed selection (`MOCK_SEED`).
+- `docs/mock-statefulness.md` — the in-memory store model and how writes persist within a run.
+- `docs/mock-docker.md` — the Docker image, `MOCK_SEED` / `MOCK_SEED_FILE` env, and the intended consumer (GitHub Actions `services:`) shape.
+- Package `README.md` — `npm run mock` (local Counterfact on `:4010`), `docker:build` / `docker:run`, and the spec→mock pipeline.
+- Source: `mock/` (`run.js`, `seeds.js`, `factories.js`, `stateful.js`, `store.js`, `quota.js`).
+- CI: `.github/workflows/{project-engine,user-manager}-client-mock-image.yaml` (publish on release), `*-mock-image-smoke.yaml` (PR smoke), `*-mock-e2e.yaml`.
+
 ## Running Locally
 
 ### Prerequisites