Add a smoke test to generated fedify init apps

changes.d/init/smoke-test.md

@@ -0,0 +1,4 @@
+ -  Added a `test` task to projects scaffolded by `fedify init`.  It starts
+    the app, waits for it to become ready, and checks that it resolves a local
+    actor, giving projects a standard smoke test to run right after scaffolding
+    and whenever the app changes afterwards.  [[#898]]

packages/init/src/action/configs.test.ts

@@ -1,15 +1,15 @@
+import { message } from "@optique/core";
 import assert from "node:assert/strict";
 import { execFile } from "node:child_process";
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import test from "node:test";
 import { promisify } from "node:util";
-import { message } from "@optique/core";
 import { kvStores, messageQueues, PACKAGE_VERSION } from "../lib.ts";
 import type { InitCommandData } from "../types.ts";
-import bareBonesDescription from "../webframeworks/bare-bones.ts";
 import astroDescription from "../webframeworks/astro.ts";
+import bareBonesDescription from "../webframeworks/bare-bones.ts";
 import nextDescription from "../webframeworks/next.ts";
 import nitroDescription from "../webframeworks/nitro.ts";
 import nuxtDescription from "../webframeworks/nuxt.ts";
@@ -35,6 +35,7 @@ function createInitData(): InitCommandData {
     initializer: {
       federationFile: "federation.ts",
       loggingFile: "logging.ts",
+      testFile: "scripts/smoke.test.ts",
       instruction: message`done`,
       tasks: {},
       compilerOptions: {},

packages/init/src/action/patch.test.ts

@@ -1,9 +1,9 @@
+import { message } from "@optique/core";
 import assert from "node:assert/strict";
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import test from "node:test";
-import { message } from "@optique/core";
 import type { InitCommandData } from "../types.ts";
 import {
   assertNoGeneratedFileConflicts,
@@ -92,6 +92,18 @@ test("patchFiles merges JSONC files containing only comments", async () => {
   });
 });
 
+test("patchFiles writes the smoke-test script", async () => {
+  await withTempDir(async (dir) => {
+    await patchFiles(createInitData(dir, false));
+
+    const testScript = await readFile(
+      join(dir, "scripts", "smoke.test.ts"),
+      "utf8",
+    );
+    assert.match(testScript, /\["npm", "run", "dev"\]/);
+  });
+});
+
 function createInitData(
   dir: string,
   allowNonEmpty: boolean,
@@ -111,6 +123,7 @@ function createInitData(
     initializer: {
       federationFile: "src/federation.ts",
       loggingFile: "src/logging.ts",
+      testFile: "scripts/smoke.test.ts",
       instruction: message`done`,
       tasks: {},
       compilerOptions: {},

packages/init/src/action/patch.ts

@@ -25,7 +25,12 @@ import {
   noticeFilesToCreate,
   noticeFilesToInsert,
 } from "./notice.ts";
-import { getImports, loadFederation, loadLogging } from "./templates.ts";
+import {
+  getImports,
+  loadFederation,
+  loadLogging,
+  loadTest,
+} from "./templates.ts";
 import { joinDir, stringifyEnvs } from "./utils.ts";
 
 const jsonsCache = new Map<string, Record<string, object>>();
@@ -133,6 +138,7 @@ const getFiles = async <
     ...data,
   }),
   [data.initializer.loggingFile]: await loadLogging(data),
+  [data.initializer.testFile]: await loadTest(data),
   ".env": stringifyEnvs(data.env),
   ...data.initializer.files,
 });
@@ -183,6 +189,7 @@ const getJsons = <
 const getGeneratedFilePaths = (data: InitCommandData): string[] => [
   data.initializer.federationFile,
   data.initializer.loggingFile,
+  data.initializer.testFile,
   ".env",
   ...Object.keys(data.initializer.files ?? {}),
   ...Object.keys(getJsons(data)),

packages/init/src/action/templates.ts

@@ -1,6 +1,6 @@
 import { concat, entries, join, map, pipe, when } from "@fxts/core";
 import { toMerged } from "es-toolkit";
-import { readTemplate } from "../lib.ts";
+import { getDevCommand, readTemplate } from "../lib.ts";
 import type { InitCommandData, PackageManager } from "../types.ts";
 import { replace } from "../utils.ts";
 import { needsDenoDotenv } from "./utils.ts";
@@ -57,6 +57,31 @@ export const loadLogging = async (
     replace(/\/\* project name \*\//, JSON.stringify(projectName)),
   );
 
+/**
+ * Loads the smoke-test script content for the initializer.
+ *
+ * Every framework shares the same *defaults/smoke.test.ts* template, so unlike
+ * {@link loadLogging} there is no per-framework template override.  The
+ * template spawns the project's own dev server, so it needs the dev command
+ * for the chosen package manager baked in at generation time.
+ *
+ * @param param0 - {@link InitCommandData} containing `packageManager`
+ * @returns The complete smoke-test script content as a string
+ */
+export const loadTest = async (
+  { packageManager }: InitCommandData,
+) =>
+  pipe(
+    await readTemplate("defaults/smoke.test.ts"),
+    replace(
+      /\/\* dev command \*\//,
+      JSON.stringify(getDevCommand(packageManager).split(" ")).replaceAll(
+        ",",
+        ", ",
+      ),
+    ),
+  );
+
 /**
  * Generates import statements for KV store and message queue dependencies.
  * Merges imports from both KV and MQ configurations and creates proper

packages/init/src/templates/defaults/smoke.test.ts.tpl

@@ -0,0 +1,170 @@
+import { getDocumentLoader } from "@fedify/fedify";
+import { type Actor, isActor, lookupObject } from "@fedify/vocab";
+import { spawn, spawnSync } from "node:child_process";
+import type { Readable } from "node:stream";
+
+const DEV_COMMAND: string[] = /* dev command */;
+const HANDLE = "john";
+const STARTUP_TIMEOUT = 15_000;
+const IS_WINDOWS = process.platform === "win32";
+
+async function main(): Promise<void> {
+  const [command, ...args] = DEV_COMMAND;
+  const server = spawn(command, args, {
+    stdio: ["ignore", "pipe", "pipe"],
+    shell: IS_WINDOWS,
+    windowsHide: true,
+    detached: !IS_WINDOWS,
+  });
+  server.on("error", () => {});
+
+  const exitOnSignal = () => {
+    stopServer(server);
+    process.exit(1);
+  };
+  process.once("SIGINT", exitOnSignal);
+  process.once("SIGTERM", exitOnSignal);
+
+  let output = "";
+  const collectOutput = (stream: Readable | null) => {
+    const decoder = new TextDecoder();
+    stream?.on("data", (chunk: Buffer) => {
+      output += decoder.decode(chunk, { stream: true });
+    });
+  };
+  collectOutput(server.stdout);
+  collectOutput(server.stderr);
+
+  try {
+    const port = await determinePort(server);
+    const target = `http://localhost:${port}/users/${HANDLE}`;
+    await waitForServer(target);
+    console.log(`Server is up at http://localhost:${port}.`);
+    const actor = await checkActor(target);
+    console.log(actor);
+    console.log(`Smoke test passed: ${target} resolved to an actor.`);
+  } catch (error) {
+    console.error("Smoke test failed:", error instanceof Error ? error.message : error);
+    if (output.trim() !== "") {
+      console.error(`\nDev server output:\n${output}`);
+    }
+    process.exitCode = 1;
+  } finally {
+    stopServer(server);
+  }
+}
+
+function stripEscape(text: string): string {
+  return text.replace(new RegExp("\\u001B\\[[0-9;]*[A-Za-z]", "g"), "");
+}
+
+function determinePort(server: ReturnType<typeof spawn>): Promise<number> {
+  const portPatterns = [
+    /listening on.*:(\d+)/i,
+    /server.*:(\d+)/i,
+    /https?:\/\/localhost:(\d+)/i,
+    /https?:\/\/0\.0\.0\.0:(\d+)/i,
+    /https?:\/\/127\.0\.0\.1:(\d+)/i,
+    /https?:\/\/[^:]+:(\d+)/i,
+  ];
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(
+        new Error(
+          `Timeout: Could not determine port from server output within ${STARTUP_TIMEOUT}ms.`,
+        ),
+      );
+    }, STARTUP_TIMEOUT);
+
+    const findPort = (text: string) => {
+      for (const pattern of portPatterns) {
+        const match = text.match(pattern);
+        if (match && match[1]) {
+          const port = Number.parseInt(match[1], 10);
+          if (port > 0 && port < 65536) return port;
+        }
+      }
+      return null;
+    };
+
+    const scan = (stream: Readable | null) => {
+      const decoder = new TextDecoder();
+      let text = "";
+      stream?.on("data", (chunk: Buffer) => {
+        text += decoder.decode(chunk, { stream: true });
+        const port = findPort(stripEscape(text));
+        if (port != null) {
+          clearTimeout(timeout);
+          resolve(port);
+        }
+      });
+    };
+
+    scan(server.stdout);
+    scan(server.stderr);
+    server.once("exit", (code) => {
+      clearTimeout(timeout);
+      reject(new Error(`The dev server exited early with code ${String(code)}.`));
+    });
+  });
+}
+
+async function waitForServer(url: string): Promise<void> {
+  const startTime = Date.now();
+  let lastStatus: number | undefined;
+
+  while (Date.now() - startTime < STARTUP_TIMEOUT) {
+    try {
+      const response = await fetch(url, {
+        headers: { Accept: "application/activity+json" },
+        signal: AbortSignal.timeout(1000),
+      });
+      await response.body?.cancel();
+      if (response.ok) return;
+      lastStatus = response.status;
+    } catch {
+      // Server not ready yet, continue waiting
+    }
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  throw new Error(
+    `The server did not become ready within ${STARTUP_TIMEOUT}ms.` +
+      (lastStatus == null ? "" : `  Last response status: ${lastStatus}.`),
+  );
+}
+
+async function checkActor(url: string): Promise<Actor> {
+  const object = await lookupObject(url, {
+    documentLoader: getDocumentLoader({ allowPrivateAddress: true }),
+  });
+  if (object == null) {
+    throw new Error(`Could not resolve an actor at ${url}.`);
+  }
+  if (!isActor(object)) {
+    throw new Error(`Expected an actor at ${url}, but got a non-actor object.`);
+  }
+  return object;
+}
+
+function stopServer(server: ReturnType<typeof spawn>): void {
+  if (server.pid == null) return;
+  if (IS_WINDOWS) {
+    spawnSync("taskkill", ["/pid", String(server.pid), "/T", "/F"], {
+      stdio: "ignore",
+      windowsHide: true,
+    });
+    return;
+  }
+  try {
+    process.kill(-server.pid, "SIGKILL");
+  } catch {
+    // Process group already exited.
+  }
+  try {
+    server.kill("SIGKILL");
+  } catch {
+    // Process already exited.
+  }
+}
+
+await main();

packages/init/src/types.ts

@@ -96,6 +96,8 @@ export interface WebFrameworkInitializer {
   federationFile: string;
   /** Relative path where the logging configuration file will be created. */
   loggingFile: string;
+  /** Relative path where the smoke-test script file will be created. */
+  testFile: string;
   /** Optional template path for the logging configuration file. */
   loggingTemplate?: string;
   /**

packages/init/src/webframeworks/astro.ts

@@ -3,7 +3,12 @@ import deps from "../json/deps.json" with { type: "json" };
 import { PACKAGE_VERSION, readTemplate } from "../lib.ts";
 import type { PackageManager, WebFrameworkDescription } from "../types.ts";
 import { defaultDenoDependencies } from "./const.ts";
-import { getInstruction, pmToRt } from "./utils.ts";
+import {
+  getInstruction,
+  getTestDependencies,
+  getTestTask,
+  pmToRt,
+} from "./utils.ts";
 
 const astroNodeBunDevDependencies = {
   "@fedify/lint": PACKAGE_VERSION,
@@ -74,9 +79,11 @@ const astroDescription: WebFrameworkDescription = {
             "@types/node": deps["npm:@types/node@22"],
           }
           : {}),
+        ...getTestDependencies(pm),
       },
       federationFile: "src/federation.ts",
       loggingFile: "src/logging.ts",
+      testFile: "scripts/smoke.test.ts",
       format: pm === "deno" ? undefined : { tool: "prettier" },
       files: {
         "astro.config.ts": await readTemplate(
@@ -131,17 +138,20 @@ const TASKS = {
     dev: `${astroDenoCommand} dev`,
     build: `${astroDenoCommand} build`,
     preview: `${astroDenoCommand} preview`,
+    test: getTestTask("deno"),
   },
   "bun": {
     dev: "bunx --bun astro dev",
     build: "bunx --bun astro build",
     preview: "bun ./dist/server/entry.mjs",
+    test: getTestTask("bun"),
     ...astroNodeBunDevToolTasks,
   },
   "node": {
     dev: "dotenvx run -- astro dev",
     build: "dotenvx run -- astro build",
     preview: "dotenvx run -- astro preview",
+    test: getTestTask("npm"),
     ...astroNodeBunDevToolTasks,
   },
 };