Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions apps/api/src/modules/backups/backup.orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ import { notification } from "../../lib/notification-dispatcher";
import { serviceHandleFor, withContainerEnv } from "./service-handle";
import { resolveSourceExecutor } from "./source-platform";
import crypto from "node:crypto";
import { safeErrorMessage } from "@repo/core";
import { detectDbImage, safeErrorMessage } from "@repo/core";
import {
boundedStorableText,
sanitizeStorableStringsExceptKeys,
Expand All @@ -81,6 +81,31 @@ const TRUNCATE_ERROR = 4096;
const TRUNCATE_HOOK_LOG = 64 * 1024;
/** Cap on waiting for a finished hook's stdout to drain (see runHook). */
const HOOK_DRAIN_TIMEOUT_MS = 500;

/**
* A service is eligible for project-level backup fan-out if:
* 1. The policy specifies custom_command or path payloads, OR
* 2. The service runs a recognized database engine (PostgreSQL, MySQL, Redis, MongoDB), OR
* 3. The service has declared volumes in its service definition.
*
* Stateless services without volumes or databases are skipped during project fan-out
* so they do not fail the backup batch (#611).
*/
export function isBackupCandidateService(
service: Pick<Service, "image" | "volumes">,
policy: Pick<BackupPolicy, "payloadKind">,
): boolean {
if (policy.payloadKind === "custom_command" || policy.payloadKind === "path") {
return true;
}
if (detectDbImage(service.image) !== null) {
return true;
}
if (Array.isArray(service.volumes) && service.volumes.length > 0) {
return true;
}
return false;
}
/** Short form for the notification payload + destination verify note. */
const TRUNCATE_ERROR_SUMMARY = 500;
/** A `PutResult.etag` in this shape is a sha256 we can compare ours against. */
Expand Down Expand Up @@ -204,8 +229,14 @@ export class BackupOrchestrator {
if (services.length === 0) {
throw new Error("Project has no services to back up — add a service or pick one.");
}
const candidates = services.filter((svc) => isBackupCandidateService(svc, policy));
if (candidates.length === 0) {
throw new Error(
"Project has no services with persistent storage (volumes or databases) to back up.",
);
}
const runIds: string[] = [];
for (const svc of services) {
for (const svc of candidates) {
try {
runIds.push(
await this.spawnRun(
Expand Down
18 changes: 11 additions & 7 deletions apps/api/src/modules/backups/triggers/cron.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,17 @@ export async function syncPolicySchedule(policyId: string): Promise<void> {
// between schedule + tick are honored.
const fresh = await repos.backupPolicy.findById(policyId);
if (!fresh || !fresh.enabled) return;
await backupOrchestrator.enqueue({
policyId,
trigger: {
source: "cron",
userId: fresh.createdBy ?? "system",
},
});
try {
await backupOrchestrator.enqueue({
policyId,
trigger: {
source: "cron",
userId: fresh.createdBy ?? "system",
},
});
} catch (err) {
console.warn(`[cron-trigger] policy ${policyId} skipped: ${safeErrorMessage(err)}`);
}
},
});
}
Expand Down
57 changes: 52 additions & 5 deletions apps/api/test/modules/backups/backup-run-durability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,15 +157,14 @@ describe("backupRun.transition — status must not ride a payload that can fail"
const h = vi.hoisted(() => ({
run: null as Record<string, unknown> | null,
policy: null as Record<string, unknown> | null,
services: [] as Array<{ id: string; enabled: boolean }>,
services: [] as Array<Record<string, unknown>>,
createdRuns: [] as Array<Record<string, unknown>>,
executionRow: null as Record<string, unknown> | null,
claimResults: ["claimed"] as Array<"claimed" | "project_unavailable" | "state_changed">,
claimCalls: 0,
acknowledgements: [] as Array<{ runId: string; status: string }>,
transition: null as
| null
| ((id: string, status: string, patch?: Record<string, unknown>) => Promise<void>),
null | ((id: string, status: string, patch?: Record<string, unknown>) => Promise<void>),
hookStdout: "hook ran\n",
hookExit: { code: 0 as number | null, stderr: "" },
artifactMetadata: {} as Record<string, unknown>,
Expand Down Expand Up @@ -360,8 +359,8 @@ describe("BackupOrchestrator.enqueue — durable batch identity", () => {
mailServerId: null,
};
h.services = [
{ id: "svc_api", enabled: true },
{ id: "svc_db", enabled: true },
{ id: "svc_api", enabled: true, volumes: ["api_data:/data"] },
{ id: "svc_db", enabled: true, image: "postgres:16" },
];
const orchestrator = new BackupOrchestrator();

Expand All @@ -386,6 +385,54 @@ describe("BackupOrchestrator.enqueue — durable batch identity", () => {
expect(new Set(secondBatch.map((run) => run.batchId)).size).toBe(1);
expect(secondBatch[0]!.batchId).not.toBe(firstBatch[0]!.batchId);
});

it("skips stateless services without volumes or database images during fan-out", async () => {
h.policy = {
...(h.policy ?? {}),
sourceKind: "service",
projectId: "proj_1",
mailServerId: null,
payloadKind: "auto",
};
h.services = [
{ id: "svc_web", enabled: true, volumes: [] },
{ id: "svc_monitor", enabled: true, volumes: [] },
{ id: "svc_db", enabled: true, image: "postgres:16" },
];
const orchestrator = new BackupOrchestrator();

await orchestrator.enqueue({
policyId: "pol_1",
trigger: { source: "cron", userId: "system" },
});

expect(h.createdRuns).toHaveLength(1);
expect(h.createdRuns[0]!.serviceId).toBe("svc_db");
});

it("throws when project has no services with persistent storage", async () => {
h.policy = {
...(h.policy ?? {}),
sourceKind: "service",
projectId: "proj_1",
mailServerId: null,
payloadKind: "auto",
};
h.services = [
{ id: "svc_web", enabled: true, volumes: [] },
{ id: "svc_monitor", enabled: true, volumes: [] },
];
const orchestrator = new BackupOrchestrator();

await expect(
orchestrator.enqueue({
policyId: "pol_1",
trigger: { source: "cron", userId: "system" },
}),
).rejects.toThrow(/no services with persistent storage/);

expect(h.createdRuns).toHaveLength(0);
});
});

describe("a terminal status is final — one owner per verdict", () => {
Expand Down