fix(mail): reconcile mounted Let's Encrypt certificates into daemon paths on boot

apps/api/test/e2e/mail-ssl-reconcile.e2e.test.ts

@@ -0,0 +1,160 @@
+import { execFile } from "node:child_process";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { afterAll, beforeAll, beforeEach, expect, it } from "vitest";
+import { shellQuote as sq } from "@repo/core";
+import { describeDockerE2E, requireDocker } from "../helpers/docker-e2e";
+
+const exec = promisify(execFile);
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../../../..");
+const RUNNER = `openship-e2e-mailtls-${process.pid.toString(36)}`;
+const IMAGE = "alpine:3.23";
+const HOST = "mail.example.test";
+const MAIL = `/test/live/${HOST}`;
+const APEX = "/test/live/example.test";
+const CERT = "/test/ssl/cert.pem";
+const KEY = "/test/ssl/key.pem";
+
+async function docker(args: string[]) {
+  const { stdout } = await exec("docker", args, { timeout: 120_000, maxBuffer: 2 * 1024 * 1024 });
+  return stdout.trim();
+}
+const run = (script: string) => docker(["exec", RUNNER, "bash", "-euo", "pipefail", "-c", script]);
+const reconcile = () =>
+  run(`bash /reconcile-ssl.sh example.test /test/live ${sq(CERT)} ${sq(KEY)}`);
+
+async function certificate(directory: string, hosts = HOST, expired = false) {
+  await run(`
+    mkdir -p ${sq(directory)}
+    openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 2 \
+      -subj '/CN=mail-test' -addext ${sq(
+        `subjectAltName=${hosts
+          .split(",")
+          .map((h) => `DNS:${h}`)
+          .join(",")}`,
+      )} \
+      -keyout ${sq(`${directory}/privkey.pem`)} -out ${sq(`${directory}/fullchain.pem`)} 2>/dev/null
+    ${expired ? `openssl x509 -in ${sq(`${directory}/fullchain.pem`)} -signkey ${sq(`${directory}/privkey.pem`)} -days 0 -out ${sq(`${directory}/fullchain.pem`)} 2>/dev/null` : ""}
+  `);
+}
+
+// The production helper runs in Linux with OpenSSL 3 (Debian 12 mail image).
+// Use those real collaborators instead of the macOS system LibreSSL or source
+// string assertions. All files and daemons are confined to this disposable runner.
+describeDockerE2E("mail TLS reconciliation on every container boot (#837)", () => {
+  beforeAll(async () => {
+    await requireDocker();
+    await docker(["pull", IMAGE]);
+    await docker(["run", "-d", "--name", RUNNER, IMAGE, "sleep", "600"]);
+    await docker(["exec", RUNNER, "apk", "add", "--no-cache", "bash", "openssl"]);
+    await docker([
+      "cp",
+      join(ROOT, "apps/email/docker/reconcile-ssl.sh"),
+      `${RUNNER}:/reconcile-ssl.sh`,
+    ]);
+  });
+
+  afterAll(async () => {
+    await docker(["rm", "-f", RUNNER]).catch(() => {});
+  });
+
+  beforeEach(async () => {
+    await run(`rm -rf /test
+      mkdir -p /test/ssl /test/live
+      printf 'fallback certificate' > ${sq(CERT)}
+      printf 'fallback key' > ${sq(KEY)}
+      chmod 600 ${sq(KEY)}`);
+  });
+
+  it("links a usable mounted pair and serves its identity over TLS", async () => {
+    await certificate(MAIL);
+    await reconcile();
+    expect(await run(`readlink ${sq(CERT)}`)).toBe(`${MAIL}/fullchain.pem`);
+    expect(await run(`readlink ${sq(KEY)}`)).toBe(`${MAIL}/privkey.pem`);
+    await run(`
+      openssl s_server -accept 127.0.0.1:10465 -cert ${sq(CERT)} -key ${sq(KEY)} -www -quiet >/test/server.log 2>&1 &
+      tls_pid=$!
+      trap 'kill "$tls_pid" 2>/dev/null || true' EXIT
+      for attempt in $(seq 1 30); do
+        if openssl s_client -brief -verify_return_error -verify_hostname ${sq(HOST)} \
+          -CAfile ${sq(`${MAIL}/fullchain.pem`)} -connect 127.0.0.1:10465 </dev/null >/test/client.log 2>&1; then
+          exit 0
+        fi
+        sleep 0.1
+      done
+      cat /test/client.log
+      exit 1
+    `);
+  });
+
+  it("repairs a stale key link even when the certificate link is already correct", async () => {
+    await certificate(MAIL);
+    await run(`rm ${sq(CERT)}; ln -s ${sq(`${MAIL}/fullchain.pem`)} ${sq(CERT)}`);
+    await reconcile();
+    expect(await run(`readlink ${sq(KEY)}`)).toBe(`${MAIL}/privkey.pem`);
+  });
+
+  it("keeps stable live links across repeated boots and certificate renewals", async () => {
+    await certificate("/test/archive/first");
+    await run(`mkdir -p ${sq(MAIL)}
+      ln -s /test/archive/first/fullchain.pem ${sq(`${MAIL}/fullchain.pem`)}
+      ln -s /test/archive/first/privkey.pem ${sq(`${MAIL}/privkey.pem`)}`);
+    await reconcile();
+    const links = await run(`stat -c '%i' ${sq(CERT)} ${sq(KEY)}`);
+    await certificate("/test/archive/renewed");
+    await run(`ln -sfn /test/archive/renewed/fullchain.pem ${sq(`${MAIL}/fullchain.pem`)}
+      ln -sfn /test/archive/renewed/privkey.pem ${sq(`${MAIL}/privkey.pem`)}`);
+    await reconcile();
+    expect(await run(`stat -c '%i' ${sq(CERT)} ${sq(KEY)}`)).toBe(links);
+    await run(`cmp ${sq(CERT)} /test/archive/renewed/fullchain.pem
+      cmp ${sq(KEY)} /test/archive/renewed/privkey.pem`);
+    expect(await run(`cat ${sq(`${CERT}.bak`)}`)).toBe("fallback certificate");
+    expect(await run(`stat -c '%a' ${sq(`${KEY}.bak`)}`)).toBe("600");
+  });
+
+  it("restores the mounted identity after the container's certificate files are recreated", async () => {
+    await certificate(MAIL);
+    await reconcile();
+    await run(`rm ${sq(CERT)} ${sq(KEY)}
+      printf 'recreated certificate' > ${sq(CERT)}
+      printf 'recreated key' > ${sq(KEY)}`);
+    await reconcile();
+    expect(await run(`readlink ${sq(CERT)}`)).toBe(`${MAIL}/fullchain.pem`);
+    expect(await run(`readlink ${sq(KEY)}`)).toBe(`${MAIL}/privkey.pem`);
+  });
+
+  it.each(["missing", "wrong hostname", "expired", "mismatched key", "malformed"])(
+    "preserves the fallback for a %s mounted certificate pair",
+    async (mode) => {
+      if (mode !== "missing") {
+        await certificate(
+          MAIL,
+          mode === "wrong hostname" ? "example.test" : HOST,
+          mode === "expired",
+        );
+        if (mode === "mismatched key") {
+          await certificate("/test/other");
+          await run(`cp /test/other/privkey.pem ${sq(`${MAIL}/privkey.pem`)}`);
+        }
+        if (mode === "malformed")
+          await run(`printf 'invalid PEM' > ${sq(`${MAIL}/fullchain.pem`)}`);
+      }
+      await reconcile();
+      expect(await run(`cat ${sq(CERT)}`)).toBe("fallback certificate");
+      expect(await run(`cat ${sq(KEY)}`)).toBe("fallback key");
+    },
+  );
+
+  it("uses an apex certificate only when it covers the mail hostname", async () => {
+    await certificate(APEX, "example.test");
+    await reconcile();
+    expect(await run(`cat ${sq(CERT)}`)).toBe("fallback certificate");
+    await certificate(APEX, "example.test,*.example.test");
+    await reconcile();
+    expect(await run(`readlink ${sq(CERT)}`)).toBe(`${APEX}/fullchain.pem`);
+    await certificate(MAIL);
+    await reconcile();
+    expect(await run(`readlink ${sq(CERT)}`)).toBe(`${MAIL}/fullchain.pem`);
+  });
+});

apps/email/docker/entrypoint.sh

@@ -143,6 +143,10 @@ if [ -n "$FIRST_DOMAIN" ]; then
   esac
 fi
 
+# 3c. /etc/ssl is in the container layer. Restore the daemon certificate links
+#     on every boot so recreating the container retains the mounted TLS identity.
+bash /opt/openship-mail/reconcile-ssl.sh "$FIRST_DOMAIN"
+
 # 4. bootstrap the mail databases (idempotent; skips if the vmail schema exists).
 #
 # The wait for the sidecar lives INSIDE db-bootstrap.sh, which polls `SELECT 1` until

apps/email/docker/reconcile-ssl.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Reconnect the mail daemons to the operator's mounted certificates before boot.
+# Optional path arguments allow the same reconciliation against an isolated root.
+set -euo pipefail
+
+MAIL_BASE_DOMAIN="${1:-}"
+MAIL_CERT_ROOT="${2:-/etc/letsencrypt/live}"
+MAIL_CERT_PATH="${3:-/etc/ssl/certs/iRedMail.crt}"
+MAIL_KEY_PATH="${4:-/etc/ssl/private/iRedMail.key}"
+
+case "$MAIL_BASE_DOMAIN" in
+  ""|*[!A-Za-z0-9.-]*) exit 0 ;;
+esac
+
+MAIL_CERT_DIR=""
+for candidate in "$MAIL_CERT_ROOT/mail.$MAIL_BASE_DOMAIN" "$MAIL_CERT_ROOT/$MAIL_BASE_DOMAIN"; do
+  [ -s "$candidate/fullchain.pem" ] && [ -s "$candidate/privkey.pem" ] || continue
+
+  # These are operator-managed certificates, not a new trust source. Validate
+  # hostname, dates and server usage against the mounted leaf itself. An apex
+  # certificate is usable only if it also covers mail.<domain> (SAN or wildcard).
+  openssl verify -partial_chain -trusted "$candidate/fullchain.pem" \
+    -purpose sslserver -verify_hostname "mail.$MAIL_BASE_DOMAIN" \
+    "$candidate/fullchain.pem" >/dev/null 2>&1 || continue
+  cert_public="$(openssl x509 -in "$candidate/fullchain.pem" -pubkey -noout 2>/dev/null)" || continue
+  key_public="$(openssl pkey -in "$candidate/privkey.pem" -pubout -passin pass: 2>/dev/null)" || continue
+  [ -n "$cert_public" ] && [ "$cert_public" = "$key_public" ] || continue
+  MAIL_CERT_DIR="$candidate"
+  break
+done
+
+# Keep the existing fallback/configuration intact when no usable pair is mounted.
+[ -n "$MAIL_CERT_DIR" ] || exit 0
+
+link_certificate() {
+  local source="$1" destination="$2"
+  # Compare the link itself, not readlink -f: Certbot's live links resolve to
+  # versioned archive files, which necessarily differ from the stable live path.
+  [ "$(readlink "$destination" 2>/dev/null || true)" != "$source" ] || return 0
+  mkdir -p "$(dirname "$destination")"
+  if [ -f "$destination" ] && [ ! -L "$destination" ] && [ ! -e "$destination.bak" ]; then
+    cp -p "$destination" "$destination.bak"
+  fi
+  ln -sfn "$source" "$destination"
+}
+
+link_certificate "$MAIL_CERT_DIR/fullchain.pem" "$MAIL_CERT_PATH"
+link_certificate "$MAIL_CERT_DIR/privkey.pem" "$MAIL_KEY_PATH"
+echo "[openship-mail] using TLS certificate for mail.$MAIL_BASE_DOMAIN from $MAIL_CERT_DIR"