diff --git a/apps/api/test/e2e/mail-ssl-reconcile.e2e.test.ts b/apps/api/test/e2e/mail-ssl-reconcile.e2e.test.ts new file mode 100644 index 000000000..765869826 --- /dev/null +++ b/apps/api/test/e2e/mail-ssl-reconcile.e2e.test.ts @@ -0,0 +1,160 @@ +import { execFile } from "node:child_process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, beforeEach, expect, it } from "vitest"; +import { shellQuote as sq } from "@repo/core"; +import { describeDockerE2E, requireDocker } from "../helpers/docker-e2e"; + +const exec = promisify(execFile); +const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../../../.."); +const RUNNER = `openship-e2e-mailtls-${process.pid.toString(36)}`; +const IMAGE = "alpine:3.23"; +const HOST = "mail.example.test"; +const MAIL = `/test/live/${HOST}`; +const APEX = "/test/live/example.test"; +const CERT = "/test/ssl/cert.pem"; +const KEY = "/test/ssl/key.pem"; + +async function docker(args: string[]) { + const { stdout } = await exec("docker", args, { timeout: 120_000, maxBuffer: 2 * 1024 * 1024 }); + return stdout.trim(); +} +const run = (script: string) => docker(["exec", RUNNER, "bash", "-euo", "pipefail", "-c", script]); +const reconcile = () => + run(`bash /reconcile-ssl.sh example.test /test/live ${sq(CERT)} ${sq(KEY)}`); + +async function certificate(directory: string, hosts = HOST, expired = false) { + await run(` + mkdir -p ${sq(directory)} + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 2 \ + -subj '/CN=mail-test' -addext ${sq( + `subjectAltName=${hosts + .split(",") + .map((h) => `DNS:${h}`) + .join(",")}`, + )} \ + -keyout ${sq(`${directory}/privkey.pem`)} -out ${sq(`${directory}/fullchain.pem`)} 2>/dev/null + ${expired ? `openssl x509 -in ${sq(`${directory}/fullchain.pem`)} -signkey ${sq(`${directory}/privkey.pem`)} -days 0 -out ${sq(`${directory}/fullchain.pem`)} 2>/dev/null` : ""} + `); +} + +// The production helper runs in Linux with OpenSSL 3 (Debian 12 mail image). +// Use those real collaborators instead of the macOS system LibreSSL or source +// string assertions. All files and daemons are confined to this disposable runner. +describeDockerE2E("mail TLS reconciliation on every container boot (#837)", () => { + beforeAll(async () => { + await requireDocker(); + await docker(["pull", IMAGE]); + await docker(["run", "-d", "--name", RUNNER, IMAGE, "sleep", "600"]); + await docker(["exec", RUNNER, "apk", "add", "--no-cache", "bash", "openssl"]); + await docker([ + "cp", + join(ROOT, "apps/email/docker/reconcile-ssl.sh"), + `${RUNNER}:/reconcile-ssl.sh`, + ]); + }); + + afterAll(async () => { + await docker(["rm", "-f", RUNNER]).catch(() => {}); + }); + + beforeEach(async () => { + await run(`rm -rf /test + mkdir -p /test/ssl /test/live + printf 'fallback certificate' > ${sq(CERT)} + printf 'fallback key' > ${sq(KEY)} + chmod 600 ${sq(KEY)}`); + }); + + it("links a usable mounted pair and serves its identity over TLS", async () => { + await certificate(MAIL); + await reconcile(); + expect(await run(`readlink ${sq(CERT)}`)).toBe(`${MAIL}/fullchain.pem`); + expect(await run(`readlink ${sq(KEY)}`)).toBe(`${MAIL}/privkey.pem`); + await run(` + openssl s_server -accept 127.0.0.1:10465 -cert ${sq(CERT)} -key ${sq(KEY)} -www -quiet >/test/server.log 2>&1 & + tls_pid=$! + trap 'kill "$tls_pid" 2>/dev/null || true' EXIT + for attempt in $(seq 1 30); do + if openssl s_client -brief -verify_return_error -verify_hostname ${sq(HOST)} \ + -CAfile ${sq(`${MAIL}/fullchain.pem`)} -connect 127.0.0.1:10465 /test/client.log 2>&1; then + exit 0 + fi + sleep 0.1 + done + cat /test/client.log + exit 1 + `); + }); + + it("repairs a stale key link even when the certificate link is already correct", async () => { + await certificate(MAIL); + await run(`rm ${sq(CERT)}; ln -s ${sq(`${MAIL}/fullchain.pem`)} ${sq(CERT)}`); + await reconcile(); + expect(await run(`readlink ${sq(KEY)}`)).toBe(`${MAIL}/privkey.pem`); + }); + + it("keeps stable live links across repeated boots and certificate renewals", async () => { + await certificate("/test/archive/first"); + await run(`mkdir -p ${sq(MAIL)} + ln -s /test/archive/first/fullchain.pem ${sq(`${MAIL}/fullchain.pem`)} + ln -s /test/archive/first/privkey.pem ${sq(`${MAIL}/privkey.pem`)}`); + await reconcile(); + const links = await run(`stat -c '%i' ${sq(CERT)} ${sq(KEY)}`); + await certificate("/test/archive/renewed"); + await run(`ln -sfn /test/archive/renewed/fullchain.pem ${sq(`${MAIL}/fullchain.pem`)} + ln -sfn /test/archive/renewed/privkey.pem ${sq(`${MAIL}/privkey.pem`)}`); + await reconcile(); + expect(await run(`stat -c '%i' ${sq(CERT)} ${sq(KEY)}`)).toBe(links); + await run(`cmp ${sq(CERT)} /test/archive/renewed/fullchain.pem + cmp ${sq(KEY)} /test/archive/renewed/privkey.pem`); + expect(await run(`cat ${sq(`${CERT}.bak`)}`)).toBe("fallback certificate"); + expect(await run(`stat -c '%a' ${sq(`${KEY}.bak`)}`)).toBe("600"); + }); + + it("restores the mounted identity after the container's certificate files are recreated", async () => { + await certificate(MAIL); + await reconcile(); + await run(`rm ${sq(CERT)} ${sq(KEY)} + printf 'recreated certificate' > ${sq(CERT)} + printf 'recreated key' > ${sq(KEY)}`); + await reconcile(); + expect(await run(`readlink ${sq(CERT)}`)).toBe(`${MAIL}/fullchain.pem`); + expect(await run(`readlink ${sq(KEY)}`)).toBe(`${MAIL}/privkey.pem`); + }); + + it.each(["missing", "wrong hostname", "expired", "mismatched key", "malformed"])( + "preserves the fallback for a %s mounted certificate pair", + async (mode) => { + if (mode !== "missing") { + await certificate( + MAIL, + mode === "wrong hostname" ? "example.test" : HOST, + mode === "expired", + ); + if (mode === "mismatched key") { + await certificate("/test/other"); + await run(`cp /test/other/privkey.pem ${sq(`${MAIL}/privkey.pem`)}`); + } + if (mode === "malformed") + await run(`printf 'invalid PEM' > ${sq(`${MAIL}/fullchain.pem`)}`); + } + await reconcile(); + expect(await run(`cat ${sq(CERT)}`)).toBe("fallback certificate"); + expect(await run(`cat ${sq(KEY)}`)).toBe("fallback key"); + }, + ); + + it("uses an apex certificate only when it covers the mail hostname", async () => { + await certificate(APEX, "example.test"); + await reconcile(); + expect(await run(`cat ${sq(CERT)}`)).toBe("fallback certificate"); + await certificate(APEX, "example.test,*.example.test"); + await reconcile(); + expect(await run(`readlink ${sq(CERT)}`)).toBe(`${APEX}/fullchain.pem`); + await certificate(MAIL); + await reconcile(); + expect(await run(`readlink ${sq(CERT)}`)).toBe(`${MAIL}/fullchain.pem`); + }); +}); diff --git a/apps/email/docker/entrypoint.sh b/apps/email/docker/entrypoint.sh index b29530d18..dbd81fc61 100644 --- a/apps/email/docker/entrypoint.sh +++ b/apps/email/docker/entrypoint.sh @@ -143,6 +143,10 @@ if [ -n "$FIRST_DOMAIN" ]; then esac fi +# 3c. /etc/ssl is in the container layer. Restore the daemon certificate links +# on every boot so recreating the container retains the mounted TLS identity. +bash /opt/openship-mail/reconcile-ssl.sh "$FIRST_DOMAIN" + # 4. bootstrap the mail databases (idempotent; skips if the vmail schema exists). # # The wait for the sidecar lives INSIDE db-bootstrap.sh, which polls `SELECT 1` until diff --git a/apps/email/docker/reconcile-ssl.sh b/apps/email/docker/reconcile-ssl.sh new file mode 100644 index 000000000..3bf9f85ce --- /dev/null +++ b/apps/email/docker/reconcile-ssl.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Reconnect the mail daemons to the operator's mounted certificates before boot. +# Optional path arguments allow the same reconciliation against an isolated root. +set -euo pipefail + +MAIL_BASE_DOMAIN="${1:-}" +MAIL_CERT_ROOT="${2:-/etc/letsencrypt/live}" +MAIL_CERT_PATH="${3:-/etc/ssl/certs/iRedMail.crt}" +MAIL_KEY_PATH="${4:-/etc/ssl/private/iRedMail.key}" + +case "$MAIL_BASE_DOMAIN" in + ""|*[!A-Za-z0-9.-]*) exit 0 ;; +esac + +MAIL_CERT_DIR="" +for candidate in "$MAIL_CERT_ROOT/mail.$MAIL_BASE_DOMAIN" "$MAIL_CERT_ROOT/$MAIL_BASE_DOMAIN"; do + [ -s "$candidate/fullchain.pem" ] && [ -s "$candidate/privkey.pem" ] || continue + + # These are operator-managed certificates, not a new trust source. Validate + # hostname, dates and server usage against the mounted leaf itself. An apex + # certificate is usable only if it also covers mail. (SAN or wildcard). + openssl verify -partial_chain -trusted "$candidate/fullchain.pem" \ + -purpose sslserver -verify_hostname "mail.$MAIL_BASE_DOMAIN" \ + "$candidate/fullchain.pem" >/dev/null 2>&1 || continue + cert_public="$(openssl x509 -in "$candidate/fullchain.pem" -pubkey -noout 2>/dev/null)" || continue + key_public="$(openssl pkey -in "$candidate/privkey.pem" -pubout -passin pass: 2>/dev/null)" || continue + [ -n "$cert_public" ] && [ "$cert_public" = "$key_public" ] || continue + MAIL_CERT_DIR="$candidate" + break +done + +# Keep the existing fallback/configuration intact when no usable pair is mounted. +[ -n "$MAIL_CERT_DIR" ] || exit 0 + +link_certificate() { + local source="$1" destination="$2" + # Compare the link itself, not readlink -f: Certbot's live links resolve to + # versioned archive files, which necessarily differ from the stable live path. + [ "$(readlink "$destination" 2>/dev/null || true)" != "$source" ] || return 0 + mkdir -p "$(dirname "$destination")" + if [ -f "$destination" ] && [ ! -L "$destination" ] && [ ! -e "$destination.bak" ]; then + cp -p "$destination" "$destination.bak" + fi + ln -sfn "$source" "$destination" +} + +link_certificate "$MAIL_CERT_DIR/fullchain.pem" "$MAIL_CERT_PATH" +link_certificate "$MAIL_CERT_DIR/privkey.pem" "$MAIL_KEY_PATH" +echo "[openship-mail] using TLS certificate for mail.$MAIL_BASE_DOMAIN from $MAIL_CERT_DIR"