Verify: decode quoted-printable bodies even when their encoding header is hidden

skills/redacted-email-proof/scripts/redacted.mjs

@@ -763,9 +763,10 @@ export function outputsOf(publicInputs, prefix, maxHeaderLength) {
   const header = latin1(headerBytes.slice(0, headerLen));
   const body = latin1(bodyBytes.slice(0, bodyLen));
   const pretty = (s) => compact(Buffer.from(s, "latin1").toString("utf8").replace(/\u0000/g, BLOCK).replace(/\r\n/g, "\n"));
-  const qpBody = hasQuotedPrintablePart(Buffer.from(body, "latin1"));
+  const qpEvidence = quotedPrintableEvidence(Buffer.from(body, "latin1"));
+  const qpBody = qpEvidence !== null;
   const bodyText = qpBody ? pretty(displayDecodeMasked(Buffer.from(body, "latin1")).toString("latin1")) : pretty(body);
-  return { header, body, headerText: pretty(header), bodyText, bodyDecodedFromQp: qpBody };
+  return { header, body, headerText: pretty(header), bodyText, bodyRawText: pretty(body), bodyDecodedFromQp: qpBody, qpEvidence };
 }
 
 // REASON: noir-bignum >= v0.9 (used by the v2 circuits) defines the Barrett parameter as
@@ -805,6 +806,37 @@ export function qpDecode(bytes) {
   return Buffer.from(out);
 }
 // Decode a masked raw body (0x00 = hidden) for display; an escape with a hidden byte -> one 0x00.
+// Same as src/utils/qp.ts quotedPrintableEvidence: "header" | "content" | null. Decide display
+// decoding from a revealed Content-Transfer-Encoding header, else from the text itself (soft
+// breaks not after "=", uppercase =XX runs that are valid UTF-8). See the REASON note there.
+export function quotedPrintableEvidence(maskedBody) {
+  const text = Buffer.from(maskedBody).toString("latin1");
+  const ctes = [...text.matchAll(/content-transfer-encoding:\s*([a-z0-9-]+)/gi)].map((m) => m[1].toLowerCase());
+  if (ctes.includes("quoted-printable")) return "header";
+  if (ctes.length) return null;
+  let softBreaks = 0;
+  for (const m of text.matchAll(/=\r\n/g)) {
+    const prev = text[m.index - 1];
+    if (prev !== "=" && prev !== "\0" && prev !== undefined) softBreaks++;
+  }
+  const escapes = (text.match(/=[0-9A-F]{2}/g) ?? []).length;
+  if (escapes) {
+    // validate the decoded text between hidden bytes (QP can split a UTF-8 char across "=\r\n")
+    const decoded = displayDecodeMasked(Buffer.from(maskedBody));
+    let start = 0;
+    for (let i = 0; i <= decoded.length; i++) {
+      if (i < decoded.length && decoded[i] !== 0) continue;
+      try {
+        new TextDecoder("utf-8", { fatal: true }).decode(decoded.subarray(start, i));
+      } catch {
+        return null;
+      }
+      start = i + 1;
+    }
+  }
+  return softBreaks >= 2 || (softBreaks >= 1 && escapes >= 1) || escapes >= 3 ? "content" : null;
+}
+
 export function displayDecodeMasked(bytes) {
   const out = [];
   for (let i = 0; i < bytes.length; i++) {
@@ -1175,6 +1207,8 @@ async function cmdVerify(ref, opts) {
     fromAlignedWithDkimDomain: binding?.fromAligned ?? null,
     maskedHeader: outs.headerText,
     maskedBody: outs.bodyText,
+    maskedBodyRaw: outs.bodyRawText ?? outs.bodyText,
+    bodyDecodedFromQp: outs.bodyDecodedFromQp ?? false,
   };
   if (opts.json) {
     console.log(JSON.stringify(result, null, 2));
@@ -1209,7 +1243,10 @@ async function cmdVerify(ref, opts) {
         );
       }
     }
-    console.log("\n----- masked header -----\n" + outs.headerText + "\n----- masked body -----\n" + outs.bodyText);
+    const qpNote = outs.bodyDecodedFromQp
+      ? ` (decoded from quoted-printable${outs.qpEvidence === "content" ? ", detected from the text" : ""}; the proof covers the encoded bytes, see maskedBodyRaw in --json)`
+      : "";
+    console.log("\n----- masked header -----\n" + outs.headerText + `\n----- masked body${qpNote} -----\n` + outs.bodyText);
   }
   if (!result.proofValid) process.exit(2);
   if (!result.keyMatches) process.exit(3);

src/lib.ts

@@ -14,7 +14,7 @@ import type { DKIMResult } from "./utils/emlParser";
 import circuitConfigs from "./circuit-configs.json";
 import { canonicalPublicInputs, decodeMaskedBytes, V2_PREFIX } from "./utils/proofOutputs";
 import { dkimFieldSequence } from "./utils/dkimFields";
-import { bodyViewFor, charMaskToByteMask, viewMaskToRawMask, displayDecodeMasked, looksQuotedPrintable } from "./utils/qp";
+import { bodyViewFor, charMaskToByteMask, viewMaskToRawMask, displayDecodeMasked, quotedPrintableEvidence } from "./utils/qp";
 
 /**
  * Circuit versions
@@ -385,6 +385,10 @@ export function extractMaskedDataFromProof(proof: ProofWithCircuit): {
   emailNullifier: Uint8Array;
   version: CircuitVersion;
   bodyDecodedFromQp: boolean;
+  /** why the body was decoded: a revealed Content-Transfer-Encoding header, or the text itself */
+  qpEvidence: "header" | "content" | null;
+  /** the proven body bytes as text, undecoded (for "show raw") */
+  maskedBodyRaw: string;
 } | null {
   try {
     // Decode exactly what handleVerifyProof verifies (throws on non-canonical inputs).
@@ -415,11 +419,14 @@ export function extractMaskedDataFromProof(proof: ProofWithCircuit): {
 
     const decoder = new TextDecoder("utf-8", { fatal: false });
     // Display-only decoding of the proven raw bytes (quoted-printable is a public function of them).
-    const bodyDecodedFromQp = looksQuotedPrintable(body);
+    const qpEvidence = quotedPrintableEvidence(body);
+    const bodyDecodedFromQp = qpEvidence !== null;
     return {
       maskedHeader: decoder.decode(header),
       maskedBody: decoder.decode(bodyDecodedFromQp ? displayDecodeMasked(body) : body),
+      maskedBodyRaw: decoder.decode(body),
       bodyDecodedFromQp,
+      qpEvidence,
       publicKeyHash,
       emailNullifier,
       version: layout.version,

src/pages/VerifyPage.tsx

@@ -48,6 +48,9 @@ export default function VerifyPage() {
   } | null>(null);
   const [maskedBody, setMaskedBody] = useState<string>('');
   const [bodyDecodedFromQp, setBodyDecodedFromQp] = useState(false);
+  const [qpEvidence, setQpEvidence] = useState<"header" | "content" | null>(null);
+  const [maskedBodyRaw, setMaskedBodyRaw] = useState("");
+  const [showRawBody, setShowRawBody] = useState(false);
   const [showVerificationModal, setShowVerificationModal] = useState(false);
 
   useEffect(() => {
@@ -183,6 +186,8 @@ export default function VerifyPage() {
         // Store the masked body
         setMaskedBody(maskedData.maskedBody);
         setBodyDecodedFromQp(maskedData.bodyDecodedFromQp);
+        setQpEvidence(maskedData.qpEvidence);
+        setMaskedBodyRaw(maskedData.maskedBodyRaw);
 
       } catch (err) {
         console.error("Error loading verification data:", err);
@@ -379,13 +384,19 @@ export default function VerifyPage() {
             <div className="p-6">
               {bodyDecodedFromQp && (
                 <p className="text-xs text-[#666] mb-3">
-                  Shown decoded from quoted-printable. The proof covers the encoded bytes; decoding
-                  them is a fixed public step anyone can repeat.
+                  {showRawBody
+                    ? "Shown as the raw quoted-printable bytes the proof covers."
+                    : qpEvidence === "content"
+                      ? "Shown decoded from quoted-printable (detected from the text; its Content-Transfer-Encoding header is hidden). The proof covers the encoded bytes; decoding them is a fixed public step anyone can repeat."
+                      : "Shown decoded from quoted-printable. The proof covers the encoded bytes; decoding them is a fixed public step anyone can repeat."}{" "}
+                  <button type="button" className="underline" onClick={() => setShowRawBody((v) => !v)}>
+                    {showRawBody ? "Show decoded" : "Show raw"}
+                  </button>
                 </p>
               )}
               <div className="prose max-w-none">
                 <MaskedText
-                  text={maskedBody}
+                  text={bodyDecodedFromQp && showRawBody ? maskedBodyRaw : maskedBody}
                   className="text-[#111314] whitespace-pre-wrap break-words"
                 />
               </div>

src/utils/qp.ts

@@ -161,6 +161,47 @@ export function displayDecodeMasked(bytes: Uint8Array): Uint8Array {
   return Uint8Array.from(out);
 }
 
-/** Should the verify page decode this (masked) body? Only if its revealed bytes say it's QP. */
-export const looksQuotedPrintable = (maskedBody: Uint8Array) =>
-  /content-transfer-encoding:\s*quoted-printable/i.test(new TextDecoder("latin1").decode(maskedBody));
+/**
+ * Should the verify page decode this (masked) body, and why: "header" when a revealed
+ * Content-Transfer-Encoding header says quoted-printable, "content" when the revealed text itself
+ * is clearly QP, else null.
+ * REASON: provers often hide the whole body and reveal a few sentences, which hides the part's
+ * Content-Transfer-Encoding header. The page then showed raw "samples=E2=80=94whether iden=" text.
+ * Decoding is display-only (the proof covers the raw bytes, and "show raw" stays available), so a
+ * content heuristic is safe as long as it doesn't fire on ordinary text:
+ *  - soft line breaks "=\r\n" not preceded by "=" (base64 padding "==" at a line end is not QP), and
+ *  - uppercase "=XX" escapes, where every revealed stretch of the decoded text is valid UTF-8
+ *    (checked after decoding, because QP may split one UTF-8 character across a soft break:
+ *    "=E2=80=\r\n=94" is an em dash),
+ *  - and no revealed Content-Transfer-Encoding header naming another encoding.
+ */
+export function quotedPrintableEvidence(maskedBody: Uint8Array): "header" | "content" | null {
+  const text = new TextDecoder("latin1").decode(maskedBody);
+  const ctes = [...text.matchAll(/content-transfer-encoding:\s*([a-z0-9-]+)/gi)].map((m) => m[1].toLowerCase());
+  if (ctes.includes("quoted-printable")) return "header";
+  if (ctes.length) return null;
+  let softBreaks = 0;
+  for (const m of text.matchAll(/=\r\n/g)) {
+    const prev = text[m.index! - 1];
+    if (prev !== "=" && prev !== "\0" && prev !== undefined) softBreaks++;
+  }
+  const escapes = (text.match(/=[0-9A-F]{2}/g) ?? []).length;
+  if (escapes) {
+    // Validate between hidden bytes (0x00): a hidden neighbour can cut a character in half.
+    const decoded = displayDecodeMasked(maskedBody);
+    let start = 0;
+    for (let i = 0; i <= decoded.length; i++) {
+      if (i < decoded.length && decoded[i] !== 0) continue;
+      try {
+        new TextDecoder("utf-8", { fatal: true }).decode(decoded.subarray(start, i));
+      } catch {
+        return null; // escapes that aren't UTF-8 text: probably not QP, don't guess
+      }
+      start = i + 1;
+    }
+  }
+  return softBreaks >= 2 || (softBreaks >= 1 && escapes >= 1) || escapes >= 3 ? "content" : null;
+}
+
+/** Should the verify page decode this (masked) body? See quotedPrintableEvidence. */
+export const looksQuotedPrintable = (maskedBody: Uint8Array) => quotedPrintableEvidence(maskedBody) !== null;

tests/qp.test.ts

@@ -5,9 +5,9 @@
  */
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { bodyViewFor, displayDecodeMasked, qpDecode, viewMaskToRawMask } from "../src/utils/qp";
+import { bodyViewFor, displayDecodeMasked, qpDecode, quotedPrintableEvidence, viewMaskToRawMask } from "../src/utils/qp";
 // @ts-expect-error - plain JS module without types
-import { qpDecode as cliQpDecode, displayDecodeMasked as cliDisplayDecodeMasked } from "../skills/redacted-email-proof/scripts/redacted.mjs";
+import { qpDecode as cliQpDecode, displayDecodeMasked as cliDisplayDecodeMasked, quotedPrintableEvidence as cliQpEvidence } from "../skills/redacted-email-proof/scripts/redacted.mjs";
 
 const RAW = Buffer.from(
   "Content-Type: text/html; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" +
@@ -64,3 +64,36 @@ test("display decoding of masked output: partly hidden escapes show as one hidde
   assert.equal(Buffer.from(cliDisplayDecodeMasked(masked)).toString("latin1"), site);
   assert.equal(site, "xâ\u0082¬ y \0 z \0!");
 });
+
+// A proof that hides the part's Content-Transfer-Encoding header (e.g. --hide-body with a few
+// sentences revealed) must still display decoded text, not "samples=E2=80=94whether iden=".
+test("QP detected from the text when the Content-Transfer-Encoding header is hidden", () => {
+  const raw = Buffer.from(
+    "Content-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" +
+      "we never sell data derived from our customers' samples=E2=80=94whether iden=\r\ntifiable or not=\r\n, ever=E2=80=\r\n=94really.\r\n",
+    "latin1"
+  );
+  const masked = Buffer.from(raw);
+  masked.fill(0, 0, raw.indexOf("\r\n\r\n") + 4); // MIME part headers hidden
+  for (const [name, evidence] of [["site", quotedPrintableEvidence], ["cli", cliQpEvidence]] as const) {
+    assert.equal(evidence(raw), "header", name);
+    assert.equal(evidence(masked), "content", name);
+  }
+  const shownText = Buffer.from(displayDecodeMasked(masked)).toString("utf8");
+  assert.ok(shownText.includes("samples—whether identifiable or not, ever—really."), shownText);
+});
+
+test("QP content detection does not fire on base64, plain text, or another declared encoding", () => {
+  const cases: [string, string][] = [
+    ["base64 padding at line ends", "SGVsbG8gd29ybGQ=\r\nSGk==\r\nQUJD==\r\n"],
+    ["plain text with = signs", "x = 1\r\nif (a == b) total = 3D printer\r\n"],
+    ["one lone soft-break-like line", "price =\r\nsee below\r\n"],
+    ["escapes that aren't UTF-8", "bytes =FF=FE=FD here\r\n"],
+    ["declared base64 part", "Content-Transfer-Encoding: base64\r\n\r\nc2Ftc=\r\nGxl=E2=80=94=\r\n"],
+  ];
+  for (const [label, text] of cases) {
+    const b = Buffer.from(text, "latin1");
+    assert.equal(quotedPrintableEvidence(b), null, `site: ${label}`);
+    assert.equal(cliQpEvidence(b), null, `cli: ${label}`);
+  }
+});