fix(collab): normalize canonical markdown so tables/structural docs don't wedge projections

scripts/heal-table-projection-wedge.ts

@@ -0,0 +1,71 @@
+#!/usr/bin/env tsx
+// One-time heal for documents wedged by the table-projection bug.
+//
+// Documents created before the write-time markdown normalization fix store raw
+// canonical markdown that never matches the collab fragment's serialization
+// (the Milkdown GFM serializer pads table columns and injects `:---` alignment
+// markers). Such docs are stuck on readSource=yjs_fallback / mutationReady=false:
+// agent mutations return 409 PROJECTION_STALE and browser edits are dropped.
+//
+// This script rewrites each affected doc's canonical markdown to the fragment
+// serialization so the projection converges. It is idempotent — already-canonical
+// docs are skipped — so it is safe to re-run.
+//
+// Usage (run with the server stopped to avoid write contention):
+//   DATABASE_PATH=$HOME/.proof/proof-share.db npx tsx scripts/heal-table-projection-wedge.ts
+//   DATABASE_PATH=... npx tsx scripts/heal-table-projection-wedge.ts --dry-run
+
+const dryRun = process.argv.includes('--dry-run');
+
+async function main(): Promise<void> {
+  const db = await import('../server/db.ts');
+  const collab = await import('../server/collab.ts');
+
+  const docs = db.listActiveDocuments();
+  console.log(`[heal] scanning ${docs.length} active document(s)${dryRun ? ' (dry run)' : ''}`);
+
+  let healed = 0;
+  let skipped = 0;
+  let failed = 0;
+
+  for (const doc of docs) {
+    try {
+      if (dryRun) {
+        // In dry-run, compute the normalized form without writing. Mirror the
+        // real heal's gate exactly, including stripEphemeralCollabSpans, so the
+        // count doesn't mislead the operator.
+        const current = collab.stripEphemeralCollabSpans(doc.markdown ?? '');
+        if (current.trim().length === 0) { skipped += 1; continue; }
+        const normalized = await collab.deriveCanonicalMarkdownForStorage(current);
+        // Match healCanonicalMarkdownForCollabFragment's gate: only structural
+        // divergence wedges; trailing-whitespace-only diffs are skipped.
+        if (normalized.trimEnd() !== current.trimEnd()) {
+          healed += 1;
+          console.log(`[heal] WOULD heal ${doc.slug} (${current.length} -> ${normalized.length} chars)`);
+        } else {
+          skipped += 1;
+        }
+        continue;
+      }
+
+      const result = await collab.healCanonicalMarkdownForCollabFragment(doc.slug);
+      if (result.healed) {
+        healed += 1;
+        console.log(`[heal] healed ${doc.slug} (${result.before} -> ${result.after} chars)`);
+      } else {
+        skipped += 1;
+      }
+    } catch (error) {
+      failed += 1;
+      console.error(`[heal] FAILED ${doc.slug}:`, error instanceof Error ? error.message : String(error));
+    }
+  }
+
+  console.log(`[heal] done — ${healed} ${dryRun ? 'would be ' : ''}healed, ${skipped} already canonical, ${failed} failed`);
+  await collab.stopCollabRuntime();
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.stack || error.message : String(error));
+  process.exit(1);
+});

server/canonical-document.ts

@@ -26,6 +26,7 @@ import {
 import {
   maybeFastQuarantineProjectionPathology,
   cloneAuthoritativeDocState,
+  deriveCanonicalMarkdownFromProseMirrorDoc,
   detectPathologicalProjectionRepeat,
   evaluateProjectionSafety,
   getCanonicalReadableDocument,
@@ -980,9 +981,20 @@ export async function mutateCanonicalDocument(args: CanonicalMutationArgs): Prom
   const nextMarksBase = hasExplicitNextMarks ? nextMarks : authoritativeMarks;
   const authoredMarks = extractAuthoredMarksFromDoc(parsedNext.doc as ProseMirrorNode, parser.schema as Schema);
   const effectiveNextMarks = synchronizeAuthoredMarks(nextMarksBase, authoredMarks);
+  // Store canonical markdown in the collab fragment's serialization (the same
+  // fixed point POST /documents uses) so the projection stays fresh after an
+  // edit. A plain parse->serialize (serializedNextMarkdown) is NOT that fixed
+  // point — it reformats GFM tables differently from the fragment round trip, so
+  // storing it re-wedges the doc (readSource=yjs_fallback) on the next read.
+  // The rich-snapshot branch keeps its raw HTML-preserving form (it intentionally
+  // holds content the fragment cannot represent). Fall back to the parse->serialize
+  // form if the fragment derivation fails.
+  const fragmentCanonicalMarkdown = shouldPreserveRichMarkdownSnapshot(sanitizedMarkdown)
+    ? null
+    : await deriveCanonicalMarkdownFromProseMirrorDoc(parsedNext.doc as ProseMirrorNode);
   const authoritativeNextMarkdown = shouldPreserveRichMarkdownSnapshot(sanitizedMarkdown)
     ? normalizeStoredMarkdownSnapshot(sanitizedMarkdown)
-    : serializedNextMarkdown;
+    : (fragmentCanonicalMarkdown ?? serializedNextMarkdown);
 
   try {
     if (liveRequired && currentMutationBase.source !== 'live_yjs') {

server/collab.ts

@@ -4487,6 +4487,111 @@ async function deriveMarkdownProjectionFromFragment(doc: Y.Doc): Promise<string
   }
 }
 
+// Normalize raw markdown into the canonical serialization the collab fragment
+// produces. Storing this at write time keeps `documents.markdown` byte-identical
+// to what the projection later derives from the seeded Yjs fragment, so the
+// projection stays fresh instead of wedging on `readSource=yjs_fallback` /
+// `mutationReady=false`.
+//
+// This must run the SAME seed -> derive pipeline the load path uses. A plain
+// parse -> serialize is NOT equivalent: the fragment round trip
+// (prosemirrorToYXmlFragment / yXmlFragmentToProseMirrorRootNode) reformats GFM
+// tables (column padding + explicit `:---` alignment markers), and only the
+// fragment-derived form is a stable fixed point. Returns the input unchanged if
+// the parser is unavailable — the load-time reconcile heals it later.
+export async function deriveCanonicalMarkdownForStorage(markdown: string): Promise<string> {
+  const input = markdown ?? '';
+  if (input.trim().length === 0) return input;
+  const ydoc = new Y.Doc();
+  try {
+    await seedFragmentFromLegacyMarkdown(ydoc, input);
+    const derived = await deriveMarkdownProjectionFromFragment(ydoc);
+    return derived ?? input;
+  } catch (error) {
+    console.warn('[collab] failed to normalize canonical markdown for storage; storing raw', {
+      error: summarizeParseError(error),
+    });
+    return input;
+  } finally {
+    ydoc.destroy();
+  }
+}
+
+// Derive the canonical (fragment-fixed-point) markdown for a ProseMirror document
+// that is about to be written into a collab fragment via prosemirrorToYXmlFragment.
+// Mutation write paths (PUT / rewrite / agent edits) build the fragment from a
+// parsed ProseMirror doc and must store the SAME serialization the fragment will
+// later derive, or the doc re-wedges on the next read exactly like a raw-markdown
+// create did. Building the fragment from the identical doc guarantees the match.
+// Returns null on failure so callers can fall back to their existing value.
+export async function deriveCanonicalMarkdownFromProseMirrorDoc(
+  doc: ProseMirrorNode,
+): Promise<string | null> {
+  const ydoc = new Y.Doc();
+  try {
+    prosemirrorToYXmlFragment(doc, ydoc.getXmlFragment('prosemirror') as any);
+    return await deriveMarkdownProjectionFromFragment(ydoc);
+  } catch (error) {
+    console.warn('[collab] failed to derive canonical markdown from ProseMirror doc', {
+      error: summarizeParseError(error),
+    });
+    return null;
+  } finally {
+    ydoc.destroy();
+  }
+}
+
+// Heal an EXISTING document whose stored canonical markdown predates the
+// write-time normalization above (e.g. a doc created with a raw GFM table before
+// this fix). Rewrites `documents.markdown` to the collab fragment's serialization
+// and refreshes the projection so it converges to fresh. Idempotent: a no-op when
+// the canonical markdown already matches the fragment serialization.
+export async function healCanonicalMarkdownForCollabFragment(
+  slug: string,
+): Promise<{ healed: boolean; reason: string; before?: number; after?: number }> {
+  const row = getDocumentBySlug(slug);
+  if (!row) return { healed: false, reason: 'missing_doc' };
+  if (row.share_state === 'DELETED') return { healed: false, reason: 'deleted' };
+  const current = stripEphemeralCollabSpans(row.markdown ?? '');
+  if (current.trim().length === 0) return { healed: false, reason: 'empty' };
+
+  // Only heal seed-only wedged documents. A doc with persisted incremental Yjs
+  // updates may hold live edits not yet compacted into the canonical row; healing
+  // re-seeds the collab room from the canonical row and clears persisted Yjs
+  // state, which would drop those edits. Docs that receive edits are kept
+  // consistent by write-path normalization instead, so skip them here.
+  if (getYUpdatesAfter(slug, 0).length > 0) {
+    return { healed: false, reason: 'has_persisted_edits' };
+  }
+
+  const normalized = await deriveCanonicalMarkdownForStorage(current);
+  if (normalized === current) return { healed: false, reason: 'already_canonical' };
+  // Only structural divergence wedges a projection — the read/snapshot path
+  // tolerates trailing-whitespace differences (a bare trailing newline never
+  // wedges). Skip cosmetic-only diffs so the heal doesn't churn healthy docs.
+  if (normalized.trimEnd() === current.trimEnd()) return { healed: false, reason: 'cosmetic_only' };
+
+  const marks = parseStoredMarks(row.marks);
+  const yStateVersion = getLatestYStateVersion(slug);
+  const now = new Date().toISOString();
+  const db = getDb();
+  const tx = db.transaction(() => {
+    db.prepare(
+      "UPDATE documents SET markdown = ?, updated_at = ? WHERE slug = ? AND share_state IN ('ACTIVE', 'PAUSED')",
+    ).run(normalized, now, slug);
+    replaceDocumentProjection(slug, normalized, marks, yStateVersion, {
+      health: 'healthy',
+      healthReason: null,
+    });
+  });
+  tx();
+  // Safe because the guard above ensures there are no persisted incremental
+  // updates: clearing lets the next load re-seed the room from the healed
+  // canonical row so the fragment, markdown mirror, and projection all agree.
+  invalidateCollabDocument(slug);
+  return { healed: true, reason: 'normalized', before: current.length, after: normalized.length };
+}
+
 let canonicalSyncPostApplyFailureForTests: string | null = null;
 let canonicalSyncForcedRefusalForTests: CanonicalCollabSyncFailureReason | null = null;
 let canonicalSyncParseFailureForTests = false;

server/routes.ts

@@ -6,6 +6,7 @@ import {
   applyAgentPresenceToLoadedCollab,
   applyCanonicalDocumentToCollab,
   buildCollabSession,
+  deriveCanonicalMarkdownForStorage,
   getCanonicalReadableDocumentSync,
   getCollabRuntime,
   invalidateCollabDocument,
@@ -784,7 +785,7 @@ function deriveShareCapabilities(role: ShareRole, shareState: string): {
 }
 
 // Create a shared document
-apiRoutes.post('/documents', (req: Request, res: Response) => {
+apiRoutes.post('/documents', async (req: Request, res: Response) => {
   const legacyCreateMode = resolveLegacyCreateMode(getPublicBaseUrl(req));
   if (legacyCreateMode === 'disabled') {
     recordLegacyCreateRouteTelemetry(req, legacyCreateMode, 'blocked_disabled');
@@ -826,7 +827,12 @@ apiRoutes.post('/documents', (req: Request, res: Response) => {
   const slug = generateSlug();
   const ownerSecret = randomUUID();
   const normalizedMarks = canonicalizeStoredMarks(marks ?? {});
-  const doc = createDocument(slug, sanitizedMarkdown, normalizedMarks, title, ownerId, ownerSecret);
+  // Store canonical markdown in the collab fragment's serialization so the
+  // projection derived on first load matches byte-for-byte. Without this, docs
+  // containing GFM tables wedge on `readSource=yjs_fallback` / `mutationReady=false`
+  // because the fragment re-serializes tables (column padding + `:---` markers).
+  const canonicalMarkdown = await deriveCanonicalMarkdownForStorage(sanitizedMarkdown);
+  const doc = createDocument(slug, canonicalMarkdown, normalizedMarks, title, ownerId, ownerSecret);
   const defaultAccess = createDocumentAccessToken(slug, 'editor');
   const links = buildShareLink(req, doc.slug);
   const shareUrlWithToken = withShareToken(links.shareUrl, defaultAccess.secret);
@@ -1078,7 +1084,11 @@ export async function handleShareMarkdown(req: Request, res: Response): Promise<
 
   const slug = generateSlug();
   const ownerSecret = randomUUID();
-  const doc = createDocument(slug, sanitizedMarkdown, marks, title, ownerId, ownerSecret);
+  // Normalize to the collab fragment's serialization so structural markdown
+  // (GFM tables, list-then-heading) doesn't wedge the projection. Same rationale
+  // as POST /documents.
+  const canonicalMarkdown = await deriveCanonicalMarkdownForStorage(sanitizedMarkdown);
+  const doc = createDocument(slug, canonicalMarkdown, marks, title, ownerId, ownerSecret);
   const access = createDocumentAccessToken(slug, requestedRole);
   const links = buildShareLink(req, doc.slug);
   const shareUrlWithToken = withShareToken(links.shareUrl, access.secret);

src/tests/agent-edit-live-viewer-regression.test.ts

@@ -179,6 +179,19 @@ async function run(): Promise<void> {
     });
     const created = await mustJson<CreateResponse>(createRes, 'create');
 
+    // Capture the canonical markdown the server actually stored. Creation
+    // normalizes markdown into the collab fragment's serialization, so the stored
+    // canonical may differ cosmetically from the raw POST body (e.g. a canonical
+    // blank line before a heading). The blocked-edit assertion below checks this
+    // stored canonical is left unchanged, not that it equals the raw input.
+    const canonicalAfterCreateRes = await fetch(`${httpBase}/api/documents/${created.slug}`, {
+      headers: {
+        ...CLIENT_HEADERS,
+        'x-share-token': created.ownerSecret,
+      },
+    });
+    const canonicalAfterCreate = (await mustJson<ReadDocResponse>(canonicalAfterCreateRes, 'read-after-create')).markdown;
+
     const collabSessionRes = await fetch(`${httpBase}/api/documents/${created.slug}/collab-session`, {
       headers: {
         ...CLIENT_HEADERS,
@@ -271,7 +284,7 @@ async function run(): Promise<void> {
       `Expected /edit/v2 guidance, got ${String(edit.recommendedEndpoint)}`,
     );
 
-    const expectedFinal = `${DOC_PREFIX}${currentSection}${DOC_SUFFIX}`;
+    const expectedFinal = canonicalAfterCreate;
     const readRes = await fetch(`${httpBase}/api/documents/${created.slug}`, {
       headers: {
         ...CLIENT_HEADERS,