Bound and exhaust Content document discovery

templates/content/actions/_document-discovery-query.ts

@@ -0,0 +1,127 @@
+import { accessFilter } from "@agent-native/core/sharing";
+import {
+  and,
+  eq,
+  exists,
+  isNotNull,
+  isNull,
+  notExists,
+  or,
+  type SQL,
+} from "drizzle-orm";
+
+import { getDb, schema } from "../server/db/index.js";
+import { documentDiscoveryFilter } from "../server/lib/documents.js";
+
+export const DOCUMENT_DISCOVERY_MAX_LIMIT = 200;
+export const DOCUMENT_DISCOVERY_DEFAULT_LIMIT = 50;
+
+export type DocumentDiscoveryType = "page" | "database";
+
+export interface DocumentDiscoveryFilters {
+  userEmail: string | null | undefined;
+  authorizedOrgIds: string[];
+  exactTitle?: string;
+  parentId?: string | null;
+  spaceId?: string;
+  documentType?: DocumentDiscoveryType;
+  additional?: SQL;
+}
+
+export function documentDiscoveryWhere({
+  userEmail,
+  authorizedOrgIds,
+  exactTitle,
+  parentId,
+  spaceId,
+  documentType,
+  additional,
+}: DocumentDiscoveryFilters) {
+  const db = getDb();
+  const accessContexts = [
+    { userEmail: userEmail ?? undefined },
+    ...authorizedOrgIds.map((orgId) => ({
+      userEmail: userEmail ?? undefined,
+      orgId,
+    })),
+  ];
+  const activeDatabaseDocument = db
+    .select({ id: schema.contentDatabases.id })
+    .from(schema.contentDatabases)
+    .where(
+      and(
+        eq(schema.contentDatabases.documentId, schema.documents.id),
+        isNull(schema.contentDatabases.deletedAt),
+      ),
+    );
+  const deletedDatabaseDocument = db
+    .select({ id: schema.contentDatabases.id })
+    .from(schema.contentDatabases)
+    .where(
+      and(
+        eq(schema.contentDatabases.documentId, schema.documents.id),
+        isNotNull(schema.contentDatabases.deletedAt),
+      ),
+    );
+  const deletedDatabaseMembership = db
+    .select({ id: schema.contentDatabaseItems.id })
+    .from(schema.contentDatabaseItems)
+    .innerJoin(
+      schema.contentDatabases,
+      eq(schema.contentDatabases.id, schema.contentDatabaseItems.databaseId),
+    )
+    .where(
+      and(
+        eq(schema.contentDatabaseItems.documentId, schema.documents.id),
+        isNotNull(schema.contentDatabases.deletedAt),
+      ),
+    );
+
+  return and(
+    or(
+      ...accessContexts.map((context) =>
+        accessFilter(schema.documents, schema.documentShares, context),
+      ),
+    ),
+    isNull(schema.documents.trashedAt),
+    documentDiscoveryFilter({
+      userEmail,
+      orgIds: authorizedOrgIds,
+    }),
+    notExists(deletedDatabaseDocument),
+    notExists(deletedDatabaseMembership),
+    exactTitle === undefined
+      ? undefined
+      : eq(schema.documents.title, exactTitle),
+    parentId === undefined
+      ? undefined
+      : parentId === null
+        ? isNull(schema.documents.parentId)
+        : eq(schema.documents.parentId, parentId),
+    spaceId === undefined ? undefined : eq(schema.documents.spaceId, spaceId),
+    documentType === "database"
+      ? exists(activeDatabaseDocument)
+      : documentType === "page"
+        ? notExists(activeDatabaseDocument)
+        : undefined,
+    additional,
+  );
+}
+
+export function documentDiscoveryPagination(args: {
+  offset: number;
+  limit: number;
+  totalItems: number;
+  returnedItems: number;
+}) {
+  const nextOffset = args.offset + args.returnedItems;
+  const hasMore = nextOffset < args.totalItems;
+  return {
+    offset: args.offset,
+    limit: args.limit,
+    totalItems: args.totalItems,
+    returnedItems: args.returnedItems,
+    hasMore,
+    nextOffset: hasMore ? nextOffset : null,
+  };
+}

templates/content/actions/document-discovery.db.test.ts

@@ -0,0 +1,205 @@
+import { rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { runWithRequestContext } from "@agent-native/core/server";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+
+const TEST_DB_PATH = join(
+  tmpdir(),
+  `content-document-discovery-${process.pid}-${Date.now()}.sqlite`,
+);
+const OWNER = "discovery-owner@example.com";
+const OUTSIDER = "discovery-outsider@example.com";
+const PARENT_ID = "bounded-discovery-parent";
+const SPACE_ID = "bounded-discovery-space";
+
+type Schema = typeof import("../server/db/schema.js");
+let getDb: () => any;
+let schema: Schema;
+let listDocuments: typeof import("./list-documents.js").default;
+let searchDocuments: typeof import("./search-documents.js").default;
+
+const asUser = <T>(userEmail: string, run: () => Promise<T>) =>
+  runWithRequestContext({ userEmail }, run);
+
+beforeAll(async () => {
+  process.env.DATABASE_URL = `file:${TEST_DB_PATH}`;
+  const dbModule = await import("../server/db/index.js");
+  getDb = dbModule.getDb;
+  schema = dbModule.schema;
+  listDocuments = (await import("./list-documents.js")).default;
+  searchDocuments = (await import("./search-documents.js")).default;
+  const plugin = (await import("../server/plugins/db.js")).default;
+  await plugin(undefined as any);
+
+  const now = new Date().toISOString();
+  await getDb().insert(schema.contentSpaces).values({
+    id: SPACE_ID,
+    name: "Bounded discovery",
+    kind: "personal",
+    ownerEmail: OWNER,
+    orgId: null,
+    filesDatabaseId: "bounded-discovery-files",
+    createdBy: OWNER,
+    createdAt: now,
+    updatedAt: now,
+  });
+  await getDb().insert(schema.documents).values({
+    id: PARENT_ID,
+    spaceId: SPACE_ID,
+    ownerEmail: OWNER,
+    orgId: null,
+    parentId: null,
+    title: "Discovery parent",
+    content: "",
+    position: 0,
+    visibility: "private",
+    createdAt: now,
+    updatedAt: now,
+  });
+  const documents = Array.from({ length: 205 }, (_, index) => ({
+    id: `bounded-discovery-document-${index.toString().padStart(3, "0")}`,
+    spaceId: SPACE_ID,
+    ownerEmail: OWNER,
+    orgId: null,
+    parentId: PARENT_ID,
+    title: index < 2 ? "Duplicate exact title" : `Bounded document ${index}`,
+    description: index === 204 ? "last page marker" : "",
+    content: `needle payload ${index}`,
+    position: index,
+    visibility: "private" as const,
+    createdAt: now,
+    updatedAt: new Date(Date.parse(now) + index).toISOString(),
+  }));
+  for (let start = 0; start < documents.length; start += 100) {
+    await getDb()
+      .insert(schema.documents)
+      .values(documents.slice(start, start + 100));
+  }
+}, 60_000);
+
+afterAll(() => {
+  for (const suffix of ["", "-shm", "-wal"])
+    rmSync(`${TEST_DB_PATH}${suffix}`, { force: true });
+});
+
+describe("bounded document discovery", () => {
+  it("returns explicit continuation metadata through a terminal list page", async () => {
+    const first = await asUser(OWNER, () =>
+      listDocuments.run({ parentId: PARENT_ID, limit: 100, offset: 0 }),
+    );
+    const second = await asUser(OWNER, () =>
+      listDocuments.run({ parentId: PARENT_ID, limit: 100, offset: 100 }),
+    );
+    const terminal = await asUser(OWNER, () =>
+      listDocuments.run({ parentId: PARENT_ID, limit: 100, offset: 200 }),
+    );
+
+    expect(first.pagination).toEqual({
+      offset: 0,
+      limit: 100,
+      totalItems: 205,
+      returnedItems: 100,
+      hasMore: true,
+      nextOffset: 100,
+    });
+    expect(second.pagination.nextOffset).toBe(200);
+    expect(terminal.pagination).toEqual({
+      offset: 200,
+      limit: 100,
+      totalItems: 205,
+      returnedItems: 5,
+      hasMore: false,
+      nextOffset: null,
+    });
+    expect(terminal.documents.at(-1)?.description).toBe("last page marker");
+  });
+
+  it("distinguishes zero, one, and multiple exact scoped title matches", async () => {
+    const none = await asUser(OWNER, () =>
+      searchDocuments.run({
+        exactTitle: "No such document",
+        parentId: PARENT_ID,
+        spaceId: SPACE_ID,
+        documentType: "page",
+        limit: 10,
+        offset: 0,
+      }),
+    );
+    const one = await asUser(OWNER, () =>
+      searchDocuments.run({
+        exactTitle: "Bounded document 204",
+        parentId: PARENT_ID,
+        spaceId: SPACE_ID,
+        documentType: "page",
+        limit: 10,
+        offset: 0,
+      }),
+    );
+    const multiple = await asUser(OWNER, () =>
+      searchDocuments.run({
+        exactTitle: "Duplicate exact title",
+        parentId: PARENT_ID,
+        spaceId: SPACE_ID,
+        documentType: "page",
+        limit: 1,
+        offset: 0,
+      }),
+    );
+
+    expect(none.pagination).toMatchObject({
+      totalItems: 0,
+      returnedItems: 0,
+      hasMore: false,
+      nextOffset: null,
+    });
+    expect(one.pagination).toMatchObject({
+      totalItems: 1,
+      returnedItems: 1,
+      hasMore: false,
+      nextOffset: null,
+    });
+    expect(multiple.pagination).toMatchObject({
+      totalItems: 2,
+      returnedItems: 1,
+      hasMore: true,
+      nextOffset: 1,
+    });
+  });
+
+  it("paginates body search and suppresses the private corpus for an outsider", async () => {
+    const ownerPage = await asUser(OWNER, () =>
+      searchDocuments.run({
+        query: "needle payload",
+        parentId: PARENT_ID,
+        limit: 200,
+        offset: 0,
+      }),
+    );
+    const outsiderPage = await asUser(OUTSIDER, () =>
+      searchDocuments.run({
+        query: "needle payload",
+        parentId: PARENT_ID,
+        limit: 200,
+        offset: 0,
+      }),
+    );
+
+    expect(ownerPage.pagination).toMatchObject({
+      totalItems: 205,
+      returnedItems: 200,
+      hasMore: true,
+      nextOffset: 200,
+    });
+    expect(outsiderPage).toMatchObject({
+      documents: [],
+      pagination: {
+        totalItems: 0,
+        returnedItems: 0,
+        hasMore: false,
+        nextOffset: null,
+      },
+    });
+  });
+});