diff --git a/templates/content/actions/_document-discovery-query.ts b/templates/content/actions/_document-discovery-query.ts new file mode 100644 index 0000000000..be9c383364 --- /dev/null +++ b/templates/content/actions/_document-discovery-query.ts @@ -0,0 +1,127 @@ +import { accessFilter } from "@agent-native/core/sharing"; +import { + and, + eq, + exists, + isNotNull, + isNull, + notExists, + or, + type SQL, +} from "drizzle-orm"; + +import { getDb, schema } from "../server/db/index.js"; +import { documentDiscoveryFilter } from "../server/lib/documents.js"; + +export const DOCUMENT_DISCOVERY_MAX_LIMIT = 200; +export const DOCUMENT_DISCOVERY_DEFAULT_LIMIT = 50; + +export type DocumentDiscoveryType = "page" | "database"; + +export interface DocumentDiscoveryFilters { + userEmail: string | null | undefined; + authorizedOrgIds: string[]; + exactTitle?: string; + parentId?: string | null; + spaceId?: string; + documentType?: DocumentDiscoveryType; + additional?: SQL; +} + +export function documentDiscoveryWhere({ + userEmail, + authorizedOrgIds, + exactTitle, + parentId, + spaceId, + documentType, + additional, +}: DocumentDiscoveryFilters) { + const db = getDb(); + const accessContexts = [ + { userEmail: userEmail ?? undefined }, + ...authorizedOrgIds.map((orgId) => ({ + userEmail: userEmail ?? undefined, + orgId, + })), + ]; + const activeDatabaseDocument = db + .select({ id: schema.contentDatabases.id }) + .from(schema.contentDatabases) + .where( + and( + eq(schema.contentDatabases.documentId, schema.documents.id), + isNull(schema.contentDatabases.deletedAt), + ), + ); + const deletedDatabaseDocument = db + .select({ id: schema.contentDatabases.id }) + .from(schema.contentDatabases) + .where( + and( + eq(schema.contentDatabases.documentId, schema.documents.id), + isNotNull(schema.contentDatabases.deletedAt), + ), + ); + const deletedDatabaseMembership = db + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.contentDatabases, + eq(schema.contentDatabases.id, schema.contentDatabaseItems.databaseId), + ) + .where( + and( + eq(schema.contentDatabaseItems.documentId, schema.documents.id), + isNotNull(schema.contentDatabases.deletedAt), + ), + ); + + return and( + or( + ...accessContexts.map((context) => + accessFilter(schema.documents, schema.documentShares, context), + ), + ), + isNull(schema.documents.trashedAt), + documentDiscoveryFilter({ + userEmail, + orgIds: authorizedOrgIds, + }), + notExists(deletedDatabaseDocument), + notExists(deletedDatabaseMembership), + exactTitle === undefined + ? undefined + : eq(schema.documents.title, exactTitle), + parentId === undefined + ? undefined + : parentId === null + ? isNull(schema.documents.parentId) + : eq(schema.documents.parentId, parentId), + spaceId === undefined ? undefined : eq(schema.documents.spaceId, spaceId), + documentType === "database" + ? exists(activeDatabaseDocument) + : documentType === "page" + ? notExists(activeDatabaseDocument) + : undefined, + additional, + ); +} + +export function documentDiscoveryPagination(args: { + offset: number; + limit: number; + totalItems: number; + returnedItems: number; +}) { + const nextOffset = args.offset + args.returnedItems; + const hasMore = nextOffset < args.totalItems; + return { + offset: args.offset, + limit: args.limit, + totalItems: args.totalItems, + returnedItems: args.returnedItems, + hasMore, + nextOffset: hasMore ? nextOffset : null, + }; +} diff --git a/templates/content/actions/document-discovery.db.test.ts b/templates/content/actions/document-discovery.db.test.ts new file mode 100644 index 0000000000..c189a2c8ea --- /dev/null +++ b/templates/content/actions/document-discovery.db.test.ts @@ -0,0 +1,205 @@ +import { rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { runWithRequestContext } from "@agent-native/core/server"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +const TEST_DB_PATH = join( + tmpdir(), + `content-document-discovery-${process.pid}-${Date.now()}.sqlite`, +); +const OWNER = "discovery-owner@example.com"; +const OUTSIDER = "discovery-outsider@example.com"; +const PARENT_ID = "bounded-discovery-parent"; +const SPACE_ID = "bounded-discovery-space"; + +type Schema = typeof import("../server/db/schema.js"); +let getDb: () => any; +let schema: Schema; +let listDocuments: typeof import("./list-documents.js").default; +let searchDocuments: typeof import("./search-documents.js").default; + +const asUser = (userEmail: string, run: () => Promise) => + runWithRequestContext({ userEmail }, run); + +beforeAll(async () => { + process.env.DATABASE_URL = `file:${TEST_DB_PATH}`; + const dbModule = await import("../server/db/index.js"); + getDb = dbModule.getDb; + schema = dbModule.schema; + listDocuments = (await import("./list-documents.js")).default; + searchDocuments = (await import("./search-documents.js")).default; + const plugin = (await import("../server/plugins/db.js")).default; + await plugin(undefined as any); + + const now = new Date().toISOString(); + await getDb().insert(schema.contentSpaces).values({ + id: SPACE_ID, + name: "Bounded discovery", + kind: "personal", + ownerEmail: OWNER, + orgId: null, + filesDatabaseId: "bounded-discovery-files", + createdBy: OWNER, + createdAt: now, + updatedAt: now, + }); + await getDb().insert(schema.documents).values({ + id: PARENT_ID, + spaceId: SPACE_ID, + ownerEmail: OWNER, + orgId: null, + parentId: null, + title: "Discovery parent", + content: "", + position: 0, + visibility: "private", + createdAt: now, + updatedAt: now, + }); + const documents = Array.from({ length: 205 }, (_, index) => ({ + id: `bounded-discovery-document-${index.toString().padStart(3, "0")}`, + spaceId: SPACE_ID, + ownerEmail: OWNER, + orgId: null, + parentId: PARENT_ID, + title: index < 2 ? "Duplicate exact title" : `Bounded document ${index}`, + description: index === 204 ? "last page marker" : "", + content: `needle payload ${index}`, + position: index, + visibility: "private" as const, + createdAt: now, + updatedAt: new Date(Date.parse(now) + index).toISOString(), + })); + for (let start = 0; start < documents.length; start += 100) { + await getDb() + .insert(schema.documents) + .values(documents.slice(start, start + 100)); + } +}, 60_000); + +afterAll(() => { + for (const suffix of ["", "-shm", "-wal"]) + rmSync(`${TEST_DB_PATH}${suffix}`, { force: true }); +}); + +describe("bounded document discovery", () => { + it("returns explicit continuation metadata through a terminal list page", async () => { + const first = await asUser(OWNER, () => + listDocuments.run({ parentId: PARENT_ID, limit: 100, offset: 0 }), + ); + const second = await asUser(OWNER, () => + listDocuments.run({ parentId: PARENT_ID, limit: 100, offset: 100 }), + ); + const terminal = await asUser(OWNER, () => + listDocuments.run({ parentId: PARENT_ID, limit: 100, offset: 200 }), + ); + + expect(first.pagination).toEqual({ + offset: 0, + limit: 100, + totalItems: 205, + returnedItems: 100, + hasMore: true, + nextOffset: 100, + }); + expect(second.pagination.nextOffset).toBe(200); + expect(terminal.pagination).toEqual({ + offset: 200, + limit: 100, + totalItems: 205, + returnedItems: 5, + hasMore: false, + nextOffset: null, + }); + expect(terminal.documents.at(-1)?.description).toBe("last page marker"); + }); + + it("distinguishes zero, one, and multiple exact scoped title matches", async () => { + const none = await asUser(OWNER, () => + searchDocuments.run({ + exactTitle: "No such document", + parentId: PARENT_ID, + spaceId: SPACE_ID, + documentType: "page", + limit: 10, + offset: 0, + }), + ); + const one = await asUser(OWNER, () => + searchDocuments.run({ + exactTitle: "Bounded document 204", + parentId: PARENT_ID, + spaceId: SPACE_ID, + documentType: "page", + limit: 10, + offset: 0, + }), + ); + const multiple = await asUser(OWNER, () => + searchDocuments.run({ + exactTitle: "Duplicate exact title", + parentId: PARENT_ID, + spaceId: SPACE_ID, + documentType: "page", + limit: 1, + offset: 0, + }), + ); + + expect(none.pagination).toMatchObject({ + totalItems: 0, + returnedItems: 0, + hasMore: false, + nextOffset: null, + }); + expect(one.pagination).toMatchObject({ + totalItems: 1, + returnedItems: 1, + hasMore: false, + nextOffset: null, + }); + expect(multiple.pagination).toMatchObject({ + totalItems: 2, + returnedItems: 1, + hasMore: true, + nextOffset: 1, + }); + }); + + it("paginates body search and suppresses the private corpus for an outsider", async () => { + const ownerPage = await asUser(OWNER, () => + searchDocuments.run({ + query: "needle payload", + parentId: PARENT_ID, + limit: 200, + offset: 0, + }), + ); + const outsiderPage = await asUser(OUTSIDER, () => + searchDocuments.run({ + query: "needle payload", + parentId: PARENT_ID, + limit: 200, + offset: 0, + }), + ); + + expect(ownerPage.pagination).toMatchObject({ + totalItems: 205, + returnedItems: 200, + hasMore: true, + nextOffset: 200, + }); + expect(outsiderPage).toMatchObject({ + documents: [], + pagination: { + totalItems: 0, + returnedItems: 0, + hasMore: false, + nextOffset: null, + }, + }); + }); +}); diff --git a/templates/content/actions/list-documents.ts b/templates/content/actions/list-documents.ts index d3e0e74b82..07c0b1fa98 100644 --- a/templates/content/actions/list-documents.ts +++ b/templates/content/actions/list-documents.ts @@ -3,22 +3,21 @@ import { getRequestOrgId, getRequestUserEmail, } from "@agent-native/core/server/request-context"; -import { - accessFilter, - ROLE_RANK, - type ShareRole, -} from "@agent-native/core/sharing"; -import { and, asc, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { ROLE_RANK, type ShareRole } from "@agent-native/core/sharing"; +import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; -import { - documentDiscoveryFilter, - parseDocumentHideFromSearch, -} from "../server/lib/documents.js"; +import { parseDocumentHideFromSearch } from "../server/lib/documents.js"; import { favoriteDocumentIds } from "./_content-favorites.js"; import { listContentOrganizationMemberships } from "./_content-space-access.js"; import { serializeDatabaseMembership } from "./_database-utils.js"; +import { + DOCUMENT_DISCOVERY_DEFAULT_LIMIT, + DOCUMENT_DISCOVERY_MAX_LIMIT, + documentDiscoveryPagination, + documentDiscoveryWhere, +} from "./_document-discovery-query.js"; import { serializeDocumentSource } from "./_document-source.js"; import { parseDatabaseViewConfig } from "./_property-utils.js"; @@ -45,10 +44,41 @@ function strongerRole(current: ShareRole | null, next: ShareRole): ShareRole { export default defineAction({ description: - "List document metadata ordered by position. Does not return full document bodies; use get-document for one document's content.", - schema: z.object({}), + "List one bounded page of access-scoped document metadata ordered by position. Returns explicit pagination; follow nextOffset until hasMore is false. Does not return full document bodies; use get-document for one document's content.", + schema: z.object({ + limit: z.coerce + .number() + .int() + .min(1) + .max(DOCUMENT_DISCOVERY_MAX_LIMIT) + .default(DOCUMENT_DISCOVERY_DEFAULT_LIMIT) + .describe("Maximum documents returned in this page"), + offset: z.coerce + .number() + .int() + .min(0) + .default(0) + .describe("Zero-based continuation offset"), + exactTitle: z + .string() + .trim() + .min(1) + .optional() + .describe("Case-sensitive exact document title"), + parentId: z + .string() + .nullable() + .optional() + .describe("Exact parent document ID; null selects roots"), + spaceId: z.string().min(1).optional().describe("Exact Content space ID"), + documentType: z + .enum(["page", "database"]) + .optional() + .describe("Only ordinary pages or database pages"), + }), http: { method: "GET" }, - run: async () => { + readOnly: true, + run: async (args) => { const db = getDb(); const userEmail = getRequestUserEmail(); const activeOrgId = getRequestOrgId(); @@ -61,13 +91,19 @@ export default defineAction({ ...(!userEmail && activeOrgId ? [activeOrgId] : []), ]), ]; - const accessContexts = [ - { userEmail: userEmail ?? undefined }, - ...authorizedOrgIds.map((orgId) => ({ - userEmail: userEmail ?? undefined, - orgId, - })), - ]; + const where = documentDiscoveryWhere({ + userEmail, + authorizedOrgIds, + exactTitle: args.exactTitle, + parentId: args.parentId, + spaceId: args.spaceId, + documentType: args.documentType, + }); + const [countRow] = await db + .select({ count: sql`count(*)` }) + .from(schema.documents) + .where(where); + const totalItems = Number(countRow?.count ?? 0); // Projection that deliberately avoids pulling the full `content` blob: // document bodies can be multi-MB, and the list/tree path only needs a // short preview plus the true length. `substr` truncates the transferred @@ -99,21 +135,10 @@ export default defineAction({ updatedAt: schema.documents.updatedAt, }) .from(schema.documents) - .where( - and( - or( - ...accessContexts.map((context) => - accessFilter(schema.documents, schema.documentShares, context), - ), - ), - isNull(schema.documents.trashedAt), - documentDiscoveryFilter({ - userEmail, - orgIds: authorizedOrgIds, - }), - ), - ) - .orderBy(asc(schema.documents.position)); + .where(where) + .orderBy(asc(schema.documents.position), asc(schema.documents.id)) + .limit(args.limit) + .offset(args.offset); const shareRoleByDocumentId = new Map(); const notionPageIdByDocumentId = new Map(); @@ -128,7 +153,6 @@ export default defineAction({ database: typeof schema.contentDatabases.$inferSelect; } >(); - const softDeletedDocumentIds = new Set(); const favoriteIds = userEmail ? await favoriteDocumentIds( db, @@ -161,89 +185,75 @@ export default defineAction({ // These queries all depend only on the initial `documents` id list // (already fetched above), not on each other's results, so they run // concurrently instead of as sequential round-trips. - const [ - notionLinks, - shareRows, - databases, - databaseMemberships, - softDeletedDatabases, - ] = await Promise.all([ - db - .select({ - documentId: schema.documentSyncLinks.documentId, - remotePageId: schema.documentSyncLinks.remotePageId, - }) - .from(schema.documentSyncLinks) - .where( - inArray(schema.documentSyncLinks.documentId, visibleDocumentIds), - ), - principalClauses.length > 0 - ? db - .select({ - resourceId: schema.documentShares.resourceId, - role: schema.documentShares.role, - }) - .from(schema.documentShares) - .where( - and( - inArray(schema.documentShares.resourceId, visibleDocumentIds), - or(...principalClauses), - ), - ) - : Promise.resolve([] as { resourceId: string; role: ShareRole }[]), - db - .select() - .from(schema.contentDatabases) - .where( - and( - inArray(schema.contentDatabases.documentId, visibleDocumentIds), - isNull(schema.contentDatabases.deletedAt), - ), - ) - .orderBy( - sql`CASE WHEN ${schema.contentDatabases.systemRole} IS NULL THEN 0 ELSE 1 END`, - sql`CASE WHEN ${schema.contentDatabases.systemRole} = 'files' THEN 0 ELSE 1 END`, - asc(schema.contentDatabases.id), - ), - db - .select({ - item: schema.contentDatabaseItems, - database: schema.contentDatabases, - }) - .from(schema.contentDatabaseItems) - .innerJoin( - schema.contentDatabases, - eq( - schema.contentDatabases.id, - schema.contentDatabaseItems.databaseId, + const [notionLinks, shareRows, databases, databaseMemberships] = + await Promise.all([ + db + .select({ + documentId: schema.documentSyncLinks.documentId, + remotePageId: schema.documentSyncLinks.remotePageId, + }) + .from(schema.documentSyncLinks) + .where( + inArray(schema.documentSyncLinks.documentId, visibleDocumentIds), ), - ) - .where( - and( - inArray( - schema.contentDatabaseItems.documentId, - visibleDocumentIds, + principalClauses.length > 0 + ? db + .select({ + resourceId: schema.documentShares.resourceId, + role: schema.documentShares.role, + }) + .from(schema.documentShares) + .where( + and( + inArray( + schema.documentShares.resourceId, + visibleDocumentIds, + ), + or(...principalClauses), + ), + ) + : Promise.resolve([] as { resourceId: string; role: ShareRole }[]), + db + .select() + .from(schema.contentDatabases) + .where( + and( + inArray(schema.contentDatabases.documentId, visibleDocumentIds), + isNull(schema.contentDatabases.deletedAt), ), - isNull(schema.contentDatabases.deletedAt), + ) + .orderBy( + sql`CASE WHEN ${schema.contentDatabases.systemRole} IS NULL THEN 0 ELSE 1 END`, + sql`CASE WHEN ${schema.contentDatabases.systemRole} = 'files' THEN 0 ELSE 1 END`, + asc(schema.contentDatabases.id), ), - ) - .orderBy( - sql`CASE WHEN ${schema.contentDatabases.systemRole} IS NULL THEN 0 ELSE 1 END`, - asc(schema.contentDatabases.id), - ), - db - .select({ - id: schema.contentDatabases.id, - documentId: schema.contentDatabases.documentId, - }) - .from(schema.contentDatabases) - .where( - and( - inArray(schema.contentDatabases.documentId, visibleDocumentIds), - isNotNull(schema.contentDatabases.deletedAt), + db + .select({ + item: schema.contentDatabaseItems, + database: schema.contentDatabases, + }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.contentDatabases, + eq( + schema.contentDatabases.id, + schema.contentDatabaseItems.databaseId, + ), + ) + .where( + and( + inArray( + schema.contentDatabaseItems.documentId, + visibleDocumentIds, + ), + isNull(schema.contentDatabases.deletedAt), + ), + ) + .orderBy( + sql`CASE WHEN ${schema.contentDatabases.systemRole} IS NULL THEN 0 ELSE 1 END`, + asc(schema.contentDatabases.id), ), - ), - ]); + ]); for (const link of notionLinks) { notionPageIdByDocumentId.set(link.documentId, link.remotePageId); @@ -268,93 +278,74 @@ export default defineAction({ databaseMembershipByDocumentId.set(row.item.documentId, row); } } - - for (const database of softDeletedDatabases) { - softDeletedDocumentIds.add(database.documentId); - } - - if (softDeletedDatabases.length > 0) { - const softDeletedItems = await db - .select({ documentId: schema.contentDatabaseItems.documentId }) - .from(schema.contentDatabaseItems) - .where( - and( - inArray( - schema.contentDatabaseItems.databaseId, - softDeletedDatabases.map((database) => database.id), - ), - inArray( - schema.contentDatabaseItems.documentId, - visibleDocumentIds, - ), - ), - ); - for (const item of softDeletedItems) { - softDeletedDocumentIds.add(item.documentId); - } - } } - const mapped = documents - .filter((d) => !softDeletedDocumentIds.has(d.id)) - .map((d) => { - let accessRole: EffectiveRole = "viewer"; - const shareRole = shareRoleByDocumentId.get(d.id) ?? null; - const database = databaseByDocumentId.get(d.id) ?? null; - const databaseMembership = - databaseMembershipByDocumentId.get(d.id) ?? null; + const mapped = documents.map((d) => { + let accessRole: EffectiveRole = "viewer"; + const shareRole = shareRoleByDocumentId.get(d.id) ?? null; + const database = databaseByDocumentId.get(d.id) ?? null; + const databaseMembership = + databaseMembershipByDocumentId.get(d.id) ?? null; - if (shareRole && ROLE_RANK[shareRole] > ROLE_RANK[accessRole]) { - accessRole = shareRole; - } - if ( - userEmail && - d.ownerEmail === userEmail && - (!d.orgId || authorizedOrgIds.includes(d.orgId)) - ) { - accessRole = "owner"; - } + if (shareRole && ROLE_RANK[shareRole] > ROLE_RANK[accessRole]) { + accessRole = shareRole; + } + if ( + userEmail && + d.ownerEmail === userEmail && + (!d.orgId || authorizedOrgIds.includes(d.orgId)) + ) { + accessRole = "owner"; + } - return { - id: d.id, - parentId: d.parentId, - title: d.title, - description: d.description, - contentPreview: contentPreview(d.contentSnippet), - contentLength: Number(d.contentLength) || 0, - icon: d.icon, - position: d.position, - isFavorite: favoriteIds.has(d.id), - hideFromSearch: parseDocumentHideFromSearch(d.hideFromSearch), - notionPageId: notionPageIdByDocumentId.get(d.id) ?? null, - notionPageUrl: notionPageIdByDocumentId.has(d.id) - ? `https://www.notion.so/${notionPageIdByDocumentId.get(d.id)!.replace(/-/g, "")}` - : null, - visibility: d.visibility, - source: serializeDocumentSource(d), - database: database - ? { - id: database.id, - documentId: database.documentId, - title: database.title, - systemRole: database.systemRole, - description: d.description, - viewConfig: parseDatabaseViewConfig(database.viewConfigJson), - createdAt: database.createdAt, - updatedAt: database.updatedAt, - } - : undefined, - databaseMembership: databaseMembership - ? serializeDatabaseMembership(databaseMembership) - : undefined, - accessRole, - canEdit: canEditRole(accessRole), - canManage: canManageRole(accessRole), - createdAt: d.createdAt, - updatedAt: d.updatedAt, - }; - }); + return { + id: d.id, + parentId: d.parentId, + title: d.title, + description: d.description, + contentPreview: contentPreview(d.contentSnippet), + contentLength: Number(d.contentLength) || 0, + icon: d.icon, + position: d.position, + isFavorite: favoriteIds.has(d.id), + hideFromSearch: parseDocumentHideFromSearch(d.hideFromSearch), + notionPageId: notionPageIdByDocumentId.get(d.id) ?? null, + notionPageUrl: notionPageIdByDocumentId.has(d.id) + ? `https://www.notion.so/${notionPageIdByDocumentId.get(d.id)!.replace(/-/g, "")}` + : null, + visibility: d.visibility, + source: serializeDocumentSource(d), + database: database + ? { + id: database.id, + documentId: database.documentId, + title: database.title, + systemRole: database.systemRole, + description: d.description, + viewConfig: parseDatabaseViewConfig(database.viewConfigJson), + createdAt: database.createdAt, + updatedAt: database.updatedAt, + } + : undefined, + databaseMembership: databaseMembership + ? serializeDatabaseMembership(databaseMembership) + : undefined, + accessRole, + canEdit: canEditRole(accessRole), + canManage: canManageRole(accessRole), + createdAt: d.createdAt, + updatedAt: d.updatedAt, + }; + }); - return { documents: mapped }; + return { + documents: mapped, + pagination: documentDiscoveryPagination({ + offset: args.offset, + limit: args.limit, + totalItems, + returnedItems: mapped.length, + }), + }; }, }); diff --git a/templates/content/actions/roadmap-capability-projection.db.test.ts b/templates/content/actions/roadmap-capability-projection.db.test.ts new file mode 100644 index 0000000000..648cf0dc71 --- /dev/null +++ b/templates/content/actions/roadmap-capability-projection.db.test.ts @@ -0,0 +1,333 @@ +import { readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path, { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { runWithRequestContext } from "@agent-native/core/server"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +const TEST_DB_PATH = join( + tmpdir(), + `content-roadmap-projection-${process.pid}-${Date.now()}.sqlite`, +); +const OWNER = "roadmap-projection-owner@example.com"; +const OUTSIDER = "roadmap-projection-outsider@example.com"; +const SOURCE_REVISION = "b5a07715c6f0240e22daa6ecaba86ecc46513b08"; +const PROJECTION_TITLE = `Content roadmap capabilities — ${SOURCE_REVISION.slice(0, 12)}`; +const CAPABILITIES_DIR = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../docs/product/capabilities", +); + +type Capability = { + id: string; + name: string; + state: string; + publicness: string; + userPromise: string; + body: string; +}; + +let getDb: () => any; +let schema: typeof import("../server/db/schema.js"); +let provisionContentSpaces: typeof import("./_content-spaces.js").provisionContentSpaces; +let createDatabase: typeof import("./create-content-database.js").default; +let configureProperty: typeof import("./configure-document-property.js").default; +let getDatabase: typeof import("./get-content-database.js").default; +let searchDocuments: typeof import("./search-documents.js").default; +let upsert: typeof import("./upsert-database-item-by-key.js").default; + +const asUser = (userEmail: string, run: () => Promise) => + runWithRequestContext({ userEmail }, run); + +function frontmatterValue(source: string, key: string) { + const frontmatter = source.split("---", 3)[1] ?? ""; + const match = frontmatter.match( + new RegExp(`^${key}:\\s*(?:"([^"]*)"|'([^']*)'|([^\\n]+))$`, "m"), + ); + if (!match) throw new Error(`Capability record is missing ${key}.`); + return (match[1] ?? match[2] ?? match[3] ?? "").trim(); +} + +function loadCapabilities(): Capability[] { + return readdirSync(CAPABILITIES_DIR) + .filter((name) => name.endsWith(".md")) + .sort() + .map((name) => { + const body = readFileSync(join(CAPABILITIES_DIR, name), "utf8"); + return { + id: frontmatterValue(body, "id"), + name: frontmatterValue(body, "name"), + state: frontmatterValue(body, "state"), + publicness: frontmatterValue(body, "publicness"), + userPromise: frontmatterValue(body, "user_promise"), + body, + }; + }); +} + +beforeAll(async () => { + process.env.DATABASE_URL = `file:${TEST_DB_PATH}`; + const dbModule = await import("../server/db/index.js"); + getDb = dbModule.getDb; + schema = dbModule.schema; + ({ provisionContentSpaces } = await import("./_content-spaces.js")); + createDatabase = (await import("./create-content-database.js")).default; + configureProperty = (await import("./configure-document-property.js")) + .default; + getDatabase = (await import("./get-content-database.js")).default; + searchDocuments = (await import("./search-documents.js")).default; + upsert = (await import("./upsert-database-item-by-key.js")).default; + const plugin = (await import("../server/plugins/db.js")).default; + await plugin(undefined as any); +}, 60_000); + +afterAll(() => { + for (const suffix of ["", "-shm", "-wal"]) + rmSync(`${TEST_DB_PATH}${suffix}`, { force: true }); +}); + +describe("private roadmap capability projection", () => { + it("projects and replays all 124 capability IDs through stable-key upsert, then exhausts paginated readback", async () => { + const capabilities = loadCapabilities(); + expect(capabilities).toHaveLength(124); + expect(new Set(capabilities.map((capability) => capability.id)).size).toBe( + 124, + ); + + const provisioned = await asUser(OWNER, () => + provisionContentSpaces(getDb(), OWNER), + ); + const absent = await asUser(OWNER, () => + searchDocuments.run({ + exactTitle: PROJECTION_TITLE, + parentId: null, + spaceId: provisioned.personalSpaceId, + documentType: "database", + limit: 10, + offset: 0, + }), + ); + expect(absent).toMatchObject({ + documents: [], + pagination: { + totalItems: 0, + returnedItems: 0, + hasMore: false, + nextOffset: null, + }, + }); + + const createdDatabase = await asUser(OWNER, () => + createDatabase.run({ + spaceId: provisioned.personalSpaceId, + parentId: null, + title: PROJECTION_TITLE, + description: `Private projection of Content capability records at ${SOURCE_REVISION}.`, + }), + ); + const databaseId = createdDatabase.database.id; + const databaseDocumentId = createdDatabase.database.documentId; + const propertyIds = new Map(); + for (const [name, type] of [ + ["Capability ID", "text"], + ["State", "text"], + ["Publicness", "text"], + ["User promise", "text"], + ["Source revision", "text"], + ] as const) { + const configured = await asUser(OWNER, () => + configureProperty.run({ + documentId: databaseDocumentId, + databaseId, + name, + type, + }), + ); + const property = configured.properties.find( + (candidate) => candidate.definition.name === name, + ); + if (!property) + throw new Error(`Projection property "${name}" was not created.`); + propertyIds.set(name, property.definition.id); + } + const keyPropertyId = propertyIds.get("Capability ID"); + if (!keyPropertyId) + throw new Error("Projection stable-key property was not created."); + + const firstReceipts = new Map< + string, + { itemId: string; documentId: string } + >(); + for (const capability of capabilities) { + const receipt = await asUser(OWNER, () => + upsert.run({ + databaseId, + keyPropertyId, + keyValue: capability.id, + title: capability.name, + body: capability.body, + propertyValues: { + [propertyIds.get("State")!]: capability.state, + [propertyIds.get("Publicness")!]: capability.publicness, + [propertyIds.get("User promise")!]: capability.userPromise, + [propertyIds.get("Source revision")!]: SOURCE_REVISION, + }, + }), + ); + expect(receipt.status).toBe("created"); + firstReceipts.set(capability.id, { + itemId: receipt.itemId, + documentId: receipt.documentId, + }); + } + + const changedCapability = capabilities[0]; + if (!changedCapability) + throw new Error("Capability projection source is unexpectedly empty."); + const changedIdentity = firstReceipts.get(changedCapability.id); + if (!changedIdentity) + throw new Error("Changed Capability is missing its first receipt."); + const changedReceipt = await asUser(OWNER, () => + upsert.run({ + databaseId, + keyPropertyId, + keyValue: changedCapability.id, + title: `${changedCapability.name} — changed`, + }), + ); + expect(changedReceipt).toMatchObject({ + status: "updated", + ...changedIdentity, + }); + const restoredReceipt = await asUser(OWNER, () => + upsert.run({ + databaseId, + keyPropertyId, + keyValue: changedCapability.id, + title: changedCapability.name, + body: changedCapability.body, + propertyValues: { + [propertyIds.get("State")!]: changedCapability.state, + [propertyIds.get("Publicness")!]: changedCapability.publicness, + [propertyIds.get("User promise")!]: changedCapability.userPromise, + [propertyIds.get("Source revision")!]: SOURCE_REVISION, + }, + }), + ); + expect(restoredReceipt).toMatchObject({ + status: "updated", + ...changedIdentity, + }); + + for (const capability of capabilities) { + const receipt = await asUser(OWNER, () => + upsert.run({ + databaseId, + keyPropertyId, + keyValue: capability.id, + title: capability.name, + body: capability.body, + propertyValues: { + [propertyIds.get("State")!]: capability.state, + [propertyIds.get("Publicness")!]: capability.publicness, + [propertyIds.get("User promise")!]: capability.userPromise, + [propertyIds.get("Source revision")!]: SOURCE_REVISION, + }, + }), + ); + expect(receipt).toMatchObject({ + status: "unchanged", + ...firstReceipts.get(capability.id), + }); + } + + const readbackIds = new Set(); + const readbackIdentity = new Map< + string, + { itemId: string; documentId: string } + >(); + const pageOffsets: number[] = []; + let offset = 0; + while (true) { + pageOffsets.push(offset); + const page = await asUser(OWNER, () => + getDatabase.run({ databaseId, limit: 37, offset }), + ); + if (!("items" in page) || !page.pagination) + throw new Error("Roadmap projection readback was unavailable."); + expect(page.pagination.offset).toBe(offset); + expect(page.pagination.returnedItems).toBe(page.items.length); + expect(page.pagination.totalItems).toBe(124); + for (const item of page.items) { + const keyProperty = item.properties.find( + (property) => property.definition.id === keyPropertyId, + ); + if (typeof keyProperty?.value !== "string") + throw new Error("Projected row is missing its Capability ID."); + expect(readbackIds.has(keyProperty.value)).toBe(false); + readbackIds.add(keyProperty.value); + readbackIdentity.set(keyProperty.value, { + itemId: item.id, + documentId: item.document.id, + }); + } + if (!page.pagination.hasMore) break; + expect(page.items.length).toBeGreaterThan(0); + offset += page.items.length; + } + + expect(pageOffsets).toEqual([0, 37, 74, 111]); + expect(readbackIds).toEqual( + new Set(capabilities.map((capability) => capability.id)), + ); + expect(readbackIdentity).toEqual(firstReceipts); + + const uniqueRoot = await asUser(OWNER, () => + searchDocuments.run({ + exactTitle: PROJECTION_TITLE, + parentId: null, + spaceId: provisioned.personalSpaceId, + documentType: "database", + limit: 10, + offset: 0, + }), + ); + expect(uniqueRoot).toMatchObject({ + documents: [{ id: databaseDocumentId }], + pagination: { + totalItems: 1, + returnedItems: 1, + hasMore: false, + nextOffset: null, + }, + }); + const outsiderRoot = await asUser(OUTSIDER, () => + searchDocuments.run({ + exactTitle: PROJECTION_TITLE, + parentId: null, + spaceId: provisioned.personalSpaceId, + documentType: "database", + limit: 10, + offset: 0, + }), + ); + expect(outsiderRoot.pagination.totalItems).toBe(0); + await expect( + asUser(OUTSIDER, () => getDatabase.run({ databaseId, limit: 1 })), + ).rejects.toThrow(); + + const [rootDocument] = await getDb() + .select({ + ownerEmail: schema.documents.ownerEmail, + visibility: schema.documents.visibility, + }) + .from(schema.documents) + .where(eq(schema.documents.id, databaseDocumentId)); + expect(rootDocument).toEqual({ + ownerEmail: OWNER, + visibility: "private", + }); + }, 240_000); +}); diff --git a/templates/content/actions/search-documents.ts b/templates/content/actions/search-documents.ts index 17850c4710..e07168ac8f 100644 --- a/templates/content/actions/search-documents.ts +++ b/templates/content/actions/search-documents.ts @@ -1,13 +1,20 @@ import { defineAction } from "@agent-native/core"; -import { accessFilter } from "@agent-native/core/sharing"; -import { and, isNull, sql } from "drizzle-orm"; +import { + getRequestOrgId, + getRequestUserEmail, +} from "@agent-native/core/server/request-context"; +import { asc, desc, sql } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; +import { parseDocumentHideFromSearch } from "../server/lib/documents.js"; +import { listContentOrganizationMemberships } from "./_content-space-access.js"; import { - documentDiscoveryFilter, - parseDocumentHideFromSearch, -} from "../server/lib/documents.js"; + DOCUMENT_DISCOVERY_DEFAULT_LIMIT, + DOCUMENT_DISCOVERY_MAX_LIMIT, + documentDiscoveryPagination, + documentDiscoveryWhere, +} from "./_document-discovery-query.js"; function escapeLike(s: string): string { return s.replace(/([\\%_])/g, "\\$1"); @@ -37,17 +44,78 @@ function makeSnippet(content: string, query: string, radius = 120) { export default defineAction({ description: - "Search documents by title and content. Returns metadata and snippets; use get-document for full content.", - schema: z.object({ - query: z.string().describe("Search text"), - limit: z.coerce.number().int().min(1).max(200).default(50), - }), + "Search one bounded page of access-scoped documents by title and content, or find an exact title within a parent, space, and document type. Returns explicit pagination; follow nextOffset until hasMore is false. Returns metadata and snippets; use get-document for full content.", + schema: z + .object({ + query: z.string().trim().min(1).optional().describe("Search text"), + exactTitle: z + .string() + .trim() + .min(1) + .optional() + .describe("Case-sensitive exact document title"), + parentId: z + .string() + .nullable() + .optional() + .describe("Exact parent document ID; null selects roots"), + spaceId: z.string().min(1).optional().describe("Exact Content space ID"), + documentType: z + .enum(["page", "database"]) + .optional() + .describe("Only ordinary pages or database pages"), + limit: z.coerce + .number() + .int() + .min(1) + .max(DOCUMENT_DISCOVERY_MAX_LIMIT) + .default(DOCUMENT_DISCOVERY_DEFAULT_LIMIT) + .describe("Maximum documents returned in this page"), + offset: z.coerce + .number() + .int() + .min(0) + .default(0) + .describe("Zero-based continuation offset"), + }) + .refine( + (args) => args.query !== undefined || args.exactTitle !== undefined, + { + message: "Provide query or exactTitle.", + }, + ), http: { method: "GET" }, + readOnly: true, run: async (args) => { - const query = args.query; - const db = getDb(); - const pattern = `%${escapeLike(query)}%`; + const userEmail = getRequestUserEmail(); + const activeOrgId = getRequestOrgId(); + const memberships = userEmail + ? await listContentOrganizationMemberships(userEmail) + : []; + const authorizedOrgIds = [ + ...new Set([ + ...memberships.map((membership) => membership.orgId), + ...(!userEmail && activeOrgId ? [activeOrgId] : []), + ]), + ]; + const pattern = args.query ? `%${escapeLike(args.query)}%` : undefined; + const where = documentDiscoveryWhere({ + userEmail, + authorizedOrgIds, + exactTitle: args.exactTitle, + parentId: args.parentId, + spaceId: args.spaceId, + documentType: args.documentType, + additional: pattern + ? sql`(${schema.documents.title} LIKE ${pattern} ESCAPE '\\' OR ${schema.documents.description} LIKE ${pattern} ESCAPE '\\' OR ${schema.documents.content} LIKE ${pattern} ESCAPE '\\')` + : undefined, + }); + const [countRow] = await db + .select({ count: sql`count(*)` }) + .from(schema.documents) + .where(where); + const totalItems = Number(countRow?.count ?? 0); // Project a bounded preview of `content` instead of the full column: // document bodies can be multi-MB, and this action only returns a short @@ -71,16 +139,10 @@ export default defineAction({ updatedAt: schema.documents.updatedAt, }) .from(schema.documents) - .where( - and( - accessFilter(schema.documents, schema.documentShares), - isNull(schema.documents.trashedAt), - documentDiscoveryFilter(), - sql`(${schema.documents.title} LIKE ${pattern} ESCAPE '\\' OR ${schema.documents.description} LIKE ${pattern} ESCAPE '\\' OR ${schema.documents.content} LIKE ${pattern} ESCAPE '\\')`, - ), - ) - .orderBy(sql`${schema.documents.updatedAt} DESC`) - .limit(args.limit); + .where(where) + .orderBy(desc(schema.documents.updatedAt), asc(schema.documents.id)) + .limit(args.limit) + .offset(args.offset); return { documents: docs.map((doc) => ({ @@ -89,11 +151,20 @@ export default defineAction({ title: doc.title, description: doc.description, icon: doc.icon, - snippet: makeSnippet(doc.contentPreview, query), + snippet: makeSnippet( + doc.contentPreview, + args.query ?? args.exactTitle ?? "", + ), contentLength: Number(doc.contentLength) || 0, hideFromSearch: parseDocumentHideFromSearch(doc.hideFromSearch), updatedAt: doc.updatedAt, })), + pagination: documentDiscoveryPagination({ + offset: args.offset, + limit: args.limit, + totalItems, + returnedItems: docs.length, + }), }; }, }); diff --git a/templates/content/app/hooks/use-documents.test.ts b/templates/content/app/hooks/use-documents.test.ts index 30d688e1f1..f0f6f8e6b1 100644 --- a/templates/content/app/hooks/use-documents.test.ts +++ b/templates/content/app/hooks/use-documents.test.ts @@ -1,11 +1,17 @@ +// @vitest-environment happy-dom + import type { ContentDatabaseItem, Document } from "@shared/api"; -import { QueryClient } from "@tanstack/react-query"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; import { describe, expect, it } from "vitest"; import { buildDocumentTree, DOCUMENT_QUERY_FRESHNESS_OPTIONS, documentUpdateSuccessPatch, + fetchCompleteDocumentList, + LIST_DOCUMENTS_QUERY_KEY, documentPropertiesQueryKey, documentQueryKey, filterDocumentTreeDocuments, @@ -20,8 +26,108 @@ import { setDocumentFavoriteInDatabaseCache, setDocumentFavoriteInListCache, seedDatabaseItemDocumentCaches, + useDocuments, } from "./use-documents"; +describe("complete document discovery", () => { + it("keeps object-shaped optimistic cache writes array-shaped for consumers", async () => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, staleTime: Infinity } }, + }); + const optimisticDocument = doc("optimistic-document", null); + queryClient.setQueryData(LIST_DOCUMENTS_QUERY_KEY, { + documents: [optimisticDocument], + }); + let consumerData: unknown; + function Consumer() { + consumerData = useDocuments().data; + return null; + } + const container = document.createElement("div"); + const root = createRoot(container); + const actEnvironment = globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; + }; + const previousActEnvironment = actEnvironment.IS_REACT_ACT_ENVIRONMENT; + actEnvironment.IS_REACT_ACT_ENVIRONMENT = true; + + try { + await act(async () => { + root.render( + createElement( + QueryClientProvider, + { client: queryClient }, + createElement(Consumer), + ), + ); + }); + + expect(Array.isArray(consumerData)).toBe(true); + expect(consumerData).toEqual([optimisticDocument]); + } finally { + await act(async () => root.unmount()); + actEnvironment.IS_REACT_ACT_ENVIRONMENT = previousActEnvironment; + queryClient.clear(); + } + }); + + it("exhausts every bounded page before returning the document tree", async () => { + const documents = Array.from({ length: 401 }, (_, index) => + doc(`document-${index}`, null, index), + ); + const offsets: number[] = []; + + const result = await fetchCompleteDocumentList(async (offset, limit) => { + offsets.push(offset); + const page = documents.slice(offset, offset + limit); + const nextOffset = offset + page.length; + return { + documents: page, + pagination: { + offset, + limit, + totalItems: documents.length, + returnedItems: page.length, + hasMore: nextOffset < documents.length, + nextOffset: nextOffset < documents.length ? nextOffset : null, + }, + }; + }); + + expect(offsets).toEqual([0, 200, 400]); + expect(result.map((document) => document.id)).toEqual( + documents.map((document) => document.id), + ); + }); + + it("rejects a response whose missing boundary could hide clipping", async () => { + await expect( + fetchCompleteDocumentList( + async () => + ({ + documents: [doc("document-1", null)], + }) as never, + ), + ).rejects.toThrow("returned no pagination boundary"); + }); + + it("rejects a non-advancing continuation", async () => { + await expect( + fetchCompleteDocumentList(async (_offset, limit) => ({ + documents: [], + pagination: { + offset: 0, + limit, + totalItems: 1, + returnedItems: 0, + hasMore: true, + nextOffset: 0, + }, + })), + ).rejects.toThrow("non-advancing continuation"); + }); +}); + describe("document query freshness", () => { it("always replaces seeded row snapshots before the editor mounts", () => { expect(DOCUMENT_QUERY_FRESHNESS_OPTIONS).toMatchObject({ diff --git a/templates/content/app/hooks/use-documents.ts b/templates/content/app/hooks/use-documents.ts index a7544a7d19..0c057b976d 100644 --- a/templates/content/app/hooks/use-documents.ts +++ b/templates/content/app/hooks/use-documents.ts @@ -1,4 +1,5 @@ import { + callAction, useActionQuery, useActionMutation, } from "@agent-native/core/client/hooks"; @@ -7,6 +8,7 @@ import type { ContentDatabaseItem, Document, DocumentCreateRequest, + DocumentListResponse, DocumentPropertiesResponse, DocumentUpdateRequest, DocumentUpdateResponse, @@ -15,7 +17,7 @@ import type { DocumentTreeNode, } from "@shared/api"; import type { QueryClient } from "@tanstack/react-query"; -import { useQueryClient } from "@tanstack/react-query"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { toast } from "sonner"; import type { DocumentUpdateConflictResponse } from "../../actions/update-document"; @@ -63,6 +65,74 @@ export const LIST_DOCUMENTS_QUERY_KEY = [ undefined, ] as const; +const DOCUMENT_LIST_PAGE_SIZE = 200; + +export async function fetchCompleteDocumentList( + fetchPage: (offset: number, limit: number) => Promise, +) { + const documents: Document[] = []; + const documentIds = new Set(); + let offset = 0; + let expectedTotal: number | null = null; + + while (true) { + const page = await fetchPage(offset, DOCUMENT_LIST_PAGE_SIZE); + const { pagination } = page; + if (!pagination) { + throw new Error( + "list-documents returned no pagination boundary; refusing to treat the result as complete.", + ); + } + if ( + pagination.offset !== offset || + pagination.limit !== DOCUMENT_LIST_PAGE_SIZE || + pagination.returnedItems !== page.documents.length + ) { + throw new Error( + "list-documents returned inconsistent pagination metadata; retry the complete read.", + ); + } + if (expectedTotal === null) expectedTotal = pagination.totalItems; + if (pagination.totalItems !== expectedTotal) { + throw new Error( + "Documents changed during paginated discovery; retry the complete read.", + ); + } + for (const document of page.documents) { + if (documentIds.has(document.id)) { + throw new Error( + `list-documents repeated document "${document.id}" across pages; refusing an ambiguous result.`, + ); + } + documentIds.add(document.id); + documents.push(document); + } + + const expectedNextOffset = offset + page.documents.length; + if (!pagination.hasMore) { + if ( + pagination.nextOffset !== null || + expectedNextOffset !== expectedTotal || + documents.length !== expectedTotal + ) { + throw new Error( + "list-documents claimed exhaustion before every declared document was returned.", + ); + } + return documents; + } + if ( + pagination.nextOffset !== expectedNextOffset || + pagination.nextOffset <= offset + ) { + throw new Error( + "list-documents returned a non-advancing continuation; refusing a clipped result.", + ); + } + offset = pagination.nextOffset; + } +} + export function documentPropertiesQueryKey( documentId: string, databaseId: string, @@ -317,11 +387,19 @@ export function seedDatabaseItemDocumentCaches( } export function useDocuments() { - return useActionQuery("list-documents", undefined, { - select: (data: any) => { - const docs = data?.documents ?? data; - return Array.isArray(docs) ? docs : []; - }, + return useQuery({ + queryKey: LIST_DOCUMENTS_QUERY_KEY, + queryFn: async ({ signal }) => ({ + documents: await fetchCompleteDocumentList((offset, limit) => + callAction( + "list-documents", + { offset, limit }, + { method: "GET", signal }, + ), + ), + }), + select: (data) => data.documents, + retry: false, }); } diff --git a/templates/content/shared/api.ts b/templates/content/shared/api.ts index e5a5f0fcb8..c084c49f59 100644 --- a/templates/content/shared/api.ts +++ b/templates/content/shared/api.ts @@ -128,6 +128,16 @@ export interface DocumentMoveRequest { export interface DocumentListResponse { documents: Document[]; + pagination: DocumentDiscoveryPagination; +} + +export interface DocumentDiscoveryPagination { + offset: number; + limit: number; + totalItems: number; + returnedItems: number; + hasMore: boolean; + nextOffset: number | null; } export interface DocumentTreeNode extends Document {