From 5ab4cb86907f8261c7c52123d248775dcd118e0f Mon Sep 17 00:00:00 2001 From: David Clark Date: Fri, 5 Jun 2026 14:45:54 -0400 Subject: [PATCH 01/25] Add partner content ingestion pipeline --- .../admin/partner-content/backfill/route.ts | 68 +++++ .../partner-content/enumerate/page/route.ts | 140 ++++++++++ .../cron/partner-content/enumerate/route.ts | 181 +++++++++++++ .../api/cron/partner-content/fetch/route.ts | 240 ++++++++++++++++++ apps/web/lib/api/scrape-creators/client.ts | 23 +- .../get-youtube-channel-videos.ts | 43 ++++ apps/web/lib/api/scrape-creators/schema.ts | 32 +++ .../lib/partner-content-search/constants.ts | 11 + .../ingestion/enqueue.ts | 112 ++++++++ .../ingestion/normalize-content.ts | 49 ++++ apps/web/lib/partner-content-search/types.ts | 10 + .../schema/partner-content-search.prisma | 84 ++++++ packages/prisma/schema/platform.prisma | 2 + 13 files changed, 994 insertions(+), 1 deletion(-) create mode 100644 apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts create mode 100644 apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts create mode 100644 apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts create mode 100644 apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts create mode 100644 apps/web/lib/api/scrape-creators/get-youtube-channel-videos.ts create mode 100644 apps/web/lib/partner-content-search/constants.ts create mode 100644 apps/web/lib/partner-content-search/ingestion/enqueue.ts create mode 100644 apps/web/lib/partner-content-search/ingestion/normalize-content.ts create mode 100644 apps/web/lib/partner-content-search/types.ts create mode 100644 packages/prisma/schema/partner-content-search.prisma diff --git a/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts b/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts new file mode 100644 index 00000000000..8bc2c3c46b5 --- /dev/null +++ b/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts @@ -0,0 +1,68 @@ +import { handleAndReturnErrorResponse } from "@/lib/api/errors"; +import { parseRequestBody } from "@/lib/api/utils"; +import { withAdmin } from "@/lib/auth"; +import { + createPartnerContentRunStamp, + enqueuePartnerContentEnumerate, + partnerContentIngestionFilterSchema, +} from "@/lib/partner-content-search/ingestion/enqueue"; +import { NextResponse } from "next/server"; +import * as z from "zod/v4"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 10; + +const backfillTriggerSchema = z.object({ + filter: partnerContentIngestionFilterSchema, + runStamp: z.string().min(1).optional(), + dryRun: z.boolean().default(false), + dispatcherDryRun: z.boolean().default(false), +}); + +// POST /api/admin/partner-content/backfill +export const POST = withAdmin( + async ({ req }) => { + try { + // Fail loud on malformed JSON instead of silently widening to a + // full backfill; parseRequestBody throws a 400 DubApiError. + const body = backfillTriggerSchema.parse(await parseRequestBody(req)); + const runStamp = body.runStamp ?? createPartnerContentRunStamp(); + + const payload = { + mode: "backfill" as const, + filter: body.filter, + runStamp, + dryRun: body.dispatcherDryRun, + }; + + if (body.dryRun) { + return NextResponse.json({ + success: true, + triggerDryRun: true, + dispatcherDryRun: payload.dryRun, + wouldEnqueue: false, + enumeratePayload: payload, + }); + } + + const qstashResponse = await enqueuePartnerContentEnumerate(payload); + + return NextResponse.json({ + success: true, + triggerDryRun: false, + dispatcherDryRun: payload.dryRun, + wouldEnqueue: true, + runStamp, + enumeratePayload: payload, + qstashResponse, + }); + } catch (error) { + // Normalize ZodError -> 422, DubApiError -> 4xx, QStash failure -> 500 + // (the withAdmin wrapper does not do this for us). + return handleAndReturnErrorResponse(error); + } + }, + { + requiredRoles: ["owner"], + }, +); diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts new file mode 100644 index 00000000000..526bf647aba --- /dev/null +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts @@ -0,0 +1,140 @@ +import { qstash } from "@/lib/cron"; +import { withCron } from "@/lib/cron/with-cron"; +import { + createPartnerContentDeduplicationId, + getPartnerContentUrl, + PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS, + PARTNER_CONTENT_SEARCH_ROUTES, + partnerContentEnumeratePagePayloadSchema, +} from "@/lib/partner-content-search/ingestion/enqueue"; +import { prisma } from "@dub/prisma"; +import { Prisma } from "@dub/prisma/client"; +import { NextResponse } from "next/server"; +import { logAndRespond } from "../../../utils"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +// POST /api/cron/partner-content/enumerate/page +export const POST = withCron(async ({ rawBody }) => { + const payload = partnerContentEnumeratePagePayloadSchema.parse( + JSON.parse(rawBody), + ); + + const partners = await prisma.partner.findMany({ + where: { + id: { + in: payload.partnerIds, + }, + }, + select: { + id: true, + platforms: { + where: buildEligiblePartnerPlatformWhere(payload), + select: { + id: true, + partnerId: true, + type: true, + identifier: true, + platformId: true, + contentLastFetchedAt: true, + latestContentUrl: true, + }, + orderBy: { + type: "asc", + }, + }, + }, + orderBy: { + id: "asc", + }, + }); + + const partnerPlatforms = partners.flatMap(({ id: partnerId, platforms }) => + platforms.map((platform) => ({ + partnerId, + partnerPlatformId: platform.id, + type: platform.type, + identifier: platform.identifier, + platformId: platform.platformId, + contentLastFetchedAt: platform.contentLastFetchedAt, + latestContentUrl: platform.latestContentUrl, + })), + ); + + if (partnerPlatforms.length === 0) { + return logAndRespond( + `[PartnerContentSearch] No eligible partner platforms found for ${payload.mode} run ${payload.runStamp}.`, + ); + } + + const fetchMessages = partnerPlatforms.map((partnerPlatform) => ({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.fetch), + method: "POST" as const, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-fetch", + payload.mode, + payload.runStamp, + partnerPlatform.partnerPlatformId, + ), + body: { + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + partnerId: partnerPlatform.partnerId, + partnerPlatformId: partnerPlatform.partnerPlatformId, + platform: partnerPlatform.type, + }, + })); + + const qstashResponses = payload.dryRun + ? [] + : await qstash.batchJSON(fetchMessages); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + partnerCount: partners.length, + partnerPlatformCount: partnerPlatforms.length, + fetchJobCount: fetchMessages.length, + qstashResponses, + partnerPlatforms, + }); +}); + +function buildEligiblePartnerPlatformWhere({ + mode, + filter, +}: { + mode: "incremental" | "backfill"; + filter: { + platforms: Array<"youtube" | "instagram" | "tiktok">; + }; +}): Prisma.PartnerPlatformWhereInput { + const incrementalCutoff = new Date( + Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, + ); + + return { + type: { + in: filter.platforms, + }, + verifiedAt: { + not: null, + }, + ...(mode === "incremental" && { + OR: [ + { + contentLastFetchedAt: null, + }, + { + contentLastFetchedAt: { + lt: incrementalCutoff, + }, + }, + ], + }), + }; +} diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts new file mode 100644 index 00000000000..b11cc403058 --- /dev/null +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts @@ -0,0 +1,181 @@ +import { qstash } from "@/lib/cron"; +import { withCron } from "@/lib/cron/with-cron"; +import { + createPartnerContentDeduplicationId, + getPartnerContentUrl, + PARTNER_CONTENT_ENUMERATE_PAGE_SIZE, + PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS, + PARTNER_CONTENT_SEARCH_ROUTES, + partnerContentEnumeratePayloadSchema, +} from "@/lib/partner-content-search/ingestion/enqueue"; +import { prisma } from "@dub/prisma"; +import { Prisma } from "@dub/prisma/client"; +import { logAndRespond } from "../../utils"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +// POST /api/cron/partner-content/enumerate +export const POST = withCron(async ({ rawBody }) => { + const payload = partnerContentEnumeratePayloadSchema.parse( + JSON.parse(rawBody), + ); + + // Process a single page per invocation and hand the next cursor back to + // this same route, rather than draining the whole partner set in one + // request (keeps each run well under maxDuration regardless of scale). + const remainingPartners = + payload.remainingPartners ?? payload.filter.limitPartners; + + const take = Math.min( + PARTNER_CONTENT_ENUMERATE_PAGE_SIZE, + remainingPartners ?? PARTNER_CONTENT_ENUMERATE_PAGE_SIZE, + ); + + const partners = await prisma.partner.findMany({ + where: buildEligiblePartnerWhere(payload), + select: { + id: true, + }, + ...(payload.startingAfter && { + cursor: { + id: payload.startingAfter, + }, + skip: 1, + }), + orderBy: { + id: "asc", + }, + take, + }); + + if (partners.length === 0) { + return logAndRespond( + `[PartnerContentSearch] No${ + payload.startingAfter ? " more" : "" + } eligible partners found for ${payload.mode} run ${payload.runStamp}.`, + ); + } + + const partnerIds = partners.map(({ id }) => id); + const lastPartnerId = partnerIds[partnerIds.length - 1]; + const nextRemainingPartners = + remainingPartners === undefined + ? undefined + : remainingPartners - partnerIds.length; + + // Only continue if this page was full and budget (if any) remains. + const hasMore = + partners.length === take && + (nextRemainingPartners === undefined || nextRemainingPartners > 0); + + const messages = [ + { + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.enumeratePage), + method: "POST" as const, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-enum-page", + payload.mode, + partnerIds[0], + payload.runStamp, + ), + body: { + mode: payload.mode, + filter: payload.filter, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + partnerIds, + }, + }, + // Self-continuation hop: re-enter this route from the last id instead of + // draining the whole partner set in a single invocation. + ...(hasMore + ? [ + { + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.enumerate), + method: "POST" as const, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-enumerate", + payload.mode, + payload.runStamp, + lastPartnerId, + ), + body: { + mode: payload.mode, + filter: payload.filter, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + startingAfter: lastPartnerId, + ...(nextRemainingPartners !== undefined && { + remainingPartners: nextRemainingPartners, + }), + }, + }, + ] + : []), + ]; + + if (!payload.dryRun) { + await qstash.batchJSON(messages); + } + + return logAndRespond( + `[PartnerContentSearch] ${ + payload.dryRun ? "Dry-run enumerated" : "Enqueued" + } ${partnerIds.length} partners for ${payload.mode} run ${ + payload.runStamp + }${hasMore ? ` (continuing after ${lastPartnerId})` : " (final page)"}.`, + ); +}); + +function buildEligiblePartnerWhere({ + mode, + filter, +}: { + mode: "incremental" | "backfill"; + filter: { + partnerId?: string; + partnerIds?: string[]; + platforms: Array<"youtube" | "instagram" | "tiktok">; + }; +}): Prisma.PartnerWhereInput { + const incrementalCutoff = new Date( + Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, + ); + + return { + networkStatus: { + in: ["approved", "trusted"], + }, + ...(filter.partnerId && { + id: filter.partnerId, + }), + ...(filter.partnerIds?.length && { + id: { + in: filter.partnerIds, + }, + }), + platforms: { + some: { + type: { + in: filter.platforms, + }, + verifiedAt: { + not: null, + }, + ...(mode === "incremental" && { + OR: [ + { + contentLastFetchedAt: null, + }, + { + contentLastFetchedAt: { + lt: incrementalCutoff, + }, + }, + ], + }), + }, + }, + }; +} diff --git a/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts b/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts new file mode 100644 index 00000000000..deabdf44884 --- /dev/null +++ b/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts @@ -0,0 +1,240 @@ +import { getYouTubeChannelVideos } from "@/lib/api/scrape-creators/get-youtube-channel-videos"; +import { withCron } from "@/lib/cron/with-cron"; +import { PARTNER_CONTENT_SEARCH_LIMITS } from "@/lib/partner-content-search/constants"; +import { partnerContentFetchPayloadSchema } from "@/lib/partner-content-search/ingestion/enqueue"; +import { + NormalizedPartnerContentItem, + normalizeYouTubeChannelVideo, +} from "@/lib/partner-content-search/ingestion/normalize-content"; +import { prisma } from "@dub/prisma"; +import { NextResponse } from "next/server"; +import { logAndRespond } from "../../utils"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +const MAX_YOUTUBE_CHANNEL_VIDEO_PAGES = 3; + +// POST /api/cron/partner-content/fetch +export const POST = withCron(async ({ rawBody }) => { + const payload = partnerContentFetchPayloadSchema.parse(JSON.parse(rawBody)); + + const partnerPlatform = await prisma.partnerPlatform.findUnique({ + where: { + id: payload.partnerPlatformId, + }, + select: { + id: true, + partnerId: true, + type: true, + identifier: true, + platformId: true, + contentLastFetchedAt: true, + latestContentUrl: true, + verifiedAt: true, + }, + }); + + if (!partnerPlatform) { + return logAndRespond( + `[PartnerContentSearch] Partner platform ${payload.partnerPlatformId} not found for ${payload.mode} run ${payload.runStamp}.`, + { status: 404, logLevel: "warn" }, + ); + } + + if ( + partnerPlatform.partnerId !== payload.partnerId || + partnerPlatform.type !== payload.platform + ) { + return logAndRespond( + `[PartnerContentSearch] Partner platform ${payload.partnerPlatformId} did not match fetch payload for ${payload.mode} run ${payload.runStamp}.`, + { status: 400, logLevel: "warn" }, + ); + } + + if (partnerPlatform.type !== "youtube") { + return logAndRespond( + `[PartnerContentSearch] Fetch and diff only supports YouTube right now. Received ${partnerPlatform.type} for ${payload.mode} run ${payload.runStamp}.`, + { status: 501, logLevel: "warn" }, + ); + } + + const fetchedContentItems = await fetchRecentYouTubeContent({ + channelId: partnerPlatform.platformId ?? undefined, + handle: partnerPlatform.platformId + ? undefined + : normalizeYouTubeHandle(partnerPlatform.identifier), + }); + + const platformContentIds = fetchedContentItems.map( + ({ platformContentId }) => platformContentId, + ); + + const existingContentItems = + platformContentIds.length === 0 + ? [] + : await prisma.partnerContentItem.findMany({ + where: { + partnerPlatformId: partnerPlatform.id, + platformContentId: { + in: platformContentIds, + }, + }, + select: { + platformContentId: true, + }, + }); + + const existingContentIdSet = new Set( + existingContentItems.map(({ platformContentId }) => platformContentId), + ); + + const newContentItems = fetchedContentItems.filter( + ({ platformContentId }) => !existingContentIdSet.has(platformContentId), + ); + + const latestContentItem = fetchedContentItems[0]; + + const writeResult = payload.dryRun + ? { + contentItemsCreated: 0, + partnerPlatformUpdated: false, + } + : await writeFetchedContentItems({ + partnerId: payload.partnerId, + partnerPlatformId: partnerPlatform.id, + contentItems: newContentItems, + latestContentUrl: latestContentItem?.url ?? null, + }); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + fetchedContentCount: fetchedContentItems.length, + existingContentCount: existingContentItems.length, + newContentCount: newContentItems.length, + writesEnabled: !payload.dryRun, + wouldWriteContentItems: newContentItems.length, + ...writeResult, + newContentItems: newContentItems.slice(0, 10), + partnerPlatform, + }); +}); + +async function fetchRecentYouTubeContent({ + channelId, + handle, +}: { + channelId?: string; + handle?: string; +}) { + const maxItems = PARTNER_CONTENT_SEARCH_LIMITS.contentItemsPerPartnerPlatform; + const recencyCutoff = getRecencyCutoff(); + const contentItems: NormalizedPartnerContentItem[] = []; + let continuationToken: string | undefined; + let page = 0; + + while ( + contentItems.length < maxItems && + page < MAX_YOUTUBE_CHANNEL_VIDEO_PAGES + ) { + const response = await getYouTubeChannelVideos({ + ...(channelId ? { channelId } : { handle: handle! }), + continuationToken, + includeExtras: false, + }); + + const normalizedVideos = response.videos + .map(normalizeYouTubeChannelVideo) + .filter((item): item is NormalizedPartnerContentItem => item !== null); + + const recentVideos = normalizedVideos.filter( + ({ publishedAt }) => !publishedAt || publishedAt >= recencyCutoff, + ); + + contentItems.push(...recentVideos); + + const oldestPublishedAt = normalizedVideos + .map(({ publishedAt }) => publishedAt) + .filter((date): date is Date => date !== null) + .sort((a, b) => a.getTime() - b.getTime())[0]; + + if ( + !response.continuationToken || + (oldestPublishedAt && oldestPublishedAt < recencyCutoff) + ) { + break; + } + + continuationToken = response.continuationToken ?? undefined; + page++; + } + + return contentItems.slice(0, maxItems); +} + +function normalizeYouTubeHandle(handle: string) { + return handle.replace(/^@/, ""); +} + +function getRecencyCutoff() { + const cutoff = new Date(); + cutoff.setMonth( + cutoff.getMonth() - PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, + ); + return cutoff; +} + +async function writeFetchedContentItems({ + partnerId, + partnerPlatformId, + contentItems, + latestContentUrl, +}: { + partnerId: string; + partnerPlatformId: string; + contentItems: NormalizedPartnerContentItem[]; + latestContentUrl: string | null; +}) { + const [createResult] = await prisma.$transaction([ + prisma.partnerContentItem.createMany({ + data: contentItems.map((item) => ({ + partnerId, + partnerPlatformId, + platformContentId: item.platformContentId, + url: item.url, + contentType: item.contentType, + title: item.title, + description: item.description, + thumbnailUrl: item.thumbnailUrl, + publishedAt: item.publishedAt, + durationMs: item.durationMs, + viewCount: + item.viewCount === null ? null : BigInt(Math.trunc(item.viewCount)), + transcriptFetchStatus: "pending", + hasTimestamps: false, + totalChunkCount: 0, + embeddedChunkCount: 0, + })), + skipDuplicates: true, + }), + prisma.partnerPlatform.update({ + where: { + id: partnerPlatformId, + }, + data: { + contentLastFetchedAt: new Date(), + ...(latestContentUrl && { + latestContentUrl, + }), + }, + }), + ]); + + return { + contentItemsCreated: createResult.count, + partnerPlatformUpdated: true, + }; +} diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.ts index 19d3a7aa304..a96f0f1207a 100644 --- a/apps/web/lib/api/scrape-creators/client.ts +++ b/apps/web/lib/api/scrape-creators/client.ts @@ -1,7 +1,11 @@ import { createFetch, createSchema } from "@better-fetch/fetch"; import { PlatformType } from "@dub/prisma/client"; import * as z from "zod/v4"; -import { socialContentSchema, socialProfileSchema } from "./schema"; +import { + socialContentSchema, + socialProfileSchema, + youtubeChannelVideosSchema, +} from "./schema"; export const scrapeCreatorsFetch = createFetch({ baseURL: "https://api.scrapecreators.com", @@ -41,6 +45,23 @@ export const scrapeCreatorsFetch = createFetch({ }), output: socialContentSchema, }, + + // Fetch recent YouTube channel videos + "/v1/youtube/channel-videos": { + method: "get", + query: z + .object({ + channelId: z.string().optional(), + handle: z.string().optional(), + sort: z.enum(["latest", "popular"]).optional(), + continuationToken: z.string().optional(), + includeExtras: z.enum(["true", "false"]).optional(), + }) + .refine((query) => query.channelId || query.handle, { + message: "Either channelId or handle is required.", + }), + output: youtubeChannelVideosSchema, + }, }, { strict: true, diff --git a/apps/web/lib/api/scrape-creators/get-youtube-channel-videos.ts b/apps/web/lib/api/scrape-creators/get-youtube-channel-videos.ts new file mode 100644 index 00000000000..04862c7c57e --- /dev/null +++ b/apps/web/lib/api/scrape-creators/get-youtube-channel-videos.ts @@ -0,0 +1,43 @@ +import { scrapeCreatorsFetch } from "./client"; + +type GetYouTubeChannelVideosParams = ( + | { + channelId: string; + handle?: string; + } + | { + channelId?: string; + handle: string; + } +) & { + continuationToken?: string; + includeExtras?: boolean; +}; + +export async function getYouTubeChannelVideos({ + channelId, + handle, + continuationToken, + includeExtras = false, +}: GetYouTubeChannelVideosParams) { + const { data, error } = await scrapeCreatorsFetch( + "/v1/youtube/channel-videos", + { + query: { + channelId, + handle, + sort: "latest", + continuationToken, + includeExtras: includeExtras ? "true" : "false", + }, + }, + ); + + if (error) { + throw new Error( + "We were unable to retrieve YouTube channel videos from ScrapeCreators.", + ); + } + + return data; +} diff --git a/apps/web/lib/api/scrape-creators/schema.ts b/apps/web/lib/api/scrape-creators/schema.ts index 8125409d023..33bee392bf1 100644 --- a/apps/web/lib/api/scrape-creators/schema.ts +++ b/apps/web/lib/api/scrape-creators/schema.ts @@ -345,3 +345,35 @@ export const socialContentSchema = z.preprocess( }), ]), ); +export const youtubeChannelVideosSchema = z.object({ + videos: z.array( + z.object({ + type: z.string(), + id: z.string(), + url: z.string(), + title: z.string().nullish(), + description: z.string().nullish(), + thumbnail: z.string().nullish(), + channel: z + .object({ + title: z.string().nullish(), + thumbnail: z.string().nullish(), + }) + .optional(), + viewCountText: z.string().nullish(), + viewCountInt: z + .number() + .nullish() + .transform((val) => val ?? 0), + publishedTimeText: z.string().nullish(), + publishedTime: z.string().nullish(), + lengthText: z.string().nullish(), + lengthSeconds: z + .number() + .nullish() + .transform((val) => val ?? 0), + badges: z.array(z.unknown()).optional(), + }), + ), + continuationToken: z.string().nullish(), +}); diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts new file mode 100644 index 00000000000..ff0c79b2bf1 --- /dev/null +++ b/apps/web/lib/partner-content-search/constants.ts @@ -0,0 +1,11 @@ +export const PARTNER_CONTENT_SEARCH_FEATURE_FLAG = + "PARTNER_CONTENT_SEARCH_ENABLED"; + +export const PARTNER_CONTENT_SEARCH_ENV_VARS = { + scrapeCreatorsApiKey: "SCRAPECREATORS_API_KEY", +} as const; + +export const PARTNER_CONTENT_SEARCH_LIMITS = { + recencyWindowMonths: 12, + contentItemsPerPartnerPlatform: 50, +} as const; diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts new file mode 100644 index 00000000000..db62ec2a44b --- /dev/null +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -0,0 +1,112 @@ +import { qstash } from "@/lib/cron"; +import { APP_DOMAIN_WITH_NGROK } from "@dub/utils"; +import * as z from "zod/v4"; +import { PARTNER_CONTENT_SEARCH_PLATFORMS } from "../types"; + +// Partners enumerated per page / per page-worker job. Defaults to 500; override +// with the PARTNER_CONTENT_ENUMERATE_PAGE_SIZE env var (e.g. 3) to exercise the +// self-continuation chain locally without seeding 500+ eligible partners. +export const PARTNER_CONTENT_ENUMERATE_PAGE_SIZE = (() => { + const raw = process.env.PARTNER_CONTENT_ENUMERATE_PAGE_SIZE; + const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; + return Number.isInteger(parsed) && parsed > 0 ? parsed : 500; +})(); +export const PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS = 7; + +export const PARTNER_CONTENT_SEARCH_ROUTES = { + enumerate: "/api/cron/partner-content/enumerate", + enumeratePage: "/api/cron/partner-content/enumerate/page", + fetch: "/api/cron/partner-content/fetch", +} as const; + +type PartnerContentSearchRoute = + (typeof PARTNER_CONTENT_SEARCH_ROUTES)[keyof typeof PARTNER_CONTENT_SEARCH_ROUTES]; + +export const partnerContentIngestionModeSchema = z.enum([ + "incremental", + "backfill", +]); + +export const partnerContentIngestionFilterSchema = z + .object({ + partnerId: z.string().optional(), + partnerIds: z.array(z.string()).max(500).optional(), + platforms: z + .array(z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS)) + .min(1) + .default([...PARTNER_CONTENT_SEARCH_PLATFORMS]), + limitPartners: z.number().int().positive().max(100_000).optional(), + }) + .default({}) + .refine((filter) => !(filter.partnerId && filter.partnerIds?.length), { + message: "Use either partnerId or partnerIds, not both.", + }); + +export const partnerContentEnumeratePayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + filter: partnerContentIngestionFilterSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + // Cursor handed back to this same route for self-continued enumeration; + // absent on the initial dispatch from the admin trigger. + startingAfter: z.string().optional(), + // Remaining partner budget carried across continuation hops. Seeded from + // filter.limitPartners on the first hop; omitted means unbounded. + remainingPartners: z.number().int().nonnegative().optional(), +}); + +export const partnerContentEnumeratePagePayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + filter: partnerContentIngestionFilterSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + partnerIds: z + .array(z.string()) + .min(1) + .max(PARTNER_CONTENT_ENUMERATE_PAGE_SIZE), +}); + +export const partnerContentFetchPayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + partnerId: z.string().min(1), + partnerPlatformId: z.string().min(1), + platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), +}); + +export type PartnerContentEnumeratePayload = z.infer< + typeof partnerContentEnumeratePayloadSchema +>; + +export function createPartnerContentRunStamp() { + return new Date().toISOString().replace(/[:.]/g, "-"); +} + +export function getPartnerContentUrl(route: PartnerContentSearchRoute) { + return `${APP_DOMAIN_WITH_NGROK}${route}`; +} + +export function createPartnerContentDeduplicationId( + ...parts: Array +) { + return parts + .filter((part): part is string | number => part !== undefined) + .map((part) => String(part).replace(/[^a-zA-Z0-9_-]/g, "-")) + .join("-"); +} + +export async function enqueuePartnerContentEnumerate( + payload: PartnerContentEnumeratePayload, +) { + return await qstash.publishJSON({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.enumerate), + method: "POST", + body: payload, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-enumerate", + payload.mode, + payload.runStamp, + ), + }); +} diff --git a/apps/web/lib/partner-content-search/ingestion/normalize-content.ts b/apps/web/lib/partner-content-search/ingestion/normalize-content.ts new file mode 100644 index 00000000000..3807d2fe481 --- /dev/null +++ b/apps/web/lib/partner-content-search/ingestion/normalize-content.ts @@ -0,0 +1,49 @@ +import { youtubeChannelVideosSchema } from "@/lib/api/scrape-creators/schema"; +import * as z from "zod/v4"; + +type YouTubeChannelVideo = z.infer< + typeof youtubeChannelVideosSchema +>["videos"][number]; + +export type NormalizedPartnerContentItem = { + platformContentId: string; + url: string; + contentType: string; + title: string | null; + description: string | null; + thumbnailUrl: string | null; + publishedAt: Date | null; + durationMs: number | null; + viewCount: number | null; +}; + +export function normalizeYouTubeChannelVideo( + video: YouTubeChannelVideo, +): NormalizedPartnerContentItem | null { + if (!video.id || !video.url) return null; + + return { + platformContentId: video.id, + url: video.url, + contentType: video.type || "video", + title: video.title ?? null, + description: video.description ?? null, + thumbnailUrl: video.thumbnail ?? null, + publishedAt: parseDate(video.publishedTime), + durationMs: + typeof video.lengthSeconds === "number" + ? Math.max(0, video.lengthSeconds * 1000) + : null, + viewCount: + typeof video.viewCountInt === "number" + ? Math.max(0, video.viewCountInt) + : null, + }; +} + +function parseDate(value?: string | null) { + if (!value) return null; + + const date = new Date(value); + return Number.isNaN(date.getTime()) ? null : date; +} diff --git a/apps/web/lib/partner-content-search/types.ts b/apps/web/lib/partner-content-search/types.ts new file mode 100644 index 00000000000..9a561e8decd --- /dev/null +++ b/apps/web/lib/partner-content-search/types.ts @@ -0,0 +1,10 @@ +import type { PlatformType } from "@dub/prisma/client"; + +export const PARTNER_CONTENT_SEARCH_PLATFORMS = [ + "youtube", + "instagram", + "tiktok", +] as const satisfies readonly PlatformType[]; + +export type PartnerContentPlatform = + (typeof PARTNER_CONTENT_SEARCH_PLATFORMS)[number]; diff --git a/packages/prisma/schema/partner-content-search.prisma b/packages/prisma/schema/partner-content-search.prisma new file mode 100644 index 00000000000..4ca05ea6062 --- /dev/null +++ b/packages/prisma/schema/partner-content-search.prisma @@ -0,0 +1,84 @@ +// Partner natural-language search schema. +// +// PlanetScale recommends using Prisma's Unsupported type for VECTOR columns, +// then using raw SQL for vector writes/searches/indexes until Prisma supports +// custom MySQL types directly: +// https://planetscale.com/docs/vitess/vectors/using-with-an-orm#prisma +// +// Keep vector fields optional in Prisma even when they are logically required +// for indexed rows. Required Unsupported fields disable generated Prisma Client +// create/update/upsert methods, and vector values still need TO_VECTOR(...), +// DISTANCE(...), and CREATE VECTOR INDEX statements outside Prisma's generated +// query API. + +enum PartnerContentTranscriptFetchStatus { + pending + fetched + notAvailable + error +} + +model PartnerContentIngestionRun { + id String @id @default(cuid()) + status String + requestedByUserId String? + platforms Json @db.Json + limitCreators Int + contentItemsPerPlatform Int + stats Json? @db.Json + error String? @db.Text + startedAt DateTime @default(now()) + completedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model PartnerContentItem { + id String @id @default(cuid()) + partnerId String + partnerPlatformId String + platformContentId String + url String @db.Text + contentType String + title String? @db.Text + description String? @db.Text + thumbnailUrl String? @db.Text // may require internal image cache for long-term storage; MVP field + publishedAt DateTime? + durationMs Int? + viewCount BigInt? + transcriptFetchStatus PartnerContentTranscriptFetchStatus @default(pending) + transcriptCreditsUsed Int? + transcriptLastAttemptedAt DateTime? + hasTimestamps Boolean @default(false) + normalizedTranscript String? @db.LongText + transcriptHash String? + embeddingModel String? + totalChunkCount Int @default(0) + embeddedChunkCount Int @default(0) + lastFetchedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([partnerPlatformId, platformContentId]) + @@index([partnerId, partnerPlatformId]) + @@index(publishedAt) +} + +model PartnerContentChunk { + id String @id @default(cuid()) + partnerContentItemId String + partnerId String + chunkIndex Int + text String @db.Text + startMs Int? + endMs Int? + transcriptHash String + embedding Unsupported("vector(1024)")? + embeddingModel String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([partnerContentItemId, chunkIndex, transcriptHash, embeddingModel]) + @@index(partnerId) + @@index(partnerContentItemId) +} diff --git a/packages/prisma/schema/platform.prisma b/packages/prisma/schema/platform.prisma index 7903a43a915..160d7337648 100644 --- a/packages/prisma/schema/platform.prisma +++ b/packages/prisma/schema/platform.prisma @@ -22,6 +22,8 @@ model PartnerPlatform { updatedAt DateTime @updatedAt verifiedAt DateTime? lastCheckedAt DateTime? + contentLastFetchedAt DateTime? // last time we fetched recent content from ScrapeCreators for this platform + latestContentUrl String? @db.Text // the most recent content URL we found for this platform partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade) From 74dfda6f8b57beba9f93a386b7d05807a8949085 Mon Sep 17 00:00:00 2001 From: David Clark Date: Fri, 5 Jun 2026 14:48:00 -0400 Subject: [PATCH 02/25] Add partner content transcript chunking --- .../partner-content/enumerate/page/route.ts | 1 + .../api/cron/partner-content/fetch/route.ts | 81 ++++++- .../cron/partner-content/transcript/route.ts | 203 ++++++++++++++++ apps/web/lib/api/scrape-creators/client.ts | 11 + .../get-youtube-video-transcript.ts | 29 +++ apps/web/lib/api/scrape-creators/schema.ts | 19 ++ .../chunk-transcript.ts | 216 ++++++++++++++++++ .../lib/partner-content-search/constants.ts | 27 +++ .../ingestion/enqueue.ts | 12 + .../ingestion/normalize-content.ts | 18 ++ apps/web/lib/partner-content-search/types.ts | 43 ++++ 11 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts create mode 100644 apps/web/lib/api/scrape-creators/get-youtube-video-transcript.ts create mode 100644 apps/web/lib/partner-content-search/chunk-transcript.ts diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts index 526bf647aba..2fa207f746e 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts @@ -81,6 +81,7 @@ export const POST = withCron(async ({ rawBody }) => { mode: payload.mode, runStamp: payload.runStamp, dryRun: payload.dryRun, + forceTranscriptJobs: false, partnerId: partnerPlatform.partnerId, partnerPlatformId: partnerPlatform.partnerPlatformId, platform: partnerPlatform.type, diff --git a/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts b/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts index deabdf44884..8152d19f3a8 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts @@ -1,7 +1,13 @@ import { getYouTubeChannelVideos } from "@/lib/api/scrape-creators/get-youtube-channel-videos"; +import { qstash } from "@/lib/cron"; import { withCron } from "@/lib/cron/with-cron"; import { PARTNER_CONTENT_SEARCH_LIMITS } from "@/lib/partner-content-search/constants"; -import { partnerContentFetchPayloadSchema } from "@/lib/partner-content-search/ingestion/enqueue"; +import { + createPartnerContentDeduplicationId, + getPartnerContentUrl, + PARTNER_CONTENT_SEARCH_ROUTES, + partnerContentFetchPayloadSchema, +} from "@/lib/partner-content-search/ingestion/enqueue"; import { NormalizedPartnerContentItem, normalizeYouTubeChannelVideo, @@ -94,16 +100,26 @@ export const POST = withCron(async ({ rawBody }) => { ); const latestContentItem = fetchedContentItems[0]; + const contentItemsForTranscriptJobs = payload.forceTranscriptJobs + ? fetchedContentItems + : newContentItems; const writeResult = payload.dryRun ? { contentItemsCreated: 0, partnerPlatformUpdated: false, + transcriptJobCount: 0, + qstashResponses: [], } : await writeFetchedContentItems({ + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, partnerId: payload.partnerId, partnerPlatformId: partnerPlatform.id, + platform: payload.platform, contentItems: newContentItems, + contentItemsForTranscriptJobs, latestContentUrl: latestContentItem?.url ?? null, }); @@ -117,6 +133,10 @@ export const POST = withCron(async ({ rawBody }) => { newContentCount: newContentItems.length, writesEnabled: !payload.dryRun, wouldWriteContentItems: newContentItems.length, + forceTranscriptJobs: payload.forceTranscriptJobs, + wouldPublishTranscriptJobs: payload.dryRun + ? 0 + : contentItemsForTranscriptJobs.length, ...writeResult, newContentItems: newContentItems.slice(0, 10), partnerPlatform, @@ -188,16 +208,30 @@ function getRecencyCutoff() { } async function writeFetchedContentItems({ + mode, + runStamp, + dryRun, partnerId, partnerPlatformId, + platform, contentItems, + contentItemsForTranscriptJobs, latestContentUrl, }: { + mode: "incremental" | "backfill"; + runStamp: string; + dryRun: boolean; partnerId: string; partnerPlatformId: string; + platform: "youtube" | "instagram" | "tiktok"; contentItems: NormalizedPartnerContentItem[]; + contentItemsForTranscriptJobs: NormalizedPartnerContentItem[]; latestContentUrl: string | null; }) { + const transcriptJobPlatformContentIds = contentItemsForTranscriptJobs.map( + ({ platformContentId }) => platformContentId, + ); + const [createResult] = await prisma.$transaction([ prisma.partnerContentItem.createMany({ data: contentItems.map((item) => ({ @@ -233,8 +267,53 @@ async function writeFetchedContentItems({ }), ]); + const transcriptContentItems = + transcriptJobPlatformContentIds.length === 0 + ? [] + : await prisma.partnerContentItem.findMany({ + where: { + partnerPlatformId, + platformContentId: { + in: transcriptJobPlatformContentIds, + }, + }, + select: { + id: true, + partnerId: true, + partnerPlatformId: true, + platformContentId: true, + }, + }); + + const transcriptMessages = transcriptContentItems.map((contentItem) => ({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.transcript), + method: "POST" as const, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-transcript", + mode, + runStamp, + contentItem.id, + ), + body: { + mode, + runStamp, + dryRun, + partnerId: contentItem.partnerId, + partnerPlatformId: contentItem.partnerPlatformId, + partnerContentItemId: contentItem.id, + platform, + }, + })); + + const qstashResponses = + transcriptMessages.length === 0 + ? [] + : await qstash.batchJSON(transcriptMessages); + return { contentItemsCreated: createResult.count, partnerPlatformUpdated: true, + transcriptJobCount: transcriptMessages.length, + qstashResponses, }; } diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts new file mode 100644 index 00000000000..be76813f8dd --- /dev/null +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -0,0 +1,203 @@ +import { getYouTubeVideoTranscript } from "@/lib/api/scrape-creators/get-youtube-video-transcript"; +import { withCron } from "@/lib/cron/with-cron"; +import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; +import { + chunkTranscriptSegments, + hashTranscript, +} from "@/lib/partner-content-search/chunk-transcript"; +import { partnerContentTranscriptPayloadSchema } from "@/lib/partner-content-search/ingestion/enqueue"; +import { normalizeYouTubeTranscriptSegments } from "@/lib/partner-content-search/ingestion/normalize-content"; +import { prisma } from "@dub/prisma"; +import { NextResponse } from "next/server"; +import { logAndRespond } from "../../utils"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +// POST /api/cron/partner-content/transcript +export const POST = withCron(async ({ rawBody }) => { + const payload = partnerContentTranscriptPayloadSchema.parse( + JSON.parse(rawBody), + ); + + const contentItem = await prisma.partnerContentItem.findUnique({ + where: { + id: payload.partnerContentItemId, + }, + select: { + id: true, + partnerId: true, + partnerPlatformId: true, + platformContentId: true, + url: true, + title: true, + transcriptFetchStatus: true, + }, + }); + + if (!contentItem) { + return logAndRespond( + `[PartnerContentSearch] Content item ${payload.partnerContentItemId} not found for ${payload.mode} run ${payload.runStamp}.`, + { status: 404, logLevel: "warn" }, + ); + } + + if ( + contentItem.partnerId !== payload.partnerId || + contentItem.partnerPlatformId !== payload.partnerPlatformId + ) { + return logAndRespond( + `[PartnerContentSearch] Content item ${payload.partnerContentItemId} did not match transcript payload for ${payload.mode} run ${payload.runStamp}.`, + { status: 400, logLevel: "warn" }, + ); + } + + if (payload.platform !== "youtube") { + return logAndRespond( + `[PartnerContentSearch] Transcript fetch only supports YouTube right now. Received ${payload.platform} for ${payload.mode} run ${payload.runStamp}.`, + { status: 501, logLevel: "warn" }, + ); + } + + try { + const transcriptResponse = await getYouTubeVideoTranscript({ + url: contentItem.url, + }); + + const transcriptSegments = normalizeYouTubeTranscriptSegments( + transcriptResponse.transcript, + ); + const normalizedTranscript = transcriptSegments + .map(({ text }) => text.trim()) + .filter(Boolean) + .join(" "); + const transcriptHash = + transcriptSegments.length > 0 ? hashTranscript(transcriptSegments) : null; + + if (!transcriptHash) { + const updatedContentItem = await prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "notAvailable", + transcriptLastAttemptedAt: new Date(), + normalizedTranscript: null, + transcriptHash: null, + hasTimestamps: false, + totalChunkCount: 0, + embeddedChunkCount: 0, + }, + select: { + id: true, + transcriptFetchStatus: true, + totalChunkCount: true, + embeddedChunkCount: true, + }, + }); + + await prisma.partnerContentChunk.deleteMany({ + where: { + partnerContentItemId: contentItem.id, + }, + }); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + writesEnabled: true, + transcriptAvailable: false, + segmentCount: 0, + normalizedTranscriptLength: 0, + transcriptHash: null, + chunkCount: 0, + chunksCreated: 0, + contentItem: updatedContentItem, + }); + } + + const chunks = chunkTranscriptSegments(transcriptSegments); + const hasTimestamps = transcriptSegments.some( + ({ startMs, endMs }) => startMs !== null || endMs !== null, + ); + const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.model; + + const [updatedContentItem, deletedChunks, createChunksResult] = + await prisma.$transaction([ + prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "fetched", + transcriptLastAttemptedAt: new Date(), + normalizedTranscript, + transcriptHash, + hasTimestamps, + embeddingModel, + totalChunkCount: chunks.length, + embeddedChunkCount: 0, + lastFetchedAt: new Date(), + }, + select: { + id: true, + transcriptFetchStatus: true, + transcriptHash: true, + totalChunkCount: true, + embeddedChunkCount: true, + hasTimestamps: true, + lastFetchedAt: true, + }, + }), + prisma.partnerContentChunk.deleteMany({ + where: { + partnerContentItemId: contentItem.id, + }, + }), + prisma.partnerContentChunk.createMany({ + data: chunks.map((chunk) => ({ + partnerContentItemId: contentItem.id, + partnerId: contentItem.partnerId, + chunkIndex: chunk.chunkIndex, + text: chunk.text, + startMs: chunk.startMs, + endMs: chunk.endMs, + transcriptHash, + embeddingModel, + })), + }), + ]); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + writesEnabled: true, + transcriptAvailable: true, + segmentCount: transcriptSegments.length, + normalizedTranscriptLength: normalizedTranscript.length, + transcriptHash, + chunkCount: chunks.length, + chunksDeleted: deletedChunks.count, + chunksCreated: createChunksResult.count, + sampleSegments: transcriptSegments.slice(0, 5), + sampleChunks: chunks.slice(0, 3), + contentItem: updatedContentItem, + }); + } catch (error) { + await prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "error", + transcriptLastAttemptedAt: new Date(), + }, + }); + + throw error; + } +}); diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.ts index a96f0f1207a..45f92548164 100644 --- a/apps/web/lib/api/scrape-creators/client.ts +++ b/apps/web/lib/api/scrape-creators/client.ts @@ -5,6 +5,7 @@ import { socialContentSchema, socialProfileSchema, youtubeChannelVideosSchema, + youtubeTranscriptSchema, } from "./schema"; export const scrapeCreatorsFetch = createFetch({ @@ -62,6 +63,16 @@ export const scrapeCreatorsFetch = createFetch({ }), output: youtubeChannelVideosSchema, }, + + // Fetch a YouTube video or short transcript + "/v1/youtube/video/transcript": { + method: "get", + query: z.object({ + url: z.string(), + language: z.string().length(2).optional(), + }), + output: youtubeTranscriptSchema, + }, }, { strict: true, diff --git a/apps/web/lib/api/scrape-creators/get-youtube-video-transcript.ts b/apps/web/lib/api/scrape-creators/get-youtube-video-transcript.ts new file mode 100644 index 00000000000..ce3598db975 --- /dev/null +++ b/apps/web/lib/api/scrape-creators/get-youtube-video-transcript.ts @@ -0,0 +1,29 @@ +import { scrapeCreatorsFetch } from "./client"; + +interface GetYouTubeVideoTranscriptParams { + url: string; + language?: string; +} + +export async function getYouTubeVideoTranscript({ + url, + language = "en", +}: GetYouTubeVideoTranscriptParams) { + const { data, error } = await scrapeCreatorsFetch( + "/v1/youtube/video/transcript", + { + query: { + url, + language, + }, + }, + ); + + if (error) { + throw new Error( + "We were unable to retrieve the YouTube transcript from ScrapeCreators.", + ); + } + + return data; +} diff --git a/apps/web/lib/api/scrape-creators/schema.ts b/apps/web/lib/api/scrape-creators/schema.ts index 33bee392bf1..1fd919cf3a7 100644 --- a/apps/web/lib/api/scrape-creators/schema.ts +++ b/apps/web/lib/api/scrape-creators/schema.ts @@ -345,6 +345,7 @@ export const socialContentSchema = z.preprocess( }), ]), ); + export const youtubeChannelVideosSchema = z.object({ videos: z.array( z.object({ @@ -377,3 +378,21 @@ export const youtubeChannelVideosSchema = z.object({ ), continuationToken: z.string().nullish(), }); + +export const youtubeTranscriptSchema = z.object({ + videoId: z.string(), + type: z.string(), + url: z.string(), + transcript: z + .array( + z.object({ + text: z.string(), + startMs: z.string(), + endMs: z.string(), + startTimeText: z.string().nullish(), + }), + ) + .nullable(), + transcript_only_text: z.string().nullish(), + language: z.string().nullish(), +}); diff --git a/apps/web/lib/partner-content-search/chunk-transcript.ts b/apps/web/lib/partner-content-search/chunk-transcript.ts new file mode 100644 index 00000000000..efe6fb06b81 --- /dev/null +++ b/apps/web/lib/partner-content-search/chunk-transcript.ts @@ -0,0 +1,216 @@ +import { createHash } from "crypto"; +import { PARTNER_CONTENT_SEARCH_LIMITS } from "./constants"; +import { + ChunkTranscriptOptions, + SentenceTimestamp, + TranscriptChunk, + TranscriptSegment, +} from "./types"; + +type SentenceUnit = SentenceTimestamp & { + tokenEstimate: number; +}; + +export function normalizeTranscriptText(text: string) { + return text.replace(/\s+/g, " ").trim(); +} + +export function normalizeTranscriptSegments(segments: TranscriptSegment[]) { + return segments + .map((segment) => ({ + text: normalizeTranscriptText(segment.text), + startMs: segment.startMs ?? null, + endMs: segment.endMs ?? null, + })) + .filter((segment) => segment.text.length > 0); +} + +export function hashTranscript(segments: TranscriptSegment[]) { + const normalized = normalizeTranscriptSegments(segments) + .map( + ({ text, startMs, endMs }) => + `${startMs ?? ""}:${endMs ?? ""}:${text}`, + ) + .join("\n"); + + return createHash("sha256").update(normalized).digest("hex"); +} + +export function estimateTokenCount(text: string) { + const normalized = normalizeTranscriptText(text); + if (!normalized) return 0; + + // A lightweight approximation keeps this utility dependency-free. The + // ingestion job can swap in a provider tokenizer later if needed. + const words = normalized.split(/\s+/).length; + return Math.max(1, Math.ceil(words * 1.35)); +} + +export function chunkTranscriptSegments( + segments: TranscriptSegment[], + options: ChunkTranscriptOptions = {}, +): TranscriptChunk[] { + const maxTokens = + options.maxTokens ?? PARTNER_CONTENT_SEARCH_LIMITS.chunkMaxTokens; + const minTokens = + options.minTokens ?? PARTNER_CONTENT_SEARCH_LIMITS.chunkMinTokens; + const overlapTokens = + options.overlapTokens ?? PARTNER_CONTENT_SEARCH_LIMITS.chunkOverlapTokens; + + if (maxTokens <= 0) throw new Error("maxTokens must be greater than 0."); + if (minTokens <= 0) throw new Error("minTokens must be greater than 0."); + if (overlapTokens < 0) throw new Error("overlapTokens cannot be negative."); + if (overlapTokens >= maxTokens) { + throw new Error("overlapTokens must be less than maxTokens."); + } + + const units = normalizeTranscriptSegments(segments).flatMap( + (segment, segmentIndex) => + splitSegmentIntoSentenceUnits(segment, segmentIndex), + ); + + if (units.length === 0) return []; + + const chunks: TranscriptChunk[] = []; + let startIndex = 0; + + while (startIndex < units.length) { + let endIndex = startIndex; + let tokenEstimate = 0; + + while (endIndex < units.length) { + const nextTokens = units[endIndex].tokenEstimate; + const wouldExceedMax = + tokenEstimate > 0 && tokenEstimate + nextTokens > maxTokens; + const meetsMinimum = tokenEstimate >= minTokens; + + if (wouldExceedMax && meetsMinimum) break; + + tokenEstimate += nextTokens; + endIndex++; + + if (tokenEstimate >= maxTokens) break; + } + + if (endIndex === startIndex) endIndex++; + + const chunkUnits = units.slice(startIndex, endIndex); + chunks.push(toTranscriptChunk(chunks.length, chunkUnits)); + + if (endIndex >= units.length) break; + + startIndex = getNextStartIndex({ + units, + startIndex, + endIndex, + overlapTokens, + }); + } + + return chunks; +} + +function splitSegmentIntoSentenceUnits( + segment: Required, + segmentIndex: number, +): SentenceUnit[] { + const sentences = splitIntoSentences(segment.text); + + if (sentences.length === 0) return []; + + const hasTiming = segment.startMs !== null && segment.endMs !== null; + const duration = hasTiming + ? Math.max(0, Number(segment.endMs) - Number(segment.startMs)) + : 0; + + return sentences.map((sentence, index) => { + const startMs = + hasTiming && duration > 0 + ? Math.round( + Number(segment.startMs) + (duration * index) / sentences.length, + ) + : segment.startMs; + const endMs = + hasTiming && duration > 0 + ? Math.round( + Number(segment.startMs) + + (duration * (index + 1)) / sentences.length, + ) + : segment.endMs; + + return { + text: sentence, + startMs, + endMs, + segmentIndex, + tokenEstimate: estimateTokenCount(sentence), + }; + }); +} + +function splitIntoSentences(text: string) { + const normalized = normalizeTranscriptText(text); + if (!normalized) return []; + + const sentences = normalized.match(/[^.!?]+[.!?]+|[^.!?]+$/g) ?? [ + normalized, + ]; + + return sentences.map(normalizeTranscriptText).filter(Boolean); +} + +function toTranscriptChunk( + chunkIndex: number, + sentenceUnits: SentenceUnit[], +): TranscriptChunk { + const sentenceTimestamps = sentenceUnits.map( + ({ tokenEstimate: _tokenEstimate, ...unit }) => unit, + ); + + return { + chunkIndex, + text: sentenceUnits.map(({ text }) => text).join(" "), + tokenEstimate: sentenceUnits.reduce( + (total, unit) => total + unit.tokenEstimate, + 0, + ), + startMs: firstNonNull(sentenceUnits.map(({ startMs }) => startMs)), + endMs: lastNonNull(sentenceUnits.map(({ endMs }) => endMs)), + sentenceTimestamps, + }; +} + +function getNextStartIndex({ + units, + startIndex, + endIndex, + overlapTokens, +}: { + units: SentenceUnit[]; + startIndex: number; + endIndex: number; + overlapTokens: number; +}) { + if (overlapTokens === 0) return endIndex; + + let overlap = 0; + let nextStartIndex = endIndex; + + while (nextStartIndex > startIndex + 1 && overlap < overlapTokens) { + nextStartIndex--; + overlap += units[nextStartIndex].tokenEstimate; + } + + return Math.max(startIndex + 1, Math.min(nextStartIndex, endIndex)); +} + +function firstNonNull(values: Array) { + return values.find((value): value is number => value !== null) ?? null; +} + +function lastNonNull(values: Array) { + for (let index = values.length - 1; index >= 0; index--) { + if (values[index] !== null) return values[index]; + } + return null; +} diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts index ff0c79b2bf1..ef66a85c5ba 100644 --- a/apps/web/lib/partner-content-search/constants.ts +++ b/apps/web/lib/partner-content-search/constants.ts @@ -3,9 +3,36 @@ export const PARTNER_CONTENT_SEARCH_FEATURE_FLAG = export const PARTNER_CONTENT_SEARCH_ENV_VARS = { scrapeCreatorsApiKey: "SCRAPECREATORS_API_KEY", + voyageApiKey: "VOYAGE_API_KEY", +} as const; + +export const PARTNER_CONTENT_SEARCH_MODELS = { + embedding: { + provider: "voyage", + model: "voyage-4", + dimensions: 1024, + outputDtype: "float", + }, + reranker: { + provider: "voyage", + model: "rerank-2.5", + }, } as const; export const PARTNER_CONTENT_SEARCH_LIMITS = { recencyWindowMonths: 12, contentItemsPerPartnerPlatform: 50, + lowTranscriptCoverageThreshold: 0.3, + maxTranscriptBytesInPlanetscale: 500_000, + chunkMinTokens: 400, + chunkMaxTokens: 800, + chunkOverlapTokens: 80, + chunkCandidateCount: 200, + rerankerCandidateCount: 150, + partnerScorePoolSize: 3, +} as const; + +export const PARTNER_CONTENT_SEARCH_FUSION_WEIGHTS = { + semantic: 0.6, + existingPartnerRank: 0.4, } as const; diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index db62ec2a44b..e4dc2320463 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -17,6 +17,7 @@ export const PARTNER_CONTENT_SEARCH_ROUTES = { enumerate: "/api/cron/partner-content/enumerate", enumeratePage: "/api/cron/partner-content/enumerate/page", fetch: "/api/cron/partner-content/fetch", + transcript: "/api/cron/partner-content/transcript", } as const; type PartnerContentSearchRoute = @@ -70,11 +71,22 @@ export const partnerContentFetchPayloadSchema = z.object({ mode: partnerContentIngestionModeSchema, runStamp: z.string().min(1), dryRun: z.boolean().default(false), + forceTranscriptJobs: z.boolean().default(false), partnerId: z.string().min(1), partnerPlatformId: z.string().min(1), platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), }); +export const partnerContentTranscriptPayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + partnerId: z.string().min(1), + partnerPlatformId: z.string().min(1), + partnerContentItemId: z.string().min(1), + platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), +}); + export type PartnerContentEnumeratePayload = z.infer< typeof partnerContentEnumeratePayloadSchema >; diff --git a/apps/web/lib/partner-content-search/ingestion/normalize-content.ts b/apps/web/lib/partner-content-search/ingestion/normalize-content.ts index 3807d2fe481..a26a6ba2517 100644 --- a/apps/web/lib/partner-content-search/ingestion/normalize-content.ts +++ b/apps/web/lib/partner-content-search/ingestion/normalize-content.ts @@ -1,4 +1,5 @@ import { youtubeChannelVideosSchema } from "@/lib/api/scrape-creators/schema"; +import { TranscriptSegment } from "@/lib/partner-content-search/types"; import * as z from "zod/v4"; type YouTubeChannelVideo = z.infer< @@ -41,9 +42,26 @@ export function normalizeYouTubeChannelVideo( }; } +export function normalizeYouTubeTranscriptSegments( + transcript: Array<{ text: string; startMs: string; endMs: string }> | null, +): TranscriptSegment[] { + return (transcript ?? []) + .map((segment) => ({ + text: segment.text, + startMs: parseMs(segment.startMs), + endMs: parseMs(segment.endMs), + })) + .filter((segment) => segment.text.trim().length > 0); +} + function parseDate(value?: string | null) { if (!value) return null; const date = new Date(value); return Number.isNaN(date.getTime()) ? null : date; } + +function parseMs(value: string) { + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : null; +} diff --git a/apps/web/lib/partner-content-search/types.ts b/apps/web/lib/partner-content-search/types.ts index 9a561e8decd..fb1cc7b6264 100644 --- a/apps/web/lib/partner-content-search/types.ts +++ b/apps/web/lib/partner-content-search/types.ts @@ -8,3 +8,46 @@ export const PARTNER_CONTENT_SEARCH_PLATFORMS = [ export type PartnerContentPlatform = (typeof PARTNER_CONTENT_SEARCH_PLATFORMS)[number]; + +export type VoyageInputType = "document" | "query"; + +export type TranscriptSegment = { + text: string; + startMs?: number | null; + endMs?: number | null; +}; + +export type SentenceTimestamp = { + text: string; + startMs: number | null; + endMs: number | null; + segmentIndex: number; +}; + +export type TranscriptChunk = { + chunkIndex: number; + text: string; + tokenEstimate: number; + startMs: number | null; + endMs: number | null; + sentenceTimestamps: SentenceTimestamp[]; +}; + +export type ChunkTranscriptOptions = { + minTokens?: number; + maxTokens?: number; + overlapTokens?: number; +}; + +export type VoyageEmbeddingMetadata = { + provider: "voyage"; + model: string; + dimensions: number; + inputType: VoyageInputType; +}; + +export type VoyageRerankResult = { + index: number; + relevanceScore: number; + document?: string; +}; From 25004feb81cd1943fd8bb336b9dedc4ae674abf6 Mon Sep 17 00:00:00 2001 From: David Clark Date: Fri, 5 Jun 2026 14:57:20 -0400 Subject: [PATCH 03/25] Add partner content embedding job --- .../api/cron/partner-content/embed/route.ts | 295 ++++++++++++++++++ .../cron/partner-content/transcript/route.ts | 26 +- .../ingestion/enqueue.ts | 19 ++ apps/web/lib/partner-content-search/voyage.ts | 157 ++++++++++ 4 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 apps/web/app/(ee)/api/cron/partner-content/embed/route.ts create mode 100644 apps/web/lib/partner-content-search/voyage.ts diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts new file mode 100644 index 00000000000..9aaeea6ab02 --- /dev/null +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -0,0 +1,295 @@ +import { qstash } from "@/lib/cron"; +import { withCron } from "@/lib/cron/with-cron"; +import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; +import { + createPartnerContentDeduplicationId, + getPartnerContentUrl, + PARTNER_CONTENT_SEARCH_ROUTES, + partnerContentEmbedPayloadSchema, +} from "@/lib/partner-content-search/ingestion/enqueue"; +import { embedPartnerContentTexts } from "@/lib/partner-content-search/voyage"; +import { prisma } from "@dub/prisma"; +import { NextResponse } from "next/server"; +import { logAndRespond } from "../../utils"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +type UnembeddedChunk = { + id: string; + text: string; +}; + +type EmbeddedChunkCount = { + embeddedChunkCount: bigint; +}; + +// POST /api/cron/partner-content/embed +export const POST = withCron(async ({ rawBody }) => { + const payload = partnerContentEmbedPayloadSchema.parse(JSON.parse(rawBody)); + + if (!payload.partnerContentItemId) { + return enqueueEmbedJobsForPartnerPlatform(payload); + } + + const contentItem = await prisma.partnerContentItem.findUnique({ + where: { + id: payload.partnerContentItemId, + }, + select: { + id: true, + partnerId: true, + transcriptFetchStatus: true, + totalChunkCount: true, + embeddedChunkCount: true, + }, + }); + + if (!contentItem) { + return logAndRespond( + `[PartnerContentSearch] Content item ${payload.partnerContentItemId} not found for ${payload.mode} embed run ${payload.runStamp}.`, + { status: 404, logLevel: "warn" }, + ); + } + + if (contentItem.partnerId !== payload.partnerId) { + return logAndRespond( + `[PartnerContentSearch] Content item ${payload.partnerContentItemId} did not match embed payload for ${payload.mode} run ${payload.runStamp}.`, + { status: 400, logLevel: "warn" }, + ); + } + + if ( + contentItem.transcriptFetchStatus !== "fetched" || + contentItem.totalChunkCount === 0 + ) { + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + skipped: true, + reason: "Content item has no fetched transcript chunks to embed.", + contentItem, + }); + } + + const chunks = await prisma.$queryRaw` + SELECT id, text + FROM PartnerContentChunk + WHERE partnerContentItemId = ${contentItem.id} + AND embedding IS NULL + ORDER BY chunkIndex ASC + LIMIT ${payload.maxChunks} + `; + + if (chunks.length === 0) { + const embeddedChunkCount = await refreshEmbeddedChunkCount(contentItem.id); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + skipped: true, + reason: "All chunks are already embedded.", + embeddedChunkCount, + contentItem: { + ...contentItem, + embeddedChunkCount, + }, + }); + } + + const embeddings = await embedPartnerContentTexts({ + input: chunks.map(({ text }) => text), + inputType: "document", + }); + + if (embeddings.length !== chunks.length) { + throw new Error( + `Voyage returned ${embeddings.length} embeddings for ${chunks.length} chunks.`, + ); + } + + await Promise.all( + chunks.map((chunk, index) => + prisma.$executeRaw` + UPDATE PartnerContentChunk + SET embedding = TO_VECTOR(${serializeEmbedding(embeddings[index])}), + embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.model} + WHERE id = ${chunk.id} + `, + ), + ); + + const embeddedChunkCount = await refreshEmbeddedChunkCount(contentItem.id); + const remainingChunkCount = Math.max( + 0, + contentItem.totalChunkCount - embeddedChunkCount, + ); + + let continuationMessageId: string | undefined; + + if (remainingChunkCount > 0) { + const response = await qstash.publishJSON({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), + method: "POST", + body: { + mode: payload.mode, + runStamp: payload.runStamp, + partnerId: payload.partnerId, + partnerContentItemId: contentItem.id, + maxChunks: payload.maxChunks, + }, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-embed", + payload.mode, + payload.runStamp, + contentItem.id, + embeddedChunkCount, + ), + }); + + continuationMessageId = response.messageId; + } + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + embeddedNow: chunks.length, + embeddedChunkCount, + totalChunkCount: contentItem.totalChunkCount, + remainingChunkCount, + continuationMessageId, + contentItem: { + id: contentItem.id, + transcriptFetchStatus: contentItem.transcriptFetchStatus, + totalChunkCount: contentItem.totalChunkCount, + embeddedChunkCount, + }, + }); +}); + +async function enqueueEmbedJobsForPartnerPlatform({ + mode, + runStamp, + partnerId, + partnerPlatformId, + limitContentItems, + maxChunks, +}: { + mode: "incremental" | "backfill"; + runStamp: string; + partnerId: string; + partnerPlatformId?: string; + limitContentItems: number; + maxChunks: number; +}) { + if (!partnerPlatformId) { + return logAndRespond( + `[PartnerContentSearch] Embed enqueue requires partnerPlatformId for ${mode} run ${runStamp}.`, + { status: 400, logLevel: "warn" }, + ); + } + + const contentItems = await prisma.partnerContentItem.findMany({ + where: { + partnerId, + partnerPlatformId, + transcriptFetchStatus: "fetched", + totalChunkCount: { + gt: 0, + }, + }, + select: { + id: true, + totalChunkCount: true, + embeddedChunkCount: true, + }, + orderBy: { + id: "asc", + }, + take: limitContentItems, + }); + + const pendingContentItems = contentItems.filter( + ({ totalChunkCount, embeddedChunkCount }) => + embeddedChunkCount < totalChunkCount, + ); + + const messages = pendingContentItems.map((contentItem) => ({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), + method: "POST" as const, + body: { + mode, + runStamp, + partnerId, + partnerContentItemId: contentItem.id, + maxChunks, + }, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-embed", + mode, + runStamp, + contentItem.id, + ), + })); + + const qstashResponses = + messages.length === 0 ? [] : await qstash.batchJSON(messages); + + return NextResponse.json({ + success: true, + mode, + runStamp, + partnerId, + partnerPlatformId, + inspectedContentItemCount: contentItems.length, + embedJobCount: messages.length, + qstashResponses, + contentItems: pendingContentItems, + }); +} + +async function refreshEmbeddedChunkCount(partnerContentItemId: string) { + const [result] = await prisma.$queryRaw` + SELECT COUNT(*) AS embeddedChunkCount + FROM PartnerContentChunk + WHERE partnerContentItemId = ${partnerContentItemId} + AND embedding IS NOT NULL + `; + + const embeddedChunkCount = Number(result?.embeddedChunkCount ?? 0); + + await prisma.partnerContentItem.update({ + where: { + id: partnerContentItemId, + }, + data: { + embeddedChunkCount, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.model, + }, + }); + + return embeddedChunkCount; +} + +function serializeEmbedding(embedding: number[]) { + const expectedDimensions = PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions; + + if (embedding.length !== expectedDimensions) { + throw new Error( + `Expected ${expectedDimensions} embedding dimensions, received ${embedding.length}.`, + ); + } + + return JSON.stringify( + embedding.map((value) => { + if (!Number.isFinite(value)) { + throw new Error("Voyage returned a non-finite embedding value."); + } + + return value; + }), + ); +} diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts index be76813f8dd..5f49d2f8cbb 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -1,11 +1,17 @@ import { getYouTubeVideoTranscript } from "@/lib/api/scrape-creators/get-youtube-video-transcript"; +import { qstash } from "@/lib/cron"; import { withCron } from "@/lib/cron/with-cron"; import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; import { chunkTranscriptSegments, hashTranscript, } from "@/lib/partner-content-search/chunk-transcript"; -import { partnerContentTranscriptPayloadSchema } from "@/lib/partner-content-search/ingestion/enqueue"; +import { + createPartnerContentDeduplicationId, + getPartnerContentUrl, + PARTNER_CONTENT_SEARCH_ROUTES, + partnerContentTranscriptPayloadSchema, +} from "@/lib/partner-content-search/ingestion/enqueue"; import { normalizeYouTubeTranscriptSegments } from "@/lib/partner-content-search/ingestion/normalize-content"; import { prisma } from "@dub/prisma"; import { NextResponse } from "next/server"; @@ -170,6 +176,23 @@ export const POST = withCron(async ({ rawBody }) => { }), ]); + const embedJob = await qstash.publishJSON({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), + method: "POST", + body: { + mode: payload.mode, + runStamp: payload.runStamp, + partnerId: contentItem.partnerId, + partnerContentItemId: contentItem.id, + }, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-embed", + payload.mode, + payload.runStamp, + contentItem.id, + ), + }); + return NextResponse.json({ success: true, mode: payload.mode, @@ -183,6 +206,7 @@ export const POST = withCron(async ({ rawBody }) => { chunkCount: chunks.length, chunksDeleted: deletedChunks.count, chunksCreated: createChunksResult.count, + embedJob, sampleSegments: transcriptSegments.slice(0, 5), sampleChunks: chunks.slice(0, 3), contentItem: updatedContentItem, diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index e4dc2320463..c8aa61e6779 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -18,6 +18,7 @@ export const PARTNER_CONTENT_SEARCH_ROUTES = { enumeratePage: "/api/cron/partner-content/enumerate/page", fetch: "/api/cron/partner-content/fetch", transcript: "/api/cron/partner-content/transcript", + embed: "/api/cron/partner-content/embed", } as const; type PartnerContentSearchRoute = @@ -87,6 +88,24 @@ export const partnerContentTranscriptPayloadSchema = z.object({ platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), }); +export const partnerContentEmbedPayloadSchema = z + .object({ + mode: partnerContentIngestionModeSchema, + runStamp: z.string().min(1), + partnerId: z.string().min(1), + partnerPlatformId: z.string().min(1).optional(), + partnerContentItemId: z.string().min(1).optional(), + limitContentItems: z.number().int().positive().max(500).default(100), + maxChunks: z.number().int().positive().max(128).default(96), + }) + .refine( + ({ partnerPlatformId, partnerContentItemId }) => + Boolean(partnerPlatformId) !== Boolean(partnerContentItemId), + { + message: "Use exactly one of partnerPlatformId or partnerContentItemId.", + }, + ); + export type PartnerContentEnumeratePayload = z.infer< typeof partnerContentEnumeratePayloadSchema >; diff --git a/apps/web/lib/partner-content-search/voyage.ts b/apps/web/lib/partner-content-search/voyage.ts new file mode 100644 index 00000000000..6b87b95cbff --- /dev/null +++ b/apps/web/lib/partner-content-search/voyage.ts @@ -0,0 +1,157 @@ +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MODELS, +} from "./constants"; +import { + VoyageEmbeddingMetadata, + VoyageInputType, + VoyageRerankResult, +} from "./types"; + +const VOYAGE_API_BASE_URL = "https://api.voyageai.com/v1"; + +type VoyageEmbeddingResponse = { + data: Array<{ + embedding: number[]; + index: number; + }>; +}; + +type VoyageRerankResponse = { + data: Array<{ + index: number; + relevance_score: number; + document?: string; + }>; +}; + +type VoyageFetch = typeof fetch; + +export function buildVoyageEmbeddingRequest({ + input, + inputType, +}: { + input: string[]; + inputType: VoyageInputType; +}) { + return { + input, + model: PARTNER_CONTENT_SEARCH_MODELS.embedding.model, + input_type: inputType, + output_dimension: PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions, + output_dtype: PARTNER_CONTENT_SEARCH_MODELS.embedding.outputDtype, + truncation: true, + }; +} + +export function buildVoyageRerankRequest({ + query, + documents, + topK, + returnDocuments = false, +}: { + query: string; + documents: string[]; + topK?: number; + returnDocuments?: boolean; +}) { + return { + query, + documents, + model: PARTNER_CONTENT_SEARCH_MODELS.reranker.model, + top_k: topK ?? PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, + return_documents: returnDocuments, + truncation: true, + }; +} + +export function getVoyageEmbeddingMetadata( + inputType: VoyageInputType, +): VoyageEmbeddingMetadata { + return { + provider: "voyage", + model: PARTNER_CONTENT_SEARCH_MODELS.embedding.model, + dimensions: PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions, + inputType, + }; +} + +export async function embedPartnerContentTexts({ + input, + inputType, + apiKey = process.env.VOYAGE_API_KEY, + fetchImpl = fetch, +}: { + input: string[]; + inputType: VoyageInputType; + apiKey?: string; + fetchImpl?: VoyageFetch; +}) { + if (!apiKey) throw new Error("VOYAGE_API_KEY is not set."); + if (input.length === 0) return []; + + const response = await fetchImpl(`${VOYAGE_API_BASE_URL}/embeddings`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(buildVoyageEmbeddingRequest({ input, inputType })), + }); + + if (!response.ok) { + throw new Error(`Voyage embeddings request failed: ${response.status}`); + } + + const result = (await response.json()) as VoyageEmbeddingResponse; + + return result.data + .sort((a, b) => a.index - b.index) + .map(({ embedding }) => embedding); +} + +export async function rerankPartnerContent({ + query, + documents, + topK, + apiKey = process.env.VOYAGE_API_KEY, + fetchImpl = fetch, +}: { + query: string; + documents: string[]; + topK?: number; + apiKey?: string; + fetchImpl?: VoyageFetch; +}): Promise { + // AI SDK Core supports embeddings, but not Voyage reranking. Keep this direct + // wrapper until Dub has a shared reranker abstraction. + if (!apiKey) throw new Error("VOYAGE_API_KEY is not set."); + if (documents.length === 0) return []; + + const response = await fetchImpl(`${VOYAGE_API_BASE_URL}/rerank`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify( + buildVoyageRerankRequest({ + query, + documents, + topK, + }), + ), + }); + + if (!response.ok) { + throw new Error(`Voyage rerank request failed: ${response.status}`); + } + + const result = (await response.json()) as VoyageRerankResponse; + + return result.data.map(({ relevance_score, ...rest }) => ({ + index: rest.index, + document: rest.document, + relevanceScore: relevance_score, + })); +} From 5c01a49b1fcc7ae162ac9bc259310677500da3ab Mon Sep 17 00:00:00 2001 From: David Clark Date: Fri, 5 Jun 2026 16:26:13 -0400 Subject: [PATCH 04/25] Tune partner content embedding fanout --- .../api/cron/partner-content/embed/route.ts | 73 ++++++++++++++++++- .../cron/partner-content/transcript/route.ts | 2 + .../ingestion/enqueue.ts | 7 ++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts index 9aaeea6ab02..d42abc78247 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -4,6 +4,7 @@ import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/cons import { createPartnerContentDeduplicationId, getPartnerContentUrl, + PARTNER_CONTENT_EMBED_FLOW_CONTROL, PARTNER_CONTENT_SEARCH_ROUTES, partnerContentEmbedPayloadSchema, } from "@/lib/partner-content-search/ingestion/enqueue"; @@ -15,6 +16,8 @@ import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; export const maxDuration = 60; +const EMBED_RATE_LIMIT_RETRY_DELAY_SECONDS = 120; + type UnembeddedChunk = { id: string; text: string; @@ -99,10 +102,51 @@ export const POST = withCron(async ({ rawBody }) => { }); } - const embeddings = await embedPartnerContentTexts({ - input: chunks.map(({ text }) => text), - inputType: "document", - }); + let embeddings: number[][]; + + try { + embeddings = await embedPartnerContentTexts({ + input: chunks.map(({ text }) => text), + inputType: "document", + }); + } catch (error) { + if (!isVoyageRateLimitError(error)) throw error; + + const response = await qstash.publishJSON({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), + method: "POST", + body: { + mode: payload.mode, + runStamp: payload.runStamp, + partnerId: payload.partnerId, + partnerContentItemId: contentItem.id, + maxChunks: payload.maxChunks, + }, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, + delay: getRateLimitRetryDelaySeconds(contentItem.id), + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-embed-rate-limit-retry", + payload.mode, + payload.runStamp, + contentItem.id, + ), + }); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + rateLimited: true, + retryScheduled: true, + retryMessageId: response.messageId, + contentItem: { + id: contentItem.id, + transcriptFetchStatus: contentItem.transcriptFetchStatus, + totalChunkCount: contentItem.totalChunkCount, + embeddedChunkCount: contentItem.embeddedChunkCount, + }, + }); + } if (embeddings.length !== chunks.length) { throw new Error( @@ -220,6 +264,7 @@ async function enqueueEmbedJobsForPartnerPlatform({ const messages = pendingContentItems.map((contentItem) => ({ url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), method: "POST" as const, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, body: { mode, runStamp, @@ -246,6 +291,8 @@ async function enqueueEmbedJobsForPartnerPlatform({ partnerPlatformId, inspectedContentItemCount: contentItems.length, embedJobCount: messages.length, + pendingContentItemCount: pendingContentItems.length, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, qstashResponses, contentItems: pendingContentItems, }); @@ -293,3 +340,21 @@ function serializeEmbedding(embedding: number[]) { }), ); } + +function isVoyageRateLimitError(error: unknown) { + return error instanceof Error && error.message.includes("failed: 429"); +} + +function getRateLimitRetryDelaySeconds(contentItemId: string) { + return ( + EMBED_RATE_LIMIT_RETRY_DELAY_SECONDS + + (getStableNumericHash(contentItemId) % 120) + ); +} + +function getStableNumericHash(value: string) { + return Array.from(value).reduce( + (hash, char) => (hash * 31 + char.charCodeAt(0)) >>> 0, + 0, + ); +} diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts index 5f49d2f8cbb..197e5ae0f67 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -9,6 +9,7 @@ import { import { createPartnerContentDeduplicationId, getPartnerContentUrl, + PARTNER_CONTENT_EMBED_FLOW_CONTROL, PARTNER_CONTENT_SEARCH_ROUTES, partnerContentTranscriptPayloadSchema, } from "@/lib/partner-content-search/ingestion/enqueue"; @@ -185,6 +186,7 @@ export const POST = withCron(async ({ rawBody }) => { partnerId: contentItem.partnerId, partnerContentItemId: contentItem.id, }, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, deduplicationId: createPartnerContentDeduplicationId( "partner-content-embed", payload.mode, diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index c8aa61e6779..82e4c3a7bbe 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -13,6 +13,13 @@ export const PARTNER_CONTENT_ENUMERATE_PAGE_SIZE = (() => { })(); export const PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS = 7; +export const PARTNER_CONTENT_EMBED_FLOW_CONTROL = { + key: "partner-content-embed-voyage", + parallelism: 5, + rate: 120, + period: "1m", +} as const; + export const PARTNER_CONTENT_SEARCH_ROUTES = { enumerate: "/api/cron/partner-content/enumerate", enumeratePage: "/api/cron/partner-content/enumerate/page", From 08325cdd3e4040c003703e5e8b1b91dcc0c4c676 Mon Sep 17 00:00:00 2001 From: David Clark Date: Sat, 6 Jun 2026 20:37:27 -0400 Subject: [PATCH 05/25] Add partner content search, redrive safeguards --- .../api/admin/partner-content/search/route.ts | 286 ++++++++++++++++ .../api/cron/partner-content/embed/route.ts | 1 + .../cron/partner-content/transcript/route.ts | 312 +++++++++++------- 3 files changed, 474 insertions(+), 125 deletions(-) create mode 100644 apps/web/app/(ee)/api/admin/partner-content/search/route.ts diff --git a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts new file mode 100644 index 00000000000..dcf10d5ca2f --- /dev/null +++ b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts @@ -0,0 +1,286 @@ +import { handleAndReturnErrorResponse } from "@/lib/api/errors"; +import { parseRequestBody } from "@/lib/api/utils"; +import { withAdmin } from "@/lib/auth"; +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MODELS, +} from "@/lib/partner-content-search/constants"; +import { embedPartnerContentTexts } from "@/lib/partner-content-search/voyage"; +import { prisma } from "@dub/prisma"; +import { Prisma } from "@dub/prisma/client"; +import { NextResponse } from "next/server"; +import * as z from "zod/v4"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 30; + +const DEFAULT_PARTNER_LIMIT = 10; +const DEFAULT_CHUNKS_PER_PARTNER = 3; + +const partnerContentSearchSchema = z.object({ + query: z.string().trim().min(1).max(500), + limit: z.number().int().positive().max(50).default(DEFAULT_PARTNER_LIMIT), + chunksPerPartner: z + .number() + .int() + .positive() + .max(10) + .default(DEFAULT_CHUNKS_PER_PARTNER), + candidateChunkCount: z + .number() + .int() + .positive() + .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) + .optional(), + partnerIds: z.array(z.string()).min(1).max(100).optional(), +}); + +type PartnerContentSearchRow = { + chunkId: string; + partnerContentItemId: string; + partnerId: string; + partnerName: string; + partnerUsername: string | null; + partnerImage: string | null; + partnerDescription: string | null; + platformType: string; + platformIdentifier: string; + platformContentId: string; + contentUrl: string; + contentTitle: string | null; + contentPublishedAt: Date | null; + chunkIndex: number; + chunkText: string; + startMs: number | null; + endMs: number | null; + distance: number | string; +}; + +// POST /api/admin/partner-content/search +export const POST = withAdmin( + async ({ req }) => { + try { + const body = partnerContentSearchSchema.parse( + await parseRequestBody(req), + ); + const candidateChunkCount = + body.candidateChunkCount ?? + Math.min( + PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, + Math.max(25, body.limit * body.chunksPerPartner * 5), + ); + + const [queryEmbedding] = await embedPartnerContentTexts({ + input: [body.query], + inputType: "query", + }); + + const queryVector = serializeEmbedding(queryEmbedding); + const rows = body.partnerIds?.length + ? await searchPartnerContentChunksForPartners({ + queryVector, + partnerIds: body.partnerIds, + limit: candidateChunkCount, + }) + : await searchPartnerContentChunks({ + queryVector, + limit: candidateChunkCount, + }); + + return NextResponse.json({ + success: true, + query: body.query, + candidateChunkCount, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.model, + resultCount: rows.length, + partners: groupPartnerSearchResults({ + rows, + limit: body.limit, + chunksPerPartner: body.chunksPerPartner, + }), + }); + } catch (error) { + return handleAndReturnErrorResponse(error); + } + }, + { + requiredRoles: ["owner"], + }, +); + +async function searchPartnerContentChunks({ + queryVector, + limit, +}: { + queryVector: string; + limit: number; +}) { + return await prisma.$queryRaw` + SELECT + c.id AS chunkId, + c.partnerContentItemId, + c.partnerId, + p.name AS partnerName, + p.username AS partnerUsername, + p.image AS partnerImage, + p.description AS partnerDescription, + pp.type AS platformType, + pp.identifier AS platformIdentifier, + pci.platformContentId, + pci.url AS contentUrl, + pci.title AS contentTitle, + pci.publishedAt AS contentPublishedAt, + c.chunkIndex, + c.text AS chunkText, + c.startMs, + c.endMs, + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance + FROM PartnerContentChunk c + INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId + INNER JOIN Partner p ON p.id = c.partnerId + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + WHERE c.embedding IS NOT NULL + ORDER BY distance ASC + LIMIT ${limit} + `; +} + +async function searchPartnerContentChunksForPartners({ + queryVector, + partnerIds, + limit, +}: { + queryVector: string; + partnerIds: string[]; + limit: number; +}) { + return await prisma.$queryRaw` + SELECT + c.id AS chunkId, + c.partnerContentItemId, + c.partnerId, + p.name AS partnerName, + p.username AS partnerUsername, + p.image AS partnerImage, + p.description AS partnerDescription, + pp.type AS platformType, + pp.identifier AS platformIdentifier, + pci.platformContentId, + pci.url AS contentUrl, + pci.title AS contentTitle, + pci.publishedAt AS contentPublishedAt, + c.chunkIndex, + c.text AS chunkText, + c.startMs, + c.endMs, + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance + FROM PartnerContentChunk c + INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId + INNER JOIN Partner p ON p.id = c.partnerId + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + WHERE c.embedding IS NOT NULL + AND c.partnerId IN (${Prisma.join(partnerIds)}) + ORDER BY distance ASC + LIMIT ${limit} + `; +} + +function groupPartnerSearchResults({ + rows, + limit, + chunksPerPartner, +}: { + rows: PartnerContentSearchRow[]; + limit: number; + chunksPerPartner: number; +}) { + const partners = new Map< + string, + { + partnerId: string; + name: string; + username: string | null; + image: string | null; + description: string | null; + bestDistance: number; + score: number; + chunks: ReturnType[]; + } + >(); + + for (const row of rows) { + const distance = Number(row.distance); + const partner = partners.get(row.partnerId) ?? { + partnerId: row.partnerId, + name: row.partnerName, + username: row.partnerUsername, + image: row.partnerImage, + description: row.partnerDescription, + bestDistance: distance, + score: toScore(distance), + chunks: [], + }; + + partner.bestDistance = Math.min(partner.bestDistance, distance); + partner.score = toScore(partner.bestDistance); + + if (partner.chunks.length < chunksPerPartner) { + partner.chunks.push(toChunkResult(row, distance)); + } + + partners.set(row.partnerId, partner); + } + + return Array.from(partners.values()) + .sort((a, b) => a.bestDistance - b.bestDistance) + .slice(0, limit); +} + +function toChunkResult(row: PartnerContentSearchRow, distance: number) { + return { + chunkId: row.chunkId, + partnerContentItemId: row.partnerContentItemId, + platform: { + type: row.platformType, + identifier: row.platformIdentifier, + }, + content: { + platformContentId: row.platformContentId, + url: row.contentUrl, + title: row.contentTitle, + publishedAt: row.contentPublishedAt?.toISOString() ?? null, + }, + chunk: { + index: row.chunkIndex, + text: row.chunkText, + startMs: row.startMs, + endMs: row.endMs, + }, + distance, + score: toScore(distance), + }; +} + +function toScore(distance: number) { + return Number((1 - distance).toFixed(6)); +} + +function serializeEmbedding(embedding: number[]) { + const expectedDimensions = PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions; + + if (embedding.length !== expectedDimensions) { + throw new Error( + `Expected ${expectedDimensions} embedding dimensions, received ${embedding.length}.`, + ); + } + + return JSON.stringify( + embedding.map((value) => { + if (!Number.isFinite(value)) { + throw new Error("Voyage returned a non-finite embedding value."); + } + + return value; + }), + ); +} diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts index d42abc78247..ab28267448a 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -184,6 +184,7 @@ export const POST = withCron(async ({ rawBody }) => { partnerContentItemId: contentItem.id, maxChunks: payload.maxChunks, }, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, deduplicationId: createPartnerContentDeduplicationId( "partner-content-embed", payload.mode, diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts index 197e5ae0f67..355597247a6 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -66,164 +66,226 @@ export const POST = withCron(async ({ rawBody }) => { ); } + let transcriptWriteResult: Awaited< + ReturnType + >; + try { - const transcriptResponse = await getYouTubeVideoTranscript({ - url: contentItem.url, + transcriptWriteResult = await writeTranscriptChunks(contentItem); + } catch (error) { + await prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "error", + transcriptLastAttemptedAt: new Date(), + }, }); - const transcriptSegments = normalizeYouTubeTranscriptSegments( - transcriptResponse.transcript, - ); - const normalizedTranscript = transcriptSegments - .map(({ text }) => text.trim()) - .filter(Boolean) - .join(" "); - const transcriptHash = - transcriptSegments.length > 0 ? hashTranscript(transcriptSegments) : null; - - if (!transcriptHash) { - const updatedContentItem = await prisma.partnerContentItem.update({ + throw error; + } + + if (!transcriptWriteResult.transcriptAvailable) { + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + writesEnabled: true, + ...transcriptWriteResult, + }); + } + + const embedJob = await enqueueEmbedJob({ + mode: payload.mode, + runStamp: payload.runStamp, + partnerId: contentItem.partnerId, + partnerContentItemId: contentItem.id, + }); + + return NextResponse.json({ + success: true, + mode: payload.mode, + runStamp: payload.runStamp, + dryRun: payload.dryRun, + writesEnabled: true, + ...transcriptWriteResult, + ...embedJob, + }); +}); + +async function writeTranscriptChunks(contentItem: { + id: string; + partnerId: string; + url: string; +}) { + const transcriptResponse = await getYouTubeVideoTranscript({ + url: contentItem.url, + }); + + const transcriptSegments = normalizeYouTubeTranscriptSegments( + transcriptResponse.transcript, + ); + const normalizedTranscript = transcriptSegments + .map(({ text }) => text.trim()) + .filter(Boolean) + .join(" "); + const transcriptHash = + transcriptSegments.length > 0 ? hashTranscript(transcriptSegments) : null; + + if (!transcriptHash) { + const updatedContentItem = await prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "notAvailable", + transcriptLastAttemptedAt: new Date(), + normalizedTranscript: null, + transcriptHash: null, + hasTimestamps: false, + totalChunkCount: 0, + embeddedChunkCount: 0, + }, + select: { + id: true, + transcriptFetchStatus: true, + totalChunkCount: true, + embeddedChunkCount: true, + }, + }); + + await prisma.partnerContentChunk.deleteMany({ + where: { + partnerContentItemId: contentItem.id, + }, + }); + + return { + transcriptAvailable: false, + segmentCount: 0, + normalizedTranscriptLength: 0, + transcriptHash: null, + chunkCount: 0, + chunksCreated: 0, + contentItem: updatedContentItem, + }; + } + + const chunks = chunkTranscriptSegments(transcriptSegments); + const hasTimestamps = transcriptSegments.some( + ({ startMs, endMs }) => startMs !== null || endMs !== null, + ); + const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.model; + + const [updatedContentItem, deletedChunks, createChunksResult] = + await prisma.$transaction([ + prisma.partnerContentItem.update({ where: { id: contentItem.id, }, data: { - transcriptFetchStatus: "notAvailable", + transcriptFetchStatus: "fetched", transcriptLastAttemptedAt: new Date(), - normalizedTranscript: null, - transcriptHash: null, - hasTimestamps: false, - totalChunkCount: 0, + normalizedTranscript, + transcriptHash, + hasTimestamps, + embeddingModel, + totalChunkCount: chunks.length, embeddedChunkCount: 0, + lastFetchedAt: new Date(), }, select: { id: true, transcriptFetchStatus: true, + transcriptHash: true, totalChunkCount: true, embeddedChunkCount: true, + hasTimestamps: true, + lastFetchedAt: true, }, - }); - - await prisma.partnerContentChunk.deleteMany({ + }), + prisma.partnerContentChunk.deleteMany({ where: { partnerContentItemId: contentItem.id, }, - }); - - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - dryRun: payload.dryRun, - writesEnabled: true, - transcriptAvailable: false, - segmentCount: 0, - normalizedTranscriptLength: 0, - transcriptHash: null, - chunkCount: 0, - chunksCreated: 0, - contentItem: updatedContentItem, - }); - } - - const chunks = chunkTranscriptSegments(transcriptSegments); - const hasTimestamps = transcriptSegments.some( - ({ startMs, endMs }) => startMs !== null || endMs !== null, - ); - const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.model; - - const [updatedContentItem, deletedChunks, createChunksResult] = - await prisma.$transaction([ - prisma.partnerContentItem.update({ - where: { - id: contentItem.id, - }, - data: { - transcriptFetchStatus: "fetched", - transcriptLastAttemptedAt: new Date(), - normalizedTranscript, - transcriptHash, - hasTimestamps, - embeddingModel, - totalChunkCount: chunks.length, - embeddedChunkCount: 0, - lastFetchedAt: new Date(), - }, - select: { - id: true, - transcriptFetchStatus: true, - transcriptHash: true, - totalChunkCount: true, - embeddedChunkCount: true, - hasTimestamps: true, - lastFetchedAt: true, - }, - }), - prisma.partnerContentChunk.deleteMany({ - where: { - partnerContentItemId: contentItem.id, - }, - }), - prisma.partnerContentChunk.createMany({ - data: chunks.map((chunk) => ({ - partnerContentItemId: contentItem.id, - partnerId: contentItem.partnerId, - chunkIndex: chunk.chunkIndex, - text: chunk.text, - startMs: chunk.startMs, - endMs: chunk.endMs, - transcriptHash, - embeddingModel, - })), - }), - ]); + }), + prisma.partnerContentChunk.createMany({ + data: chunks.map((chunk) => ({ + partnerContentItemId: contentItem.id, + partnerId: contentItem.partnerId, + chunkIndex: chunk.chunkIndex, + text: chunk.text, + startMs: chunk.startMs, + endMs: chunk.endMs, + transcriptHash, + embeddingModel, + })), + }), + ]); + return { + transcriptAvailable: true, + segmentCount: transcriptSegments.length, + normalizedTranscriptLength: normalizedTranscript.length, + transcriptHash, + chunkCount: chunks.length, + chunksDeleted: deletedChunks.count, + chunksCreated: createChunksResult.count, + sampleSegments: transcriptSegments.slice(0, 5), + sampleChunks: chunks.slice(0, 3), + contentItem: updatedContentItem, + }; +} + +async function enqueueEmbedJob({ + mode, + runStamp, + partnerId, + partnerContentItemId, +}: { + mode: "incremental" | "backfill"; + runStamp: string; + partnerId: string; + partnerContentItemId: string; +}) { + try { const embedJob = await qstash.publishJSON({ url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), method: "POST", body: { - mode: payload.mode, - runStamp: payload.runStamp, - partnerId: contentItem.partnerId, - partnerContentItemId: contentItem.id, + mode, + runStamp, + partnerId, + partnerContentItemId, }, flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, deduplicationId: createPartnerContentDeduplicationId( "partner-content-embed", - payload.mode, - payload.runStamp, - contentItem.id, + mode, + runStamp, + partnerContentItemId, ), }); - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - dryRun: payload.dryRun, - writesEnabled: true, - transcriptAvailable: true, - segmentCount: transcriptSegments.length, - normalizedTranscriptLength: normalizedTranscript.length, - transcriptHash, - chunkCount: chunks.length, - chunksDeleted: deletedChunks.count, - chunksCreated: createChunksResult.count, + return { + embedEnqueueStatus: "enqueued" as const, embedJob, - sampleSegments: transcriptSegments.slice(0, 5), - sampleChunks: chunks.slice(0, 3), - contentItem: updatedContentItem, - }); + }; } catch (error) { - await prisma.partnerContentItem.update({ - where: { - id: contentItem.id, - }, - data: { - transcriptFetchStatus: "error", - transcriptLastAttemptedAt: new Date(), - }, + console.error("[PartnerContentSearch] Failed to enqueue embed job", { + error, + mode, + runStamp, + partnerId, + partnerContentItemId, }); - throw error; + return { + embedEnqueueStatus: "failed" as const, + embedEnqueueError: + error instanceof Error ? error.message : "Unknown QStash error", + }; } -}); +} From bcd7a1d58cf80768cab865c0db82f5cbc7caf37d Mon Sep 17 00:00:00 2001 From: David Clark Date: Sun, 7 Jun 2026 21:59:48 -0400 Subject: [PATCH 06/25] normalize partner content schema and ingestion routes --- .../api/admin/partner-content/search/route.ts | 108 ++++--------- .../api/cron/partner-content/embed/route.ts | 140 +++++------------ .../partner-content/enumerate/page/route.ts | 77 +++------- .../cron/partner-content/enumerate/route.ts | 43 ++---- .../api/cron/partner-content/fetch/route.ts | 61 +++----- .../cron/partner-content/transcript/route.ts | 145 +++++++----------- apps/web/lib/api/scrape-creators/client.ts | 6 - .../ingestion/enqueue.ts | 67 +++++++- apps/web/lib/partner-content-search/voyage.ts | 38 ++++- .../partner-content-search-vector-index.sql | 27 ++++ .../schema/partner-content-search.prisma | 26 ++-- packages/prisma/schema/partner.prisma | 2 + packages/prisma/schema/platform.prisma | 3 +- 13 files changed, 328 insertions(+), 415 deletions(-) create mode 100644 packages/prisma/manual/partner-content-search-vector-index.sql diff --git a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts index dcf10d5ca2f..8c6dec50569 100644 --- a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts +++ b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts @@ -5,9 +5,12 @@ import { PARTNER_CONTENT_SEARCH_LIMITS, PARTNER_CONTENT_SEARCH_MODELS, } from "@/lib/partner-content-search/constants"; -import { embedPartnerContentTexts } from "@/lib/partner-content-search/voyage"; +import { + embedPartnerContentTexts, + serializeEmbeddingForVector, +} from "@/lib/partner-content-search/voyage"; import { prisma } from "@dub/prisma"; -import { Prisma } from "@dub/prisma/client"; +import { PlatformType, Prisma } from "@dub/prisma/client"; import { NextResponse } from "next/server"; import * as z from "zod/v4"; @@ -33,6 +36,7 @@ const partnerContentSearchSchema = z.object({ .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) .optional(), partnerIds: z.array(z.string()).min(1).max(100).optional(), + platform: z.enum(PlatformType).optional(), }); type PartnerContentSearchRow = { @@ -48,6 +52,7 @@ type PartnerContentSearchRow = { platformContentId: string; contentUrl: string; contentTitle: string | null; + contentThumbnailUrl: string | null; contentPublishedAt: Date | null; chunkIndex: number; chunkText: string; @@ -75,17 +80,13 @@ export const POST = withAdmin( inputType: "query", }); - const queryVector = serializeEmbedding(queryEmbedding); - const rows = body.partnerIds?.length - ? await searchPartnerContentChunksForPartners({ - queryVector, - partnerIds: body.partnerIds, - limit: candidateChunkCount, - }) - : await searchPartnerContentChunks({ - queryVector, - limit: candidateChunkCount, - }); + const queryVector = serializeEmbeddingForVector(queryEmbedding); + const rows = await searchPartnerContentChunks({ + queryVector, + limit: candidateChunkCount, + partnerIds: body.partnerIds, + platform: body.platform, + }); return NextResponse.json({ success: true, @@ -111,50 +112,22 @@ export const POST = withAdmin( async function searchPartnerContentChunks({ queryVector, limit, -}: { - queryVector: string; - limit: number; -}) { - return await prisma.$queryRaw` - SELECT - c.id AS chunkId, - c.partnerContentItemId, - c.partnerId, - p.name AS partnerName, - p.username AS partnerUsername, - p.image AS partnerImage, - p.description AS partnerDescription, - pp.type AS platformType, - pp.identifier AS platformIdentifier, - pci.platformContentId, - pci.url AS contentUrl, - pci.title AS contentTitle, - pci.publishedAt AS contentPublishedAt, - c.chunkIndex, - c.text AS chunkText, - c.startMs, - c.endMs, - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance - FROM PartnerContentChunk c - INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId - INNER JOIN Partner p ON p.id = c.partnerId - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - WHERE c.embedding IS NOT NULL - ORDER BY distance ASC - LIMIT ${limit} - `; -} - -async function searchPartnerContentChunksForPartners({ - queryVector, partnerIds, - limit, + platform, }: { queryVector: string; - partnerIds: string[]; limit: number; + partnerIds?: string[]; + platform?: PlatformType; }) { - return await prisma.$queryRaw` + const partnerFilter = partnerIds?.length + ? Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})` + : Prisma.empty; + const platformFilter = platform + ? Prisma.sql`AND pp.type = ${platform}` + : Prisma.empty; + + return await prisma.$queryRaw(Prisma.sql` SELECT c.id AS chunkId, c.partnerContentItemId, @@ -168,9 +141,10 @@ async function searchPartnerContentChunksForPartners({ pci.platformContentId, pci.url AS contentUrl, pci.title AS contentTitle, + pci.thumbnailUrl AS contentThumbnailUrl, pci.publishedAt AS contentPublishedAt, c.chunkIndex, - c.text AS chunkText, + c.chunkText, c.startMs, c.endMs, DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance @@ -179,10 +153,11 @@ async function searchPartnerContentChunksForPartners({ INNER JOIN Partner p ON p.id = c.partnerId INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId WHERE c.embedding IS NOT NULL - AND c.partnerId IN (${Prisma.join(partnerIds)}) + ${partnerFilter} + ${platformFilter} ORDER BY distance ASC LIMIT ${limit} - `; + `); } function groupPartnerSearchResults({ @@ -248,11 +223,12 @@ function toChunkResult(row: PartnerContentSearchRow, distance: number) { platformContentId: row.platformContentId, url: row.contentUrl, title: row.contentTitle, + thumbnailUrl: row.contentThumbnailUrl, publishedAt: row.contentPublishedAt?.toISOString() ?? null, }, chunk: { index: row.chunkIndex, - text: row.chunkText, + chunkText: row.chunkText, startMs: row.startMs, endMs: row.endMs, }, @@ -264,23 +240,3 @@ function toChunkResult(row: PartnerContentSearchRow, distance: number) { function toScore(distance: number) { return Number((1 - distance).toFixed(6)); } - -function serializeEmbedding(embedding: number[]) { - const expectedDimensions = PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions; - - if (embedding.length !== expectedDimensions) { - throw new Error( - `Expected ${expectedDimensions} embedding dimensions, received ${embedding.length}.`, - ); - } - - return JSON.stringify( - embedding.map((value) => { - if (!Number.isFinite(value)) { - throw new Error("Voyage returned a non-finite embedding value."); - } - - return value; - }), - ); -} diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts index ab28267448a..ca6e291957c 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -6,11 +6,16 @@ import { getPartnerContentUrl, PARTNER_CONTENT_EMBED_FLOW_CONTROL, PARTNER_CONTENT_SEARCH_ROUTES, + parsePartnerContentCronPayload, partnerContentEmbedPayloadSchema, + PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; -import { embedPartnerContentTexts } from "@/lib/partner-content-search/voyage"; +import { + embedPartnerContentTexts, + serializeEmbeddingForVector, + VoyageApiError, +} from "@/lib/partner-content-search/voyage"; import { prisma } from "@dub/prisma"; -import { NextResponse } from "next/server"; import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; @@ -20,7 +25,7 @@ const EMBED_RATE_LIMIT_RETRY_DELAY_SECONDS = 120; type UnembeddedChunk = { id: string; - text: string; + chunkText: string; }; type EmbeddedChunkCount = { @@ -29,7 +34,11 @@ type EmbeddedChunkCount = { // POST /api/cron/partner-content/embed export const POST = withCron(async ({ rawBody }) => { - const payload = partnerContentEmbedPayloadSchema.parse(JSON.parse(rawBody)); + const payload = parsePartnerContentCronPayload( + partnerContentEmbedPayloadSchema, + rawBody, + ); + if (payload instanceof Response) return payload; if (!payload.partnerContentItemId) { return enqueueEmbedJobsForPartnerPlatform(payload); @@ -66,18 +75,13 @@ export const POST = withCron(async ({ rawBody }) => { contentItem.transcriptFetchStatus !== "fetched" || contentItem.totalChunkCount === 0 ) { - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - skipped: true, - reason: "Content item has no fetched transcript chunks to embed.", - contentItem, - }); + return logAndRespond( + `[PartnerContentSearch] Skipping embed for content item ${contentItem.id}: no fetched transcript chunks for ${payload.mode} run ${payload.runStamp}.`, + ); } const chunks = await prisma.$queryRaw` - SELECT id, text + SELECT id, chunkText FROM PartnerContentChunk WHERE partnerContentItemId = ${contentItem.id} AND embedding IS NULL @@ -88,31 +92,22 @@ export const POST = withCron(async ({ rawBody }) => { if (chunks.length === 0) { const embeddedChunkCount = await refreshEmbeddedChunkCount(contentItem.id); - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - skipped: true, - reason: "All chunks are already embedded.", - embeddedChunkCount, - contentItem: { - ...contentItem, - embeddedChunkCount, - }, - }); + return logAndRespond( + `[PartnerContentSearch] Content item ${contentItem.id} already fully embedded (${embeddedChunkCount}/${contentItem.totalChunkCount}) for ${payload.mode} run ${payload.runStamp}.`, + ); } let embeddings: number[][]; try { embeddings = await embedPartnerContentTexts({ - input: chunks.map(({ text }) => text), + input: chunks.map(({ chunkText }) => chunkText), inputType: "document", }); } catch (error) { if (!isVoyageRateLimitError(error)) throw error; - const response = await qstash.publishJSON({ + await qstash.publishJSON({ url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), method: "POST", body: { @@ -132,20 +127,9 @@ export const POST = withCron(async ({ rawBody }) => { ), }); - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - rateLimited: true, - retryScheduled: true, - retryMessageId: response.messageId, - contentItem: { - id: contentItem.id, - transcriptFetchStatus: contentItem.transcriptFetchStatus, - totalChunkCount: contentItem.totalChunkCount, - embeddedChunkCount: contentItem.embeddedChunkCount, - }, - }); + return logAndRespond( + `[PartnerContentSearch] Voyage rate-limited embed for content item ${contentItem.id}; retry scheduled for ${payload.mode} run ${payload.runStamp}.`, + ); } if (embeddings.length !== chunks.length) { @@ -158,7 +142,7 @@ export const POST = withCron(async ({ rawBody }) => { chunks.map((chunk, index) => prisma.$executeRaw` UPDATE PartnerContentChunk - SET embedding = TO_VECTOR(${serializeEmbedding(embeddings[index])}), + SET embedding = TO_VECTOR(${serializeEmbeddingForVector(embeddings[index])}), embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.model} WHERE id = ${chunk.id} `, @@ -171,10 +155,8 @@ export const POST = withCron(async ({ rawBody }) => { contentItem.totalChunkCount - embeddedChunkCount, ); - let continuationMessageId: string | undefined; - if (remainingChunkCount > 0) { - const response = await qstash.publishJSON({ + await qstash.publishJSON({ url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), method: "POST", body: { @@ -193,26 +175,11 @@ export const POST = withCron(async ({ rawBody }) => { embeddedChunkCount, ), }); - - continuationMessageId = response.messageId; } - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - embeddedNow: chunks.length, - embeddedChunkCount, - totalChunkCount: contentItem.totalChunkCount, - remainingChunkCount, - continuationMessageId, - contentItem: { - id: contentItem.id, - transcriptFetchStatus: contentItem.transcriptFetchStatus, - totalChunkCount: contentItem.totalChunkCount, - embeddedChunkCount, - }, - }); + return logAndRespond( + `[PartnerContentSearch] Embedded ${chunks.length} chunks for content item ${contentItem.id} (${embeddedChunkCount}/${contentItem.totalChunkCount}, ${remainingChunkCount} remaining) on ${payload.mode} run ${payload.runStamp}.`, + ); }); async function enqueueEmbedJobsForPartnerPlatform({ @@ -223,7 +190,7 @@ async function enqueueEmbedJobsForPartnerPlatform({ limitContentItems, maxChunks, }: { - mode: "incremental" | "backfill"; + mode: PartnerContentIngestionMode; runStamp: string; partnerId: string; partnerPlatformId?: string; @@ -281,22 +248,13 @@ async function enqueueEmbedJobsForPartnerPlatform({ ), })); - const qstashResponses = - messages.length === 0 ? [] : await qstash.batchJSON(messages); - - return NextResponse.json({ - success: true, - mode, - runStamp, - partnerId, - partnerPlatformId, - inspectedContentItemCount: contentItems.length, - embedJobCount: messages.length, - pendingContentItemCount: pendingContentItems.length, - flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, - qstashResponses, - contentItems: pendingContentItems, - }); + if (messages.length > 0) { + await qstash.batchJSON(messages); + } + + return logAndRespond( + `[PartnerContentSearch] Enqueued ${messages.length} embed jobs (${pendingContentItems.length} pending of ${contentItems.length} inspected) for partner platform ${partnerPlatformId} on ${mode} run ${runStamp}.`, + ); } async function refreshEmbeddedChunkCount(partnerContentItemId: string) { @@ -322,28 +280,8 @@ async function refreshEmbeddedChunkCount(partnerContentItemId: string) { return embeddedChunkCount; } -function serializeEmbedding(embedding: number[]) { - const expectedDimensions = PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions; - - if (embedding.length !== expectedDimensions) { - throw new Error( - `Expected ${expectedDimensions} embedding dimensions, received ${embedding.length}.`, - ); - } - - return JSON.stringify( - embedding.map((value) => { - if (!Number.isFinite(value)) { - throw new Error("Voyage returned a non-finite embedding value."); - } - - return value; - }), - ); -} - function isVoyageRateLimitError(error: unknown) { - return error instanceof Error && error.message.includes("failed: 429"); + return error instanceof VoyageApiError && error.status === 429; } function getRateLimitRetryDelaySeconds(contentItemId: string) { diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts index 2fa207f746e..00e6c547c5a 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/page/route.ts @@ -1,15 +1,14 @@ import { qstash } from "@/lib/cron"; import { withCron } from "@/lib/cron/with-cron"; import { + buildEligiblePartnerPlatformWhere, createPartnerContentDeduplicationId, getPartnerContentUrl, - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS, PARTNER_CONTENT_SEARCH_ROUTES, + parsePartnerContentCronPayload, partnerContentEnumeratePagePayloadSchema, } from "@/lib/partner-content-search/ingestion/enqueue"; import { prisma } from "@dub/prisma"; -import { Prisma } from "@dub/prisma/client"; -import { NextResponse } from "next/server"; import { logAndRespond } from "../../../utils"; export const dynamic = "force-dynamic"; @@ -17,9 +16,11 @@ export const maxDuration = 60; // POST /api/cron/partner-content/enumerate/page export const POST = withCron(async ({ rawBody }) => { - const payload = partnerContentEnumeratePagePayloadSchema.parse( - JSON.parse(rawBody), + const payload = parsePartnerContentCronPayload( + partnerContentEnumeratePagePayloadSchema, + rawBody, ); + if (payload instanceof Response) return payload; const partners = await prisma.partner.findMany({ where: { @@ -30,7 +31,10 @@ export const POST = withCron(async ({ rawBody }) => { select: { id: true, platforms: { - where: buildEligiblePartnerPlatformWhere(payload), + where: buildEligiblePartnerPlatformWhere({ + mode: payload.mode, + platforms: payload.filter.platforms, + }), select: { id: true, partnerId: true, @@ -88,54 +92,17 @@ export const POST = withCron(async ({ rawBody }) => { }, })); - const qstashResponses = payload.dryRun - ? [] - : await qstash.batchJSON(fetchMessages); - - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - dryRun: payload.dryRun, - partnerCount: partners.length, - partnerPlatformCount: partnerPlatforms.length, - fetchJobCount: fetchMessages.length, - qstashResponses, - partnerPlatforms, - }); -}); + if (!payload.dryRun) { + await qstash.batchJSON(fetchMessages); + } -function buildEligiblePartnerPlatformWhere({ - mode, - filter, -}: { - mode: "incremental" | "backfill"; - filter: { - platforms: Array<"youtube" | "instagram" | "tiktok">; - }; -}): Prisma.PartnerPlatformWhereInput { - const incrementalCutoff = new Date( - Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, + return logAndRespond( + `[PartnerContentSearch] ${ + payload.dryRun ? "Dry-run enumerated" : "Enqueued" + } ${fetchMessages.length} fetch jobs across ${ + partnerPlatforms.length + } platforms (${partners.length} partners) for ${payload.mode} run ${ + payload.runStamp + }.`, ); - - return { - type: { - in: filter.platforms, - }, - verifiedAt: { - not: null, - }, - ...(mode === "incremental" && { - OR: [ - { - contentLastFetchedAt: null, - }, - { - contentLastFetchedAt: { - lt: incrementalCutoff, - }, - }, - ], - }), - }; -} +}); diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts index b11cc403058..4646d44170e 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts @@ -1,13 +1,16 @@ import { qstash } from "@/lib/cron"; import { withCron } from "@/lib/cron/with-cron"; import { + buildEligiblePartnerPlatformWhere, createPartnerContentDeduplicationId, getPartnerContentUrl, PARTNER_CONTENT_ENUMERATE_PAGE_SIZE, - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS, PARTNER_CONTENT_SEARCH_ROUTES, + parsePartnerContentCronPayload, partnerContentEnumeratePayloadSchema, + PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; +import { PartnerContentPlatform } from "@/lib/partner-content-search/types"; import { prisma } from "@dub/prisma"; import { Prisma } from "@dub/prisma/client"; import { logAndRespond } from "../../utils"; @@ -17,9 +20,11 @@ export const maxDuration = 60; // POST /api/cron/partner-content/enumerate export const POST = withCron(async ({ rawBody }) => { - const payload = partnerContentEnumeratePayloadSchema.parse( - JSON.parse(rawBody), + const payload = parsePartnerContentCronPayload( + partnerContentEnumeratePayloadSchema, + rawBody, ); + if (payload instanceof Response) return payload; // Process a single page per invocation and hand the next cursor back to // this same route, rather than draining the whole partner set in one @@ -132,17 +137,13 @@ function buildEligiblePartnerWhere({ mode, filter, }: { - mode: "incremental" | "backfill"; + mode: PartnerContentIngestionMode; filter: { partnerId?: string; partnerIds?: string[]; - platforms: Array<"youtube" | "instagram" | "tiktok">; + platforms: PartnerContentPlatform[]; }; }): Prisma.PartnerWhereInput { - const incrementalCutoff = new Date( - Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, - ); - return { networkStatus: { in: ["approved", "trusted"], @@ -156,26 +157,10 @@ function buildEligiblePartnerWhere({ }, }), platforms: { - some: { - type: { - in: filter.platforms, - }, - verifiedAt: { - not: null, - }, - ...(mode === "incremental" && { - OR: [ - { - contentLastFetchedAt: null, - }, - { - contentLastFetchedAt: { - lt: incrementalCutoff, - }, - }, - ], - }), - }, + some: buildEligiblePartnerPlatformWhere({ + mode, + platforms: filter.platforms, + }), }, }; } diff --git a/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts b/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts index 8152d19f3a8..3f8ca7edda9 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/fetch/route.ts @@ -6,14 +6,16 @@ import { createPartnerContentDeduplicationId, getPartnerContentUrl, PARTNER_CONTENT_SEARCH_ROUTES, + parsePartnerContentCronPayload, partnerContentFetchPayloadSchema, + PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; import { NormalizedPartnerContentItem, normalizeYouTubeChannelVideo, } from "@/lib/partner-content-search/ingestion/normalize-content"; +import { PartnerContentPlatform } from "@/lib/partner-content-search/types"; import { prisma } from "@dub/prisma"; -import { NextResponse } from "next/server"; import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; @@ -23,7 +25,11 @@ const MAX_YOUTUBE_CHANNEL_VIDEO_PAGES = 3; // POST /api/cron/partner-content/fetch export const POST = withCron(async ({ rawBody }) => { - const payload = partnerContentFetchPayloadSchema.parse(JSON.parse(rawBody)); + const payload = parsePartnerContentCronPayload( + partnerContentFetchPayloadSchema, + rawBody, + ); + if (payload instanceof Response) return payload; const partnerPlatform = await prisma.partnerPlatform.findUnique({ where: { @@ -104,13 +110,8 @@ export const POST = withCron(async ({ rawBody }) => { ? fetchedContentItems : newContentItems; - const writeResult = payload.dryRun - ? { - contentItemsCreated: 0, - partnerPlatformUpdated: false, - transcriptJobCount: 0, - qstashResponses: [], - } + const { contentItemsCreated, transcriptJobCount } = payload.dryRun + ? { contentItemsCreated: 0, transcriptJobCount: 0 } : await writeFetchedContentItems({ mode: payload.mode, runStamp: payload.runStamp, @@ -123,24 +124,15 @@ export const POST = withCron(async ({ rawBody }) => { latestContentUrl: latestContentItem?.url ?? null, }); - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - dryRun: payload.dryRun, - fetchedContentCount: fetchedContentItems.length, - existingContentCount: existingContentItems.length, - newContentCount: newContentItems.length, - writesEnabled: !payload.dryRun, - wouldWriteContentItems: newContentItems.length, - forceTranscriptJobs: payload.forceTranscriptJobs, - wouldPublishTranscriptJobs: payload.dryRun - ? 0 - : contentItemsForTranscriptJobs.length, - ...writeResult, - newContentItems: newContentItems.slice(0, 10), - partnerPlatform, - }); + return logAndRespond( + `[PartnerContentSearch] ${ + payload.dryRun ? "Dry-run fetched" : "Fetched" + } ${fetchedContentItems.length} items (${newContentItems.length} new, ${ + existingContentItems.length + } existing) for partner platform ${partnerPlatform.id}: created ${contentItemsCreated}, enqueued ${transcriptJobCount} transcript jobs on ${ + payload.mode + } run ${payload.runStamp}.`, + ); }); async function fetchRecentYouTubeContent({ @@ -218,12 +210,12 @@ async function writeFetchedContentItems({ contentItemsForTranscriptJobs, latestContentUrl, }: { - mode: "incremental" | "backfill"; + mode: PartnerContentIngestionMode; runStamp: string; dryRun: boolean; partnerId: string; partnerPlatformId: string; - platform: "youtube" | "instagram" | "tiktok"; + platform: PartnerContentPlatform; contentItems: NormalizedPartnerContentItem[]; contentItemsForTranscriptJobs: NormalizedPartnerContentItem[]; latestContentUrl: string | null; @@ -248,7 +240,7 @@ async function writeFetchedContentItems({ viewCount: item.viewCount === null ? null : BigInt(Math.trunc(item.viewCount)), transcriptFetchStatus: "pending", - hasTimestamps: false, + transcriptHasTimestamps: false, totalChunkCount: 0, embeddedChunkCount: 0, })), @@ -305,15 +297,12 @@ async function writeFetchedContentItems({ }, })); - const qstashResponses = - transcriptMessages.length === 0 - ? [] - : await qstash.batchJSON(transcriptMessages); + if (transcriptMessages.length > 0) { + await qstash.batchJSON(transcriptMessages); + } return { contentItemsCreated: createResult.count, - partnerPlatformUpdated: true, transcriptJobCount: transcriptMessages.length, - qstashResponses, }; } diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts index 355597247a6..80b2bb5d59d 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -11,11 +11,12 @@ import { getPartnerContentUrl, PARTNER_CONTENT_EMBED_FLOW_CONTROL, PARTNER_CONTENT_SEARCH_ROUTES, + parsePartnerContentCronPayload, partnerContentTranscriptPayloadSchema, + PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; import { normalizeYouTubeTranscriptSegments } from "@/lib/partner-content-search/ingestion/normalize-content"; import { prisma } from "@dub/prisma"; -import { NextResponse } from "next/server"; import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; @@ -23,9 +24,11 @@ export const maxDuration = 60; // POST /api/cron/partner-content/transcript export const POST = withCron(async ({ rawBody }) => { - const payload = partnerContentTranscriptPayloadSchema.parse( - JSON.parse(rawBody), + const payload = parsePartnerContentCronPayload( + partnerContentTranscriptPayloadSchema, + rawBody, ); + if (payload instanceof Response) return payload; const contentItem = await prisma.partnerContentItem.findUnique({ where: { @@ -87,32 +90,21 @@ export const POST = withCron(async ({ rawBody }) => { } if (!transcriptWriteResult.transcriptAvailable) { - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - dryRun: payload.dryRun, - writesEnabled: true, - ...transcriptWriteResult, - }); + return logAndRespond( + `[PartnerContentSearch] No transcript available for content item ${contentItem.id} on ${payload.mode} run ${payload.runStamp}.`, + ); } - const embedJob = await enqueueEmbedJob({ + const { embedEnqueueStatus } = await enqueueEmbedJob({ mode: payload.mode, runStamp: payload.runStamp, partnerId: contentItem.partnerId, partnerContentItemId: contentItem.id, }); - return NextResponse.json({ - success: true, - mode: payload.mode, - runStamp: payload.runStamp, - dryRun: payload.dryRun, - writesEnabled: true, - ...transcriptWriteResult, - ...embedJob, - }); + return logAndRespond( + `[PartnerContentSearch] Transcribed content item ${contentItem.id}: ${transcriptWriteResult.chunkCount} chunks (${transcriptWriteResult.chunksCreated} created), embed enqueue ${embedEnqueueStatus} for ${payload.mode} run ${payload.runStamp}.`, + ); }); async function writeTranscriptChunks(contentItem: { @@ -135,7 +127,7 @@ async function writeTranscriptChunks(contentItem: { transcriptSegments.length > 0 ? hashTranscript(transcriptSegments) : null; if (!transcriptHash) { - const updatedContentItem = await prisma.partnerContentItem.update({ + await prisma.partnerContentItem.update({ where: { id: contentItem.id, }, @@ -144,16 +136,10 @@ async function writeTranscriptChunks(contentItem: { transcriptLastAttemptedAt: new Date(), normalizedTranscript: null, transcriptHash: null, - hasTimestamps: false, + transcriptHasTimestamps: false, totalChunkCount: 0, embeddedChunkCount: 0, }, - select: { - id: true, - transcriptFetchStatus: true, - totalChunkCount: true, - embeddedChunkCount: true, - }, }); await prisma.partnerContentChunk.deleteMany({ @@ -169,61 +155,50 @@ async function writeTranscriptChunks(contentItem: { transcriptHash: null, chunkCount: 0, chunksCreated: 0, - contentItem: updatedContentItem, }; } const chunks = chunkTranscriptSegments(transcriptSegments); - const hasTimestamps = transcriptSegments.some( + const transcriptHasTimestamps = transcriptSegments.some( ({ startMs, endMs }) => startMs !== null || endMs !== null, ); const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.model; - const [updatedContentItem, deletedChunks, createChunksResult] = - await prisma.$transaction([ - prisma.partnerContentItem.update({ - where: { - id: contentItem.id, - }, - data: { - transcriptFetchStatus: "fetched", - transcriptLastAttemptedAt: new Date(), - normalizedTranscript, - transcriptHash, - hasTimestamps, - embeddingModel, - totalChunkCount: chunks.length, - embeddedChunkCount: 0, - lastFetchedAt: new Date(), - }, - select: { - id: true, - transcriptFetchStatus: true, - transcriptHash: true, - totalChunkCount: true, - embeddedChunkCount: true, - hasTimestamps: true, - lastFetchedAt: true, - }, - }), - prisma.partnerContentChunk.deleteMany({ - where: { - partnerContentItemId: contentItem.id, - }, - }), - prisma.partnerContentChunk.createMany({ - data: chunks.map((chunk) => ({ - partnerContentItemId: contentItem.id, - partnerId: contentItem.partnerId, - chunkIndex: chunk.chunkIndex, - text: chunk.text, - startMs: chunk.startMs, - endMs: chunk.endMs, - transcriptHash, - embeddingModel, - })), - }), - ]); + const [, deletedChunks, createChunksResult] = await prisma.$transaction([ + prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "fetched", + transcriptLastAttemptedAt: new Date(), + normalizedTranscript, + transcriptHash, + transcriptHasTimestamps, + embeddingModel, + totalChunkCount: chunks.length, + embeddedChunkCount: 0, + lastFetchedAt: new Date(), + }, + }), + prisma.partnerContentChunk.deleteMany({ + where: { + partnerContentItemId: contentItem.id, + }, + }), + prisma.partnerContentChunk.createMany({ + data: chunks.map((chunk) => ({ + partnerContentItemId: contentItem.id, + partnerId: contentItem.partnerId, + chunkIndex: chunk.chunkIndex, + chunkText: chunk.text, + startMs: chunk.startMs, + endMs: chunk.endMs, + transcriptHash, + embeddingModel, + })), + }), + ]); return { transcriptAvailable: true, @@ -233,9 +208,6 @@ async function writeTranscriptChunks(contentItem: { chunkCount: chunks.length, chunksDeleted: deletedChunks.count, chunksCreated: createChunksResult.count, - sampleSegments: transcriptSegments.slice(0, 5), - sampleChunks: chunks.slice(0, 3), - contentItem: updatedContentItem, }; } @@ -245,13 +217,13 @@ async function enqueueEmbedJob({ partnerId, partnerContentItemId, }: { - mode: "incremental" | "backfill"; + mode: PartnerContentIngestionMode; runStamp: string; partnerId: string; partnerContentItemId: string; }) { try { - const embedJob = await qstash.publishJSON({ + await qstash.publishJSON({ url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), method: "POST", body: { @@ -269,10 +241,7 @@ async function enqueueEmbedJob({ ), }); - return { - embedEnqueueStatus: "enqueued" as const, - embedJob, - }; + return { embedEnqueueStatus: "enqueued" as const }; } catch (error) { console.error("[PartnerContentSearch] Failed to enqueue embed job", { error, @@ -282,10 +251,6 @@ async function enqueueEmbedJob({ partnerContentItemId, }); - return { - embedEnqueueStatus: "failed" as const, - embedEnqueueError: - error instanceof Error ? error.message : "Unknown QStash error", - }; + return { embedEnqueueStatus: "failed" as const }; } } diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.ts index 45f92548164..0b0710d87ba 100644 --- a/apps/web/lib/api/scrape-creators/client.ts +++ b/apps/web/lib/api/scrape-creators/client.ts @@ -81,10 +81,4 @@ export const scrapeCreatorsFetch = createFetch({ onError: ({ error }) => { console.error("[ScrapeCreators] Error", error); }, - // onResponse: async ({ response }) => { - // console.log( - // "[ScrapeCreators] Response", - // prettyPrint(await response.clone().json()), - // ); - // }, }); diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index 82e4c3a7bbe..a3965e33fc4 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -1,7 +1,9 @@ import { qstash } from "@/lib/cron"; import { APP_DOMAIN_WITH_NGROK } from "@dub/utils"; +import { logAndRespond } from "app/(ee)/api/cron/utils"; +import type { Prisma } from "@dub/prisma/client"; import * as z from "zod/v4"; -import { PARTNER_CONTENT_SEARCH_PLATFORMS } from "../types"; +import { PartnerContentPlatform, PARTNER_CONTENT_SEARCH_PLATFORMS } from "../types"; // Partners enumerated per page / per page-worker job. Defaults to 500; override // with the PARTNER_CONTENT_ENUMERATE_PAGE_SIZE env var (e.g. 3) to exercise the @@ -36,6 +38,10 @@ export const partnerContentIngestionModeSchema = z.enum([ "backfill", ]); +export type PartnerContentIngestionMode = z.infer< + typeof partnerContentIngestionModeSchema +>; + export const partnerContentIngestionFilterSchema = z .object({ partnerId: z.string().optional(), @@ -46,7 +52,7 @@ export const partnerContentIngestionFilterSchema = z .default([...PARTNER_CONTENT_SEARCH_PLATFORMS]), limitPartners: z.number().int().positive().max(100_000).optional(), }) - .default({}) + .prefault({}) .refine((filter) => !(filter.partnerId && filter.partnerIds?.length), { message: "Use either partnerId or partnerIds, not both.", }); @@ -148,3 +154,60 @@ export async function enqueuePartnerContentEnumerate( ), }); } + +export function getIncrementalRefreshCutoff() { + return new Date( + Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, + ); +} + +// Shared platform-eligibility predicate for the enumerate fan-out. The +// partner-level (enumerate) and platform-level (enumerate/page) routes both +// filter platforms on the same verified + incremental-recency rules. +export function buildEligiblePartnerPlatformWhere({ + mode, + platforms, +}: { + mode: PartnerContentIngestionMode; + platforms: PartnerContentPlatform[]; +}): Prisma.PartnerPlatformWhereInput { + return { + type: { + in: platforms, + }, + verifiedAt: { + not: null, + }, + ...(mode === "incremental" && { + OR: [ + { + contentLastFetchedAt: null, + }, + { + contentLastFetchedAt: { + lt: getIncrementalRefreshCutoff(), + }, + }, + ], + }), + }; +} + +// Parse a QStash payload for a cron route. A malformed payload is a permanent +// error QStash should not retry, so respond 400 rather than letting it bubble +// to withCron's 500 + error alert. Returns the parsed payload, or a Response to +// return directly from the handler. +export function parsePartnerContentCronPayload( + schema: T, + rawBody: string, +): z.infer | Response { + try { + return schema.parse(JSON.parse(rawBody)) as z.infer; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return logAndRespond( + `[PartnerContentSearch] Ignoring invalid cron payload: ${message}`, + { status: 400, logLevel: "warn" }, + ); + } +} diff --git a/apps/web/lib/partner-content-search/voyage.ts b/apps/web/lib/partner-content-search/voyage.ts index 6b87b95cbff..4040de9acda 100644 --- a/apps/web/lib/partner-content-search/voyage.ts +++ b/apps/web/lib/partner-content-search/voyage.ts @@ -27,6 +27,40 @@ type VoyageRerankResponse = { type VoyageFetch = typeof fetch; +// Typed error so callers can branch on the HTTP status (e.g. 429 rate limits) +// instead of string-matching the message. +export class VoyageApiError extends Error { + constructor( + readonly status: number, + readonly endpoint: string, + ) { + super(`Voyage ${endpoint} request failed: ${status}`); + this.name = "VoyageApiError"; + } +} + +// Serialize an embedding for use in raw SQL TO_VECTOR(...) statements. Shared by +// the embed cron route (writes) and the admin search route (query vector). +export function serializeEmbeddingForVector(embedding: number[]) { + const expectedDimensions = PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions; + + if (embedding.length !== expectedDimensions) { + throw new Error( + `Expected ${expectedDimensions} embedding dimensions, received ${embedding.length}.`, + ); + } + + return JSON.stringify( + embedding.map((value) => { + if (!Number.isFinite(value)) { + throw new Error("Voyage returned a non-finite embedding value."); + } + + return value; + }), + ); +} + export function buildVoyageEmbeddingRequest({ input, inputType, @@ -100,7 +134,7 @@ export async function embedPartnerContentTexts({ }); if (!response.ok) { - throw new Error(`Voyage embeddings request failed: ${response.status}`); + throw new VoyageApiError(response.status, "embeddings"); } const result = (await response.json()) as VoyageEmbeddingResponse; @@ -144,7 +178,7 @@ export async function rerankPartnerContent({ }); if (!response.ok) { - throw new Error(`Voyage rerank request failed: ${response.status}`); + throw new VoyageApiError(response.status, "rerank"); } const result = (await response.json()) as VoyageRerankResponse; diff --git a/packages/prisma/manual/partner-content-search-vector-index.sql b/packages/prisma/manual/partner-content-search-vector-index.sql new file mode 100644 index 00000000000..375fa87eede --- /dev/null +++ b/packages/prisma/manual/partner-content-search-vector-index.sql @@ -0,0 +1,27 @@ +-- Manual PlanetScale DDL for partner natural-language search. +-- +-- Prisma can create the PartnerContentChunk.embedding VECTOR(1024) column via +-- `prisma db push`, but it cannot express PlanetScale VECTOR INDEX DDL. Run +-- this after the PartnerContentChunk table and embedding column exist on the +-- target PlanetScale branch. +-- +-- Keep the index distance in sync with the search query in: +-- apps/web/app/(ee)/api/admin/partner-content/search/route.ts +-- That query currently uses DISTANCE(..., 'cosine'); if these metrics differ, +-- PlanetScale cannot use the vector index and will fall back to a full scan. + +CREATE /*vt+ QUERY_TIMEOUT_MS=0 */ + VECTOR INDEX partner_content_chunk_embedding_cosine_idx + ON PartnerContentChunk(embedding) + SECONDARY_ENGINE_ATTRIBUTE='{"type":"spann", "distance":"cosine"}'; + +-- Optional verification once the table has enough rows for the planner to use +-- the vector index: +-- +-- EXPLAIN +-- SELECT c.id, +-- DISTANCE(TO_VECTOR('[0, ...]'), c.embedding, 'cosine') AS distance +-- FROM PartnerContentChunk c +-- WHERE c.embedding IS NOT NULL +-- ORDER BY distance ASC +-- LIMIT 10; diff --git a/packages/prisma/schema/partner-content-search.prisma b/packages/prisma/schema/partner-content-search.prisma index 4ca05ea6062..1f8c5506fa6 100644 --- a/packages/prisma/schema/partner-content-search.prisma +++ b/packages/prisma/schema/partner-content-search.prisma @@ -18,21 +18,6 @@ enum PartnerContentTranscriptFetchStatus { error } -model PartnerContentIngestionRun { - id String @id @default(cuid()) - status String - requestedByUserId String? - platforms Json @db.Json - limitCreators Int - contentItemsPerPlatform Int - stats Json? @db.Json - error String? @db.Text - startedAt DateTime @default(now()) - completedAt DateTime? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt -} - model PartnerContentItem { id String @id @default(cuid()) partnerId String @@ -49,7 +34,7 @@ model PartnerContentItem { transcriptFetchStatus PartnerContentTranscriptFetchStatus @default(pending) transcriptCreditsUsed Int? transcriptLastAttemptedAt DateTime? - hasTimestamps Boolean @default(false) + transcriptHasTimestamps Boolean @default(false) normalizedTranscript String? @db.LongText transcriptHash String? embeddingModel String? @@ -59,6 +44,10 @@ model PartnerContentItem { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade) + partnerPlatform PartnerPlatform @relation(fields: [partnerPlatformId], references: [id], onDelete: Cascade) + chunks PartnerContentChunk[] + @@unique([partnerPlatformId, platformContentId]) @@index([partnerId, partnerPlatformId]) @@index(publishedAt) @@ -69,7 +58,7 @@ model PartnerContentChunk { partnerContentItemId String partnerId String chunkIndex Int - text String @db.Text + chunkText String @db.Text startMs Int? endMs Int? transcriptHash String @@ -78,6 +67,9 @@ model PartnerContentChunk { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + partnerContentItem PartnerContentItem @relation(fields: [partnerContentItemId], references: [id], onDelete: Cascade) + partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade) + @@unique([partnerContentItemId, chunkIndex, transcriptHash, embeddingModel]) @@index(partnerId) @@index(partnerContentItemId) diff --git a/packages/prisma/schema/partner.prisma b/packages/prisma/schema/partner.prisma index b8528214173..9bcbab45ba6 100644 --- a/packages/prisma/schema/partner.prisma +++ b/packages/prisma/schema/partner.prisma @@ -109,6 +109,8 @@ model Partner { platforms PartnerPlatform[] referredApplications ProgramApplicationEvent[] submittedLeads SubmittedLead[] + partnerContentItems PartnerContentItem[] + partnerContentChunks PartnerContentChunk[] @@index(country) @@index(networkStatus) diff --git a/packages/prisma/schema/platform.prisma b/packages/prisma/schema/platform.prisma index 160d7337648..3d28778ed45 100644 --- a/packages/prisma/schema/platform.prisma +++ b/packages/prisma/schema/platform.prisma @@ -25,7 +25,8 @@ model PartnerPlatform { contentLastFetchedAt DateTime? // last time we fetched recent content from ScrapeCreators for this platform latestContentUrl String? @db.Text // the most recent content URL we found for this platform - partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade) + partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade) + partnerContentItems PartnerContentItem[] @@unique([partnerId, type]) @@index(type) From 260d3e316674cb62ee8304ca95a134bdc406e466 Mon Sep 17 00:00:00 2001 From: David Clark Date: Mon, 8 Jun 2026 20:55:10 -0400 Subject: [PATCH 07/25] add draft UI as shared in video 6-8-26 --- .../network/partners/content-search/route.ts | 352 ++++++++++++++++++ .../network-content-search-results.tsx | 285 ++++++++++++++ .../(ee)/program/network/page-client.tsx | 222 +++++++++-- 3 files changed, 829 insertions(+), 30 deletions(-) create mode 100644 apps/web/app/(ee)/api/network/partners/content-search/route.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts new file mode 100644 index 00000000000..ab5fd43dcc3 --- /dev/null +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -0,0 +1,352 @@ +import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; +import { parseRequestBody } from "@/lib/api/utils"; +import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw"; +import { withWorkspace } from "@/lib/auth"; +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MODELS, +} from "@/lib/partner-content-search/constants"; +import { + embedPartnerContentTexts, + serializeEmbeddingForVector, +} from "@/lib/partner-content-search/voyage"; +import { prisma } from "@dub/prisma"; +import { PlatformType, Prisma } from "@dub/prisma/client"; +import { NextResponse } from "next/server"; +import * as z from "zod/v4"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 30; + +const DEFAULT_PARTNER_LIMIT = 20; +const DEFAULT_CHUNKS_PER_PARTNER = 2; + +const partnerNetworkContentSearchSchema = z.object({ + query: z.string().trim().max(500).optional(), + platform: z.enum(PlatformType), + starred: z.boolean().optional(), + limit: z.number().int().positive().max(50).default(DEFAULT_PARTNER_LIMIT), + chunksPerPartner: z + .number() + .int() + .positive() + .max(4) + .default(DEFAULT_CHUNKS_PER_PARTNER), + candidateChunkCount: z + .number() + .int() + .positive() + .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) + .optional(), +}); + +type PartnerNetworkContentSearchRow = { + chunkId: string; + partnerContentItemId: string; + partnerId: string; + partnerName: string; + partnerUsername: string | null; + partnerImage: string | null; + partnerDescription: string | null; + platformType: string; + platformIdentifier: string; + platformContentId: string; + contentUrl: string; + contentTitle: string | null; + contentThumbnailUrl: string | null; + contentPublishedAt: Date | null; + chunkText: string; + startMs: number | null; + endMs: number | null; + distance: number | string; +}; + +// POST /api/network/partners/content-search - semantic search over indexed partner content +export const POST = withWorkspace( + async ({ workspace, req }) => { + try { + const programId = getDefaultProgramIdOrThrow(workspace); + + const { partnerNetworkEnabledAt } = + await prisma.program.findUniqueOrThrow({ + select: { + partnerNetworkEnabledAt: true, + }, + where: { + id: programId, + }, + }); + + if (!partnerNetworkEnabledAt) { + throw new DubApiError({ + code: "forbidden", + message: "Partner network is not enabled for this program.", + }); + } + + const body = partnerNetworkContentSearchSchema.parse( + await parseRequestBody(req), + ); + const candidateChunkCount = body.query + ? (body.candidateChunkCount ?? + Math.min( + PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, + Math.max(25, body.limit * body.chunksPerPartner * 6), + )) + : body.limit * body.chunksPerPartner * 2; + + const rows = body.query + ? await searchPartnerNetworkContent({ + programId, + query: body.query, + platform: body.platform, + starred: body.starred, + limit: candidateChunkCount, + }) + : await listPartnerNetworkContent({ + programId, + platform: body.platform, + starred: body.starred, + limit: candidateChunkCount, + }); + + return NextResponse.json({ + success: true, + query: body.query ?? null, + platform: body.platform, + candidateChunkCount, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.model, + resultCount: rows.length, + partners: groupPartnerSearchResults({ + rows, + limit: body.limit, + chunksPerPartner: body.chunksPerPartner, + }), + }); + } catch (error) { + return handleAndReturnErrorResponse(error); + } + }, + { + requiredPlan: ["enterprise", "advanced"], + }, +); + +async function searchPartnerNetworkContent({ + programId, + query, + platform, + starred, + limit, +}: { + programId: string; + query: string; + platform: PlatformType; + starred?: boolean; + limit: number; +}) { + const [queryEmbedding] = await embedPartnerContentTexts({ + input: [query], + inputType: "query", + }); + const queryVector = serializeEmbeddingForVector(queryEmbedding); + const starredFilter = + starred === true + ? Prisma.sql`AND dp.starredAt IS NOT NULL` + : starred === false + ? Prisma.sql`AND (dp.starredAt IS NULL OR dp.id IS NULL)` + : Prisma.empty; + + return await prisma.$queryRaw(Prisma.sql` + SELECT + c.id AS chunkId, + c.partnerContentItemId, + c.partnerId, + p.name AS partnerName, + p.username AS partnerUsername, + p.image AS partnerImage, + p.description AS partnerDescription, + pp.type AS platformType, + pp.identifier AS platformIdentifier, + pci.platformContentId, + pci.url AS contentUrl, + pci.title AS contentTitle, + pci.thumbnailUrl AS contentThumbnailUrl, + pci.publishedAt AS contentPublishedAt, + c.text AS chunkText, + c.startMs, + c.endMs, + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance + FROM PartnerContentChunk c + INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId + INNER JOIN Partner p ON p.id = c.partnerId + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + LEFT JOIN ProgramEnrollment enrolled + ON enrolled.partnerId = p.id + AND enrolled.programId = ${programId} + LEFT JOIN DiscoveredPartner dp + ON dp.partnerId = p.id + AND dp.programId = ${programId} + WHERE c.embedding IS NOT NULL + AND pp.type = ${platform} + AND p.networkStatus IN ("approved", "trusted") + AND enrolled.id IS NULL + AND (dp.ignoredAt IS NULL OR dp.id IS NULL) + ${starredFilter} + ORDER BY distance ASC + LIMIT ${limit} + `); +} + +async function listPartnerNetworkContent({ + programId, + platform, + starred, + limit, +}: { + programId: string; + platform: PlatformType; + starred?: boolean; + limit: number; +}) { + const starredFilter = + starred === true + ? Prisma.sql`AND dp.starredAt IS NOT NULL` + : starred === false + ? Prisma.sql`AND (dp.starredAt IS NULL OR dp.id IS NULL)` + : Prisma.empty; + + return await prisma.$queryRaw(Prisma.sql` + SELECT + MIN(c.id) AS chunkId, + pci.id AS partnerContentItemId, + pci.partnerId, + p.name AS partnerName, + p.username AS partnerUsername, + p.image AS partnerImage, + p.description AS partnerDescription, + pp.type AS platformType, + pp.identifier AS platformIdentifier, + pci.platformContentId, + pci.url AS contentUrl, + pci.title AS contentTitle, + pci.thumbnailUrl AS contentThumbnailUrl, + pci.publishedAt AS contentPublishedAt, + "" AS chunkText, + NULL AS startMs, + NULL AS endMs, + 0 AS distance + FROM PartnerContentItem pci + INNER JOIN PartnerContentChunk c ON c.partnerContentItemId = pci.id + INNER JOIN Partner p ON p.id = pci.partnerId + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + LEFT JOIN ProgramEnrollment enrolled + ON enrolled.partnerId = p.id + AND enrolled.programId = ${programId} + LEFT JOIN DiscoveredPartner dp + ON dp.partnerId = p.id + AND dp.programId = ${programId} + WHERE c.embedding IS NOT NULL + AND pp.type = ${platform} + AND p.networkStatus IN ("approved", "trusted") + AND enrolled.id IS NULL + AND (dp.ignoredAt IS NULL OR dp.id IS NULL) + ${starredFilter} + GROUP BY + pci.id, + pci.partnerId, + p.name, + p.username, + p.image, + p.description, + pp.type, + pp.identifier, + pci.platformContentId, + pci.url, + pci.title, + pci.thumbnailUrl, + pci.publishedAt + ORDER BY pci.publishedAt DESC, pci.id ASC + LIMIT ${limit} + `); +} + +function groupPartnerSearchResults({ + rows, + limit, + chunksPerPartner, +}: { + rows: PartnerNetworkContentSearchRow[]; + limit: number; + chunksPerPartner: number; +}) { + const partners = new Map< + string, + { + partnerId: string; + name: string; + username: string | null; + image: string | null; + description: string | null; + bestDistance: number; + score: number; + chunks: ReturnType[]; + } + >(); + + for (const row of rows) { + const distance = Number(row.distance); + const partner = partners.get(row.partnerId) ?? { + partnerId: row.partnerId, + name: row.partnerName, + username: row.partnerUsername, + image: row.partnerImage, + description: row.partnerDescription, + bestDistance: distance, + score: toScore(distance), + chunks: [], + }; + + partner.bestDistance = Math.min(partner.bestDistance, distance); + partner.score = toScore(partner.bestDistance); + + if (partner.chunks.length < chunksPerPartner) { + partner.chunks.push(toChunkResult(row, distance)); + } + + partners.set(row.partnerId, partner); + } + + return Array.from(partners.values()) + .sort((a, b) => a.bestDistance - b.bestDistance) + .slice(0, limit); +} + +function toChunkResult(row: PartnerNetworkContentSearchRow, distance: number) { + return { + chunkId: row.chunkId, + partnerContentItemId: row.partnerContentItemId, + platform: { + type: row.platformType, + identifier: row.platformIdentifier, + }, + content: { + platformContentId: row.platformContentId, + url: row.contentUrl, + title: row.contentTitle, + thumbnailUrl: row.contentThumbnailUrl, + publishedAt: row.contentPublishedAt?.toISOString() ?? null, + }, + chunk: { + text: row.chunkText, + startMs: row.startMs, + endMs: row.endMs, + }, + distance, + score: toScore(distance), + }; +} + +function toScore(distance: number) { + return Number((1 - distance).toFixed(6)); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx new file mode 100644 index 00000000000..bbe2e71e989 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -0,0 +1,285 @@ +"use client"; + +import { EnvelopeArrowRight, Star } from "@dub/ui/icons"; + +export type PartnerContentSearchResponse = { + success: boolean; + partners: PartnerContentSearchPartner[]; +}; + +export type PartnerContentSearchPartner = { + partnerId: string; + name: string; + username: string | null; + image: string | null; + description: string | null; + score: number; + chunks: { + chunkId: string; + platform: { + type: string; + identifier: string; + }; + content: { + platformContentId: string; + url: string; + title: string | null; + thumbnailUrl: string | null; + publishedAt: string | null; + }; + chunk: { + text: string; + startMs: number | null; + endMs: number | null; + }; + score: number; + }[]; +}; + +export function NetworkContentSearchResults({ + error, + hasQuery, + isLoading, + partners, + onOpenPartner, +}: { + error: unknown; + hasQuery: boolean; + isLoading: boolean; + partners?: PartnerContentSearchPartner[]; + onOpenPartner: (partnerId: string) => void; +}) { + if (error) { + return ( +
+ Failed to search partner content +
+ ); + } + + if (!partners && isLoading) { + return ( +
+ {[...Array(8)].map((_, idx) => ( +
+
+
+
+
+
+
+
+
+ ))} +
+ ); + } + + if (!partners?.length) { + return ( +
+

+ {hasQuery + ? "No matching YouTube content found" + : "No indexed YouTube content found"} +

+

+ {hasQuery + ? "Try a broader search or select a different platform." + : "Try another platform or index more partner content."} +

+
+ ); + } + + return ( +
+ {partners.map((partner) => ( + + ))} +
+ ); +} + +function NetworkContentSearchCard({ + hasQuery, + partner, + onOpenPartner, +}: { + hasQuery: boolean; + partner: PartnerContentSearchPartner; + onOpenPartner: (partnerId: string) => void; +}) { + const contentPreviews = Array.from( + new Map( + partner.chunks.map((chunk) => [chunk.content.platformContentId, chunk]), + ).values(), + ).slice(0, 2); + const handle = + partner.username ?? + contentPreviews.find(({ platform }) => platform.identifier)?.platform + .identifier; + + return ( +
+
+ + +
+ + {handle && ( +
+ @{handle.replace(/^@/, "")} +
+ )} +
+ +
+ {hasQuery ? ( + <> + + {Math.round(partner.score * 100)}% + + content match + + ) : ( + Recent YouTube content + )} +
+ +
+ {contentPreviews.map((chunk) => ( + + + {chunk.chunk.text && ( +
+ + {formatChunkTimeRange(chunk.chunk)} + +

+ {chunk.chunk.text} +

+
+ )} + + ))} + {contentPreviews.length === 1 && ( +
+ )} +
+ + {contentPreviews[0] && ( + + {partner.image ? ( + + ) : ( + + +
+ + +
+
+ ); +} + +function formatChunkTimeRange({ + startMs, + endMs, +}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { + if (startMs === null && endMs === null) return "Transcript match"; + + if (startMs !== null && endMs !== null) { + return `${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; + } + + return formatTimestamp(startMs ?? endMs ?? 0); +} + +function formatTimestamp(ms: number) { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +function getYouTubePreviewThumbnail( + chunk: PartnerContentSearchPartner["chunks"][number], +) { + return ( + chunk.content.thumbnailUrl ?? + `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg` + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index 2372f89da43..47c66b8fc14 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -1,24 +1,32 @@ "use client"; import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner"; +import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms"; import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; import useWorkspace from "@/lib/swr/use-workspace"; import { NetworkPartnerProps } from "@/lib/types"; import { PARTNER_NETWORK_MAX_PAGE_SIZE } from "@/lib/zod/schemas/partner-network"; import { NetworkPartnerSheet } from "@/ui/partners/partner-network/network-partner-sheet"; +import { SearchBoxPersisted } from "@/ui/shared/search-box"; +import type { PlatformType } from "@dub/prisma/client"; import { AnimatedSizeContainer, Filter, PaginationControls, - Switch, usePagination, useRouterStuff, } from "@dub/ui"; +import type { Icon } from "@dub/ui/icons"; +import { Star, StarFill } from "@dub/ui/icons"; import { cn, fetcher } from "@dub/utils"; import { useAction } from "next-safe-action/hooks"; import { useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import useSWR from "swr"; +import { + NetworkContentSearchResults, + type PartnerContentSearchResponse, +} from "./network-content-search-results"; import { NetworkEmptyState } from "./network-empty-state"; import { NetworkPartnerCard } from "./network-partner-card"; import { usePartnerNetworkFilters } from "./use-partner-network-filters"; @@ -38,6 +46,20 @@ const tabs = [ }, ] as const; +const platformFilters = PARTNER_PLATFORM_FIELDS.filter( + ({ label }) => label !== "Website", +).map(({ label, icon: Icon }) => ({ + label, + value: label === "X/Twitter" ? "twitter" : label.toLowerCase(), + icon: Icon, +})) as { + label: string; + value: PlatformType; + icon: Icon; +}[]; + +const PARTNER_CONTENT_SEARCH_PARTNER_LIMIT = 50; + type ProgramPartnerNetworkPageClientProps = { variant?: "default" | "ignored"; }; @@ -47,21 +69,71 @@ export function ProgramPartnerNetworkPageClient({ }: ProgramPartnerNetworkPageClientProps = {}) { const { id: workspaceId } = useWorkspace(); const { searchParams, getQueryString, queryParams } = useRouterStuff(); + const selectedPlatform = searchParams.get("platform") as PlatformType | null; + const search = searchParams.get("search")?.trim() ?? ""; + const starred = searchParams.get("starred") === "true"; const status = variant === "ignored" ? "ignored" : tabs.find(({ id }) => id === searchParams.get("tab"))?.id || "discover"; + const isYouTubeContentSearchMode = + status === "discover" && selectedPlatform === "youtube"; const { data: partnerCounts, error: countError } = useNetworkPartnersCount(); + const { + data: contentSearchResults, + error: contentSearchError, + isLoading: isSearchingContent, + } = useSWR( + workspaceId && isYouTubeContentSearchMode + ? ["partner-content-search", search, workspaceId, selectedPlatform, starred] + : null, + async ([, query]: readonly [ + string, + string, + string, + PlatformType | null, + boolean, + ]) => { + const response = await fetch( + `/api/network/partners/content-search?workspaceId=${workspaceId}`, + { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify({ + query, + limit: PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, + chunksPerPartner: 2, + platform: "youtube", + starred: starred || undefined, + }), + }, + ); + + if (!response.ok) { + throw new Error("Failed to search partner content"); + } + + return response.json(); + }, + { + keepPreviousData: true, + revalidateOnFocus: false, + }, + ); + const { data: partners, error, mutate: mutatePartners, isValidating, } = useSWR( - workspaceId && + !isYouTubeContentSearchMode && + workspaceId && `/api/network/partners${getQueryString( { workspaceId, @@ -70,8 +142,8 @@ export function ProgramPartnerNetworkPageClient({ { exclude: variant === "ignored" - ? ["tab", "partnerId", "starred"] - : ["tab", "partnerId"], + ? ["tab", "partnerId", "starred", "search"] + : ["tab", "partnerId", "search"], }, )}`, fetcher, @@ -88,6 +160,10 @@ export function ProgramPartnerNetworkPageClient({ const { filters, activeFilters, onSelect, onRemove, onRemoveAll } = usePartnerNetworkFilters({ status }); + const nonPlatformFilters = filters.filter(({ key }) => key !== "platform"); + const listedActiveFilters = activeFilters.filter( + ({ key }) => key !== "platform", + ); const [detailsSheetState, setDetailsSheetState] = useState< | { open: false; partnerId: string | null } @@ -187,35 +263,51 @@ export function ProgramPartnerNetworkPageClient({ {status === "discover" && (
-
- -
+
+ ); +} + +function ContentMatchSkeletons({ count }: { count: number }) { + return ( + <> + {[...Array(count)].map((_, index) => ( +
+
+
+
+
+
+
+
+ ))} + + ); +} + +// A matched post for the detail-pane lists. Built from the cached summary's +// content bars (complete + instant) and enriched with a loaded chunk (snippet, +// timed transcript, richer thumbnail) when one is available. +type MatchedContentItem = { + contentItemId: string; + platform: string; + platformContentId: string; + title: string | null; + url: string | null; + durationMs: number | null; + publishedAt: string | null; + viewCount: number | null; + // The displayed relevance rating (0-1); also feeds the blend. + relevance: number; + // Relevance + reach blend, used only to order the Top content list. + blendedScore: number; + matchEvidence: PartnerContentMatchEvidence; + chunk?: PartnerContentSearchPartner["chunks"][number]; +}; + +function buildMatchedContentItems( + summary: PartnerContentSearchPartner["matchSummary"] | undefined, + chunks: PartnerContentSearchPartner["chunks"], +): MatchedContentItem[] { + const bars = summary?.contentBars ?? []; + + // Best loaded chunk per content item, for snippet/thumbnail enrichment. + const chunkByContentItemId = new Map< + string, + PartnerContentSearchPartner["chunks"][number] + >(); + for (const chunk of chunks) { + const current = chunkByContentItemId.get(chunk.partnerContentItemId); + if (!current || chunk.score > current.score) { + chunkByContentItemId.set(chunk.partnerContentItemId, chunk); + } + } + + // Per-creator engagement baseline: median views across all recent posts + // (matched + unmatched), so a viral hit can't skew the normalization. + const baselineViews = getViewBaseline(bars.map((bar) => bar.viewCount)); + + return bars + .filter((bar) => bar.matched) + .map((bar) => { + const relevance = + getEvidenceDisplayScore(bar.matchEvidence) ?? bar.matchScore ?? 0; + + return { + contentItemId: bar.partnerContentItemId, + platform: bar.platform, + platformContentId: bar.platformContentId, + title: bar.title, + url: bar.url, + durationMs: bar.durationMs, + publishedAt: bar.publishedAt, + viewCount: bar.viewCount, + relevance, + blendedScore: getBlendedTopContentScore({ + relevance, + views: bar.viewCount, + baselineViews, + }), + matchEvidence: bar.matchEvidence, + chunk: chunkByContentItemId.get(bar.partnerContentItemId), + }; + }); +} + +function publishedAtMs(iso: string | null) { + if (!iso) return -Infinity; + const ms = new Date(iso).getTime(); + return Number.isNaN(ms) ? -Infinity : ms; +} + +function ContentMatchRow({ item }: { item: MatchedContentItem }) { + const { chunk } = item; + const evidence = item.matchEvidence; + const isTimedTranscriptMatch = chunk + ? hasTimedTranscriptMatch(chunk.chunk) + : false; + const timeLabel = + chunk && isTimedTranscriptMatch + ? formatChunkTimeRange(chunk.chunk) + : formatDuration(item.durationMs); + const dateLabel = formatPublishedDate(item.publishedAt); + const matchTags = getMatchTags( + evidence, + chunk?.chunk.source ?? + (evidence.primarySource === "creatorText" ? "metadata" : "transcript"), + ); + const snippet = chunk ? getMatchSnippet(chunk) : null; + const excerpt = snippet ? `"…${snippet.slice(0, 130).trimEnd()}…"` : null; + const thumbnail = getItemThumbnail(item); + const meta = [timeLabel, dateLabel].filter(Boolean).join(" · "); + const score = item.relevance; + const title = getItemTitle(item); + const viewCount = item.viewCount; + + return ( + + {/* Preview image */} +
+ {thumbnail ? ( + + ) : ( +
+ +
+ )} +
+
+
+ + + {title} + +
+
+ {meta && {meta}} + {matchTags.length > 0 && ( + <> + {meta && ·} + + {matchTags.map((tag) => ( + + {tag.label} + + ))} + + + )} + {viewCount != null && viewCount > 0 && ( + <> + · + {nFormatter(viewCount)} views + + )} +
+ {excerpt && ( +

+ {excerpt} +

+ )} +
+
+ + {formatMatchPercent(score)} + + + match + +
+ + ); +} + +function getEvidenceDisplayScore( + evidence: PartnerContentMatchEvidence | undefined, +) { + if (!evidence || evidence.sources.length === 0) return null; + + return Math.max( + evidence.transcriptScore ?? 0, + evidence.creatorTextScore ?? 0, + ); +} + +function getMatchTags( + evidence: PartnerContentMatchEvidence | undefined, + fallbackSource: string, +) { + const sources = evidence?.sources.length + ? evidence.sources + : fallbackSource === "metadata" + ? ["creatorText" as const] + : ["transcript" as const]; + + return sources.map((source) => ({ + source, + label: source === "transcript" ? "Transcript" : "Creator text", + })); +} + +// The displayed snippet. Transcript chunks are real prose; creator-text chunks +// store the raw embedding input ("Content type: video Title: … Description: …"), +// so we surface just the creator-entered text for a cleaner preview. +function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { + const text = (chunk.chunk.text ?? "").trim(); + if (!text) return null; + if (chunk.chunk.source !== "metadata") return text; + + const description = text.match(/Description:\s*([\s\S]+)$/i)?.[1]; + return ( + ( + description ?? text.replace(/^Content type:[\s\S]*?Title:\s*/i, "") + ).trim() || null + ); +} + +function PlatformIcon({ + platform, + className, +}: { + platform: string; + className?: string; +}) { + const Icon = + platform === "youtube" + ? YouTube + : platform === "instagram" + ? Instagram + : platform === "tiktok" + ? TikTok + : null; + + return Icon ? : null; +} + +function getPreviewThumbnail( + chunk: PartnerContentSearchPartner["chunks"][number], +) { + if (chunk.content.thumbnailUrl) return chunk.content.thumbnailUrl; + if (chunk.platform.type === "youtube") { + return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; + } + return null; +} + +// Thumbnail for a matched item: the loaded chunk's preview when enriched, +// otherwise a YouTube thumbnail derived from the content id (the only platform +// with a stable URL pattern from the bar data alone). +function getItemThumbnail(item: MatchedContentItem) { + if (item.chunk) return getPreviewThumbnail(item.chunk); + if (item.platform === "youtube" && item.platformContentId) { + return `https://i.ytimg.com/vi/${item.platformContentId}/hqdefault.jpg`; + } + return null; +} + +function getItemTitle(item: MatchedContentItem) { + if (item.chunk) return getContentTitle(item.chunk); + return item.title?.trim() || "Untitled content"; +} + +// Link for a matched item: the chunk-aware href (YouTube deep-link timestamp, +// Instagram normalization) when enriched, otherwise the bar's canonical URL. +function getItemHref(item: MatchedContentItem) { + if (item.chunk) return getContentHref(item.chunk); + return item.url ?? "#"; +} + +// A noun phrase describing exactly what window the ranks are computed over, for +// composing into the caption. The window is time-based (recencyWindowMonths) but +// capped per partner (recentContentMaxPerPartner); when that cap bites, say so +// explicitly instead of implying full coverage. +function formatRankWindowPhrase( + summary: PartnerContentSearchPartner["matchSummary"] | undefined, +) { + if (!summary || !summary.recentContentCount) return null; + + const { recentContentCount, oldestPublishedAt, newestPublishedAt } = summary; + const oldest = formatMonthYear(oldestPublishedAt); + const newest = formatMonthYear(newestPublishedAt); + const countCapped = + recentContentCount >= PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner; + + if (countCapped) { + return oldest + ? `the ${recentContentCount} most recent posts, back to ${oldest}` + : `the ${recentContentCount} most recent posts`; + } + + if (oldest && newest) { + return oldest === newest + ? `${recentContentCount} ${ + recentContentCount === 1 ? "post" : "posts" + } from ${oldest}` + : `${recentContentCount} posts, ${oldest} – ${newest}`; + } + + return `${recentContentCount} recent ${ + recentContentCount === 1 ? "post" : "posts" + }`; +} + +function formatMonthYear(iso: string | null) { + if (!iso) return null; + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return null; + + return new Intl.DateTimeFormat(undefined, { + month: "short", + year: "numeric", + }).format(date); +} + +function getContentTitle(chunk: PartnerContentSearchPartner["chunks"][number]) { + return ( + chunk.content.title?.trim() || + chunk.content.description?.trim().split(/\r?\n/)[0] || + "Untitled content" + ); +} + +function getContentHref(chunk: PartnerContentSearchPartner["chunks"][number]) { + if (chunk.platform.type === "instagram") { + return getInstagramContentHref(chunk); + } + + if (chunk.platform.type !== "youtube" || chunk.chunk.startMs === null) { + return chunk.content.url; + } + + try { + const url = new URL(chunk.content.url); + url.searchParams.set("t", `${Math.floor(chunk.chunk.startMs / 1000)}s`); + return url.toString(); + } catch { + return chunk.content.url; + } +} + +function getInstagramContentHref( + chunk: PartnerContentSearchPartner["chunks"][number], +) { + const shortcode = + extractInstagramShortcode(chunk.content.url) || + chunk.content.platformContentId; + + return `https://www.instagram.com/${chunk.content.type === "reel" ? "reel" : "p"}/${shortcode}/`; +} + +function extractInstagramShortcode(url: string) { + try { + return ( + new URL(url).pathname.match( + /^\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1] ?? null + ); + } catch { + return ( + url.match( + /instagram\.com\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1] ?? null + ); + } +} + +function hasTimedTranscriptMatch({ + source, + startMs, + endMs, +}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { + return source !== "metadata" && (startMs !== null || endMs !== null); +} + +function formatChunkTimeRange({ + source, + startMs, + endMs, +}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { + if (source === "metadata") return "Creator text match"; + if (startMs === null && endMs === null) return "Transcript match"; + if (startMs !== null && endMs !== null) { + return `${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; + } + return formatTimestamp(startMs ?? endMs ?? 0); +} + +function formatTimestamp(ms: number) { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +function formatDuration(durationMs: number | null) { + if (!durationMs || durationMs <= 0) return null; + + const totalSeconds = Math.floor(durationMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds + .toString() + .padStart(2, "0")}`; + } + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +function formatPublishedDate(publishedAt: string | null) { + if (!publishedAt) return null; + + const date = new Date(publishedAt); + if (Number.isNaN(date.getTime())) return null; + + return new Intl.DateTimeFormat(undefined, { + month: "short", + day: "numeric", + year: "numeric", + }).format(date); +} + +function formatMatchPercent(score: number) { + return `${Math.round(Math.min(1, Math.max(0, score)) * 100)}%`; +} + +function getBackQueryString(searchParams: { toString(): string }) { + const params = new URLSearchParams(searchParams.toString()); + const queryString = params.toString(); + + return queryString ? `?${queryString}` : ""; +} + +function getPartnerApiPath({ + workspaceId, + partnerStatus, + partnerId, + country, +}: { + workspaceId: string; + partnerStatus: string; + partnerId: string; + country?: string; +}) { + const params = new URLSearchParams({ + workspaceId, + status: partnerStatus, + partnerIds: partnerId, + pageSize: "1", + }); + + if (country) params.set("country", country); + + return `/api/network/partners?${params}`; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page.tsx new file mode 100644 index 00000000000..d403eeeed8c --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page.tsx @@ -0,0 +1,19 @@ +import { PageContent } from "@/ui/layout/page-content"; +import { PageWidthWrapper } from "@/ui/layout/page-width-wrapper"; +import { NetworkPartnerDetailPageClient } from "./page-client"; + +export default async function NetworkPartnerDetailPage({ + params, +}: { + params: Promise<{ partnerId: string }>; +}) { + const { partnerId } = await params; + + return ( + + + + + + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index c88c43eeb72..cd3d131e28c 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -1,10 +1,19 @@ "use client"; +import type { PartnerContentTopicFitBand } from "@/lib/partner-content-search/constants"; import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { + Globe, + Instagram, + LinkedIn, + TikTok, + Twitter, + User, + YouTube, +} from "@dub/ui/icons"; +import { cn, nFormatter } from "@dub/utils"; import type { PlatformType } from "@prisma/client"; -import { Button } from "@dub/ui"; -import { ArrowUpRight, EnvelopeArrowRight } from "@dub/ui/icons"; -import { memo } from "react"; +import { NetworkPartnerCard } from "./network-partner-card"; const PLATFORM_LABELS: Partial> = { youtube: "YouTube", @@ -16,22 +25,26 @@ function platformLabel(platform: PlatformType) { return PLATFORM_LABELS[platform] ?? "partner"; } +function contentLabel(platform?: PlatformType) { + return platform ? platformLabel(platform) : "indexed"; +} + export function NetworkContentSearchResults({ error, hasQuery, isLoading, partners, platform, - onOpenPartner, + onToggleStarred, }: { error: unknown; hasQuery: boolean; isLoading: boolean; partners?: PartnerContentSearchPartner[]; - platform: PlatformType; - onOpenPartner: (partnerId: string) => void; + platform?: PlatformType; + onToggleStarred?: (partnerId: string, starred: boolean) => void; }) { - const label = platformLabel(platform); + const label = contentLabel(platform); if (error) { return ( @@ -47,14 +60,30 @@ export function NetworkContentSearchResults({ {[...Array(8)].map((_, idx) => (
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {[...Array(12)].map((_, index) => ( +
+ ))} +
))} @@ -82,208 +111,279 @@ export function NetworkContentSearchResults({ return (
{partners.map((partner) => ( - onToggleStarred(partner.partnerId, starred) + : undefined + } + bottomContent={ + + } /> ))}
); } -const NetworkContentSearchCard = memo(function NetworkContentSearchCard({ +function NetworkPartnerContentMatch({ hasQuery, partner, platform, - onOpenPartner, }: { hasQuery: boolean; partner: PartnerContentSearchPartner; - platform: PlatformType; - onOpenPartner: (partnerId: string) => void; + platform?: PlatformType; }) { - const label = platformLabel(platform); - const contentPreviews = Array.from( - new Map( - partner.chunks.map((chunk) => [chunk.content.platformContentId, chunk]), - ).values(), - ).slice(0, 2); - const handle = - partner.username ?? - contentPreviews.find(({ platform }) => platform.identifier)?.platform - .identifier; - - return ( -
-
- + const summary = partner.matchSummary; + const platforms = ( + summary?.topPlatforms?.length + ? summary.topPlatforms + : summary?.platforms ?? [platform].filter(Boolean) + ) as string[]; -
- - {handle && ( -
- @{handle.replace(/^@/, "")} -
- )} + // List mode (no query): there's no topic fit to score, so show the partner's + // platforms + how recently they've published. + if (!hasQuery) { + return ( +
+
+ + Recent content + + + {formatPublishedWindow(summary)} +
- -
- {hasQuery ? ( - <> - - {Math.round(partner.score * 100)}% - - content match - - ) : ( - Recent {label} content - )} +
+ + {formatPlatformNames(platforms)}
+
+ ); + } -
- {contentPreviews.map((chunk) => { - const thumbnail = getPreviewThumbnail(chunk, platform); - - return ( - - {thumbnail ? ( - - ) : ( -
- )} - {chunk.chunk.text && ( -
- - {formatChunkTimeRange(chunk.chunk)} - -

- {chunk.chunk.text} -

-
- )} - - ); - })} - {contentPreviews.length === 1 && ( -
+ const band = summary?.band ?? "none"; + const styles = BAND_STYLES[band]; + const followers = summary?.followers ?? null; + const medianViews = summary?.medianViews ?? null; + const matchLabel = formatMatchEvidenceLabel(summary); + const lastOnTopic = lastPublishedLabel(summary?.lastOnTopicAt); + + return ( +
+ + Topic fit + +
+ + {summary?.topicFit ?? 0} + + + {BAND_LABELS[band]} + +
+ + {(followers || medianViews) && ( +
+ {followers ? ( + {nFormatter(followers)} followers + ) : null} + {followers && medianViews ? ( + · + ) : null} + {medianViews ? ( + + {nFormatter(medianViews)} median views + + ) : null}
- - {contentPreviews[0] && ( - - {partner.image ? ( - - ) : ( - + ); +} -
- -
+function ContentMatchScoreDebug({ + partner, +}: { + partner: PartnerContentSearchPartner; +}) { + const { cosineScore, rerankScore } = partner; + + if (cosineScore == null && rerankScore == null) return null; + + return ( +
+ + cos {cosineScore == null ? "-" : formatMatchPercent(cosineScore)} + + {"->"} + + rerank {rerankScore == null ? "-" : formatMatchPercent(rerankScore)} +
); -}); - -function formatChunkTimeRange({ - source, - startMs, - endMs, -}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { - if (source === "metadata") return "Description match"; - if (startMs === null && endMs === null) return "Transcript match"; - - if (startMs !== null && endMs !== null) { - return `${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; +} + +const BAND_LABELS: Record = { + consistent: "Consistent", + frequent: "Frequent", + occasional: "Occasional", + "one-off": "One-off", + none: "No recent match", +}; + +// Number + chip colors per band. Number color follows the band tier so the score +// reads at a glance (green = consistent, down to gray = one-off/none). +const BAND_STYLES: Record< + PartnerContentTopicFitBand, + { number: string; chip: string } +> = { + consistent: { + number: "text-green-600", + chip: "border-green-100 bg-green-50 text-green-700", + }, + frequent: { + number: "text-blue-600", + chip: "border-blue-100 bg-blue-50 text-blue-700", + }, + occasional: { + number: "text-amber-600", + chip: "border-amber-100 bg-amber-50 text-amber-700", + }, + "one-off": { + number: "text-neutral-500", + chip: "border-neutral-200 bg-neutral-100 text-neutral-600", + }, + none: { + number: "text-neutral-400", + chip: "border-neutral-200 bg-neutral-100 text-neutral-500", + }, +}; + +const PLATFORM_ICONS: Partial> = { + youtube: YouTube, + instagram: Instagram, + tiktok: TikTok, + twitter: Twitter, + linkedin: LinkedIn, + website: Globe, +}; + +function PlatformIcons({ platforms }: { platforms: string[] }) { + const icons = platforms + .map((platform) => PLATFORM_ICONS[platform as PlatformType]) + .filter((icon): icon is typeof User => Boolean(icon)); + + if (!icons.length) { + return ; } - return formatTimestamp(startMs ?? endMs ?? 0); + return ( + // No text-color override here: the YouTube/TikTok/Instagram icons carry their + // own brand colors, so we let them render in color rather than desaturating + // them to the surrounding muted text color. + + {icons.map((Icon, index) => ( + + ))} + + ); } -function formatTimestamp(ms: number) { - const totalSeconds = Math.max(0, Math.floor(ms / 1000)); - const minutes = Math.floor(totalSeconds / 60); - const seconds = totalSeconds % 60; +// Coarse "Nd / Nw / Nmo ago" label for the last publish date (timeAgo from +// @dub/utils switches to absolute dates past ~23h, which we don't want here). +function lastPublishedLabel(iso: string | null | undefined) { + if (!iso) return null; - return `${minutes}:${seconds.toString().padStart(2, "0")}`; + const days = Math.floor((Date.now() - new Date(iso).getTime()) / 86_400_000); + if (days <= 0) return "today"; + if (days < 7) return `${days}d ago`; + if (days < 8 * 7) return `${Math.floor(days / 7)}w ago`; + if (days < 365) return `${Math.floor(days / 30)}mo ago`; + return `${Math.floor(days / 365)}y ago`; } -function getPreviewThumbnail( - chunk: PartnerContentSearchPartner["chunks"][number], - platform: PlatformType, +function clampScore(score: number) { + return Math.min(1, Math.max(0, score)); +} + +function formatMatchPercent(score: number) { + return `${Math.round(clampScore(score) * 100)}%`; +} + +function formatMatchEvidenceLabel( + summary: PartnerContentSearchPartner["matchSummary"] | undefined, ) { - if (chunk.content.thumbnailUrl) return chunk.content.thumbnailUrl; + const matchingPosts = summary?.matchedContentCount ?? 0; + const recentPosts = summary?.recentContentCount ?? 0; - // YouTube exposes a deterministic thumbnail URL from the video id; other - // platforms don't, so fall back to a placeholder when there's no stored URL. - if (platform === "youtube") { - return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; + // Aggregate coverage rather than splitting evidence by source (transcript vs + // creator text), which read as noisy on the compact card. + if (recentPosts > 0) { + return `${matchingPosts} of ${recentPosts} matching`; } - return null; + return `${matchingPosts} matching ${matchingPosts === 1 ? "post" : "posts"}`; +} + +function formatPlatformNames(platforms: Array) { + const uniquePlatforms = Array.from(new Set(platforms.filter(Boolean))); + + if (uniquePlatforms.length === 0) return "indexed content"; + + return uniquePlatforms + .map((platform) => platformLabel(platform as PlatformType)) + .join(", "); +} + +function formatPublishedWindow( + summary: PartnerContentSearchPartner["matchSummary"], +) { + if (!summary?.oldestPublishedAt || !summary.newestPublishedAt) { + return "recent content"; + } + + const oldest = new Date(summary.oldestPublishedAt); + const newest = new Date(summary.newestPublishedAt); + const monthDiff = Math.max( + 1, + (newest.getFullYear() - oldest.getFullYear()) * 12 + + newest.getMonth() - + oldest.getMonth() + + 1, + ); + + return `past ${monthDiff} ${monthDiff === 1 ? "month" : "months"}`; } diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx index e672a3e29ca..ea62d127161 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx @@ -30,12 +30,21 @@ import { import { EmailContent } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-email-preview"; import { InviteNetworkPartnerSheet } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-network-partner-sheet"; import { usePathname } from "next/navigation"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { type ReactNode, useEffect, useMemo, useRef, useState } from "react"; + +type PartnerPlatformDisplayData = ReturnType< + (typeof PARTNER_PLATFORM_FIELDS)[number]["data"] +> & { + label: string; + icon: Icon; +}; export function NetworkPartnerCard({ + bottomContent, partner, onToggleStarred, }: { + bottomContent?: ReactNode; partner?: NetworkPartnerProps; onToggleStarred?: (starred: boolean) => void; }) { @@ -115,7 +124,9 @@ export function NetworkPartnerCard({ "_blank", ); } else { - queryParams({ set: { partnerId: partner.id } }); + queryParams({ + set: { partnerId: partner.id }, + }); } }} > @@ -191,47 +202,64 @@ export function NetworkPartnerCard({
-
- - Audience - + {bottomContent ?? ( + + )} +
+
+ ); +} -
- {partnerPlatformsData - ? partnerPlatformsData.map( - ({ - label, - icon: PlatformIcon, - verified, - stat, - value, - href, - info, - verifiedAt, - }) => ( - - ), - ) - : [...Array(6)].map((_, idx) => ( -
- ))} -
-
+function NetworkPartnerAudienceSection({ + partner, + partnerPlatformsData, +}: { + partner?: NetworkPartnerProps; + partnerPlatformsData: PartnerPlatformDisplayData[] | null; +}) { + return ( +
+ + Audience + + +
+ {partnerPlatformsData + ? partnerPlatformsData.map( + ({ + label, + icon: PlatformIcon, + verified, + stat, + value, + href, + info, + verifiedAt, + }) => ( + + ), + ) + : [...Array(6)].map((_, idx) => ( +
+ ))}
); @@ -297,14 +325,6 @@ function NetworkPartnerCardActions({ })} />
- {onToggleStarred && ( - - )} {showInvite && (
); diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-detail-sheet.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-detail-sheet.tsx new file mode 100644 index 00000000000..a8c3d3f7f0d --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-detail-sheet.tsx @@ -0,0 +1,89 @@ +"use client"; + +import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { X } from "@/ui/shared/icons"; +import { + Button, + ChevronLeft, + ChevronRight, + Sheet, + useRouterStuff, +} from "@dub/ui"; +import { NetworkPartnerDetailContent } from "./[partnerId]/page-client"; + +type NetworkPartnerDetailSheetProps = { + isOpen: boolean; + onNext?: () => void; + onPrevious?: () => void; + partnerId: string; + partnerStatus: string; + searchPartner?: PartnerContentSearchPartner; + setIsOpen: (open: boolean) => void; +}; + +export function NetworkPartnerDetailSheet({ + isOpen, + onNext, + onPrevious, + partnerId, + partnerStatus, + searchPartner, + setIsOpen, +}: NetworkPartnerDetailSheetProps) { + const { queryParams } = useRouterStuff(); + + return ( + queryParams({ del: "partnerId" })} + contentProps={{ + className: "md:w-[max(min(calc(100vw-334px),1170px),540px)]", + }} + > +
+
+ + Partner network + +
+
+
+ +
+
+ +
+ +
+
+ + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index 31a7f80b6a0..59911c22c62 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -1,12 +1,12 @@ "use client"; import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner"; +import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; import usePartnerContentSearch from "@/lib/swr/use-partner-content-search"; import useWorkspace from "@/lib/swr/use-workspace"; import { NetworkPartnerProps } from "@/lib/types"; import { PARTNER_NETWORK_MAX_PAGE_SIZE } from "@/lib/zod/schemas/partner-network"; -import { NetworkPartnerSheet } from "@/ui/partners/partner-network/network-partner-sheet"; import { SearchBoxPersisted } from "@/ui/shared/search-box"; import { AnimatedSizeContainer, @@ -37,6 +37,7 @@ import useSWR from "swr"; import { NetworkContentSearchResults } from "./network-content-search-results"; import { NetworkEmptyState } from "./network-empty-state"; import { NetworkPartnerCard } from "./network-partner-card"; +import { NetworkPartnerDetailSheet } from "./network-partner-detail-sheet"; import { PartnerNetworkSort } from "./partner-network-sort"; import { usePartnerNetworkFilters } from "./use-partner-network-filters"; @@ -80,14 +81,22 @@ export function ProgramPartnerNetworkPageClient({ const selectedPlatform = (searchParams.get("platform") as PlatformType | null) ?? "all"; const search = searchParams.get("search")?.trim() ?? ""; + const country = searchParams.get("country") ?? undefined; const starred = searchParams.get("starred") === "true"; const status = variant === "ignored" ? "ignored" : tabs.find(({ id }) => id === searchParams.get("tab"))?.id || "discover"; - const isYouTubeContentSearchMode = - status === "discover" && selectedPlatform === "youtube"; + const contentSearchPlatform = + selectedPlatform !== "all" && + isPartnerContentSearchPlatform(selectedPlatform) + ? selectedPlatform + : undefined; + const isContentSearchMode = + status === "discover" && + search.length > 0 && + (selectedPlatform === "all" || Boolean(contentSearchPlatform)); const { data: partnerCounts, error: countError } = useNetworkPartnersCount(); @@ -95,10 +104,12 @@ export function ProgramPartnerNetworkPageClient({ data: contentSearchResults, error: contentSearchError, isLoading: isSearchingContent, + mutate: mutateContentSearch, } = usePartnerContentSearch({ - enabled: isYouTubeContentSearchMode, + enabled: isContentSearchMode, query: search, - platform: "youtube", + platform: contentSearchPlatform, + country, starred, }); @@ -108,7 +119,7 @@ export function ProgramPartnerNetworkPageClient({ mutate: mutatePartners, isValidating, } = useSWR( - !isYouTubeContentSearchMode && + !isContentSearchMode && workspaceId && `/api/network/partners${getQueryString( { @@ -152,38 +163,56 @@ export function ProgramPartnerNetworkPageClient({ useEffect(() => { const partnerId = searchParams.get("partnerId"); - if (partnerId) setDetailsSheetState({ open: true, partnerId }); + if (partnerId) { + setDetailsSheetState({ open: true, partnerId }); + } else { + setDetailsSheetState({ open: false, partnerId: null }); + } }, [searchParams]); - const { currentPartner } = useCurrentPartner({ - partners, - partnerId: detailsSheetState.partnerId, - partnerListStatus: status, - }); + const sheetPartnerIds = useMemo( + () => + isContentSearchMode + ? contentSearchResults?.partners.map(({ partnerId }) => partnerId) + : partners?.map(({ id }) => id), + [contentSearchResults?.partners, isContentSearchMode, partners], + ); const [previousPartnerId, nextPartnerId] = useMemo(() => { - if (!partners || !detailsSheetState.partnerId) return [null, null]; + if (!sheetPartnerIds?.length || !detailsSheetState.partnerId) { + return [null, null]; + } - const currentIndex = partners.findIndex( - ({ id }) => id === detailsSheetState.partnerId, + const currentIndex = sheetPartnerIds.findIndex( + (id) => id === detailsSheetState.partnerId, ); if (currentIndex === -1) return [null, null]; return [ - currentIndex > 0 ? partners[currentIndex - 1].id : null, - currentIndex < partners.length - 1 ? partners[currentIndex + 1].id : null, + currentIndex > 0 ? sheetPartnerIds[currentIndex - 1] : null, + currentIndex < sheetPartnerIds.length - 1 + ? sheetPartnerIds[currentIndex + 1] + : null, ]; - }, [partners, detailsSheetState.partnerId]); + }, [detailsSheetState.partnerId, sheetPartnerIds]); return (
- {detailsSheetState.partnerId && currentPartner && ( - - setDetailsSheetState((s) => ({ ...s, open }) as any) + setDetailsSheetState((state) => + open && state.partnerId + ? { open: true, partnerId: state.partnerId } + : { open: false, partnerId: state.partnerId }, + ) } - partner={currentPartner} + partnerId={detailsSheetState.partnerId} + partnerStatus={status} + searchPartner={contentSearchResults?.partners.find( + ({ partnerId }) => partnerId === detailsSheetState.partnerId, + )} onPrevious={ previousPartnerId ? () => @@ -324,18 +353,71 @@ export function ProgramPartnerNetworkPageClient({
)} - {isYouTubeContentSearchMode ? ( + {isContentSearchMode ? ( 0} isLoading={isSearchingContent} partners={contentSearchResults?.partners} - platform="youtube" - onOpenPartner={(partnerId) => { - queryParams({ - set: { partnerId }, - }); - }} + platform={contentSearchPlatform} + onToggleStarred={ + variant === "ignored" + ? undefined + : (partnerId, starred) => { + mutateContentSearch( + // @ts-ignore SWR doesn't seem to have proper typing for partial data results w/ `populateCache` + async () => { + const result = await updateDiscoveredPartner({ + workspaceId: workspaceId!, + partnerId, + starred, + }); + if (!result?.data) { + toast.error("Failed to star partner"); + throw new Error("Failed to star partner"); + } + + return result.data; + }, + { + optimisticData: (data) => + data && { + ...data, + partners: data.partners.map((p) => + p.partnerId === partnerId + ? { + ...p, + partner: { + ...p.partner, + starredAt: starred ? new Date() : null, + }, + } + : p, + ), + }, + populateCache: ( + result: { starredAt: Date | null }, + data, + ) => + data && { + ...data, + partners: data.partners.map((p) => + p.partnerId === partnerId + ? { + ...p, + partner: { + ...p.partner, + starredAt: result.starredAt, + }, + } + : p, + ), + }, + revalidate: false, + }, + ); + } + } /> ) : error || countError ? (
@@ -429,36 +511,3 @@ export function ProgramPartnerNetworkPageClient({
); } - -/** Gets the current partner from the loaded partners array if available, or a separate fetch if not */ -function useCurrentPartner({ - partners, - partnerId, - partnerListStatus, -}: { - partners?: NetworkPartnerProps[]; - partnerId: string | null; - partnerListStatus: string; -}) { - const { id: workspaceId } = useWorkspace(); - - let currentPartner = partnerId - ? partners?.find(({ id }) => id === partnerId) - : null; - - const fetchPartnerId = partnerId && !currentPartner ? partnerId : null; - - const { data: fetchedPartners, isLoading } = useSWR( - fetchPartnerId && - `/api/network/partners?workspaceId=${workspaceId}&partnerIds=${fetchPartnerId}&status=${partnerListStatus}`, - fetcher, - { - keepPreviousData: true, - }, - ); - - if (!currentPartner && fetchedPartners?.[0]?.id === partnerId) - currentPartner = fetchedPartners[0]; - - return { currentPartner, isLoading }; -} diff --git a/apps/web/lib/api/network/normalize-ranked-network-partner.ts b/apps/web/lib/api/network/normalize-ranked-network-partner.ts new file mode 100644 index 00000000000..459c0781868 --- /dev/null +++ b/apps/web/lib/api/network/normalize-ranked-network-partner.ts @@ -0,0 +1,44 @@ +import { NetworkPartnerSchema } from "@/lib/zod/schemas/partner-network"; +import { PreferredEarningStructure, SalesChannel } from "@prisma/client"; +import * as z from "zod/v4"; + +type RankedNetworkPartnerRow = Record & { + starredAt?: Date | string | null; + ignoredAt?: Date | string | null; + invitedAt?: Date | string | null; + identityVerificationStatus?: string | null; + identityVerifiedAt?: Date | string | null; + categories?: string | null; + preferredEarningStructures?: string | null; + salesChannels?: string | null; +}; + +export function parseRankedNetworkPartners( + partners: RankedNetworkPartnerRow[], +) { + return z.array(NetworkPartnerSchema).parse( + partners.map((partner) => ({ + ...partner, + starredAt: toNullableDate(partner.starredAt), + ignoredAt: toNullableDate(partner.ignoredAt), + invitedAt: toNullableDate(partner.invitedAt), + identityVerificationStatus: partner.identityVerificationStatus ?? null, + identityVerifiedAt: toNullableDate(partner.identityVerifiedAt), + categories: splitCommaSeparated(partner.categories), + preferredEarningStructures: splitCommaSeparated( + partner.preferredEarningStructures, + ) as PreferredEarningStructure[], + salesChannels: splitCommaSeparated( + partner.salesChannels, + ) as SalesChannel[], + })), + ); +} + +function toNullableDate(value: Date | string | null | undefined) { + return value ? new Date(value) : null; +} + +function splitCommaSeparated(value: string | null | undefined) { + return value ? value.split(",").map((item) => item.trim()) : []; +} diff --git a/apps/web/lib/api/scrape-creators/get-tiktok-video-transcript.ts b/apps/web/lib/api/scrape-creators/get-tiktok-video-transcript.ts index 3168fc62e02..7808d1662bb 100644 --- a/apps/web/lib/api/scrape-creators/get-tiktok-video-transcript.ts +++ b/apps/web/lib/api/scrape-creators/get-tiktok-video-transcript.ts @@ -23,6 +23,14 @@ export async function getTikTokVideoTranscript({ ); if (error) { + if (isTikTokTranscriptUnavailableError(error)) { + return { + id: null, + url, + transcript: null, + }; + } + throw new Error( "We were unable to retrieve the TikTok transcript from ScrapeCreators.", ); @@ -30,3 +38,27 @@ export async function getTikTokVideoTranscript({ return data; } + +function isTikTokTranscriptUnavailableError(error: unknown) { + if (!error || typeof error !== "object") return false; + + const status = + "status" in error && typeof error.status === "number" + ? error.status + : "errorStatus" in error && typeof error.errorStatus === "number" + ? error.errorStatus + : null; + + if (status !== 400) return false; + + const message = + "message" in error && typeof error.message === "string" + ? error.message.toLowerCase() + : ""; + + return ( + message.includes("not a video") || + message.includes("photo carousel") || + message.includes("no transcript") + ); +} diff --git a/apps/web/lib/api/scrape-creators/schema.ts b/apps/web/lib/api/scrape-creators/schema.ts index 46902b36819..5069da45c05 100644 --- a/apps/web/lib/api/scrape-creators/schema.ts +++ b/apps/web/lib/api/scrape-creators/schema.ts @@ -366,6 +366,8 @@ export const youtubeChannelVideosSchema = z.object({ .number() .nullish() .transform((val) => val ?? 0), + likeCountInt: z.number().nullish(), + commentCountInt: z.number().nullish(), publishedTimeText: z.string().nullish(), publishedTime: z.string().nullish(), lengthText: z.string().nullish(), @@ -403,6 +405,22 @@ const urlListSchema = z }) .nullish(); +const stringFromStringOrNumberSchema = z + .union([z.string(), z.number()]) + .nullish() + .transform((value) => (value == null ? value : String(value))); + +const booleanFromBooleanOrNumberSchema = z + .union([z.boolean(), z.number()]) + .nullish() + .transform((value) => { + if (typeof value === "number") { + return value !== 0; + } + + return value; + }); + export const tiktokProfileVideosSchema = z.object({ aweme_list: z.array( z.object({ @@ -431,8 +449,8 @@ export const tiktokProfileVideosSchema = z.object({ .nullish(), }), ), - max_cursor: z.string().nullish(), - has_more: z.boolean().nullish(), + max_cursor: stringFromStringOrNumberSchema, + has_more: booleanFromBooleanOrNumberSchema, }); export const tiktokTranscriptSchema = z.object({ diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts index 26c0d693c8f..72313c8b849 100644 --- a/apps/web/lib/partner-content-search/constants.ts +++ b/apps/web/lib/partner-content-search/constants.ts @@ -29,7 +29,21 @@ export const PARTNER_CONTENT_SEARCH_LIMITS = { chunkMaxTokens: 800, chunkOverlapTokens: 80, chunkCandidateCount: 200, + // Safety cap on recent posts per partner feeding the topic-fit window. The + // window itself is time-based (recencyWindowMonths); this just bounds the + // per-partner item set so the exact per-item scoring query stays cheap for + // extremely prolific creators. + recentContentMaxPerPartner: 200, + // Pre-dedup ANN over-fetch for retrieval. We pull this many raw chunks via the + // vector index (flat `ORDER BY DISTANCE ... LIMIT`, which stays index-friendly), + // then collapse to the best chunk per content item in app code. Larger = better + // video/partner diversity when one high-volume creator dominates a query; + // bounded so the ANN fetch stays fast. + vectorSearchChunkPoolSize: 1000, rerankerCandidateCount: 150, + // Cap each document sent to the reranker. Chunks are <=800 tokens; trimming the + // tail keeps the rerank payload (and thus latency) down with little quality loss. + rerankerMaxDocChars: 2_000, partnerScorePoolSize: 3, } as const; @@ -38,6 +52,68 @@ export const PARTNER_CONTENT_SEARCH_FUSION_WEIGHTS = { existingPartnerRank: 0.4, } as const; +// "Topic fit" is the aggregate card score. Reranker relevance gates which recent +// posts count as on-topic, then the displayed score is based on coverage. +// `bandThresholds` are coverage cutoffs that drive the qualitative label + color. +export const PARTNER_CONTENT_SEARCH_TOPIC_FIT = { + // A recent post counts as "on topic" when its reranker relevance clears this. + // Reranker scores for a broad keyword query top out well below 1 (~0.6-0.8 for + // a clearly on-topic post), so magnitude gates membership rather than scaling + // the score. Tune against a known off-topic creator to set the floor. + rerankMatchThreshold: 0.4, + // Topic fit = 100 * adjustedCoverage^exponent, where adjusted coverage is + // strength-weighted evidence coverage after the small-sample confidence + // adjustment. The <1 exponent is a concave lift so mid-coverage creators don't + // read punishingly low. + coverageCurveExponent: 0.5, + // A matched source at or above this score gets full per-source topic credit. + // Scores below this but above the match threshold still count, but only + // partially. This prevents reranker magnitude from capping creators who are + // clearly and consistently on-topic. + strongMatchScore: 0.7, + // A just-over-threshold source gets this much per-source topic credit. + minimumMatchedEvidenceStrength: 0.35, + // Small recent-content samples should not rank as confidently as larger + // bodies of work with the same match ratio. Keep this light so a creator with + // 25-50 consistently on-topic posts can still reach the high 90s. + sampleConfidenceContentCount: 2, + // Creator-entered text on videos/reels (titles, descriptions, captions) is + // useful evidence, but weaker than transcript evidence because it can contain + // SEO copy, link blocks, or repeated channel boilerplate. Static posts get full + // credit because creator text is the main searchable content there. + creatorTextOnlyVideoWeight: 0.4, + bandThresholds: { + consistent: 0.5, + frequent: 0.25, + }, +} as const; + +// Ranking for the detail-pane "Top content" section: a relevance-led blend of +// how on-topic a matched post is with how well it performed (views), normalized +// per creator and robust to a single viral outlier. This only orders that list; +// it never feeds Topic Fit, the bars, or partner ordering. +export const PARTNER_CONTENT_SEARCH_TOP_CONTENT = { + // How many matched posts the "Top content" section highlights. + topContentCount: 5, + // Blend weights. Relevance leads because this is a topic search; views break + // ties and surface the strongest proof points without overriding relevance. + relevanceWeight: 0.7, + engagementWeight: 0.3, + // Spread (in log10 view units) of the logistic that maps a post's views to a + // [0,1] engagement score relative to the creator's median. ~0.5 means roughly + // a 3x swing in views moves the score one logistic unit. Median post -> ~0.5, + // a viral outlier saturates toward 1 (never dominates), a below-median post + // lands near 0 but is never punished into negative territory. + engagementLogSpread: 0.5, +} as const; + +export type PartnerContentTopicFitBand = + | "consistent" + | "frequent" + | "occasional" + | "one-off" + | "none"; + // Max number of partners returned by the network content search (and the cap the // route enforces). Shared so the client request and the route stay in sync. export const PARTNER_CONTENT_SEARCH_PARTNER_LIMIT = 50; @@ -45,7 +121,16 @@ export const PARTNER_CONTENT_SEARCH_PARTNER_LIMIT = 50; // Default number of chunks surfaced per partner in the network content search. export const PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER = 2; +// Max number of content-level matches surfaced for one partner. This lines up +// with the card's "last 28 posts" content match visualization. +export const PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER = 28; + // Timeout for the query-embedding call on user-facing search routes. Fails fast // through the normal error path instead of hanging until the function's // maxDuration (30s) is hit. export const PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS = 10_000; + +// Timeout for the second-stage reranker call. Unlike the embedding timeout this +// fails *soft*: on timeout the search falls back to cosine ordering so a slow +// rerank never breaks results, it just doesn't improve them. +export const PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS = 4_000; diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index 41702535590..535d37cf32a 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -90,6 +90,9 @@ export const partnerContentFetchPayloadSchema = z.object({ runStamp: z.string().min(1), dryRun: z.boolean().default(false), forceTranscriptJobs: z.boolean().default(false), + // Manual/direct fetch escape hatch for linked-but-unverified platforms. + // Enumerated backfills still only enqueue verified platforms. + ignoreUnverified: z.boolean().default(false), partnerId: z.string().min(1), partnerPlatformId: z.string().min(1), platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), diff --git a/apps/web/lib/partner-content-search/ingestion/normalize-content.ts b/apps/web/lib/partner-content-search/ingestion/normalize-content.ts index 03ac620823c..a0df9816ab7 100644 --- a/apps/web/lib/partner-content-search/ingestion/normalize-content.ts +++ b/apps/web/lib/partner-content-search/ingestion/normalize-content.ts @@ -13,7 +13,9 @@ type YouTubeChannelVideo = z.infer< type TikTokProfileVideo = z.infer< typeof tiktokProfileVideosSchema >["aweme_list"][number]; -type InstagramUserPost = z.infer["items"][number]; +type InstagramUserPost = z.infer< + typeof instagramUserPostsSchema +>["items"][number]; type InstagramMediaTranscript = z.infer; export type NormalizedPartnerContentItem = { @@ -26,6 +28,10 @@ export type NormalizedPartnerContentItem = { publishedAt: Date | null; durationMs: number | null; viewCount: number | null; + likeCount: number | null; + commentCount: number | null; + shareCount: number | null; + saveCount: number | null; transcriptEligible?: boolean; }; @@ -50,6 +56,10 @@ export function normalizeYouTubeChannelVideo( typeof video.viewCountInt === "number" ? Math.max(0, video.viewCountInt) : null, + likeCount: normalizeMetricCount(video.likeCountInt), + commentCount: normalizeMetricCount(video.commentCountInt), + shareCount: null, + saveCount: null, }; } @@ -97,6 +107,10 @@ export function normalizeTikTokProfileVideo( typeof video.statistics?.play_count === "number" ? Math.max(0, video.statistics.play_count) : null, + likeCount: normalizeMetricCount(video.statistics?.digg_count), + commentCount: normalizeMetricCount(video.statistics?.comment_count), + shareCount: normalizeMetricCount(video.statistics?.share_count), + saveCount: normalizeMetricCount(video.statistics?.collect_count), transcriptEligible: durationMs === null || durationMs <= 120_000, }; } @@ -110,13 +124,19 @@ export function normalizeTikTokTranscriptSegments( export function normalizeInstagramUserPost( post: InstagramUserPost, ): NormalizedPartnerContentItem | null { - const platformContentId = post.code ?? post.pk ?? post.id; + const shortcode = post.code ?? extractInstagramShortcode(post.url); + const platformContentId = shortcode ?? post.pk ?? post.id; if (!platformContentId) return null; const contentType = getInstagramContentType(post); - const url = - post.url ?? - `https://www.instagram.com/${contentType === "reel" ? "reel" : "p"}/${platformContentId}/`; + const url = getInstagramContentUrl({ + contentType, + fallbackUrl: post.url, + shortcode, + }); + if (!url) return null; + + const caption = post.caption?.text?.trim() || null; const durationMs = typeof post.video_duration === "number" ? Math.max(0, Math.round(post.video_duration * 1000)) @@ -126,8 +146,8 @@ export function normalizeInstagramUserPost( platformContentId, url, contentType, - title: null, - description: post.caption?.text ?? null, + title: normalizeCaptionTitle(caption), + description: caption, thumbnailUrl: post.display_uri ?? firstUrl(post.image_versions2?.candidates), publishedAt: @@ -139,6 +159,10 @@ export function normalizeInstagramUserPost( : typeof post.ig_play_count === "number" ? Math.max(0, post.ig_play_count) : null, + likeCount: normalizeMetricCount(post.like_count), + commentCount: normalizeMetricCount(post.comment_count), + shareCount: null, + saveCount: null, transcriptEligible: isInstagramVideoContent(post) && (durationMs === null || durationMs <= 120_000), @@ -176,11 +200,15 @@ function normalizeTikTokDurationMs(value?: number | null) { return duration > 0 && duration < 1000 ? duration * 1000 : duration; } +function normalizeMetricCount(value?: number | null) { + return typeof value === "number" ? Math.max(0, value) : null; +} + function firstUrl(values?: Array<{ url?: string | null } | string> | null) { const first = values?.[0]; if (!first) return null; - return typeof first === "string" ? first : (first.url ?? null); + return typeof first === "string" ? first : first.url ?? null; } function getInstagramContentType(post: InstagramUserPost) { @@ -208,6 +236,50 @@ function isInstagramVideoContent(post: InstagramUserPost) { ); } +function extractInstagramShortcode(url?: string | null) { + if (!url) return null; + + try { + const pathname = new URL(url).pathname; + const shortcode = pathname.match( + /^\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1]; + return shortcode || null; + } catch { + const shortcode = url.match( + /instagram\.com\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1]; + return shortcode || null; + } +} + +function getInstagramContentUrl({ + contentType, + fallbackUrl, + shortcode, +}: { + contentType: string; + fallbackUrl?: string | null; + shortcode?: string | null; +}) { + if (shortcode) { + return `https://www.instagram.com/${contentType === "reel" ? "reel" : "p"}/${shortcode}/`; + } + + return fallbackUrl ?? null; +} + +function normalizeCaptionTitle(caption: string | null) { + if (!caption) return null; + + const firstLine = caption.split(/\r?\n/)[0]?.trim(); + if (!firstLine) return null; + + return firstLine.length > 140 + ? `${firstLine.slice(0, 137).trimEnd()}...` + : firstLine; +} + function parseWebVttTranscript(transcript: string): TranscriptSegment[] { const lines = transcript.replace(/\r\n/g, "\n").split("\n"); const segments: TranscriptSegment[] = []; diff --git a/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts b/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts index 0d8ab99e055..cf06b963559 100644 --- a/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts +++ b/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts @@ -1,10 +1,10 @@ import "server-only"; -import { logger } from "@/lib/axiom/server"; import { createId } from "@/lib/api/create-id"; +import { logger } from "@/lib/axiom/server"; import { qstash } from "@/lib/cron"; -import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; import { hashText } from "@/lib/partner-content-search/chunk-transcript"; +import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; import { refreshPartnerContentItemChunkCountsBulk } from "@/lib/partner-content-search/ingestion/chunk-counts"; import { createPartnerContentDeduplicationId, @@ -69,8 +69,11 @@ export async function writeFetchedContentItems({ thumbnailUrl: item.thumbnailUrl, publishedAt: item.publishedAt, durationMs: item.durationMs, - viewCount: - item.viewCount === null ? null : BigInt(Math.trunc(item.viewCount)), + viewCount: toNullableBigInt(item.viewCount), + likeCount: toNullableBigInt(item.likeCount), + commentCount: toNullableBigInt(item.commentCount), + shareCount: toNullableBigInt(item.shareCount), + saveCount: toNullableBigInt(item.saveCount), transcriptFetchStatus: "pending", transcriptHasTimestamps: false, totalChunkCount: 0, @@ -325,15 +328,17 @@ async function writeMetadataChunks({ id, }, data: { + url: sourceItem.url, title: sourceItem.title, description: sourceItem.description, thumbnailUrl: sourceItem.thumbnailUrl, publishedAt: sourceItem.publishedAt, durationMs: sourceItem.durationMs, - viewCount: - sourceItem.viewCount === null - ? null - : BigInt(Math.trunc(sourceItem.viewCount)), + viewCount: toNullableBigInt(sourceItem.viewCount), + likeCount: toNullableBigInt(sourceItem.likeCount), + commentCount: toNullableBigInt(sourceItem.commentCount), + shareCount: toNullableBigInt(sourceItem.shareCount), + saveCount: toNullableBigInt(sourceItem.saveCount), }, }), ), @@ -366,3 +371,7 @@ function createMetadataChunkText(item: NormalizedPartnerContentItem) { return lines.join("\n"); } + +function toNullableBigInt(value: number | null) { + return value === null ? null : BigInt(Math.trunc(value)); +} diff --git a/apps/web/lib/partner-content-search/ranking.ts b/apps/web/lib/partner-content-search/ranking.ts new file mode 100644 index 00000000000..7c6e45d53f5 --- /dev/null +++ b/apps/web/lib/partner-content-search/ranking.ts @@ -0,0 +1,445 @@ +import type { PartnerContentTopicFitBand } from "./constants"; +import { PARTNER_CONTENT_SEARCH_TOPIC_FIT } from "./constants"; +import { toScore, type PartnerContentSearchRow } from "./search-utils"; + +export type PartnerContentMatchSource = "transcript" | "creatorText"; + +export type PartnerContentSearchQueryIntent = "entity" | "semantic"; + +export type PartnerContentSearchQuerySignals = { + normalizedQuery: string; + intent: PartnerContentSearchQueryIntent; +}; + +export type PartnerContentMatchEvidence = { + primarySource: PartnerContentMatchSource | null; + sources: PartnerContentMatchSource[]; + transcriptScore: number | null; + creatorTextScore: number | null; + creatorTextWeight: number; + weight: number; +}; + +const ENTITY_EXACT_MATCH_SCORE = 0.95; + +const SEMANTIC_QUERY_TERMS = new Set([ + "athlete", + "athletes", + "beauty", + "blogger", + "bloggers", + "coach", + "coaches", + "cooking", + "creator", + "creators", + "dad", + "fashion", + "finance", + "fitness", + "food", + "gamer", + "gamers", + "gaming", + "health", + "influencer", + "influencers", + "lifestyle", + "mom", + "music", + "nutrition", + "outdoor", + "outdoors", + "parenting", + "personal", + "photography", + "podcast", + "podcaster", + "recipe", + "recipes", + "running", + "skincare", + "sports", + "travel", + "wellness", + "yoga", +]); + +export function deriveTopicFit({ + matchedContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + recentContentCount, +}: { + matchedContentCount: number; + weightedMatchedContentCount: number; + weightedMatchedContentScore: number; + recentContentCount: number; +}): { topicFit: number; band: PartnerContentTopicFitBand } { + if (matchedContentCount === 0 || recentContentCount === 0) { + return { topicFit: 0, band: "none" }; + } + + // Topic fit is driven by strength-adjusted weighted coverage. Transcript + // evidence and creator text on static posts get full credit; creator text on + // videos is weighted by source context so repeated descriptions stay weaker + // than transcript evidence while exact captions can still count. Small samples + // are discounted so 3/3 weak matches do not read as equivalent to 30/30 strong + // matches. + const rawCoverage = weightedMatchedContentScore / recentContentCount; + const { + coverageCurveExponent, + bandThresholds, + sampleConfidenceContentCount, + } = PARTNER_CONTENT_SEARCH_TOPIC_FIT; + const sampleConfidence = Math.sqrt( + recentContentCount / (recentContentCount + sampleConfidenceContentCount), + ); + const coverage = Math.min(1, Math.max(0, rawCoverage * sampleConfidence)); + const topicFit = Math.round(100 * Math.pow(coverage, coverageCurveExponent)); + + const band: PartnerContentTopicFitBand = + coverage >= bandThresholds.consistent + ? "consistent" + : coverage >= bandThresholds.frequent + ? "frequent" + : matchedContentCount === 1 + ? "one-off" + : "occasional"; + + return { topicFit, band }; +} + +export function getRowRelevanceScore(row: PartnerContentSearchRow) { + return row.rerankScore ?? toScore(Number(row.distance)); +} + +export function getSourceAwareRowScore(row: PartnerContentSearchRow) { + const score = getRowRelevanceScore(row); + const source = getEvidenceSource(row.chunkSource); + + if (source === "transcript") return score; + return Number((score * getCreatorTextWeight(row.contentType)).toFixed(6)); +} + +export function sortRowsByRelevanceScore(rows: PartnerContentSearchRow[]) { + return [...rows].sort( + (a, b) => getRowRelevanceScore(b) - getRowRelevanceScore(a), + ); +} + +export function sortRowsBySourceAwareScore(rows: PartnerContentSearchRow[]) { + return [...rows].sort( + (a, b) => getSourceAwareRowScore(b) - getSourceAwareRowScore(a), + ); +} + +export function getEvidenceSource(source: string): PartnerContentMatchSource { + return source === "transcript" ? "transcript" : "creatorText"; +} + +export function getPartnerContentSearchQuerySignals( + query?: string | null, +): PartnerContentSearchQuerySignals { + const normalizedQuery = normalizeSearchText(query); + const terms = normalizedQuery.split(" ").filter(Boolean); + const intent: PartnerContentSearchQueryIntent = + normalizedQuery.length === 0 || + terms.some((term) => SEMANTIC_QUERY_TERMS.has(term)) || + terms.length > 3 + ? "semantic" + : "entity"; + + return { + normalizedQuery, + intent, + }; +} + +export function normalizeSearchText(value?: string | null) { + return (value ?? "") + .toLowerCase() + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/[^a-z0-9]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +export function hasExactQueryMention( + value: string | null | undefined, + querySignals: PartnerContentSearchQuerySignals, +) { + return ( + querySignals.normalizedQuery.length > 0 && + normalizeSearchText(value).includes(querySignals.normalizedQuery) + ); +} + +export function getEntityTranscriptScore({ + currentScore, + hasExactQueryMention, + queryIntent, +}: { + currentScore: number | null; + hasExactQueryMention: boolean; + queryIntent: PartnerContentSearchQueryIntent; +}) { + if (queryIntent !== "entity" || !hasExactQueryMention) return currentScore; + return maxNullableScore(currentScore, ENTITY_EXACT_MATCH_SCORE); +} + +export function getEntityCreatorTextBoost({ + platformType, + contentType, + transcriptFetchStatus, + titleHasExactQueryMention, + descriptionHasExactQueryMention, + chunkHasExactQueryMention, + transcriptScore, + queryIntent, +}: { + platformType: string; + contentType: string; + transcriptFetchStatus?: string | null; + titleHasExactQueryMention: boolean; + descriptionHasExactQueryMention: boolean; + chunkHasExactQueryMention: boolean; + transcriptScore: number | null; + queryIntent: PartnerContentSearchQueryIntent; +}): { score: number; weight: number } | null { + if ( + queryIntent !== "entity" || + (!titleHasExactQueryMention && + !descriptionHasExactQueryMention && + !chunkHasExactQueryMention) + ) { + return null; + } + + if (isStaticContentType(contentType)) { + return { + score: ENTITY_EXACT_MATCH_SCORE, + weight: 1, + }; + } + + const platform = platformType.toLowerCase(); + const transcriptUnavailable = + transcriptScore == null || + isUnavailableTranscriptStatus(transcriptFetchStatus); + + if (platform === "tiktok" || platform === "instagram") { + return transcriptUnavailable + ? { + score: ENTITY_EXACT_MATCH_SCORE, + weight: 0.95, + } + : { + score: 0.9, + weight: 0.85, + }; + } + + if (platform === "youtube") { + return titleHasExactQueryMention + ? { + score: 0.9, + weight: 0.85, + } + : { + score: 0.65, + weight: 0.35, + }; + } + + return titleHasExactQueryMention + ? { + score: 0.9, + weight: 0.85, + } + : { + score: 0.75, + weight: 0.6, + }; +} + +export function maxNullableScore(...scores: Array) { + const numericScores = scores.filter( + (score): score is number => typeof score === "number", + ); + + return numericScores.length ? Math.max(...numericScores) : null; +} + +function getCreatorTextWeight(contentType: string) { + return isVideoLikeContentType(contentType) + ? PARTNER_CONTENT_SEARCH_TOPIC_FIT.creatorTextOnlyVideoWeight + : 1; +} + +function isVideoLikeContentType(contentType: string) { + return ["video", "reel", "short", "shorts"].includes( + contentType.toLowerCase(), + ); +} + +function isStaticContentType(contentType: string) { + return ["carousel", "image", "photo", "post"].includes( + contentType.toLowerCase(), + ); +} + +function isUnavailableTranscriptStatus(status?: string | null) { + return ( + status === "pending" || status === "notAvailable" || status === "error" + ); +} + +export function createContentMatchEvidence({ + contentType, + transcriptScore, + creatorTextScore, + creatorTextWeightOverride, +}: { + contentType: string; + transcriptScore: number | null; + creatorTextScore: number | null; + creatorTextWeightOverride?: number | null; +}): PartnerContentMatchEvidence { + const sources: PartnerContentMatchSource[] = []; + if (transcriptScore != null) sources.push("transcript"); + if (creatorTextScore != null) sources.push("creatorText"); + + const primarySource: PartnerContentMatchSource | null = + transcriptScore != null + ? "transcript" + : creatorTextScore != null + ? "creatorText" + : null; + const creatorTextWeight = + creatorTextWeightOverride ?? getCreatorTextWeight(contentType); + const weight = + primarySource === null + ? 0 + : primarySource === "transcript" + ? 1 + : creatorTextWeight; + + return { + primarySource, + sources, + transcriptScore, + creatorTextScore, + creatorTextWeight, + weight, + }; +} + +export function getEvidenceMatchScore(evidence: PartnerContentMatchEvidence) { + if (evidence.sources.length === 0) return null; + + return Number( + Math.max( + getEvidenceStrength(evidence.transcriptScore), + getEvidenceStrength(evidence.creatorTextScore) * + evidence.creatorTextWeight, + ).toFixed(6), + ); +} + +export function getEvidenceTopicScore(evidence: PartnerContentMatchEvidence) { + if (evidence.sources.length === 0) return null; + + return Number( + Math.max( + // A transcript/source-content match has already passed the relevance gate, + // so it gets full topic credit. Creator text uses the context-aware weight + // chosen when the evidence was created. + evidence.transcriptScore != null ? 1 : 0, + getEvidenceStrength(evidence.creatorTextScore) * + evidence.creatorTextWeight, + ).toFixed(6), + ); +} + +function getEvidenceStrength(score: number | null) { + if (score == null) return 0; + + const { + rerankMatchThreshold, + strongMatchScore, + minimumMatchedEvidenceStrength, + } = PARTNER_CONTENT_SEARCH_TOPIC_FIT; + + if (score >= strongMatchScore) return 1; + if (score <= rerankMatchThreshold) return minimumMatchedEvidenceStrength; + + const progress = + (score - rerankMatchThreshold) / (strongMatchScore - rerankMatchThreshold); + + return ( + minimumMatchedEvidenceStrength + + progress * (1 - minimumMatchedEvidenceStrength) + ); +} + +export function getMatchedSourceScore({ + rerankScore, + bestDistance, + cutoffDistance, +}: { + rerankScore?: number; + bestDistance?: number; + cutoffDistance?: number | null; +}) { + if (rerankScore !== undefined) { + return rerankScore >= PARTNER_CONTENT_SEARCH_TOPIC_FIT.rerankMatchThreshold + ? rerankScore + : null; + } + + return bestDistance !== undefined && + cutoffDistance != null && + bestDistance <= cutoffDistance + ? toScore(bestDistance) + : null; +} + +export function getSourceScore( + sourceScoresByItemId: Map>, + itemId: string, + source: PartnerContentMatchSource, +) { + return sourceScoresByItemId.get(itemId)?.get(source); +} + +export function setSourceScore( + sourceScoresByItemId: Map>, + itemId: string, + source: PartnerContentMatchSource, + score: number, +) { + const itemScores = sourceScoresByItemId.get(itemId) ?? new Map(); + const current = itemScores.get(source); + + if (current === undefined || score > current) { + itemScores.set(source, score); + sourceScoresByItemId.set(itemId, itemScores); + } +} + +export function setSourceDistance( + sourceDistancesByItemId: Map>, + itemId: string, + source: PartnerContentMatchSource, + distance: number, +) { + const itemDistances = sourceDistancesByItemId.get(itemId) ?? new Map(); + const current = itemDistances.get(source); + + if (current === undefined || distance < current) { + itemDistances.set(source, distance); + sourceDistancesByItemId.set(itemId, itemDistances); + } +} diff --git a/apps/web/lib/partner-content-search/search-utils.ts b/apps/web/lib/partner-content-search/search-utils.ts index 07bb9b5a38b..aa0d0ffcf11 100644 --- a/apps/web/lib/partner-content-search/search-utils.ts +++ b/apps/web/lib/partner-content-search/search-utils.ts @@ -2,6 +2,16 @@ // Both routes run the same raw vector query and group rows by partner; only the // per-chunk shape differs, so each route passes its own `toChunkResult` mapper. +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS, +} from "./constants"; +import { + rerankPartnerContent, + VoyageApiError, + VoyageTimeoutError, +} from "./voyage"; + export type PartnerContentSearchRow = { chunkId: string; partnerContentItemId: string; @@ -14,9 +24,12 @@ export type PartnerContentSearchRow = { platformIdentifier: string; platformContentId: string; contentUrl: string; + contentType: string; contentTitle: string | null; + contentDescription?: string | null; contentThumbnailUrl: string | null; contentPublishedAt: Date | null; + contentDurationMs: number | null; chunkSource: string; // Only the admin query selects chunkIndex; the network query omits it. chunkIndex?: number; @@ -24,22 +37,69 @@ export type PartnerContentSearchRow = { startMs: number | null; endMs: number | null; distance: number | string; + // Set by the reranker (second stage) when it runs; absent for cosine-only + // results. `rerankPartnerSearchRows` mutates rows in place to attach it. + rerankScore?: number | null; }; export function toScore(distance: number) { return Number((1 - distance).toFixed(6)); } +// The score used for ranking + display: the reranker's relevance score when +// present, otherwise cosine similarity (1 - cosine distance). +export function effectiveRowScore(row: PartnerContentSearchRow) { + return row.rerankScore ?? toScore(Number(row.distance)); +} + +// Collapse a flat, distance-ascending chunk pool to the single best chunk per +// content item. Because rows are sorted best-first, the first occurrence of each +// content item is its best chunk. This turns a top-k *chunk* pool into a per-video +// candidate set in app code, avoiding a SQL window-function dedup that would force +// a full distance scan and defeat the ANN vector index. +export function dedupeBestChunkPerContentItem(rows: PartnerContentSearchRow[]) { + const seen = new Set(); + const deduped: PartnerContentSearchRow[] = []; + + for (const row of rows) { + if (seen.has(row.partnerContentItemId)) continue; + seen.add(row.partnerContentItemId); + deduped.push(row); + } + + return deduped; +} + +export function dedupeBestChunkPerContentItemSource( + rows: PartnerContentSearchRow[], +) { + const seen = new Set(); + const deduped: PartnerContentSearchRow[] = []; + + for (const row of rows) { + const key = `${row.partnerContentItemId}:${row.chunkSource}`; + if (seen.has(key)) continue; + seen.add(key); + deduped.push(row); + } + + return deduped; +} + export function groupPartnerSearchResults({ rows, limit, chunksPerPartner, toChunkResult, + dedupeKey, + getRowScore = effectiveRowScore, }: { rows: PartnerContentSearchRow[]; limit: number; chunksPerPartner: number; toChunkResult: (row: PartnerContentSearchRow, distance: number) => TChunk; + dedupeKey?: (row: PartnerContentSearchRow) => string; + getRowScore?: (row: PartnerContentSearchRow) => number; }) { const partners = new Map< string, @@ -49,36 +109,121 @@ export function groupPartnerSearchResults({ username: string | null; image: string | null; description: string | null; - bestDistance: number; score: number; + cosineScore: number; + rerankScore: number | null; chunks: TChunk[]; + chunkKeys: Set; } >(); for (const row of rows) { const distance = Number(row.distance); - const partner = partners.get(row.partnerId) ?? { + const cosineScore = toScore(distance); + const rerankScore = row.rerankScore ?? null; + const effectiveScore = getRowScore(row); + + const existing = partners.get(row.partnerId); + const partner = existing ?? { partnerId: row.partnerId, name: row.partnerName, username: row.partnerUsername, image: row.partnerImage, description: row.partnerDescription, - bestDistance: distance, - score: toScore(distance), + score: effectiveScore, + cosineScore, + rerankScore, chunks: [] as TChunk[], + chunkKeys: new Set(), }; - partner.bestDistance = Math.min(partner.bestDistance, distance); - partner.score = toScore(partner.bestDistance); + if (existing && effectiveScore > partner.score) { + partner.score = effectiveScore; + partner.cosineScore = cosineScore; + partner.rerankScore = rerankScore; + } + + const chunkKey = dedupeKey?.(row); + const shouldAddChunk = + partner.chunks.length < chunksPerPartner && + (!chunkKey || !partner.chunkKeys.has(chunkKey)); - if (partner.chunks.length < chunksPerPartner) { + if (shouldAddChunk) { partner.chunks.push(toChunkResult(row, distance)); + if (chunkKey) partner.chunkKeys.add(chunkKey); } partners.set(row.partnerId, partner); } return Array.from(partners.values()) - .sort((a, b) => a.bestDistance - b.bestDistance) - .slice(0, limit); + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .map(({ chunkKeys, ...partner }) => partner); +} + +// Re-score the top vector-search candidates with the Voyage reranker. Mutates +// rows in place to attach `rerankScore`, then returns them sorted by effective +// score (rerank when present) descending. Fails *soft*: on a reranker timeout or +// API error it logs and returns the original cosine ordering, so the second-stage +// call can never break search; it can only improve it. +export async function rerankPartnerSearchRows({ + query, + rows, +}: { + query: string; + rows: PartnerContentSearchRow[]; +}): Promise<{ rows: PartnerContentSearchRow[]; reranked: boolean }> { + if (rows.length === 0) return { rows, reranked: false }; + + // Only the top-N cosine candidates are reranked, to bound the rerank payload + // (and latency). Rows beyond the cap keep their cosine score. + const candidates = rows.slice( + 0, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, + ); + const documents = candidates.map((row) => + (row.chunkText ?? "").slice( + 0, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerMaxDocChars, + ), + ); + + try { + const results = await rerankPartnerContent({ + query, + documents, + topK: candidates.length, + timeoutMs: PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS, + }); + + for (const { index, relevanceScore } of results) { + const row = candidates[index]; + if (row) row.rerankScore = relevanceScore; + } + } catch (error) { + if ( + error instanceof VoyageTimeoutError || + error instanceof VoyageApiError + ) { + console.error( + "[partner-content-search] reranker failed, falling back to cosine:", + error.message, + ); + return { rows, reranked: false }; + } + throw error; + } + + // Reranked candidates always sort ahead of the cosine-only tail (rows beyond + // the rerank cap): their cosine score was already higher, and we don't want a + // weak un-reranked row leapfrogging a reranked one on a different score scale. + const reranked = [...rows].sort((a, b) => { + const aReranked = a.rerankScore != null; + const bReranked = b.rerankScore != null; + if (aReranked !== bReranked) return aReranked ? -1 : 1; + return effectiveRowScore(b) - effectiveRowScore(a); + }); + + return { rows: reranked, reranked: true }; } diff --git a/apps/web/lib/partner-content-search/top-content-ranking.ts b/apps/web/lib/partner-content-search/top-content-ranking.ts new file mode 100644 index 00000000000..311f40d96f0 --- /dev/null +++ b/apps/web/lib/partner-content-search/top-content-ranking.ts @@ -0,0 +1,80 @@ +// Pure ranking helpers for the partner detail-pane "Top content" section. +// +// This module is intentionally dependency-free (only constants) so it can run on +// both the server and the client without dragging server-only code (voyage, db) +// into the client bundle. It blends a matched post's relevance with how well it +// performed (views), normalized per creator and robust to a single viral post. +// +// Scope note: this orders the "Top content" list only. It does not feed Topic +// Fit, the content-match bars, the X-of-Y counts, or partner ordering. + +import { PARTNER_CONTENT_SEARCH_TOP_CONTENT } from "./constants"; + +// Median of a numeric list (robust center, unlike a mean which a viral outlier +// would drag upward). Returns null for an empty list. +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[mid - 1] + sorted[mid]) / 2 + : sorted[mid]; +} + +// The creator's "typical" view count, used as the engagement baseline. Computed +// over all of their recent posts (matched + unmatched) with real view data, so a +// single viral hit can't move it and posts aren't normalized against each other. +export function getViewBaseline( + views: Array, +): number | null { + return median( + views.filter((value): value is number => value != null && value > 0), + ); +} + +// Maps a post's views to a [0,1] engagement score relative to the creator's +// median, on a log scale through a logistic squash: +// - a median post scores ~0.5, +// - a viral outlier saturates toward 1 (bounded, never dominates), +// - a below-median post lands near 0 but is never negative (not punished). +// Returns null when there's no usable signal (no baseline, or no views on the +// post) so callers can fall back to pure relevance instead of guessing. +export function getEngagementScore({ + views, + baselineViews, +}: { + views: number | null | undefined; + baselineViews: number | null | undefined; +}): number | null { + if (baselineViews == null || baselineViews <= 0) return null; + if (views == null || views <= 0) return null; + + const { engagementLogSpread } = PARTNER_CONTENT_SEARCH_TOP_CONTENT; + const z = + (Math.log10(1 + views) - Math.log10(1 + baselineViews)) / + engagementLogSpread; + + return 1 / (1 + Math.exp(-z)); +} + +// Relevance-led blend used to order the "Top content" list. When there's no +// usable engagement signal (no creator baseline or no views on the post), this +// degrades gracefully to pure relevance so ranking never depends on view data +// being present. +export function getBlendedTopContentScore({ + relevance, + views, + baselineViews, +}: { + relevance: number; + views: number | null | undefined; + baselineViews: number | null | undefined; +}): number { + const engagement = getEngagementScore({ views, baselineViews }); + if (engagement == null) return relevance; + + const { relevanceWeight, engagementWeight } = + PARTNER_CONTENT_SEARCH_TOP_CONTENT; + + return relevanceWeight * relevance + engagementWeight * engagement; +} diff --git a/apps/web/lib/partner-content-search/types.ts b/apps/web/lib/partner-content-search/types.ts index 489726cea85..f66f399259f 100644 --- a/apps/web/lib/partner-content-search/types.ts +++ b/apps/web/lib/partner-content-search/types.ts @@ -9,6 +9,16 @@ export const PARTNER_CONTENT_SEARCH_PLATFORMS = [ export type PartnerContentPlatform = (typeof PARTNER_CONTENT_SEARCH_PLATFORMS)[number]; +const PARTNER_CONTENT_SEARCH_PLATFORM_SET = new Set( + PARTNER_CONTENT_SEARCH_PLATFORMS, +); + +export function isPartnerContentSearchPlatform( + platform: string | null | undefined, +): platform is PartnerContentPlatform { + return Boolean(platform && PARTNER_CONTENT_SEARCH_PLATFORM_SET.has(platform)); +} + export type VoyageInputType = "document" | "query"; export type TranscriptSegment = { diff --git a/apps/web/lib/partner-content-search/voyage.ts b/apps/web/lib/partner-content-search/voyage.ts index f2edb023c16..f3a884f4099 100644 --- a/apps/web/lib/partner-content-search/voyage.ts +++ b/apps/web/lib/partner-content-search/voyage.ts @@ -185,32 +185,45 @@ export async function rerankPartnerContent({ topK, apiKey = process.env.VOYAGE_API_KEY, fetchImpl = fetch, + timeoutMs, }: { query: string; documents: string[]; topK?: number; apiKey?: string; fetchImpl?: VoyageFetch; + // When set, aborts the request after this many ms. User-facing search routes + // pass it so a slow rerank fails fast and falls back to cosine ordering. + timeoutMs?: number; }): Promise { // AI SDK Core supports embeddings, but not Voyage reranking. Keep this direct // wrapper until Dub has a shared reranker abstraction. if (!apiKey) throw new Error("VOYAGE_API_KEY is not set."); if (documents.length === 0) return []; - const response = await fetchImpl(`${VOYAGE_API_BASE_URL}/rerank`, { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify( - buildVoyageRerankRequest({ - query, - documents, - topK, - }), - ), - }); + let response: Awaited>; + try { + response = await fetchImpl(`${VOYAGE_API_BASE_URL}/rerank`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify( + buildVoyageRerankRequest({ + query, + documents, + topK, + }), + ), + signal: timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined, + }); + } catch (error) { + if (timeoutMs && isAbortOrTimeoutError(error)) { + throw new VoyageTimeoutError("rerank", timeoutMs); + } + throw error; + } if (!response.ok) { throw new VoyageApiError(response.status, "rerank"); diff --git a/apps/web/lib/swr/use-partner-content-search.ts b/apps/web/lib/swr/use-partner-content-search.ts index 37ffa569c70..13eb3004a4b 100644 --- a/apps/web/lib/swr/use-partner-content-search.ts +++ b/apps/web/lib/swr/use-partner-content-search.ts @@ -1,11 +1,24 @@ import { PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, + type PartnerContentTopicFitBand, } from "@/lib/partner-content-search/constants"; +import type { NetworkPartnerProps } from "@/lib/types"; import type { PlatformType } from "@prisma/client"; import useSWR from "swr"; import useWorkspace from "./use-workspace"; +export type PartnerContentMatchSource = "transcript" | "creatorText"; + +export type PartnerContentMatchEvidence = { + primarySource: PartnerContentMatchSource | null; + sources: PartnerContentMatchSource[]; + transcriptScore: number | null; + creatorTextScore: number | null; + creatorTextWeight: number; + weight: number; +}; + export type PartnerContentSearchPartner = { partnerId: string; name: string; @@ -13,8 +26,13 @@ export type PartnerContentSearchPartner = { image: string | null; description: string | null; score: number; + // Debug comparison for validating reranker impact against raw vector search. + cosineScore?: number | null; + rerankScore?: number | null; + partner: NetworkPartnerProps; chunks: { chunkId: string; + partnerContentItemId: string; platform: { type: string; identifier: string; @@ -22,9 +40,12 @@ export type PartnerContentSearchPartner = { content: { platformContentId: string; url: string; + type: string; title: string | null; + description: string | null; thumbnailUrl: string | null; publishedAt: string | null; + durationMs: number | null; }; chunk: { source: "metadata" | "transcript" | string; @@ -33,11 +54,51 @@ export type PartnerContentSearchPartner = { endMs: number | null; }; score: number; + cosineScore?: number; + rerankScore?: number | null; + matchEvidence?: PartnerContentMatchEvidence; }[]; + matchSummary: { + matchedContentCount: number; + transcriptMatchedContentCount: number; + creatorTextMatchedContentCount: number; + creatorTextOnlyContentCount: number; + weightedMatchedContentCount: number; + weightedMatchedContentScore: number; + recentContentCount: number; + totalContentCount: number; + // Aggregate card score (0-100) + its coverage-based band, and the partner's + // platforms ranked by matched-post frequency. + topicFit: number; + band: PartnerContentTopicFitBand; + topPlatforms: string[]; + // Brand-facing reach/activity signals over the on-topic posts. + followers: number | null; + medianViews: number | null; + lastOnTopicAt: string | null; + platforms: string[]; + sources: string[]; + oldestPublishedAt: string | null; + newestPublishedAt: string | null; + contentBars: { + partnerContentItemId: string; + platform: string; + platformContentId: string; + title: string | null; + url: string | null; + durationMs: number | null; + publishedAt: string | null; + viewCount: number | null; + matched: boolean; + matchScore: number | null; + matchEvidence: PartnerContentMatchEvidence; + }[]; + } | null; }; export type PartnerContentSearchResponse = { success: boolean; + reranked?: boolean; partners: PartnerContentSearchPartner[]; }; @@ -45,18 +106,39 @@ export default function usePartnerContentSearch({ enabled, query, platform, + country, starred, + partnerIds, + candidateChunkCount, + limit = PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, + chunksPerPartner = PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, }: { enabled: boolean; query: string; - platform: PlatformType; + platform?: PlatformType; + country?: string; starred: boolean; + partnerIds?: string[]; + candidateChunkCount?: number; + limit?: number; + chunksPerPartner?: number; }) { const { id: workspaceId } = useWorkspace(); - const { data, error, isLoading } = useSWR( + const { data, error, isLoading, mutate } = useSWR( enabled && workspaceId - ? ["partner-content-search", workspaceId, query, platform, starred] + ? [ + "partner-content-search", + workspaceId, + query, + platform, + country, + starred, + partnerIds?.join(",") ?? "", + candidateChunkCount, + limit, + chunksPerPartner, + ] : null, // The shared `fetcher` from @dub/utils is GET-only; this endpoint takes a // POST body, so it needs its own fetcher. @@ -70,9 +152,12 @@ export default function usePartnerContentSearch({ }, body: JSON.stringify({ query, - limit: PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, - chunksPerPartner: PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, - platform, + limit, + chunksPerPartner, + ...(candidateChunkCount && { candidateChunkCount }), + ...(platform && { platform }), + ...(country && { country }), + ...(partnerIds?.length && { partnerIds }), starred: starred || undefined, }), }, @@ -91,5 +176,5 @@ export default function usePartnerContentSearch({ }, ); - return { data, error, isLoading }; + return { data, error, isLoading, mutate }; } diff --git a/apps/web/prisma/manual/partner-content-search-vector-index.sql b/apps/web/prisma/manual/partner-content-search-vector-index.sql index c07584cfb5a..fdd61f6a671 100644 --- a/apps/web/prisma/manual/partner-content-search-vector-index.sql +++ b/apps/web/prisma/manual/partner-content-search-vector-index.sql @@ -1,18 +1,17 @@ --- Manual PlanetScale DDL for partner natural-language search. +-- Manual PlanetScale index DDL for partner natural-language search. -- --- Prisma intentionally does not manage the PartnerContentChunk.embedding --- VECTOR(1024) column because plain MySQL cannot parse VECTOR DDL in CI/local --- `prisma db push` jobs. Run this after Prisma has created the --- PartnerContentChunk table on the target PlanetScale branch. +-- Prisma manages the PartnerContentChunk.embedding VECTOR(1024) column through +-- Unsupported("vector(1024)") in apps/web/prisma/schema/partner-content-search.prisma. +-- Run this after Prisma has created the column on the target PlanetScale branch. +-- The schema includes a same-named @@index only so Prisma db push preserves this +-- manual index; Prisma cannot define the VECTOR INDEX distance/options below. -- -- Keep the index distance in sync with the search query in: --- apps/web/app/(ee)/api/admin/partner-content/search/route.ts +-- apps/web/app/(ee)/api/admin/partner-content/search/route.ts and +-- apps/web/app/(ee)/api/network/partners/content-search/route.ts. -- That query currently uses DISTANCE(..., 'cosine'); if these metrics differ, -- PlanetScale cannot use the vector index and will fall back to a full scan. -ALTER TABLE PartnerContentChunk - ADD COLUMN embedding VECTOR(1024) NULL; - CREATE /*vt+ QUERY_TIMEOUT_MS=0 */ VECTOR INDEX partner_content_chunk_embedding_cosine_idx ON PartnerContentChunk(embedding) diff --git a/apps/web/prisma/schema/partner-content-search.prisma b/apps/web/prisma/schema/partner-content-search.prisma index d753ecd4aa6..abca05ec2b0 100644 --- a/apps/web/prisma/schema/partner-content-search.prisma +++ b/apps/web/prisma/schema/partner-content-search.prisma @@ -1,11 +1,12 @@ // Partner natural-language search schema. // -// The PartnerContentChunk.embedding VECTOR(1024) column is intentionally not in -// this Prisma schema. Plain MySQL cannot parse VECTOR DDL, so keeping it here -// breaks CI/local db push jobs that use non-PlanetScale databases. Create and -// index that column with apps/web/prisma/manual/partner-content-search-vector-index.sql -// on PlanetScale branches that support vectors, and keep all vector writes/search -// operations in raw SQL. +// Prisma does not expose PartnerContentChunk.embedding in Prisma Client because +// VECTOR is modeled as an unsupported database type. Prisma still manages the +// column for schema drift, while all vector reads/writes stay in raw SQL. +// Keep the matching @@index below so Prisma db push preserves the manual index, +// but create/recreate the real cosine vector index with +// apps/web/prisma/manual/partner-content-search-vector-index.sql because Prisma +// cannot express PlanetScale VECTOR INDEX options. enum PartnerContentTranscriptFetchStatus { pending @@ -32,6 +33,10 @@ model PartnerContentItem { publishedAt DateTime? durationMs Int? viewCount BigInt? + likeCount BigInt? + commentCount BigInt? + shareCount BigInt? + saveCount BigInt? transcriptFetchStatus PartnerContentTranscriptFetchStatus @default(pending) transcriptCreditsUsed Int? transcriptLastAttemptedAt DateTime? @@ -55,18 +60,19 @@ model PartnerContentItem { } model PartnerContentChunk { - id String @id + id String @id partnerContentItemId String partnerId String - source PartnerContentChunkSource @default(transcript) + source PartnerContentChunkSource @default(transcript) chunkIndex Int - chunkText String @db.Text + chunkText String @db.Text startMs Int? endMs Int? textHash String + embedding Unsupported("vector(1024)")? embeddingModel String - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt partnerContentItem PartnerContentItem @relation(fields: [partnerContentItemId], references: [id], onDelete: Cascade) partner Partner @relation(fields: [partnerId], references: [id], onDelete: Cascade) @@ -78,4 +84,7 @@ model PartnerContentChunk { // Covers the empty-query "recent content" list path, which filters on // embeddingModel while joining chunks by partnerContentItemId. @@index([partnerContentItemId, embeddingModel]) + // Drift guard only: Prisma cannot express PlanetScale VECTOR INDEX options. + // Create/recreate the real cosine ANN index with prisma/manual/partner-content-search-vector-index.sql. + @@index([embedding], map: "partner_content_chunk_embedding_cosine_idx") } diff --git a/apps/web/tests/partner-content-search/normalize-content.test.ts b/apps/web/tests/partner-content-search/normalize-content.test.ts new file mode 100644 index 00000000000..a151c76328b --- /dev/null +++ b/apps/web/tests/partner-content-search/normalize-content.test.ts @@ -0,0 +1,43 @@ +import { normalizeInstagramUserPost } from "@/lib/partner-content-search/ingestion/normalize-content"; +import { describe, expect, test } from "vitest"; + +describe("partner content normalization", () => { + test("canonicalizes Instagram reel URLs and uses captions as titles", () => { + const item = normalizeInstagramUserPost({ + id: "3590000000000000000", + pk: "3590000000000000000", + code: "DZqeWQvJ1Fk", + url: "https://www.instagram.com/andresthedesigner/p/DZqeWQvJ1Fk/", + media_type: 2, + product_type: "clips", + taken_at: 1781659229, + caption: { + text: "Framer 3.0 is pretty nice imo\n#framer #webdesign", + }, + play_count: 1234, + ig_play_count: null, + like_count: 123, + comment_count: 12, + display_uri: null, + image_versions2: { + candidates: [{ url: "https://cdn.example.com/thumbnail.jpg" }], + }, + video_versions: [{ url: "https://cdn.example.com/video.mp4" }], + video_duration: 12.3, + has_audio: true, + }); + + expect(item).toMatchObject({ + platformContentId: "DZqeWQvJ1Fk", + url: "https://www.instagram.com/reel/DZqeWQvJ1Fk/", + contentType: "reel", + title: "Framer 3.0 is pretty nice imo", + description: "Framer 3.0 is pretty nice imo\n#framer #webdesign", + thumbnailUrl: "https://cdn.example.com/thumbnail.jpg", + viewCount: 1234, + likeCount: 123, + commentCount: 12, + transcriptEligible: true, + }); + }); +}); diff --git a/apps/web/tests/partner-content-search/ranking.test.ts b/apps/web/tests/partner-content-search/ranking.test.ts new file mode 100644 index 00000000000..18e26de8109 --- /dev/null +++ b/apps/web/tests/partner-content-search/ranking.test.ts @@ -0,0 +1,147 @@ +import { + createContentMatchEvidence, + deriveTopicFit, + getEntityCreatorTextBoost, + getEvidenceMatchScore, + getEvidenceTopicScore, + getPartnerContentSearchQuerySignals, +} from "@/lib/partner-content-search/ranking"; +import { describe, expect, test, vi } from "vitest"; + +// Mock server-only module (ranking.ts imports search-utils.ts -> voyage.ts). +vi.mock("server-only", () => ({})); + +describe("partner content ranking", () => { + test("weights creator-entered video text below transcript evidence", () => { + const creatorTextOnlyVideo = createContentMatchEvidence({ + contentType: "video", + transcriptScore: null, + creatorTextScore: 0.8, + }); + const transcriptVideo = createContentMatchEvidence({ + contentType: "video", + transcriptScore: 0.8, + creatorTextScore: null, + }); + const creatorTextOnlyPhoto = createContentMatchEvidence({ + contentType: "photo", + transcriptScore: null, + creatorTextScore: 0.8, + }); + + expect(creatorTextOnlyVideo.weight).toBe(0.4); + expect(getEvidenceMatchScore(creatorTextOnlyVideo)).toBe(0.4); + expect(getEvidenceMatchScore(transcriptVideo)).toBe(1); + expect(getEvidenceMatchScore(creatorTextOnlyPhoto)).toBe(1); + expect(getEvidenceTopicScore(creatorTextOnlyVideo)).toBe(0.4); + expect(getEvidenceTopicScore(transcriptVideo)).toBe(1); + }); + + test("lets exact entity captions override the default creator text video discount", () => { + const evidence = createContentMatchEvidence({ + contentType: "video", + transcriptScore: null, + creatorTextScore: 0.95, + creatorTextWeightOverride: 0.95, + }); + + expect(evidence.creatorTextWeight).toBe(0.95); + expect(evidence.weight).toBe(0.95); + expect(getEvidenceMatchScore(evidence)).toBe(0.95); + expect(getEvidenceTopicScore(evidence)).toBe(0.95); + }); + + test("only boosts exact creator text for entity-style queries", () => { + const entityQuery = getPartnerContentSearchQuerySignals("Framer"); + const semanticQuery = + getPartnerContentSearchQuerySignals("fitness influencer"); + + expect(entityQuery.intent).toBe("entity"); + expect(semanticQuery.intent).toBe("semantic"); + + expect( + getEntityCreatorTextBoost({ + platformType: "tiktok", + contentType: "video", + transcriptFetchStatus: "pending", + titleHasExactQueryMention: false, + descriptionHasExactQueryMention: true, + chunkHasExactQueryMention: true, + transcriptScore: null, + queryIntent: entityQuery.intent, + }), + ).toEqual({ + score: 0.95, + weight: 0.95, + }); + expect( + getEntityCreatorTextBoost({ + platformType: "tiktok", + contentType: "video", + transcriptFetchStatus: "pending", + titleHasExactQueryMention: false, + descriptionHasExactQueryMention: true, + chunkHasExactQueryMention: true, + transcriptScore: null, + queryIntent: semanticQuery.intent, + }), + ).toBeNull(); + }); + + test("does not let high creator text override weaker transcript evidence at full weight", () => { + const evidence = createContentMatchEvidence({ + contentType: "video", + transcriptScore: 0.6, + creatorTextScore: 0.9, + }); + + expect(evidence.primarySource).toBe("transcript"); + expect(getEvidenceMatchScore(evidence)).toBeCloseTo(0.783, 3); + expect(getEvidenceTopicScore(evidence)).toBe(1); + }); + + test("topic fit rewards stronger evidence for the same matched coverage", () => { + const weak = deriveTopicFit({ + matchedContentCount: 20, + weightedMatchedContentCount: 20, + weightedMatchedContentScore: 7.433, + recentContentCount: 20, + }); + const strong = deriveTopicFit({ + matchedContentCount: 20, + weightedMatchedContentCount: 20, + weightedMatchedContentScore: 20, + recentContentCount: 20, + }); + + expect(strong.topicFit).toBeGreaterThan(weak.topicFit); + }); + + test("topic fit discounts tiny samples with the same coverage and strength", () => { + const smallSample = deriveTopicFit({ + matchedContentCount: 3, + weightedMatchedContentCount: 3, + weightedMatchedContentScore: 3, + recentContentCount: 3, + }); + const largeSample = deriveTopicFit({ + matchedContentCount: 30, + weightedMatchedContentCount: 30, + weightedMatchedContentScore: 30, + recentContentCount: 30, + }); + + expect(largeSample.topicFit).toBeGreaterThan(smallSample.topicFit); + }); + + test("topic fit lets broad strong coverage reach the high 90s", () => { + const topicFit = deriveTopicFit({ + matchedContentCount: 28, + weightedMatchedContentCount: 28, + weightedMatchedContentScore: 28, + recentContentCount: 28, + }); + + expect(topicFit.topicFit).toBeGreaterThanOrEqual(98); + }); +}); diff --git a/apps/web/tests/partner-content-search/top-content-ranking.test.ts b/apps/web/tests/partner-content-search/top-content-ranking.test.ts new file mode 100644 index 00000000000..b08bda6042c --- /dev/null +++ b/apps/web/tests/partner-content-search/top-content-ranking.test.ts @@ -0,0 +1,100 @@ +import { PARTNER_CONTENT_SEARCH_TOP_CONTENT } from "@/lib/partner-content-search/constants"; +import { + getBlendedTopContentScore, + getEngagementScore, + getViewBaseline, +} from "@/lib/partner-content-search/top-content-ranking"; +import { describe, expect, test } from "vitest"; + +describe("top content ranking", () => { + test("view baseline is the median and ignores zero/missing views", () => { + expect(getViewBaseline([100, 200, 300, null, 0, undefined])).toBe(200); + }); + + test("view baseline is robust to a single viral outlier", () => { + const typical = [100, 120, 90, 110, 105]; + const withViral = [...typical, 5_000_000]; + + // One viral post barely moves the median (a mean would jump to ~830k). + expect(getViewBaseline(withViral)!).toBeLessThan(150); + expect(getViewBaseline(withViral)!).toBeGreaterThan(100); + }); + + test("a median post scores about 0.5 engagement", () => { + const score = getEngagementScore({ views: 1000, baselineViews: 1000 }); + expect(score).toBeCloseTo(0.5, 5); + }); + + test("a viral post saturates toward 1 without running away", () => { + const big = getEngagementScore({ views: 1_000_000, baselineViews: 1000 })!; + const huge = getEngagementScore({ views: 100_000_000, baselineViews: 1000 })!; + + expect(big).toBeGreaterThan(0.9); + expect(big).toBeLessThan(1); + // 100x more views than the already-viral post barely moves the score: the + // outlier is bounded, it can't dominate the blend. + expect(huge - big).toBeLessThan(0.05); + expect(huge).toBeLessThan(1); + }); + + test("a below-median post lands near 0 but is never negative", () => { + const score = getEngagementScore({ views: 10, baselineViews: 10_000 })!; + expect(score).toBeGreaterThan(0); + expect(score).toBeLessThan(0.5); + }); + + test("engagement is unavailable without a usable signal", () => { + expect(getEngagementScore({ views: 1000, baselineViews: null })).toBeNull(); + expect(getEngagementScore({ views: 1000, baselineViews: 0 })).toBeNull(); + expect(getEngagementScore({ views: null, baselineViews: 1000 })).toBeNull(); + expect(getEngagementScore({ views: 0, baselineViews: 1000 })).toBeNull(); + }); + + test("a high-relevance typical post outranks a low-relevance viral post", () => { + const baselineViews = 1000; + const relevantTypical = getBlendedTopContentScore({ + relevance: 0.9, + views: 1000, + baselineViews, + }); + const irrelevantViral = getBlendedTopContentScore({ + relevance: 0.3, + views: 10_000_000, + baselineViews, + }); + + expect(relevantTypical).toBeGreaterThan(irrelevantViral); + }); + + test("among similarly-relevant posts the higher-view one wins", () => { + const baselineViews = 1000; + const lowViews = getBlendedTopContentScore({ + relevance: 0.7, + views: 200, + baselineViews, + }); + const highViews = getBlendedTopContentScore({ + relevance: 0.7, + views: 50_000, + baselineViews, + }); + + expect(highViews).toBeGreaterThan(lowViews); + }); + + test("falls back to pure relevance when there's no engagement signal", () => { + expect( + getBlendedTopContentScore({ + relevance: 0.42, + views: null, + baselineViews: null, + }), + ).toBe(0.42); + }); + + test("blend weights sum to one so a perfect post scores one", () => { + const { relevanceWeight, engagementWeight } = + PARTNER_CONTENT_SEARCH_TOP_CONTENT; + expect(relevanceWeight + engagementWeight).toBeCloseTo(1, 5); + }); +}); diff --git a/apps/web/ui/partners/partner-about.tsx b/apps/web/ui/partners/partner-about.tsx index 2af8482a711..cbfd9a700b7 100644 --- a/apps/web/ui/partners/partner-about.tsx +++ b/apps/web/ui/partners/partner-about.tsx @@ -13,6 +13,9 @@ import { Icon, InfoTooltip } from "@dub/ui"; export function PartnerAbout({ partner, error, + hideSocials = false, + hideDescription = false, + hideMonthlyTraffic = false, }: { partner?: Pick< EnrolledPartnerExtendedProps, @@ -25,32 +28,41 @@ export function PartnerAbout({ | "platforms" >; error?: any; + // Network-browse sheets surface website/socials, description, and traffic in the + // right sidebar instead, so they hide them here to avoid duplication. + hideSocials?: boolean; + hideDescription?: boolean; + hideMonthlyTraffic?: boolean; }) { return partner ? ( <> -
-

- Description -

-

- {partner.description || ( - - No description provided - - )} -

-
+ {!hideDescription && ( +
+

+ Description +

+

+ {partner.description || ( + + No description provided + + )} +

+
+ )} -
-

- Website and socials -

- -
+ {!hideSocials && ( +
+

+ Website and socials +

+ +
+ )} {Boolean(partner.industryInterests?.length) && (
@@ -97,7 +109,7 @@ export function PartnerAbout({
)} - {Boolean(partner.monthlyTraffic) && ( + {!hideMonthlyTraffic && Boolean(partner.monthlyTraffic) && (

diff --git a/apps/web/ui/partners/partner-info-cards.tsx b/apps/web/ui/partners/partner-info-cards.tsx index e385386af80..31afe13ccf9 100644 --- a/apps/web/ui/partners/partner-info-cards.tsx +++ b/apps/web/ui/partners/partner-info-cards.tsx @@ -19,6 +19,7 @@ import { CopyButton, Globe, Heart, + InfoTooltip, OfficeBuilding, TimestampTooltip, Trophy, @@ -46,6 +47,8 @@ import { } from "./fraud-risks/partner-risk-banner"; import { PartnerRiskIndicator } from "./fraud-risks/partner-risk-indicator"; import { PartnerAvatar } from "./partner-avatar"; +import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms"; +import { monthlyTrafficAmountsMap } from "@/lib/partners/partner-profile"; import { PartnerInfoGroup } from "./partner-info-group"; import { PartnerNetworkStatusBadge } from "./partner-network/partner-network-status-badge"; import { PartnerStarButton } from "./partner-star-button"; @@ -72,6 +75,13 @@ type PartnerInfoCardsProps = { // Only used for a controlled group selector that doesn't persist the selection itself selectedGroupId?: string | null; setSelectedGroupId?: (groupId: string) => void; + + /** + * Network-browse mode (marketplace discovery): the partner isn't enrolled in a + * group, so hide the group/rewards/bounties block and surface website & socials + * in the sidebar instead. + */ + browseMode?: boolean; } & ( | { type?: "enrolled"; partner?: EnrolledPartnerExtendedProps } | { type: "network"; partner?: NetworkPartnerProps } @@ -97,6 +107,7 @@ export function PartnerInfoCards({ setSelectedGroupId, showFraudIndicator = true, showApplicationRiskAnalysis = false, + browseMode = false, }: PartnerInfoCardsProps) { const { id: workspaceId, slug: workspaceSlug, plan } = useWorkspace(); @@ -241,6 +252,30 @@ export function PartnerInfoCards({ ]); } + // Browse mode: website/socials rendered as compact rows (matching the detail + // rows above) instead of the bordered platform cards. + const socialFields = + browseMode && partner && "platforms" in partner + ? PARTNER_PLATFORM_FIELDS.map((field) => ({ + label: field.label, + icon: field.icon, + ...field.data(partner.platforms), + })).filter((field) => field.value && field.href) + : []; + + // Browse mode: description + monthly traffic move into the sidebar profile card. + const description = + browseMode && partner && "description" in partner + ? partner.description + : null; + const monthlyTrafficLabel = + browseMode && + partner && + "monthlyTraffic" in partner && + partner.monthlyTraffic + ? (monthlyTrafficAmountsMap[partner.monthlyTraffic]?.label ?? null) + : null; + return (
@@ -368,6 +403,60 @@ export function PartnerInfoCards({ ); })}
+ + {description && ( +
+

+ About +

+

+ {description} +

+
+ )} + + {socialFields.length > 0 && ( +
+

+ Website and socials +

+
+ {socialFields.map(({ label, icon: Icon, value, href, info }) => ( + + +
+
{value}
+ {info && info.length > 0 && ( +
+ {info.join(" · ")} +
+ )} +
+ + ))} +
+
+ )} + + {monthlyTrafficLabel && ( +
+
+

+ Monthly traffic +

+ +
+ + {monthlyTrafficLabel} + +
+ )} {isEnrolled && partner && } {partner && isEnrolled && showApplicationRiskAnalysis && ( @@ -376,7 +465,7 @@ export function PartnerInfoCards({

- {!isAdmin && ( + {!isAdmin && !browseMode && (
{/* Group */}
diff --git a/apps/web/ui/partners/partner-sheet-tabs.tsx b/apps/web/ui/partners/partner-sheet-tabs.tsx index 5332ed527ff..75fb3b7a528 100644 --- a/apps/web/ui/partners/partner-sheet-tabs.tsx +++ b/apps/web/ui/partners/partner-sheet-tabs.tsx @@ -8,10 +8,13 @@ export function PartnerSheetTabs({ partnerId, currentTabId, setCurrentTabId, + aboutLabel = "About", }: { partnerId: string; currentTabId: string; setCurrentTabId: Dispatch>; + // Override the first tab's label (e.g. "Content matches" in network browse). + aboutLabel?: string; }) { const { count: commentsCount } = usePartnerCommentsCount( { @@ -26,7 +29,7 @@ export function PartnerSheetTabs({ () => [ { id: "about", - label: "About", + label: aboutLabel, icon: User, }, { @@ -40,7 +43,7 @@ export function PartnerSheetTabs({ icon: Msg, }, ], - [commentsCount], + [commentsCount, aboutLabel], ); const layoutGroupId = useId(); From d107064ddb3b58d3b27edaf712e3bca822016e77 Mon Sep 17 00:00:00 2001 From: David Clark Date: Sun, 21 Jun 2026 10:43:12 -0400 Subject: [PATCH 11/25] WIP - adjust filters + add performance tracking --- .../network/partners/content-search/route.ts | 366 ++++++++++++++---- .../(ee)/api/network/partners/count/route.ts | 33 +- .../app/(ee)/api/network/partners/route.ts | 4 +- .../api/partner-content/thumbnail/route.ts | 37 ++ .../network/[partnerId]/page-client.tsx | 147 ++++--- .../network/network-country-filter.tsx | 136 +++++++ .../network/network-platform-filter.tsx | 196 ++++++++++ .../program/network/network-reach-filter.tsx | 129 ++++++ .../(ee)/program/network/page-client.tsx | 272 ++++++------- .../program/network/partner-network-sort.tsx | 106 ----- .../program/network/platform-filter-utils.ts | 53 +++ .../network/use-partner-network-filters.tsx | 28 +- .../api/network/calculate-partner-ranking.ts | 79 ++-- .../network/partner-network-listing-where.ts | 6 +- apps/web/lib/api/network/reach-tiers.ts | 70 ++++ apps/web/lib/api/scrape-creators/client.ts | 4 +- .../get-instagram-user-posts.ts | 50 ++- .../ingestion/enqueue.ts | 4 +- .../partner-content-search/thumbnail-url.ts | 20 + .../web/lib/swr/use-partner-content-search.ts | 34 +- apps/web/lib/zod/schemas/partner-network.ts | 13 +- 21 files changed, 1343 insertions(+), 444 deletions(-) create mode 100644 apps/web/app/api/partner-content/thumbnail/route.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx delete mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/partner-network-sort.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts create mode 100644 apps/web/lib/api/network/reach-tiers.ts create mode 100644 apps/web/lib/partner-content-search/thumbnail-url.ts diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index 1db2af9403d..e2a02cbda4c 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -45,6 +45,7 @@ import { serializeEmbeddingForVector, VoyageTimeoutError, } from "@/lib/partner-content-search/voyage"; +import { REACH_TIER_KEYS, type ReachTier } from "@/lib/api/network/reach-tiers"; import { prisma } from "@/lib/prisma"; import { PlatformType, Prisma } from "@prisma/client"; import { NextResponse } from "next/server"; @@ -55,9 +56,15 @@ export const maxDuration = 30; const DEFAULT_PARTNER_LIMIT = 20; +type PartnerContentSearchTimingLogger = ( + stage: string, + metadata?: Record, +) => void; + const partnerNetworkContentSearchSchema = z.object({ query: z.string().trim().max(500).optional(), - platform: z.enum(PlatformType).optional(), + platforms: z.array(z.enum(PlatformType)).min(1).optional(), + reach: z.array(z.enum(REACH_TIER_KEYS)).min(1).optional(), country: z.string().trim().min(1).optional(), partnerIds: z.array(z.string()).min(1).max(100).optional(), starred: z.boolean().optional(), @@ -114,22 +121,39 @@ export const POST = withWorkspace( Math.max(25, body.limit * body.chunksPerPartner * 6), ) : body.limit * body.chunksPerPartner * 2; + const logTiming = createPartnerContentSearchTimingLogger({ + workspaceId: workspace.id, + programId, + hasQuery: Boolean(body.query), + queryLength: body.query?.length ?? 0, + platforms: body.platforms ?? null, + reach: body.reach ?? null, + country: body.country ?? null, + starred: body.starred ?? null, + limit: body.limit, + chunksPerPartner: body.chunksPerPartner, + candidateChunkCount, + rerank: body.rerank, + }); + + logTiming("request-parsed"); const { rows, reranked, queryVector, cutoffDistance } = body.query ? await searchPartnerNetworkContent({ programId, query: body.query, - platform: body.platform, + platforms: body.platforms, country: body.country, partnerIds: body.partnerIds, starred: body.starred, limit: candidateChunkCount, rerank: body.rerank, + logTiming, }) : { rows: await listPartnerNetworkContent({ programId, - platform: body.platform, + platforms: body.platforms, country: body.country, partnerIds: body.partnerIds, starred: body.starred, @@ -139,6 +163,11 @@ export const POST = withWorkspace( queryVector: null, cutoffDistance: null, }; + logTiming("content-rows-loaded", { + rowCount: rows.length, + reranked, + cutoffDistance, + }); const partnerCandidateLimit = body.query ? Math.min( PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, @@ -153,21 +182,39 @@ export const POST = withWorkspace( dedupeKey: ({ partnerContentItemId }) => partnerContentItemId, getRowScore: getRowRelevanceScore, }); + logTiming("partner-candidates-grouped", { + partnerCandidateCount: partnerCandidates.length, + partnerCandidateLimit, + }); + logTiming("match-summaries-start", { + partnerCandidateCount: partnerCandidates.length, + }); const matchSummaries = await getPartnerMatchSummaries({ rows, partnerIds: partnerCandidates.map(({ partnerId }) => partnerId), - platform: body.platform, + platforms: body.platforms, query: body.query, queryVector, cutoffDistance, + logTiming, + }); + logTiming("match-summaries-complete", { + summaryCount: matchSummaries.size, + }); + logTiming("partner-hydration-start", { + partnerCandidateCount: partnerCandidates.length, }); const networkPartners = await getNetworkPartnersById({ programId, partnerIds: partnerCandidates.map(({ partnerId }) => partnerId), - platform: body.platform, + platforms: body.platforms, + reach: body.reach, country: body.country, starred: body.starred, }); + logTiming("partner-hydration-complete", { + hydratedPartnerCount: networkPartners.size, + }); const partners = partnerCandidates .map((partner) => { const networkPartner = networkPartners.get(partner.partnerId); @@ -183,11 +230,14 @@ export const POST = withWorkspace( const sortedPartners = body.query ? sortPartnersByTopicFit(partners) : partners; + logTiming("response-ready", { + returnedPartnerCount: Math.min(sortedPartners.length, body.limit), + }); return NextResponse.json({ success: true, query: body.query ?? null, - platform: body.platform ?? null, + platforms: body.platforms ?? null, country: body.country ?? null, candidateChunkCount, embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, @@ -204,6 +254,26 @@ export const POST = withWorkspace( }, ); +function createPartnerContentSearchTimingLogger( + context: Record, +): PartnerContentSearchTimingLogger { + const startedAt = Date.now(); + let previousAt = startedAt; + + return (stage, metadata = {}) => { + const now = Date.now(); + + console.info("[partner-content-search:timing]", { + stage, + elapsedMs: now - startedAt, + deltaMs: now - previousAt, + ...context, + ...metadata, + }); + previousAt = now; + }; +} + function isNonNull(value: T | null): value is T { return value !== null; } @@ -262,13 +332,15 @@ function sortPartnersByTopicFit< async function getNetworkPartnersById({ programId, partnerIds, - platform, + platforms, + reach, country, starred, }: { programId: string; partnerIds: string[]; - platform?: PlatformType; + platforms?: PlatformType[]; + reach?: ReachTier[]; country?: string; starred?: boolean; }) { @@ -278,12 +350,12 @@ async function getNetworkPartnersById({ programId, partnerIds, status: "discover", - sortBy: "relevance", page: 1, pageSize: partnerIds.length, country, starred: starred ?? undefined, - platform, + platform: platforms, + reach, }); const partners = parseRankedNetworkPartners(rankedPartners); @@ -293,23 +365,26 @@ async function getNetworkPartnersById({ async function searchPartnerNetworkContent({ programId, query, - platform, + platforms, country, partnerIds, starred, limit, rerank, + logTiming, }: { programId: string; query: string; - platform?: PlatformType; + platforms?: PlatformType[]; country?: string; partnerIds?: string[]; starred?: boolean; limit: number; rerank: boolean; + logTiming?: PartnerContentSearchTimingLogger; }) { let queryEmbedding: number[]; + logTiming?.("query-embedding-start"); try { [queryEmbedding] = await embedPartnerContentTexts({ input: [query], @@ -325,6 +400,9 @@ async function searchPartnerNetworkContent({ } throw error; } + logTiming?.("query-embedding-complete", { + embeddingDimensions: queryEmbedding.length, + }); const queryVector = serializeEmbeddingForVector(queryEmbedding); const starredFilter = starred === true @@ -332,8 +410,8 @@ async function searchPartnerNetworkContent({ : starred === false ? Prisma.sql`AND (dp.starredAt IS NULL OR dp.id IS NULL)` : Prisma.empty; - const platformFilter = platform - ? Prisma.sql`AND pp.type = ${platform}` + const platformFilter = platforms?.length + ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` : Prisma.empty; const countryFilter = country ? Prisma.sql`AND p.country = ${country}` @@ -353,6 +431,9 @@ async function searchPartnerNetworkContent({ limit, PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolSize, ); + logTiming?.("vector-search-start", { + poolSize, + }); const poolRows = await prisma.$queryRaw(Prisma.sql` SELECT c.id AS chunkId, @@ -373,7 +454,7 @@ async function searchPartnerNetworkContent({ pci.publishedAt AS contentPublishedAt, pci.durationMs AS contentDurationMs, c.source AS chunkSource, - c.chunkText, + "" AS chunkText, c.startMs, c.endMs, DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance @@ -399,6 +480,9 @@ async function searchPartnerNetworkContent({ ORDER BY distance ASC LIMIT ${poolSize} `); + logTiming?.("vector-search-complete", { + poolRowCount: poolRows.length, + }); // Collapse to one best chunk per content item for the item-level cutoff, and // one best chunk per content item + source for reranking/source-aware evidence. @@ -417,17 +501,47 @@ async function searchPartnerNetworkContent({ const cutoffDistance = itemRows.length ? Number(itemRows[itemRows.length - 1].distance) : null; + logTiming?.("candidate-dedupe-complete", { + itemRowCount: itemRows.length, + sourceRowCount: rows.length, + cutoffDistance, + }); + + const rowsWithChunkText = await hydratePartnerContentChunkText({ + rows, + maxRows: rerank + ? PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount + : rows.length, + logTiming, + }); if (!rerank) { return { - rows: sortRowsByRelevanceScore(rows), + rows: sortRowsByRelevanceScore(rowsWithChunkText), reranked: false, queryVector, cutoffDistance, }; } - const rerankResult = await rerankPartnerSearchRows({ query, rows }); + logTiming?.("rerank-start", { + rowCount: rowsWithChunkText.length, + rerankerCandidateCount: Math.min( + rowsWithChunkText.length, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, + ), + }); + const rerankResult = await rerankPartnerSearchRows({ + query, + rows: rowsWithChunkText, + }); + logTiming?.("rerank-complete", { + reranked: rerankResult.reranked, + rowCount: rerankResult.rows.length, + rerankedRowCount: rerankResult.rows.filter( + ({ rerankScore }) => rerankScore != null, + ).length, + }); return { ...rerankResult, rows: sortRowsByRelevanceScore(rerankResult.rows), @@ -436,16 +550,62 @@ async function searchPartnerNetworkContent({ }; } +async function hydratePartnerContentChunkText({ + rows, + maxRows, + logTiming, +}: { + rows: PartnerContentSearchRow[]; + maxRows: number; + logTiming?: PartnerContentSearchTimingLogger; +}) { + const rowsToHydrate = rows.slice(0, maxRows); + const chunkIds = rowsToHydrate.map(({ chunkId }) => chunkId); + + if (chunkIds.length === 0) return rows; + + logTiming?.("chunk-text-hydration-start", { + requestedRowCount: rowsToHydrate.length, + totalRowCount: rows.length, + }); + const chunkTextRows = await prisma.partnerContentChunk.findMany({ + where: { + id: { + in: chunkIds, + }, + }, + select: { + id: true, + chunkText: true, + }, + }); + const chunkTextById = new Map( + chunkTextRows.map((row) => [row.id, row.chunkText]), + ); + logTiming?.("chunk-text-hydration-complete", { + hydratedRowCount: chunkTextById.size, + }); + + return rows.map((row, index) => + index < maxRows + ? { + ...row, + chunkText: chunkTextById.get(row.chunkId) ?? row.chunkText, + } + : row, + ); +} + async function listPartnerNetworkContent({ programId, - platform, + platforms, country, partnerIds, starred, limit, }: { programId: string; - platform?: PlatformType; + platforms?: PlatformType[]; country?: string; partnerIds?: string[]; starred?: boolean; @@ -499,9 +659,9 @@ async function listPartnerNetworkContent({ embeddedChunkCount: { gt: 0, }, - ...(platform && { + ...(platforms?.length && { partnerPlatform: { - type: platform, + type: { in: platforms }, }, }), partner: { @@ -628,14 +788,15 @@ function median(values: number[]): number | null { async function getPartnerMatchSummaries({ rows, partnerIds, - platform, + platforms, query, queryVector, cutoffDistance, + logTiming, }: { rows: PartnerContentSearchRow[]; partnerIds: string[]; - platform?: PlatformType; + platforms?: PlatformType[]; query?: string | null; // Present only in query mode. When set, each shown partner's recent videos are // scored exactly against the query (bounded by their ids) and counted as matched @@ -643,6 +804,7 @@ async function getPartnerMatchSummaries({ // (list mode) we fall back to the retrieval rows for the match determination. queryVector?: string | null; cutoffDistance?: number | null; + logTiming?: PartnerContentSearchTimingLogger; }) { if (partnerIds.length === 0) return new Map(); @@ -651,8 +813,8 @@ async function getPartnerMatchSummaries({ querySignals.intent === "entity" && querySignals.normalizedQuery ? `%${querySignals.normalizedQuery}%` : null; - const platformFilter = platform - ? Prisma.sql`AND pp.type = ${platform}` + const platformFilter = platforms?.length + ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` : Prisma.empty; // "Recent" is a time window (last recencyWindowMonths), not a fixed post count. @@ -681,9 +843,9 @@ async function getPartnerMatchSummaries({ embeddedChunkCount: { gt: 0, }, - ...(platform && { + ...(platforms?.length && { partnerPlatform: { - type: platform, + type: { in: platforms }, }, }), }, @@ -760,8 +922,8 @@ async function getPartnerMatchSummaries({ partnerId: { in: partnerIds, }, - ...(platform && { - type: platform, + ...(platforms?.length && { + type: { in: platforms }, }), }, _sum: { @@ -769,11 +931,19 @@ async function getPartnerMatchSummaries({ }, }); + logTiming?.("match-summary-base-queries-start", { + partnerCount: partnerIds.length, + }); const [contentCounts, recentContentRows, followerRows] = await Promise.all([ contentCountsPromise, recentContentRowsPromise, followerRowsPromise, ]); + logTiming?.("match-summary-base-queries-complete", { + contentCountRows: contentCounts.length, + recentContentRows: recentContentRows.length, + followerRows: followerRows.length, + }); const followersByPartner = new Map( followerRows.map((row) => [ row.partnerId, @@ -816,62 +986,101 @@ async function getPartnerMatchSummaries({ ({ partnerContentItemId }) => partnerContentItemId, ); if (recentItemIds.length > 0) { - const itemScores = await prisma.$queryRaw< - { - partnerContentItemId: string; - source: string; - bestDistance: number | string; - }[] - >(Prisma.sql` - SELECT - c.partnerContentItemId, - c.source, - MIN( - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') - ) AS bestDistance - FROM PartnerContentChunk c - WHERE c.partnerContentItemId IN (${Prisma.join(recentItemIds)}) - AND c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - GROUP BY c.partnerContentItemId, c.source - `); - for (const row of itemScores) { - setSourceDistance( - recentItemSourceBestDistance, - row.partnerContentItemId, - getEvidenceSource(row.source), - Number(row.bestDistance), - ); - } - - if (exactMentionPattern) { - const exactMentionRows = await prisma.$queryRaw< + logTiming?.("match-summary-item-vector-start", { + recentItemCount: recentItemIds.length, + }); + const itemScoresPromise = prisma + .$queryRaw< { partnerContentItemId: string; source: string; - exactMentionCount: bigint | number; + bestDistance: number | string; }[] - >(Prisma.sql` + >( + Prisma.sql` SELECT c.partnerContentItemId, c.source, - COUNT(*) AS exactMentionCount + MIN( + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') + ) AS bestDistance FROM PartnerContentChunk c WHERE c.partnerContentItemId IN (${Prisma.join(recentItemIds)}) AND c.embedding IS NOT NULL AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - AND LOWER(c.chunkText) LIKE ${exactMentionPattern} GROUP BY c.partnerContentItemId, c.source - `); - - for (const row of exactMentionRows) { - if (Number(row.exactMentionCount) === 0) continue; - setSourceExactMention( - recentItemSourceExactMentions, - row.partnerContentItemId, - getEvidenceSource(row.source), - ); - } + `, + ) + .then((itemScores) => { + logTiming?.("match-summary-item-vector-complete", { + itemScoreRows: itemScores.length, + }); + return itemScores; + }); + + const exactMentionRowsPromise = exactMentionPattern + ? (() => { + logTiming?.("match-summary-exact-mentions-start", { + recentItemCount: recentItemIds.length, + queryIntent: querySignals.intent, + }); + return prisma + .$queryRaw< + { + partnerContentItemId: string; + source: string; + exactMentionCount: bigint | number; + }[] + >( + Prisma.sql` + SELECT + c.partnerContentItemId, + c.source, + COUNT(*) AS exactMentionCount + FROM PartnerContentChunk c + WHERE c.partnerContentItemId IN (${Prisma.join(recentItemIds)}) + AND c.embedding IS NOT NULL + AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + AND LOWER(c.chunkText) LIKE ${exactMentionPattern} + GROUP BY c.partnerContentItemId, c.source + `, + ) + .then((exactMentionRows) => { + logTiming?.("match-summary-exact-mentions-complete", { + exactMentionRows: exactMentionRows.length, + }); + return exactMentionRows; + }); + })() + : Promise.resolve([]); + + if (!exactMentionPattern) { + logTiming?.("match-summary-exact-mentions-skipped", { + queryIntent: querySignals.intent, + }); + } + + const [itemScores, exactMentionRows] = await Promise.all([ + itemScoresPromise, + exactMentionRowsPromise, + ]); + + for (const row of itemScores) { + setSourceDistance( + recentItemSourceBestDistance, + row.partnerContentItemId, + getEvidenceSource(row.source), + Number(row.bestDistance), + ); + } + + for (const row of exactMentionRows) { + if (Number(row.exactMentionCount) === 0) continue; + setSourceExactMention( + recentItemSourceExactMentions, + row.partnerContentItemId, + getEvidenceSource(row.source), + ); } } } @@ -893,7 +1102,11 @@ async function getPartnerMatchSummaries({ } } - return new Map( + logTiming?.("match-summary-aggregation-start", { + partnerCount: partnerIds.length, + recentContentRows: recentContentRows.length, + }); + const summaries = new Map( partnerIds.map((partnerId) => { const partnerRows = rowsByPartnerId.get(partnerId) ?? []; const recentRows = recentRowsByPartnerId.get(partnerId) ?? []; @@ -1156,6 +1369,11 @@ async function getPartnerMatchSummaries({ ] as const; }), ); + logTiming?.("match-summary-aggregation-complete", { + summaryCount: summaries.size, + }); + + return summaries; } function toChunkResult(row: PartnerContentSearchRow, distance: number) { diff --git a/apps/web/app/(ee)/api/network/partners/count/route.ts b/apps/web/app/(ee)/api/network/partners/count/route.ts index 62a4fdf35eb..508a813e476 100644 --- a/apps/web/app/(ee)/api/network/partners/count/route.ts +++ b/apps/web/app/(ee)/api/network/partners/count/route.ts @@ -3,10 +3,12 @@ import { partnerNetworkListingParts, partnerWhereFromListingParts, } from "@/lib/api/network/partner-network-listing-where"; +import { reachTiersToRanges } from "@/lib/api/network/reach-tiers"; import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw"; import { withWorkspace } from "@/lib/auth"; import { prisma } from "@/lib/prisma"; import { getNetworkPartnersCountQuerySchema } from "@/lib/zod/schemas/partner-network"; +import { Prisma } from "@prisma/client"; import { NextResponse } from "next/server"; // GET /api/network/partners/count - get the number of available partners in the network @@ -30,7 +32,7 @@ export const GET = withWorkspace( }); } - const { partnerIds, status, groupBy, country, starred, platform } = + const { partnerIds, status, groupBy, country, starred, platform, reach } = getNetworkPartnersCountQuerySchema.parse(searchParams); const listingParts = partnerNetworkListingParts({ @@ -41,6 +43,28 @@ export const GET = withWorkspace( const commonWhere = partnerWhereFromListingParts(listingParts); + // Reach is a discover-only filter. Approximate the ranking's "max subscribers + // across selected platforms in tier" with a `some` test (a selected platform + // whose subscribers fall in a chosen tier) so pagination totals track the + // filtered discover results. Applied only to discover-scoped counts below. + const reachRanges = reach?.length ? reachTiersToRanges(reach) : []; + const reachWhere: Prisma.PartnerWhereInput = reachRanges.length + ? { + platforms: { + some: { + verifiedAt: { not: null }, + ...(platform?.length && { type: { in: platform } }), + OR: reachRanges.map(({ min, max }) => ({ + subscribers: { + gte: BigInt(min), + ...(max != null && { lt: BigInt(max) }), + }, + })), + }, + }, + } + : {}; + const statusWheres = { discover: { programs: { none: { programId } }, @@ -104,6 +128,7 @@ export const GET = withWorkspace( where: { ...commonWhere, ...statusWheres.discover, + ...reachWhere, }, }) : undefined, @@ -143,7 +168,11 @@ export const GET = withWorkspace( const countries = await prisma.partner.groupBy({ by: ["country"], _count: true, - where: { ...commonWhere, ...statusWhereForFacet }, + where: { + ...commonWhere, + ...statusWhereForFacet, + ...(!status || status === "discover" ? reachWhere : {}), + }, orderBy: { _count: { country: "desc", diff --git a/apps/web/app/(ee)/api/network/partners/route.ts b/apps/web/app/(ee)/api/network/partners/route.ts index 255b6114d74..0cc30353a8b 100644 --- a/apps/web/app/(ee)/api/network/partners/route.ts +++ b/apps/web/app/(ee)/api/network/partners/route.ts @@ -50,8 +50,8 @@ export const GET = withWorkspace( pageSize, country, starred, - sortBy, platform, + reach, } = getNetworkPartnersQuerySchema.parse(searchParams); if (status !== "discover") { @@ -124,8 +124,8 @@ export const GET = withWorkspace( page, pageSize, starred: starred ?? undefined, - sortBy, platform: platform ?? undefined, + reach: reach ?? undefined, similarPrograms, }); console.timeEnd("calculatePartnerRanking"); diff --git a/apps/web/app/api/partner-content/thumbnail/route.ts b/apps/web/app/api/partner-content/thumbnail/route.ts new file mode 100644 index 00000000000..4d88f241cc0 --- /dev/null +++ b/apps/web/app/api/partner-content/thumbnail/route.ts @@ -0,0 +1,37 @@ +import { isInstagramCdnUrl } from "@/lib/partner-content-search/thumbnail-url"; +import { NextRequest, NextResponse } from "next/server"; + +export async function GET(req: NextRequest) { + const url = req.nextUrl.searchParams.get("url"); + + if (!url || !isInstagramCdnUrl(url)) { + return new NextResponse("Invalid thumbnail URL", { status: 400 }); + } + + const response = await fetch(url, { + headers: { + Accept: + "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "User-Agent": + "Mozilla/5.0 (compatible; DubPartnerContentPreview/1.0; +https://dub.co)", + }, + redirect: "error", + }); + + if (!response.ok || !response.body) { + return new NextResponse("Failed to fetch thumbnail", { status: 502 }); + } + + const contentType = response.headers.get("content-type") ?? ""; + + if (!contentType.startsWith("image/")) { + return new NextResponse("Invalid thumbnail content type", { status: 502 }); + } + + return new NextResponse(response.body, { + headers: { + "Cache-Control": "public, max-age=3600, s-maxage=86400", + "Content-Type": contentType, + }, + }); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 48db690d4e2..61a7a162563 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -10,7 +10,7 @@ import { getBlendedTopContentScore, getViewBaseline, } from "@/lib/partner-content-search/top-content-ranking"; -import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; +import { getPartnerContentThumbnailUrl } from "@/lib/partner-content-search/thumbnail-url"; import { mutatePrefix } from "@/lib/swr/mutate"; import usePartnerContentSearch, { type PartnerContentMatchEvidence, @@ -28,6 +28,11 @@ import { cn, fetcher, nFormatter } from "@dub/utils"; import { EmailContent } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-email-preview"; import { InviteNetworkPartnerSheet } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-network-partner-sheet"; import Link from "next/link"; +import { + getContentSearchPlatforms, + parseSelectedPlatforms, + platformFilterParam, +} from "../platform-filter-utils"; import { useSearchParams } from "next/navigation"; import { ArrowLeft } from "lucide-react"; import { useState } from "react"; @@ -68,12 +73,12 @@ export function NetworkPartnerDetailContent({ const searchParams = useSearchParams(); const search = searchParams.get("search")?.trim() ?? ""; const country = searchParams.get("country") ?? undefined; - const selectedPlatform = searchParams.get("platform"); - const contentSearchPlatform = isPartnerContentSearchPlatform(selectedPlatform) - ? selectedPlatform - : undefined; + const selectedPlatforms = parseSelectedPlatforms( + searchParams.get("platform"), + ); + const contentSearchPlatforms = getContentSearchPlatforms(selectedPlatforms); const hasContentSearch = - search.length > 0 && (!selectedPlatform || Boolean(contentSearchPlatform)); + search.length > 0 && contentSearchPlatforms.length > 0; const backHref = workspaceSlug ? `/${workspaceSlug}/program/network${getBackQueryString(searchParams)}` : "#"; @@ -90,20 +95,20 @@ export function NetworkPartnerDetailContent({ const partner = partners?.[0]; // The results page already ran the (global) Voyage search and handed us this - // partner's match data on click, so render straight from it — opening the - // detail is instant and never re-runs Voyage. Re-running scoped to a single - // partner would also recompute the candidate cutoff against a one-partner set, - // drifting the bars/counts away from the list. We only hit Voyage again when - // there's a genuine reason to: - // • a cold deep-link (?partnerId=… with no results-page data in hand), or - // • the user explicitly asks to see the full matched-content list (the list - // only ships the top couple of matched snippets per partner to stay light). - // Either way the displayed Topic Fit / bars / counts stay sourced from the - // cached summary, so the on-demand fetch can add cards but never shift scores. - const [showAllMatches, setShowAllMatches] = useState(false); - const needsSearchFallback = hasContentSearch && !initialSearchPartner; - const shouldFetchSearch = - needsSearchFallback || (hasContentSearch && showAllMatches); + // partner's match data on click, so we render straight from it — opening the + // detail is instant. In the background we also run a scoped, single-partner + // search whenever there's a content-search context, for one reason: to put the + // matched-content list on a SINGLE relevance scale. The global search only + // reranks items that made its top-N candidate pool, so a creator's other matched + // posts fall back to raw cosine — two incomparable scales mixed in one list (the + // tell-tale "65% vs 95%" banding). A per-partner rerank covers all of this + // creator's items, so every row shares the reranker scale. + // + // This is non-blocking: the cached summary paints immediately and the unified + // scores swap in when the scoped run resolves. Topic Fit / bars / counts stay + // sourced from the cached summary (re-running scoped would recompute the cutoff + // against a one-partner set and drift them), so the headline never shifts. + const shouldFetchSearch = hasContentSearch; const { data: searchResults, error: searchError, @@ -111,7 +116,7 @@ export function NetworkPartnerDetailContent({ } = usePartnerContentSearch({ enabled: Boolean(workspaceId && shouldFetchSearch), query: search, - platform: contentSearchPlatform, + platforms: platformFilterParam(selectedPlatforms), country, starred: false, partnerIds: [partnerId], @@ -126,10 +131,16 @@ export function NetworkPartnerDetailContent({ const searchPartner = fetchedPartner ?? initialSearchPartner; const searchSummary = initialSearchPartner?.matchSummary ?? fetchedPartner?.matchSummary; - // A deep-link has nothing cached → full skeleton. A "show all" fetch already - // has cards on screen → append-only loading, don't blank what's shown. + // The matched-content list's per-row relevance comes from the scoped, fully + // reranked summary (one scale) once it lands; null until then, so the list + // shows cached scores and quietly upgrades. Everything else stays on the + // cached summary above. + const relevanceSummary = fetchedPartner?.matchSummary ?? null; + // A deep-link has nothing cached → full skeleton. With cached data in hand the + // scoped run is a silent, non-blocking relevance refinement. const isFallbackLoading = isLoadingSearch && !initialSearchPartner; - const isLoadingMoreMatches = isLoadingSearch && Boolean(initialSearchPartner); + const isRefiningRelevance = + isLoadingSearch && Boolean(initialSearchPartner) && !relevanceSummary; if (!isLoadingPartner && !partner) { return ( @@ -203,12 +214,9 @@ export function NetworkPartnerDetailContent({ setShowAllMatches(true)} + isRefining={isRefiningRelevance} summary={searchSummary} + relevanceSummary={relevanceSummary} searchPartner={searchPartner} /> )} @@ -436,29 +444,37 @@ function BarTooltip({ function SearchFitPanel({ error, isLoading, - isLoadingMore = false, - hasLoadedAllMatches = false, - onLoadAllMatches, + isRefining = false, summary: initialSummary, + relevanceSummary, searchPartner, }: { error: unknown; isLoading: boolean; - // The on-demand "show all" fetch is in flight (cards already on screen). - isLoadingMore?: boolean; - // The full matched-content set has been fetched (or we never needed to). - hasLoadedAllMatches?: boolean; - onLoadAllMatches?: () => void; + // The scoped single-partner rerank is in flight (cached scores still on screen); + // when it lands, every row's relevance moves onto one consistent scale. + isRefining?: boolean; summary?: PartnerContentSearchPartner["matchSummary"]; + // Scoped, fully reranked summary used only to put the list's per-row relevance + // on a single scale. Null until the background rerank resolves. + relevanceSummary?: PartnerContentSearchPartner["matchSummary"] | null; searchPartner?: PartnerContentSearchPartner; }) { const [visibleAllCount, setVisibleAllCount] = useState( DETAIL_CONTENT_INITIAL_MATCH_COUNT, ); const summary = initialSummary ?? searchPartner?.matchSummary; + // Per-item relevance on a single scale, from the scoped reranked summary. Until + // it lands this is empty and rows show the cached (possibly mixed-scale) score. + const unifiedRelevanceByItemId = buildUnifiedRelevanceMap(relevanceSummary); // Both lists come from the cached summary's full matched set (instant, complete); - // loaded chunks only enrich a row with its snippet/thumbnail when available. - const items = buildMatchedContentItems(summary, searchPartner?.chunks ?? []); + // loaded chunks only enrich a row with its snippet/thumbnail when available, and + // the unified-relevance map upgrades each row's score in place once available. + const items = buildMatchedContentItems( + summary, + searchPartner?.chunks ?? [], + unifiedRelevanceByItemId, + ); // Top content: the relevance-led blend of relevance + reach (robust to a single // viral post). All content: the same matched set, simply newest-first. @@ -474,13 +490,6 @@ function SearchFitPanel({ // the top set; with ≤ topContentCount matches the top list already shows them all. const showAllSection = allContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; - // Some rows may still be missing their snippet preview (the list page only ships - // the top couple of chunks). The lists are already complete from the summary, so - // this fetch only enriches previews — it never changes ordering or scores. - const canLoadPreviews = - !hasLoadedAllMatches && - Boolean(onLoadAllMatches) && - items.some((item) => !item.chunk); const band = summary?.band ?? "none"; const bandStyles = TOPIC_FIT_BAND_STYLES[band]; @@ -576,6 +585,9 @@ function SearchFitPanel({

{topContentCaption} + {isRefining && ( + · refining match… + )}

@@ -613,9 +625,6 @@ function SearchFitPanel({ {visibleAll.map((item) => ( ))} - {/* Preview-enrichment fetch in flight: append skeletons rather than - blanking the cards already on screen. */} - {isLoadingMore && }
{hiddenAllCount > 0 ? ( @@ -635,19 +644,6 @@ function SearchFitPanel({ className="h-9 rounded-lg px-4" />
- ) : canLoadPreviews ? ( - // Fills in snippet previews for the rest of the matched set on - // demand (the one path that re-runs Voyage from the detail view). -
-
) : null}
)} @@ -693,9 +689,25 @@ type MatchedContentItem = { chunk?: PartnerContentSearchPartner["chunks"][number]; }; +// Per-item relevance from the scoped, fully reranked summary, keyed by content +// item. Used to upgrade each list row onto a single reranker scale (the cached +// global summary mixes reranker and cosine scores). Includes any item with +// evidence — not just `matched` ones — so boundary items still get unified. +function buildUnifiedRelevanceMap( + relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, +) { + const map = new Map(); + for (const bar of relevanceSummary?.contentBars ?? []) { + const score = getEvidenceDisplayScore(bar.matchEvidence); + if (score != null) map.set(bar.partnerContentItemId, score); + } + return map; +} + function buildMatchedContentItems( summary: PartnerContentSearchPartner["matchSummary"] | undefined, chunks: PartnerContentSearchPartner["chunks"], + unifiedRelevanceByItemId?: Map, ): MatchedContentItem[] { const bars = summary?.contentBars ?? []; @@ -718,8 +730,13 @@ function buildMatchedContentItems( return bars .filter((bar) => bar.matched) .map((bar) => { + // Prefer the unified (single-scale) relevance when the scoped rerank has + // landed; otherwise fall back to the cached score so rows render instantly. const relevance = - getEvidenceDisplayScore(bar.matchEvidence) ?? bar.matchScore ?? 0; + unifiedRelevanceByItemId?.get(bar.partnerContentItemId) ?? + getEvidenceDisplayScore(bar.matchEvidence) ?? + bar.matchScore ?? + 0; return { contentItemId: bar.partnerContentItemId, @@ -911,7 +928,9 @@ function PlatformIcon({ function getPreviewThumbnail( chunk: PartnerContentSearchPartner["chunks"][number], ) { - if (chunk.content.thumbnailUrl) return chunk.content.thumbnailUrl; + if (chunk.content.thumbnailUrl) { + return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); + } if (chunk.platform.type === "youtube") { return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; } diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx new file mode 100644 index 00000000000..ab18e271bf3 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx @@ -0,0 +1,136 @@ +import { Popover } from "@dub/ui"; +import { FlagWavy } from "@dub/ui/icons"; +import { cn } from "@dub/utils"; +import { Check, ChevronDown } from "lucide-react"; +import { ReactNode, useMemo, useState } from "react"; + +type CountryOption = { value: string; label: string; right?: ReactNode }; + +export function NetworkCountryFilter({ + options, + getOptionIcon, + selectedValue, + onSelect, + onClear, + className, +}: { + options: CountryOption[]; + getOptionIcon?: (value: string) => ReactNode; + selectedValue?: string; + onSelect: (value: string) => void; + onClear: () => void; + className?: string; +}) { + const [openPopover, setOpenPopover] = useState(false); + const [search, setSearch] = useState(""); + + const selectedOption = options.find((o) => o.value === selectedValue); + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!q) return options; + return options.filter((o) => o.label.toLowerCase().includes(q)); + }, [options, search]); + + return ( + +
+ setSearch(e.target.value)} + placeholder="Search countries…" + className="h-8 w-full rounded-md border-none bg-transparent px-2 text-sm outline-none placeholder:text-neutral-400" + /> +
+
+ {selectedValue && ( + + )} + {filtered.length === 0 ? ( +
+ No countries found +
+ ) : ( + filtered.map((option) => { + const isSelected = option.value === selectedValue; + + return ( + + ); + }) + )} +
+
+ } + openPopover={openPopover} + setOpenPopover={setOpenPopover} + align="start" + > + + + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx new file mode 100644 index 00000000000..015b7c3d753 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx @@ -0,0 +1,196 @@ +import { Popover } from "@dub/ui"; +import { + Globe, + Instagram, + LinkedIn, + TikTok, + Twitter, + User, + YouTube, +} from "@dub/ui/icons"; +import { cn } from "@dub/utils"; +import { PlatformType } from "@prisma/client"; +import { ChevronDown } from "lucide-react"; +import { useState } from "react"; +import { + isAllPlatformsSelected, + NETWORK_FILTER_PLATFORMS, +} from "./platform-filter-utils"; + +const PLATFORM_META: Record< + (typeof NETWORK_FILTER_PLATFORMS)[number], + { label: string; icon: typeof User } +> = { + website: { label: "Website", icon: Globe }, + youtube: { label: "YouTube", icon: YouTube }, + twitter: { label: "X", icon: Twitter }, + linkedin: { label: "LinkedIn", icon: LinkedIn }, + instagram: { label: "Instagram", icon: Instagram }, + tiktok: { label: "TikTok", icon: TikTok }, +}; + +export function NetworkPlatformFilter({ + selectedPlatforms, + onChange, + className, +}: { + selectedPlatforms: PlatformType[]; + onChange: (platforms: PlatformType[]) => void; + className?: string; +}) { + const [openPopover, setOpenPopover] = useState(false); + + const selectedSet = new Set(selectedPlatforms); + const isFiltered = !isAllPlatformsSelected(selectedPlatforms); + + const toggle = (platform: PlatformType) => { + if (selectedSet.has(platform)) { + // Never let the user clear the last platform — an empty selection would + // show nothing. Deselecting the last one is a no-op. + if (selectedSet.size === 1) return; + // Remove from the CURRENT selection, not the full list — otherwise a second + // deselect would silently re-add the first. + onChange( + NETWORK_FILTER_PLATFORMS.filter( + (p) => selectedSet.has(p) && p !== platform, + ), + ); + } else { + onChange( + NETWORK_FILTER_PLATFORMS.filter( + (p) => selectedSet.has(p) || p === platform, + ), + ); + } + }; + + return ( + + {NETWORK_FILTER_PLATFORMS.map((platform) => { + const { label, icon: Icon } = PLATFORM_META[platform]; + const isSelected = selectedSet.has(platform); + + return ( + + ); + })} +
+ +
+ } + openPopover={openPopover} + setOpenPopover={setOpenPopover} + align="end" + > + + + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx new file mode 100644 index 00000000000..60647ecae28 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx @@ -0,0 +1,129 @@ +import { + REACH_TIER_KEYS, + REACH_TIERS, + type ReachTier, +} from "@/lib/api/network/reach-tiers"; +import { Popover } from "@dub/ui"; +import { cn } from "@dub/utils"; +import { ChevronDown, Users } from "lucide-react"; +import { useState } from "react"; + +export function NetworkReachFilter({ + selectedTiers, + onChange, + className, +}: { + selectedTiers: ReachTier[]; + onChange: (tiers: ReachTier[]) => void; + className?: string; +}) { + const [openPopover, setOpenPopover] = useState(false); + + const selectedSet = new Set(selectedTiers); + const isFiltered = selectedTiers.length > 0; + + const toggle = (tier: ReachTier) => + onChange( + selectedSet.has(tier) + ? REACH_TIER_KEYS.filter((t) => t !== tier && selectedSet.has(t)) + : REACH_TIER_KEYS.filter((t) => selectedSet.has(t) || t === tier), + ); + + const label = !isFiltered + ? "Audience" + : selectedTiers.length === 1 + ? REACH_TIERS[selectedTiers[0]].range + : `Audience · ${selectedTiers.length}`; + + return ( + + {REACH_TIER_KEYS.map((tier) => { + const { range, descriptor } = REACH_TIERS[tier]; + const isSelected = selectedSet.has(tier); + + return ( + + ); + })} + {isFiltered && ( + <> +
+ + + )} +
+ } + openPopover={openPopover} + setOpenPopover={setOpenPopover} + align="start" + > + + + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index 59911c22c62..a728acd9646 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -1,7 +1,10 @@ "use client"; import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner"; -import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; +import { + parseReachTiers, + type ReachTier, +} from "@/lib/api/network/reach-tiers"; import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; import usePartnerContentSearch from "@/lib/swr/use-partner-content-search"; import useWorkspace from "@/lib/swr/use-workspace"; @@ -9,38 +12,40 @@ import { NetworkPartnerProps } from "@/lib/types"; import { PARTNER_NETWORK_MAX_PAGE_SIZE } from "@/lib/zod/schemas/partner-network"; import { SearchBoxPersisted } from "@/ui/shared/search-box"; import { - AnimatedSizeContainer, Button, - Filter, PaginationControls, - ToggleGroup, usePagination, useRouterStuff, } from "@dub/ui"; -import { - Globe, - Instagram, - LinkedIn, - Star, - StarFill, - TikTok, - Twitter, - User, - YouTube, -} from "@dub/ui/icons"; +import { Star, StarFill } from "@dub/ui/icons"; import { cn, fetcher } from "@dub/utils"; import { PlatformType } from "@prisma/client"; import { useAction } from "next-safe-action/hooks"; import { useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; +import { useDebounce } from "use-debounce"; import useSWR from "swr"; import { NetworkContentSearchResults } from "./network-content-search-results"; +import { NetworkCountryFilter } from "./network-country-filter"; import { NetworkEmptyState } from "./network-empty-state"; import { NetworkPartnerCard } from "./network-partner-card"; import { NetworkPartnerDetailSheet } from "./network-partner-detail-sheet"; -import { PartnerNetworkSort } from "./partner-network-sort"; +import { NetworkPlatformFilter } from "./network-platform-filter"; +import { NetworkReachFilter } from "./network-reach-filter"; +import { + getContentSearchPlatforms, + isAllPlatformsSelected, + parseSelectedPlatforms, + platformFilterParam, +} from "./platform-filter-utils"; import { usePartnerNetworkFilters } from "./use-partner-network-filters"; +// Filter changes update the URL instantly (snappy controls) but the data fetch is +// debounced so a rapid burst of toggles collapses into a single request — sparing +// the heavy ranking SQL and (in search mode) the billed Voyage embed+rerank call. +// keepPreviousData holds the prior results on screen during the brief wait. +const FILTER_FETCH_DEBOUNCE_MS = 250; + const tabs = [ { label: "Discover", @@ -56,19 +61,6 @@ const tabs = [ }, ] as const; -const PLATFORM_TOGGLE_OPTIONS: { - value: PlatformType | "all"; - icon: typeof User; -}[] = [ - { value: "all", icon: User }, - { value: "website", icon: Globe }, - { value: "youtube", icon: YouTube }, - { value: "twitter", icon: Twitter }, - { value: "linkedin", icon: LinkedIn }, - { value: "instagram", icon: Instagram }, - { value: "tiktok", icon: TikTok }, -]; - type ProgramPartnerNetworkPageClientProps = { variant?: "default" | "ignored"; }; @@ -78,8 +70,20 @@ export function ProgramPartnerNetworkPageClient({ }: ProgramPartnerNetworkPageClientProps = {}) { const { id: workspaceId } = useWorkspace(); const { searchParams, getQueryString, queryParams } = useRouterStuff(); - const selectedPlatform = - (searchParams.get("platform") as PlatformType | null) ?? "all"; + const selectedPlatforms = useMemo( + () => parseSelectedPlatforms(searchParams.get("platform")), + [searchParams], + ); + const platformFilter = platformFilterParam(selectedPlatforms); + // The content-searchable subset of the selection. Semantic search runs only + // when at least one searchable platform is selected; otherwise we fall back to + // the ranked partner list (filtered to the chosen platforms). + const contentSearchPlatforms = getContentSearchPlatforms(selectedPlatforms); + const selectedReach = useMemo( + () => parseReachTiers(searchParams.get("reach")), + [searchParams], + ); + const reachFilter = selectedReach.length ? selectedReach : undefined; const search = searchParams.get("search")?.trim() ?? ""; const country = searchParams.get("country") ?? undefined; const starred = searchParams.get("starred") === "true"; @@ -88,15 +92,37 @@ export function ProgramPartnerNetworkPageClient({ variant === "ignored" ? "ignored" : tabs.find(({ id }) => id === searchParams.get("tab"))?.id || "discover"; - const contentSearchPlatform = - selectedPlatform !== "all" && - isPartnerContentSearchPlatform(selectedPlatform) - ? selectedPlatform - : undefined; const isContentSearchMode = status === "discover" && search.length > 0 && - (selectedPlatform === "all" || Boolean(contentSearchPlatform)); + contentSearchPlatforms.length > 0; + + // Update filter params via the History API instead of router.push. These are + // query-only changes that drive client-side SWR; page.tsx reads no searchParams, + // so a full RSC navigation per click only adds latency before useSearchParams() + // (and thus the control's checked state) updates. pushState updates it + // synchronously — instant feedback — while SWR still refetches on key change. + const updateSearchParams = (opts: { + set?: Record; + del?: string | string[]; + }) => { + const newPath = queryParams({ ...opts, getNewPath: true }) as string; + window.history.pushState(null, "", newPath); + }; + + const onPlatformsChange = (platforms: PlatformType[]) => + updateSearchParams( + isAllPlatformsSelected(platforms) + ? { del: ["platform", "page"] } + : { set: { platform: platforms.join(",") }, del: "page" }, + ); + + const onReachChange = (tiers: ReachTier[]) => + updateSearchParams( + tiers.length + ? { set: { reach: tiers.join(",") }, del: "page" } + : { del: ["reach", "page"] }, + ); const { data: partnerCounts, error: countError } = useNetworkPartnersCount(); @@ -108,34 +134,43 @@ export function ProgramPartnerNetworkPageClient({ } = usePartnerContentSearch({ enabled: isContentSearchMode, query: search, - platform: contentSearchPlatform, + platforms: platformFilter, + reach: reachFilter, country, starred, + debounceMs: FILTER_FETCH_DEBOUNCE_MS, }); + const partnersKey = + !isContentSearchMode && workspaceId + ? `/api/network/partners${getQueryString( + { + workspaceId, + status, + }, + { + exclude: + variant === "ignored" + ? ["tab", "partnerId", "starred", "search"] + : ["tab", "partnerId", "search"], + }, + )}` + : null; + // Debounce the fetch key so rapid filter toggles coalesce into one request. + const [debouncedPartnersKey] = useDebounce( + partnersKey, + FILTER_FETCH_DEBOUNCE_MS, + ); + const { data: partners, error, mutate: mutatePartners, isValidating, - } = useSWR( - !isContentSearchMode && - workspaceId && - `/api/network/partners${getQueryString( - { - workspaceId, - status, - }, - { - exclude: - variant === "ignored" - ? ["tab", "partnerId", "starred", "search"] - : ["tab", "partnerId", "search"], - }, - )}`, - fetcher, - { revalidateOnFocus: false, keepPreviousData: true }, - ); + } = useSWR(debouncedPartnersKey, fetcher, { + revalidateOnFocus: false, + keepPreviousData: true, + }); const { executeAsync: updateDiscoveredPartner } = useAction( updateDiscoveredPartnerAction, @@ -145,14 +180,10 @@ export function ProgramPartnerNetworkPageClient({ PARTNER_NETWORK_MAX_PAGE_SIZE, ); - const { - filters, - activeFilters, - isFiltered, - onSelect, - onRemove, - onRemoveAll, - } = usePartnerNetworkFilters({ status }); + const { filters, isFiltered, onSelect, onRemove, onRemoveAll } = + usePartnerNetworkFilters({ status }); + + const countryFilter = filters.find((f) => f.key === "country"); const isStarred = searchParams.get("starred") === "true"; @@ -250,7 +281,7 @@ export function ProgramPartnerNetworkPageClient({ onClick={() => { queryParams({ set: { tab: tab.id }, - del: ["page", "starred", "sortBy"], + del: ["page", "starred"], }); }} > @@ -272,84 +303,49 @@ export function ProgramPartnerNetworkPageClient({ {status === "discover" && (
-
-
- +
+ onSelect("country", value)} + onClear={() => country && onRemove("country", country)} + /> +
- -
+
- - {activeFilters.length > 0 && ( -
- -
- )} -
)} @@ -359,7 +355,11 @@ export function ProgramPartnerNetworkPageClient({ hasQuery={search.length > 0} isLoading={isSearchingContent} partners={contentSearchResults?.partners} - platform={contentSearchPlatform} + platform={ + contentSearchPlatforms.length === 1 + ? contentSearchPlatforms[0] + : undefined + } onToggleStarred={ variant === "ignored" ? undefined diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/partner-network-sort.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/partner-network-sort.tsx deleted file mode 100644 index a063ba525d9..00000000000 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/partner-network-sort.tsx +++ /dev/null @@ -1,106 +0,0 @@ -import { IconMenu, Popover, Tick, useRouterStuff } from "@dub/ui"; -import { ChartActivity2, Megaphone } from "@dub/ui/icons"; -import { cn } from "@dub/utils"; -import { PlatformType } from "@prisma/client"; -import { ChevronDown } from "lucide-react"; -import { useEffect, useMemo, useState } from "react"; - -function getPlatformSortLabel(platform: PlatformType) { - switch (platform) { - case "website": - return "Domain Ranking"; - case "youtube": - return "Subscribers"; - default: - return "Followers"; - } -} - -export function PartnerNetworkSort({ - selectedPlatform, - className, -}: { - selectedPlatform: PlatformType | "all"; - className?: string; -}) { - const { queryParams, searchParams } = useRouterStuff(); - const [openPopover, setOpenPopover] = useState(false); - - const sortBy = searchParams.get("sortBy") || "relevance"; - - useEffect(() => { - if (selectedPlatform === "all" && sortBy === "subscribers") { - queryParams({ del: ["sortBy"] }); - } - }, [selectedPlatform, sortBy, queryParams]); - - const sortOptions = useMemo(() => { - const options = [ - { - value: "relevance", - label: "Relevance", - icon: ChartActivity2, - }, - ]; - - if (selectedPlatform !== "all") { - options.push({ - value: "subscribers", - label: getPlatformSortLabel(selectedPlatform), - icon: Megaphone, - }); - } - - return options; - }, [selectedPlatform]); - - const selectedSort = - sortOptions.find((option) => option.value === sortBy) ?? sortOptions[0]; - - return ( - - {sortOptions.map(({ label, value, icon: Icon }) => ( - - ))} -
- } - openPopover={openPopover} - setOpenPopover={setOpenPopover} - > - - - ); -} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts new file mode 100644 index 00000000000..9c9ab94b372 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts @@ -0,0 +1,53 @@ +import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; +import type { PlatformType } from "@prisma/client"; + +// The platforms a partner can be filtered by, in display order. This is the +// inclusion set: with all of them selected the filter is a no-op (we omit the +// `platform` param entirely), and deselecting narrows results to partners +// present on any of the still-selected platforms. +export const NETWORK_FILTER_PLATFORMS = [ + "youtube", + "instagram", + "tiktok", + "website", + "twitter", + "linkedin", +] as const satisfies readonly PlatformType[]; + +const PLATFORM_SET = new Set(NETWORK_FILTER_PLATFORMS); + +// Parse the comma-separated `platform` query param. Absent/empty means "all +// platforms" (no filter), which we represent as the full list so callers always +// get a concrete selection to render. +export function parseSelectedPlatforms( + param: string | null | undefined, +): PlatformType[] { + if (!param) return [...NETWORK_FILTER_PLATFORMS]; + + const selected = param + .split(",") + .map((value) => value.trim()) + .filter((value): value is PlatformType => PLATFORM_SET.has(value)); + + return selected.length ? selected : [...NETWORK_FILTER_PLATFORMS]; +} + +export function isAllPlatformsSelected(selected: PlatformType[]): boolean { + return selected.length >= NETWORK_FILTER_PLATFORMS.length; +} + +// What to actually send to the API: `undefined` when everything is selected +// (keeps the default ranking path and clean URLs), otherwise the selection. +export function platformFilterParam( + selected: PlatformType[], +): PlatformType[] | undefined { + return isAllPlatformsSelected(selected) ? undefined : selected; +} + +// Semantic content search only covers a subset of platforms; this intersection +// decides whether content search runs at all for the current selection. +export function getContentSearchPlatforms( + selected: PlatformType[], +): PlatformType[] { + return selected.filter(isPartnerContentSearchPlatform); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx index e104d620dec..b4cb8b14933 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx @@ -12,6 +12,17 @@ export function usePartnerNetworkFilters({ }) { const { searchParamsObj, queryParams } = useRouterStuff(); + // Apply filter changes via the History API (instant) rather than router.push, + // which would trigger a full RSC navigation per click. See page-client for the + // rationale — all data here is client-side SWR keyed off useSearchParams. + const updateSearchParams = useCallback( + (opts: { set?: Record; del?: string | string[] }) => { + const newPath = queryParams({ ...opts, getNewPath: true }) as string; + window.history.pushState(null, "", newPath); + }, + [queryParams], + ); + const { data: countriesCount } = useNetworkPartnersCount< | { country: string; @@ -65,7 +76,7 @@ export function usePartnerNetworkFilters({ const onSelect = useCallback( (key: string, value: any) => - queryParams({ + updateSearchParams({ set: Object.keys(multiFilters).includes(key) ? { [key]: multiFilters[key].concat(value).join(","), @@ -75,7 +86,7 @@ export function usePartnerNetworkFilters({ }, del: "page", }), - [queryParams, multiFilters], + [updateSearchParams, multiFilters], ); const onRemove = useCallback( @@ -84,32 +95,33 @@ export function usePartnerNetworkFilters({ Object.keys(multiFilters).includes(key) && !(multiFilters[key].length === 1 && multiFilters[key][0] === value) ) { - queryParams({ + updateSearchParams({ set: { [key]: multiFilters[key].filter((id) => id !== value).join(","), }, del: "page", }); } else { - queryParams({ + updateSearchParams({ del: [key, "page"], }); } }, - [queryParams, multiFilters], + [updateSearchParams, multiFilters], ); const onRemoveAll = useCallback( () => - queryParams({ - del: ["country", "starred", "platform", "sortBy"], + updateSearchParams({ + del: ["country", "starred", "platform", "reach"], }), - [queryParams], + [updateSearchParams], ); const isFiltered = activeFilters.length > 0 || !!searchParamsObj.platform || + !!searchParamsObj.reach || !!searchParamsObj.search; return { diff --git a/apps/web/lib/api/network/calculate-partner-ranking.ts b/apps/web/lib/api/network/calculate-partner-ranking.ts index 72c8049be6e..99cf2283917 100644 --- a/apps/web/lib/api/network/calculate-partner-ranking.ts +++ b/apps/web/lib/api/network/calculate-partner-ranking.ts @@ -1,8 +1,9 @@ import { prisma } from "@/lib/prisma"; import { getNetworkPartnersQuerySchema } from "@/lib/zod/schemas/partner-network"; import { ACME_PROGRAM_ID } from "@dub/utils"; -import { PlatformType, Prisma } from "@prisma/client"; +import { Prisma } from "@prisma/client"; import * as z from "zod/v4"; +import { reachTiersToRanges } from "./reach-tiers"; type PartnerRankingFilters = z.infer; @@ -37,29 +38,11 @@ export interface PartnerRankingParams extends PartnerRankingFilters { * - clickToConversionRate: Average click-to-conversion rate across ALL programs the partner is enrolled in * - lastConversionAt: Most recent conversion date across ALL programs the partner is enrolled in */ -function buildOrderByClause({ - starred, - sortBy, - platform, -}: { - starred?: boolean | null; - sortBy?: "relevance" | "subscribers"; - platform?: PlatformType; -}) { +function buildOrderByClause({ starred }: { starred?: boolean | null }) { if (starred === true) { return Prisma.sql`dp.starredAt ASC`; } - if (sortBy === "subscribers" && platform) { - return Prisma.sql`( - SELECT COALESCE(MAX(pp_sort.subscribers), 0) - FROM PartnerPlatform pp_sort - WHERE pp_sort.partnerId = p.id - AND pp_sort.type = ${platform} - AND pp_sort.verifiedAt IS NOT NULL - ) DESC, p.id ASC`; - } - return Prisma.sql`finalScore DESC, p.id ASC`; } @@ -68,8 +51,8 @@ export async function calculatePartnerRanking({ partnerIds, country, starred, - sortBy = "relevance", platform, + reach, page = 1, pageSize, similarPrograms = [], @@ -89,21 +72,16 @@ export async function calculatePartnerRanking({ conditions.push(Prisma.sql`p.country = ${country}`); } - // Filter by platform type (must have verified platform) - // Combine both filters into a single EXISTS clause so they apply to the same platform - if (platform) { - const platformConditions: Prisma.Sql[] = [ - Prisma.sql`pp_filter.partnerId = p.id`, - Prisma.sql`pp_filter.verifiedAt IS NOT NULL`, - ]; - - platformConditions.push(Prisma.sql`pp_filter.type = ${platform}`); - + // Filter by platform type (must have a verified platform of one of the selected + // types). Inclusion semantics: a partner qualifies if present on any selection. + if (platform && platform.length > 0) { conditions.push( Prisma.sql`EXISTS ( - SELECT 1 - FROM PartnerPlatform pp_filter - WHERE ${Prisma.join(platformConditions, " AND ")} + SELECT 1 + FROM PartnerPlatform pp_filter + WHERE pp_filter.partnerId = p.id + AND pp_filter.verifiedAt IS NOT NULL + AND pp_filter.type IN (${Prisma.join(platform)}) )`, ); } @@ -114,6 +92,37 @@ export async function calculatePartnerRanking({ conditions.push(Prisma.sql`(dp.starredAt IS NULL OR dp.id IS NULL)`); } + // Filter by audience-size tier(s): the partner's MAX subscriber count across + // the selected platforms (their headline reach) must fall in one of the chosen + // tiers. Scoped to the same platform selection so reach reflects the audience + // the brand is actually filtering for. + if (reach && reach.length > 0) { + const ranges = reachTiersToRanges(reach); + const reachPlatformScope = + platform && platform.length > 0 + ? Prisma.sql`AND pp_reach.type IN (${Prisma.join(platform)})` + : Prisma.empty; + const rangeConditions = ranges.map(({ min, max }) => + max == null + ? Prisma.sql`reach_r.maxSubscribers >= ${min}` + : Prisma.sql`(reach_r.maxSubscribers >= ${min} AND reach_r.maxSubscribers < ${max})`, + ); + + conditions.push( + Prisma.sql`EXISTS ( + SELECT 1 + FROM ( + SELECT MAX(pp_reach.subscribers) AS maxSubscribers + FROM PartnerPlatform pp_reach + WHERE pp_reach.partnerId = p.id + AND pp_reach.verifiedAt IS NOT NULL + ${reachPlatformScope} + ) reach_r + WHERE ${Prisma.join(rangeConditions, " OR ")} + )`, + ); + } + const whereClause = Prisma.join(conditions, " AND "); // Rank partners with no platforms lower @@ -123,7 +132,7 @@ export async function calculatePartnerRanking({ WHERE pp.partnerId = p.id )`; - const orderByClause = buildOrderByClause({ starred, sortBy, platform }); + const orderByClause = buildOrderByClause({ starred }); const offset = (page - 1) * pageSize; diff --git a/apps/web/lib/api/network/partner-network-listing-where.ts b/apps/web/lib/api/network/partner-network-listing-where.ts index 50f26e95b5a..8ad90a1f341 100644 --- a/apps/web/lib/api/network/partner-network-listing-where.ts +++ b/apps/web/lib/api/network/partner-network-listing-where.ts @@ -4,7 +4,7 @@ import { PlatformType, Prisma } from "@prisma/client"; export type PartnerNetworkListingParams = { partnerIds?: string[]; country?: string; - platform?: PlatformType; + platform?: PlatformType[]; }; export type PartnerNetworkListingParts = { @@ -17,10 +17,10 @@ function listingPlatformSomeFromParams({ }: Pick): | Prisma.PartnerPlatformWhereInput | undefined { - return platform + return platform && platform.length ? { verifiedAt: { not: null }, - ...(platform && { type: platform }), + type: { in: platform }, } : undefined; } diff --git a/apps/web/lib/api/network/reach-tiers.ts b/apps/web/lib/api/network/reach-tiers.ts new file mode 100644 index 00000000000..ebde6f66830 --- /dev/null +++ b/apps/web/lib/api/network/reach-tiers.ts @@ -0,0 +1,70 @@ +// Audience-size tiers for partner discovery. Reach is gauged as the partner's +// MAX subscriber count across the *selected* platforms (their headline audience), +// so the tier filter composes with topic relevance instead of overriding it and +// stays well-defined for any platform selection (unlike a per-platform sort). + +export const REACH_TIER_KEYS = [ + "nano", + "micro", + "mid", + "macro", + "mega", +] as const; + +export type ReachTier = (typeof REACH_TIER_KEYS)[number]; + +// Display: the numeric `range` leads (objective, no "small = lesser" baggage) and +// the `descriptor` is a muted, all-positive secondary handle. The keys above stay +// as the stable URL/param values. +export const REACH_TIERS: Record< + ReachTier, + { range: string; descriptor: string; min: number; max: number | null } +> = { + nano: { range: "Under 10K", descriptor: "Emerging", min: 0, max: 10_000 }, + micro: { range: "10K – 100K", descriptor: "Rising", min: 10_000, max: 100_000 }, + mid: { + range: "100K – 500K", + descriptor: "Established", + min: 100_000, + max: 500_000, + }, + macro: { + range: "500K – 1M", + descriptor: "Leading", + min: 500_000, + max: 1_000_000, + }, + mega: { range: "1M+", descriptor: "Flagship", min: 1_000_000, max: null }, +}; + +const REACH_TIER_SET = new Set(REACH_TIER_KEYS); + +export function isReachTier(value: string | null | undefined): value is ReachTier { + return Boolean(value && REACH_TIER_SET.has(value)); +} + +// Parse the comma-separated `reach` query param into valid tier keys (in canonical +// order, deduped). Absent/invalid → empty (no reach filter). +export function parseReachTiers(param: string | null | undefined): ReachTier[] { + if (!param) return []; + + const selected = new Set( + param + .split(",") + .map((value) => value.trim()) + .filter(isReachTier), + ); + + return REACH_TIER_KEYS.filter((key) => selected.has(key)); +} + +// Subscriber ranges (min inclusive, max exclusive; null max = unbounded) for the +// selected tiers — one entry per tier, OR'd together by callers. +export function reachTiersToRanges( + tiers: ReachTier[], +): { min: number; max: number | null }[] { + return tiers.map((tier) => { + const { min, max } = REACH_TIERS[tier]; + return { min, max }; + }); +} diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.ts index 3746e354442..de1a2fccc15 100644 --- a/apps/web/lib/api/scrape-creators/client.ts +++ b/apps/web/lib/api/scrape-creators/client.ts @@ -111,7 +111,9 @@ export const scrapeCreatorsFetch = createFetch({ next_max_id: z.string().optional(), trim: z.boolean().optional(), }), - output: instagramUserPostsSchema, + // Parsed in getInstagramUserPosts so account-unavailable responses can + // be classified as empty results instead of strict-schema 500s. + output: z.unknown(), }, // Fetch an Instagram video/reel transcript diff --git a/apps/web/lib/api/scrape-creators/get-instagram-user-posts.ts b/apps/web/lib/api/scrape-creators/get-instagram-user-posts.ts index a492f90763f..cde3e8fc98b 100644 --- a/apps/web/lib/api/scrape-creators/get-instagram-user-posts.ts +++ b/apps/web/lib/api/scrape-creators/get-instagram-user-posts.ts @@ -1,4 +1,5 @@ import { scrapeCreatorsFetch } from "./client"; +import { instagramUserPostsSchema } from "./schema"; interface GetInstagramUserPostsParams { handle: string; @@ -28,5 +29,52 @@ export async function getInstagramUserPosts({ ); } - return data; + const parsed = instagramUserPostsSchema.safeParse(data); + + if (!parsed.success && isMissingInstagramUserPostsItems(data)) { + console.warn("[ScrapeCreators] Instagram user posts unavailable", { + handle, + nextMaxId, + issues: parsed.error.issues, + response: summarizeScrapeCreatorsResponse(data), + }); + + return { + items: [], + next_max_id: null, + more_available: false, + }; + } + + if (!parsed.success) { + throw new Error( + `Unexpected Instagram user posts response from ScrapeCreators: ${JSON.stringify({ + issues: parsed.error.issues, + response: summarizeScrapeCreatorsResponse(data), + })}`, + ); + } + + return parsed.data; +} + +function isMissingInstagramUserPostsItems(data: unknown) { + if (!data || typeof data !== "object" || Array.isArray(data)) return false; + return !("items" in data); +} + +function summarizeScrapeCreatorsResponse(data: unknown) { + if (!data || typeof data !== "object" || Array.isArray(data)) return data; + + const response = data as Record; + + return { + keys: Object.keys(response).slice(0, 20), + success: response.success, + error: response.error, + errorStatus: response.errorStatus, + message: response.message, + status: response.status, + statusText: response.statusText, + }; } diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index 535d37cf32a..fa80d76860e 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -21,8 +21,8 @@ export const PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS = 7; export const PARTNER_CONTENT_EMBED_FLOW_CONTROL = { key: "partner-content-embed-voyage", - parallelism: 5, - rate: 120, + parallelism: 20, + rate: 600, period: "1m", } as const; diff --git a/apps/web/lib/partner-content-search/thumbnail-url.ts b/apps/web/lib/partner-content-search/thumbnail-url.ts new file mode 100644 index 00000000000..ef527b6709d --- /dev/null +++ b/apps/web/lib/partner-content-search/thumbnail-url.ts @@ -0,0 +1,20 @@ +export function getPartnerContentThumbnailUrl(thumbnailUrl: string | null) { + if (!thumbnailUrl) return null; + + if (isInstagramCdnUrl(thumbnailUrl)) { + return `/api/partner-content/thumbnail?url=${encodeURIComponent(thumbnailUrl)}`; + } + + return thumbnailUrl; +} + +export function isInstagramCdnUrl(url: string) { + try { + const { hostname } = new URL(url); + return ( + hostname === "cdninstagram.com" || hostname.endsWith(".cdninstagram.com") + ); + } catch { + return false; + } +} diff --git a/apps/web/lib/swr/use-partner-content-search.ts b/apps/web/lib/swr/use-partner-content-search.ts index 13eb3004a4b..120d0790e39 100644 --- a/apps/web/lib/swr/use-partner-content-search.ts +++ b/apps/web/lib/swr/use-partner-content-search.ts @@ -1,3 +1,4 @@ +import type { ReachTier } from "@/lib/api/network/reach-tiers"; import { PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, @@ -6,6 +7,7 @@ import { import type { NetworkPartnerProps } from "@/lib/types"; import type { PlatformType } from "@prisma/client"; import useSWR from "swr"; +import { useDebounce } from "use-debounce"; import useWorkspace from "./use-workspace"; export type PartnerContentMatchSource = "transcript" | "creatorText"; @@ -105,40 +107,55 @@ export type PartnerContentSearchResponse = { export default function usePartnerContentSearch({ enabled, query, - platform, + platforms, + reach, country, starred, partnerIds, candidateChunkCount, limit = PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, chunksPerPartner = PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, + debounceMs = 0, }: { enabled: boolean; query: string; - platform?: PlatformType; + platforms?: PlatformType[]; + reach?: ReachTier[]; country?: string; starred: boolean; partnerIds?: string[]; candidateChunkCount?: number; limit?: number; chunksPerPartner?: number; + // Coalesce rapid filter changes into one request. Voyage search is billed and + // ~seconds, so callers driving it from fast-changing filters pass a small delay. + debounceMs?: number; }) { const { id: workspaceId } = useWorkspace(); - const { data, error, isLoading, mutate } = useSWR( + // Stable string signature (not a fresh array each render) so the debounce timer + // only resets when the inputs actually change. The fetcher below reads the live + // values; by the time the debounced key settles they equal this signature. + const keySignature = enabled && workspaceId - ? [ - "partner-content-search", + ? JSON.stringify([ workspaceId, query, - platform, + platforms?.join(",") ?? "", + reach?.join(",") ?? "", country, starred, partnerIds?.join(",") ?? "", candidateChunkCount, limit, chunksPerPartner, - ] + ]) + : null; + const [debouncedKeySignature] = useDebounce(keySignature, debounceMs); + + const { data, error, isLoading, mutate } = useSWR( + debouncedKeySignature + ? ["partner-content-search", debouncedKeySignature] : null, // The shared `fetcher` from @dub/utils is GET-only; this endpoint takes a // POST body, so it needs its own fetcher. @@ -155,7 +172,8 @@ export default function usePartnerContentSearch({ limit, chunksPerPartner, ...(candidateChunkCount && { candidateChunkCount }), - ...(platform && { platform }), + ...(platforms?.length && { platforms }), + ...(reach?.length && { reach }), ...(country && { country }), ...(partnerIds?.length && { partnerIds }), starred: starred || undefined, diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index e69ec255d79..ab21daef080 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -1,3 +1,4 @@ +import { REACH_TIER_KEYS } from "@/lib/api/network/reach-tiers"; import { processKey } from "@/lib/api/links/utils"; import { PlatformType } from "@prisma/client"; import * as z from "zod/v4"; @@ -41,8 +42,16 @@ export const getNetworkPartnersQuerySchema = z status: NetworkPartnersStatusSchema.default("discover"), country: z.string().optional(), starred: booleanQuerySchema.nullish(), - sortBy: z.enum(["relevance", "subscribers"]).default("relevance"), - platform: z.enum(PlatformType).optional(), + platform: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .pipe(z.array(z.enum(PlatformType))) + .optional(), + reach: z + .union([z.string(), z.array(z.string())]) + .transform((v) => (Array.isArray(v) ? v : v.split(","))) + .pipe(z.array(z.enum(REACH_TIER_KEYS))) + .optional(), partnerIds: z .union([z.string(), z.array(z.string())]) .transform((v) => (Array.isArray(v) ? v : v.split(","))) From fd0580f7ed53fef5cb412345495ea81c05928cb7 Mon Sep 17 00:00:00 2001 From: David Clark Date: Mon, 22 Jun 2026 16:53:05 -0400 Subject: [PATCH 12/25] Refactor partner content search route and network filters; add performance trace info --- .../api/admin/partner-content/search/route.ts | 3 +- .../api/cron/partner-content/embed/route.ts | 28 +- .../network/partners/content-search/route.ts | 1368 +++++++++++------ .../network/[partnerId]/page-client.tsx | 122 +- .../network-content-search-results.tsx | 428 +++++- .../network/network-country-filter.tsx | 136 -- .../network/network-platform-filter.tsx | 372 +++-- .../program/network/network-reach-filter.tsx | 129 -- .../(ee)/program/network/page-client.tsx | 71 +- .../program/network/platform-filter-utils.ts | 106 +- .../network/use-partner-network-filters.tsx | 290 ++-- .../lib/partner-content-search/constants.ts | 22 +- .../partner-content-search/search-utils.ts | 23 +- .../partner-content-search/thumbnail-url.ts | 4 +- .../web/lib/swr/use-partner-content-search.ts | 9 + .../partner-content-search-vector-index.sql | 12 +- .../schema/partner-content-search.prisma | 10 + .../vector-index-sync.test.ts | 32 + packages/ui/src/tooltip.tsx | 22 +- 19 files changed, 1907 insertions(+), 1280 deletions(-) delete mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx delete mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx create mode 100644 apps/web/tests/partner-content-search/vector-index-sync.test.ts diff --git a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts index 28d328319cf..6c9f1a24454 100644 --- a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts +++ b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts @@ -2,6 +2,7 @@ import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; import { parseRequestBody } from "@/lib/api/utils"; import { withAdmin } from "@/lib/auth"; import { + PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, PARTNER_CONTENT_SEARCH_LIMITS, PARTNER_CONTENT_SEARCH_MODELS, PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, @@ -162,7 +163,7 @@ async function searchPartnerContentChunks({ c.chunkText, c.startMs, c.endMs, - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance FROM PartnerContentChunk c INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId INNER JOIN Partner p ON p.id = c.partnerId diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts index 5b7d5a18205..d30f2449766 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -52,6 +52,20 @@ export const POST = withCron(async ({ rawBody }) => { partnerId: true, totalChunkCount: true, embeddedChunkCount: true, + // Source values for the denormalized search pre-filter columns on + // PartnerContentChunk. The embed write is the single choke point that makes + // a chunk searchable, so we stamp these here to keep new chunks correctly + // filterable the moment they gain an embedding. + partner: { + select: { + country: true, + }, + }, + partnerPlatform: { + select: { + type: true, + }, + }, }, }); @@ -134,12 +148,24 @@ export const POST = withCron(async ({ rawBody }) => { ); } + // Denormalized search pre-filter values, stamped alongside the embedding so a + // chunk is correctly filterable as soon as it becomes searchable. All chunks in + // this job share one content item, hence one partner/platform, so these are + // constant across the batch. Both are effectively static (platformType immutable, + // country ~never changes); a periodic reconcile catches the rare country edit. + // networkStatus is intentionally not denormalized — it stays a post-filter in the + // search query (ingestion already gates content on approved/trusted partners). + const country = contentItem.partner.country; + const platformType = contentItem.partnerPlatform.type; + const updates = chunks.map( (contentChunk, index) => prisma.$executeRaw` UPDATE PartnerContentChunk SET embedding = TO_VECTOR(${serializeEmbeddingForVector(embeddings[index])}), - embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id}, + country = ${country}, + platformType = ${platformType} WHERE id = ${contentChunk.id} `, ); diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index e2a02cbda4c..a1a9c8f1af6 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -5,6 +5,8 @@ import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-progr import { parseRequestBody } from "@/lib/api/utils"; import { withWorkspace } from "@/lib/auth"; import { + PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, + PARTNER_CONTENT_CHUNK_VECTOR_INDEX, PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_LIMITS, PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER, @@ -31,6 +33,7 @@ import { sortRowsByRelevanceScore, type PartnerContentMatchEvidence, type PartnerContentMatchSource, + type PartnerContentSearchQuerySignals, } from "@/lib/partner-content-search/ranking"; import { dedupeBestChunkPerContentItem, @@ -55,12 +58,47 @@ export const dynamic = "force-dynamic"; export const maxDuration = 30; const DEFAULT_PARTNER_LIMIT = 20; +const MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS = 5; +const PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES = new Set([ + "query-embedding-complete", + "partner-eligibility-resolved", + "vector-candidate-search-complete", + "vector-candidate-hydration-complete", + "vector-search-pool-expanded", + "vector-search-complete", + "candidate-dedupe-complete", + "chunk-text-hydration-complete", + "rerank-complete", + "partner-candidates-grouped", + "match-summary-base-queries-complete", + "match-summary-content-counts-complete", + "match-summary-recent-content-complete", + "match-summary-followers-complete", + "match-summary-aggregation-complete", + "partner-hydration-complete", + "response-ready", +]); type PartnerContentSearchTimingLogger = ( stage: string, metadata?: Record, ) => void; +type SourceScoreByContentItemId = Map< + string, + Map +>; + +type PartnerContentSearchCandidateRow = Pick< + PartnerContentSearchRow, + "chunkId" | "partnerContentItemId" | "chunkSource" | "distance" +>; + +type PartnerContentSearchHydrationRow = Omit< + PartnerContentSearchRow, + "distance" +>; + const partnerNetworkContentSearchSchema = z.object({ query: z.string().trim().max(500).optional(), platforms: z.array(z.enum(PlatformType)).min(1).optional(), @@ -138,31 +176,33 @@ export const POST = withWorkspace( logTiming("request-parsed"); - const { rows, reranked, queryVector, cutoffDistance } = body.query - ? await searchPartnerNetworkContent({ - programId, - query: body.query, - platforms: body.platforms, - country: body.country, - partnerIds: body.partnerIds, - starred: body.starred, - limit: candidateChunkCount, - rerank: body.rerank, - logTiming, - }) - : { - rows: await listPartnerNetworkContent({ + const { rows, reranked, queryVector, cutoffDistance, itemSourceBestDistance } = + body.query + ? await searchPartnerNetworkContent({ programId, + query: body.query, platforms: body.platforms, country: body.country, partnerIds: body.partnerIds, starred: body.starred, limit: candidateChunkCount, - }), - reranked: false, - queryVector: null, - cutoffDistance: null, - }; + rerank: body.rerank, + logTiming, + }) + : { + rows: await listPartnerNetworkContent({ + programId, + platforms: body.platforms, + country: body.country, + partnerIds: body.partnerIds, + starred: body.starred, + limit: candidateChunkCount, + }), + reranked: false, + queryVector: null, + cutoffDistance: null, + itemSourceBestDistance: undefined, + }; logTiming("content-rows-loaded", { rowCount: rows.length, reranked, @@ -196,6 +236,7 @@ export const POST = withWorkspace( query: body.query, queryVector, cutoffDistance, + itemSourceBestDistance, logTiming, }); logTiming("match-summaries-complete", { @@ -262,15 +303,24 @@ function createPartnerContentSearchTimingLogger( return (stage, metadata = {}) => { const now = Date.now(); + const elapsedMs = now - startedAt; + const deltaMs = now - previousAt; + previousAt = now; + + if ( + deltaMs < MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS && + !PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES.has(stage) + ) { + return; + } console.info("[partner-content-search:timing]", { stage, - elapsedMs: now - startedAt, - deltaMs: now - previousAt, + elapsedMs, + deltaMs, ...context, ...metadata, }); - previousAt = now; }; } @@ -278,22 +328,19 @@ function isNonNull(value: T | null): value is T { return value !== null; } -function getSourceExactMention( - sourceMentionsByItemId: Map>, - itemId: string, - source: PartnerContentMatchSource, -) { - return sourceMentionsByItemId.get(itemId)?.get(source) ?? false; +function getVectorSearchChunkPoolSize(limit: number) { + return Math.max( + limit, + Math.min( + PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize, + limit * PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMultiplier, + ), + ); } -function setSourceExactMention( - sourceMentionsByItemId: Map>, - itemId: string, - source: PartnerContentMatchSource, -) { - const sourceMentions = sourceMentionsByItemId.get(itemId) ?? new Map(); - sourceMentions.set(source, true); - sourceMentionsByItemId.set(itemId, sourceMentions); +function countDistinctContentItems(rows: { partnerContentItemId: string }[]) { + return new Set(rows.map(({ partnerContentItemId }) => partnerContentItemId)) + .size; } function sortPartnersByTopicFit< @@ -362,6 +409,58 @@ async function getNetworkPartnersById({ return new Map(partners.map((partner) => [partner.id, partner])); } +// Resolves a program's partner-eligibility sets for inline ANN pre-filtering: +// the deny-set (enrolled ∪ ignored partners) excluded from every query, plus the +// starred set the starred filter includes/excludes. All are per-program and bounded +// by the program's roster, so they ride into the flat ANN as id-set predicates +// without dragging the relational joins onto the vector-index traversal path. +async function resolveProgramPartnerEligibility({ + programId, + starred, + logTiming, +}: { + programId: string; + starred?: boolean; + logTiming?: PartnerContentSearchTimingLogger; +}) { + const [enrolledRows, ignoredRows, starredRows] = await Promise.all([ + prisma.programEnrollment.findMany({ + where: { programId }, + select: { partnerId: true }, + }), + prisma.discoveredPartner.findMany({ + where: { programId, ignoredAt: { not: null } }, + select: { partnerId: true }, + }), + // Only needed when the starred filter is active. + starred !== undefined + ? prisma.discoveredPartner.findMany({ + where: { programId, starredAt: { not: null } }, + select: { partnerId: true }, + }) + : Promise.resolve<{ partnerId: string }[]>([]), + ]); + + const excludedPartnerIds = [ + ...new Set([ + ...enrolledRows.map(({ partnerId }) => partnerId), + ...ignoredRows.map(({ partnerId }) => partnerId), + ]), + ]; + const starredPartnerIds = [ + ...new Set(starredRows.map(({ partnerId }) => partnerId)), + ]; + + logTiming?.("partner-eligibility-resolved", { + enrolledCount: enrolledRows.length, + ignoredCount: ignoredRows.length, + excludedPartnerCount: excludedPartnerIds.length, + starredPartnerCount: starredPartnerIds.length, + }); + + return { excludedPartnerIds, starredPartnerIds }; +} + async function searchPartnerNetworkContent({ programId, query, @@ -383,6 +482,16 @@ async function searchPartnerNetworkContent({ rerank: boolean; logTiming?: PartnerContentSearchTimingLogger; }) { + // Resolve the program's partner-eligibility sets in parallel with the query + // embedding — they need only programId, so they run under the Voyage round-trip + // and add little to no incremental time delay. (PrismaPromise.all kicks the queries off + // synchronously here, before we await the embedding below.) + const eligibilityPromise = resolveProgramPartnerEligibility({ + programId, + starred, + logTiming, + }); + let queryEmbedding: number[]; logTiming?.("query-embedding-start"); try { @@ -404,86 +513,149 @@ async function searchPartnerNetworkContent({ embeddingDimensions: queryEmbedding.length, }); const queryVector = serializeEmbeddingForVector(queryEmbedding); - const starredFilter = - starred === true - ? Prisma.sql`AND dp.starredAt IS NOT NULL` - : starred === false - ? Prisma.sql`AND (dp.starredAt IS NULL OR dp.id IS NULL)` - : Prisma.empty; - const platformFilter = platforms?.length - ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` - : Prisma.empty; - const countryFilter = country - ? Prisma.sql`AND p.country = ${country}` - : Prisma.empty; - const partnerIdsFilter = partnerIds?.length - ? Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})` - : Prisma.empty; + const eligibility = await eligibilityPromise; + + // Pre-filter eligibility + user filters INLINE on the flat ANN (no joins): exclude + // the program's enrolled + ignored partners, honor explicit partnerIds, apply the + // starred filter, and apply the country/platform user filters via the denormalized + // c.country / c.platformType columns. Pre-filtering (vs. the old post-hydration + // filter) keeps these from biasing/starving the candidate pool — enrolled partners + // are exactly a program's strongest matches, so post-filtering them silently + // dropped the best results as a program matured. networkStatus stays a post-filter + // (non-selective: ingestion only embeds approved/trusted partners). + const annFilters: Prisma.Sql[] = []; + if (eligibility.excludedPartnerIds.length > 0) { + annFilters.push( + Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.excludedPartnerIds)})`, + ); + } + if (partnerIds?.length) { + annFilters.push(Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})`); + } + if (starred === true) { + // starred filter on with no starred partners → no eligible candidates. + annFilters.push( + eligibility.starredPartnerIds.length > 0 + ? Prisma.sql`AND c.partnerId IN (${Prisma.join(eligibility.starredPartnerIds)})` + : Prisma.sql`AND 1 = 0`, + ); + } else if (starred === false && eligibility.starredPartnerIds.length > 0) { + annFilters.push( + Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.starredPartnerIds)})`, + ); + } + if (country) { + annFilters.push(Prisma.sql`AND c.country = ${country}`); + } + if (platforms?.length) { + annFilters.push( + Prisma.sql`AND c.platformType IN (${Prisma.join(platforms)})`, + ); + } + const annFilter = + annFilters.length > 0 ? Prisma.join(annFilters, " ") : Prisma.empty; + + // Retrieval is split into two bounded phases. First, keep the ANN query flat so + // PlanetScale can use the vector index for a small chunk-candidate pool. Then + // hydrate/filter only those chunk ids through the relational partner/platform + // joins below. This keeps the expensive joins off the vector traversal path. + // The cheap phase also returns the content-item id + source so we can dedup to + // the best chunk per item+source BEFORE the join (see retrievePool). + const fetchCandidateChunks = (poolSize: number) => + prisma.$queryRaw(Prisma.sql` + SELECT + c.id AS chunkId, + c.partnerContentItemId, + c.source AS chunkSource, + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance + FROM PartnerContentChunk c FORCE INDEX (${Prisma.raw(PARTNER_CONTENT_CHUNK_VECTOR_INDEX)}) + WHERE c.embedding IS NOT NULL + AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + ${annFilter} + ORDER BY distance ASC + LIMIT ${poolSize} + `); + + // Dedup to the best chunk per content-item + source BEFORE the relational + // hydration join. The post-ANN filters are all per-item/per-partner, so the + // surviving items are identical whether we filter-then-dedup or dedup-then- + // filter; this just keeps chunk-heavy items from sending redundant rows through + // the 5-way join. itemRows (cutoff) and rows (rerank) are both derived from the + // per-item+source set below, so nothing downstream changes. + const retrievePool = async (poolSize: number) => { + const candidateRows = await fetchCandidateChunks(poolSize); + logTiming?.("vector-candidate-search-complete", { + candidateRowCount: candidateRows.length, + poolSize, + vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, + }); + const dedupedCandidates = + dedupeBestChunkPerContentItemSource(candidateRows); + const poolRows = await hydratePartnerContentSearchRows({ + candidateRows: dedupedCandidates, + logTiming, + }); + return { candidateRowCount: candidateRows.length, poolRows }; + }; - // Retrieval stays on the index-friendly flat ANN shape (`ORDER BY DISTANCE ... - // LIMIT`). A SQL window-function dedup here would force a full distance scan and - // defeat the vector index, so instead we over-fetch a chunk pool and collapse it - // to the best chunk per content item in app code (below). That keeps chunk-heavy - // videos from crowding the candidate set while preserving the ANN fast path. - // Per-video "match" coverage for the bars is computed exactly and separately in - // getPartnerMatchSummaries. - const poolSize = Math.max( - limit, - PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolSize, - ); + const initialPoolSize = getVectorSearchChunkPoolSize(limit); + const maxPoolSize = PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize; logTiming?.("vector-search-start", { - poolSize, + poolSize: initialPoolSize, + maxPoolSize, + retrievalShape: "two-phase", }); - const poolRows = await prisma.$queryRaw(Prisma.sql` - SELECT - c.id AS chunkId, - c.partnerContentItemId, - c.partnerId, - p.name AS partnerName, - p.username AS partnerUsername, - p.image AS partnerImage, - p.description AS partnerDescription, - pp.type AS platformType, - pp.identifier AS platformIdentifier, - pci.platformContentId, - pci.url AS contentUrl, - pci.contentType, - pci.title AS contentTitle, - pci.description AS contentDescription, - pci.thumbnailUrl AS contentThumbnailUrl, - pci.publishedAt AS contentPublishedAt, - pci.durationMs AS contentDurationMs, - c.source AS chunkSource, - "" AS chunkText, - c.startMs, - c.endMs, - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance - FROM PartnerContentChunk c - INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId - INNER JOIN Partner p ON p.id = c.partnerId - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - LEFT JOIN ProgramEnrollment enrolled - ON enrolled.partnerId = p.id - AND enrolled.programId = ${programId} - LEFT JOIN DiscoveredPartner dp - ON dp.partnerId = p.id - AND dp.programId = ${programId} - WHERE c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - ${platformFilter} - ${countryFilter} - ${partnerIdsFilter} - AND p.networkStatus IN ("approved", "trusted") - AND enrolled.id IS NULL - AND (dp.ignoredAt IS NULL OR dp.id IS NULL) - ${starredFilter} - ORDER BY distance ASC - LIMIT ${poolSize} - `); + + let poolSize = initialPoolSize; + let { candidateRowCount, poolRows } = await retrievePool(poolSize); + + // Eligibility + user filters run INLINE in the ANN, so the pool comes back + // already eligible; networkStatus is the only remaining post-ANN filter and it's + // non-selective (ingestion gates on approved/trusted), so the hydrated pool rarely + // shrinks. Keep a one-shot expansion as a safety net for the rare case it does + // (e.g. a cluster of demoted partners): if we under-filled and the ANN hadn't + // exhausted the index (it returned a full pool), widen to the cap once and retry. + let distinctItemCount = countDistinctContentItems(poolRows); + if ( + poolSize < maxPoolSize && + candidateRowCount === poolSize && + distinctItemCount < limit + ) { + poolSize = maxPoolSize; + logTiming?.("vector-search-pool-expanded", { + previousPoolSize: initialPoolSize, + poolSize, + distinctItemCount, + limit, + }); + ({ candidateRowCount, poolRows } = await retrievePool(poolSize)); + distinctItemCount = countDistinctContentItems(poolRows); + } logTiming?.("vector-search-complete", { + candidateRowCount, poolRowCount: poolRows.length, + distinctItemCount, + poolSize, + expanded: poolSize !== initialPoolSize, + vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, + retrievalShape: "two-phase", }); + // Best cosine distance per content-item + evidence source across the candidate + // pool. getPartnerMatchSummaries reuses this to gate which recent posts count as + // "matched" instead of recomputing DISTANCE per item in SQL: cutoffDistance is + // itself a pool item's distance, so any item that could clear it is already in + // this map (an item missing here is provably beyond the cutoff — not matched). + const itemSourceBestDistance: SourceScoreByContentItemId = new Map(); + for (const row of poolRows) { + setSourceDistance( + itemSourceBestDistance, + row.partnerContentItemId, + getEvidenceSource(row.chunkSource), + Number(row.distance), + ); + } + // Collapse to one best chunk per content item for the item-level cutoff, and // one best chunk per content item + source for reranking/source-aware evidence. // This keeps metadata and transcript evidence separable without letting one @@ -521,6 +693,7 @@ async function searchPartnerNetworkContent({ reranked: false, queryVector, cutoffDistance, + itemSourceBestDistance, }; } @@ -547,9 +720,136 @@ async function searchPartnerNetworkContent({ rows: sortRowsByRelevanceScore(rerankResult.rows), queryVector, cutoffDistance, + itemSourceBestDistance, }; } +async function hydratePartnerContentSearchRows({ + candidateRows, + logTiming, +}: { + candidateRows: PartnerContentSearchCandidateRow[]; + logTiming?: PartnerContentSearchTimingLogger; +}) { + if (candidateRows.length === 0) return []; + + const chunkIds = candidateRows.map(({ chunkId }) => chunkId); + // Eligibility + user filters (enrolled/ignored/starred/country/platform) are all + // pre-filtered in the ANN now, so hydration is a plain metadata fetch over + // already-eligible chunk ids. networkStatus is the one remaining post-filter + // (cheap and non-selective — see the search query note); the partner/platform + // joins stay only to supply display columns and that networkStatus check. + + // Plain relational hydration over an already-eligible chunk-id set: the inner + // joins in the prior raw form were pure filters (all relations are required + // FKs), and the lone post-ANN predicate, networkStatus, maps to the partner + // relation filter below. `chunkText` is deliberately not selected here (it's + // hydrated separately in hydratePartnerContentChunkText); we backfill the "" + // placeholder when flattening to the row shape. + const hydratedRows = await prisma.partnerContentChunk.findMany({ + where: { + id: { in: chunkIds }, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + partner: { + networkStatus: { in: ["approved", "trusted"] }, + }, + }, + select: { + id: true, + partnerContentItemId: true, + partnerId: true, + source: true, + startMs: true, + endMs: true, + partner: { + select: { + name: true, + username: true, + image: true, + description: true, + }, + }, + partnerContentItem: { + select: { + platformContentId: true, + url: true, + contentType: true, + title: true, + description: true, + thumbnailUrl: true, + publishedAt: true, + durationMs: true, + viewCount: true, + likeCount: true, + commentCount: true, + shareCount: true, + saveCount: true, + partnerPlatform: { + select: { + type: true, + identifier: true, + }, + }, + }, + }, + }, + }); + const hydratedByChunkId = new Map( + hydratedRows.map(({ partner, partnerContentItem: item, ...chunk }): [ + string, + PartnerContentSearchHydrationRow, + ] => [ + chunk.id, + { + chunkId: chunk.id, + partnerContentItemId: chunk.partnerContentItemId, + partnerId: chunk.partnerId, + partnerName: partner.name, + partnerUsername: partner.username, + partnerImage: partner.image, + partnerDescription: partner.description, + platformType: item.partnerPlatform.type, + platformIdentifier: item.partnerPlatform.identifier, + platformContentId: item.platformContentId, + contentUrl: item.url, + contentType: item.contentType, + contentTitle: item.title, + contentDescription: item.description, + contentThumbnailUrl: item.thumbnailUrl, + contentPublishedAt: item.publishedAt, + contentDurationMs: item.durationMs, + contentViewCount: item.viewCount, + contentLikeCount: item.likeCount, + contentCommentCount: item.commentCount, + contentShareCount: item.shareCount, + contentSaveCount: item.saveCount, + chunkSource: chunk.source, + chunkText: "", + startMs: chunk.startMs, + endMs: chunk.endMs, + }, + ]), + ); + const rows = candidateRows.flatMap((candidate) => { + const hydrated = hydratedByChunkId.get(candidate.chunkId); + if (!hydrated) return []; + + return [ + { + ...hydrated, + distance: candidate.distance, + }, + ]; + }); + logTiming?.("vector-candidate-hydration-complete", { + candidateRowCount: candidateRows.length, + hydratedRowCount: hydratedRows.length, + filteredOutRowCount: candidateRows.length - rows.length, + }); + + return rows; +} + async function hydratePartnerContentChunkText({ rows, maxRows, @@ -688,6 +988,11 @@ async function listPartnerNetworkContent({ thumbnailUrl: true, publishedAt: true, durationMs: true, + viewCount: true, + likeCount: true, + commentCount: true, + shareCount: true, + saveCount: true, partner: { select: { name: true, @@ -749,6 +1054,11 @@ async function listPartnerNetworkContent({ contentThumbnailUrl: contentItem.thumbnailUrl, contentPublishedAt: contentItem.publishedAt, contentDurationMs: contentItem.durationMs, + contentViewCount: contentItem.viewCount, + contentLikeCount: contentItem.likeCount, + contentCommentCount: contentItem.commentCount, + contentShareCount: contentItem.shareCount, + contentSaveCount: contentItem.saveCount, chunkSource: chunk.source, chunkText: "", startMs: null, @@ -771,9 +1081,49 @@ type PartnerRecentContentBarRow = { transcriptFetchStatus: string | null; publishedAt: Date | null; viewCount: bigint | number | null; + likeCount: bigint | number | null; + commentCount: bigint | number | null; + shareCount: bigint | number | null; + saveCount: bigint | number | null; rowNumber: bigint | number; }; +type PartnerContentItemBestMatch = { + transcriptScore: number | null; + creatorTextScore: number | null; +}; + +type PartnerContentMatchBar = { + partnerContentItemId: string; + platform: string; + platformContentId: string; + title: string | null; + url: string | null; + durationMs: number | null; + publishedAt: string | null; + viewCount: number | null; + likeCount: number | null; + commentCount: number | null; + shareCount: number | null; + saveCount: number | null; + matched: boolean; + matchScore: number | null; + matchEvidence: PartnerContentMatchEvidence; +}; + +type QueryContentMatchContext = { + querySignals: PartnerContentSearchQuerySignals; + cutoffDistance?: number | null; + recentItemSourceBestDistance: SourceScoreByContentItemId; + rerankByItemSource: SourceScoreByContentItemId; +}; + +type ListContentMatchContext = { + bestMatchByContentItemId: Map; +}; + +type ContentMatchContext = QueryContentMatchContext | ListContentMatchContext; + // Median of a numeric list (robust to viral outliers vs. a mean). Returns null // for an empty list so callers can omit the signal when there's no data. function median(values: number[]): number | null { @@ -785,6 +1135,226 @@ function median(values: number[]): number | null { : sorted[mid]; } +function groupRowsBy(rows: T[], getKey: (row: T) => K) { + const rowsByKey = new Map(); + + for (const row of rows) { + const key = getKey(row); + const group = rowsByKey.get(key) ?? []; + group.push(row); + rowsByKey.set(key, group); + } + + return rowsByKey; +} + +function getBestMatchByContentItemId(rows: PartnerContentSearchRow[]) { + const bestMatchByContentItemId = new Map< + string, + PartnerContentItemBestMatch + >(); + + for (const row of rows) { + // Use the effective score (rerank when present) so the content-match bars + // stay consistent with the reranked partner ordering. + const score = row.rerankScore ?? toScore(Number(row.distance)); + const evidenceSource = getEvidenceSource(row.chunkSource); + const existing = bestMatchByContentItemId.get(row.partnerContentItemId); + const next = existing ?? { + transcriptScore: null, + creatorTextScore: null, + }; + + if (evidenceSource === "transcript") { + next.transcriptScore = + next.transcriptScore == null + ? score + : Math.max(next.transcriptScore, score); + } else { + next.creatorTextScore = + next.creatorTextScore == null + ? score + : Math.max(next.creatorTextScore, score); + } + + bestMatchByContentItemId.set(row.partnerContentItemId, next); + } + + return bestMatchByContentItemId; +} + +function getQueryContentMatch({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: QueryContentMatchContext; +}) { + // On-topic gate: prefer the reranker's calibrated relevance, which cleanly + // separates on- from off-topic. Fall back to the cosine cutoff only for items + // the reranker didn't score (rerank disabled/failed, or the item never reached + // the candidate pool). + const transcriptScore = getEntityTranscriptScore({ + currentScore: getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "transcript", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "transcript", + ), + cutoffDistance: context.cutoffDistance, + }), + // Transcript-level substring matching was removed from the search path; only + // title/description exact mentions (computed in memory below) feed scoring now. + hasExactQueryMention: false, + queryIntent: context.querySignals.intent, + }); + const vectorCreatorTextScore = getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "creatorText", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "creatorText", + ), + cutoffDistance: context.cutoffDistance, + }); + const titleHasExactQueryMention = hasExactQueryMention( + row.contentTitle, + context.querySignals, + ); + const descriptionHasExactQueryMention = hasExactQueryMention( + row.contentDescription, + context.querySignals, + ); + const creatorTextBoost = getEntityCreatorTextBoost({ + platformType: row.platformType, + contentType: row.contentType, + transcriptFetchStatus: row.transcriptFetchStatus, + titleHasExactQueryMention, + descriptionHasExactQueryMention, + // Chunk-level substring matching was removed from the search path (see above). + chunkHasExactQueryMention: false, + transcriptScore, + queryIntent: context.querySignals.intent, + }); + const creatorTextScore = maxNullableScore( + vectorCreatorTextScore, + creatorTextBoost?.score, + ); + + const matchEvidence = createContentMatchEvidence({ + contentType: row.contentType, + transcriptScore, + creatorTextScore, + creatorTextWeightOverride: creatorTextBoost?.weight, + }); + + return { + matchEvidence, + matchScore: getEvidenceMatchScore(matchEvidence), + }; +} + +function getListContentMatch({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: ListContentMatchContext; +}) { + const match = context.bestMatchByContentItemId.get(row.partnerContentItemId); + const matchEvidence = createContentMatchEvidence({ + contentType: row.contentType, + transcriptScore: match?.transcriptScore ?? null, + creatorTextScore: match?.creatorTextScore ?? null, + }); + + return { + matchEvidence, + matchScore: getEvidenceMatchScore(matchEvidence), + }; +} + +function toContentMatchBar({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: ContentMatchContext; +}): PartnerContentMatchBar { + const { matchEvidence, matchScore } = + "bestMatchByContentItemId" in context + ? getListContentMatch({ row, context }) + : getQueryContentMatch({ row, context }); + const matched = matchEvidence.sources.length > 0; + + return { + partnerContentItemId: row.partnerContentItemId, + platform: row.platformType, + platformContentId: row.platformContentId, + title: row.contentTitle, + url: row.contentUrl, + durationMs: + row.contentDurationMs != null ? Number(row.contentDurationMs) : null, + publishedAt: row.publishedAt?.toISOString() ?? null, + viewCount: row.viewCount != null ? Number(row.viewCount) : null, + likeCount: row.likeCount != null ? Number(row.likeCount) : null, + commentCount: row.commentCount != null ? Number(row.commentCount) : null, + shareCount: row.shareCount != null ? Number(row.shareCount) : null, + saveCount: row.saveCount != null ? Number(row.saveCount) : null, + matched, + matchScore, + matchEvidence, + }; +} + +function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { + const matchedBars = contentBars.filter((bar) => bar.matched); + const matchedContentCount = matchedBars.length; + const transcriptMatchedContentCount = contentBars.filter( + ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), + ).length; + const creatorTextMatchedContentCount = contentBars.filter( + ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), + ).length; + const creatorTextOnlyContentCount = contentBars.filter( + ({ matchEvidence }) => + matchEvidence.primarySource === "creatorText" && + matchEvidence.sources.length === 1, + ).length; + const weightedMatchedContentCount = Number( + contentBars + .reduce((total, bar) => total + bar.matchEvidence.weight, 0) + .toFixed(3), + ); + const weightedMatchedContentScore = Number( + contentBars + .reduce( + (total, bar) => total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), + 0, + ) + .toFixed(3), + ); + + return { + matchedBars, + matchedContentCount, + transcriptMatchedContentCount, + creatorTextMatchedContentCount, + creatorTextOnlyContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + }; +} + async function getPartnerMatchSummaries({ rows, partnerIds, @@ -792,6 +1362,7 @@ async function getPartnerMatchSummaries({ query, queryVector, cutoffDistance, + itemSourceBestDistance, logTiming, }: { rows: PartnerContentSearchRow[]; @@ -799,20 +1370,21 @@ async function getPartnerMatchSummaries({ platforms?: PlatformType[]; query?: string | null; // Present only in query mode. When set, each shown partner's recent videos are - // scored exactly against the query (bounded by their ids) and counted as matched - // when at least as relevant as the weakest candidate (cutoffDistance). When null - // (list mode) we fall back to the retrieval rows for the match determination. + // counted as matched when at least as relevant as the weakest candidate + // (cutoffDistance). When null (list mode) we fall back to the retrieval rows for + // the match determination. queryVector?: string | null; cutoffDistance?: number | null; + // Best cosine distance per content-item + source across the candidate pool, + // computed once during retrieval. Reused here to gate matched recent content + // without a second per-item DISTANCE pass (any item that clears the cutoff is + // already in this map — see the producer in searchPartnerNetworkContent). + itemSourceBestDistance?: SourceScoreByContentItemId; logTiming?: PartnerContentSearchTimingLogger; }) { if (partnerIds.length === 0) return new Map(); const querySignals = getPartnerContentSearchQuerySignals(query); - const exactMentionPattern = - querySignals.intent === "entity" && querySignals.normalizedQuery - ? `%${querySignals.normalizedQuery}%` - : null; const platformFilter = platforms?.length ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` : Prisma.empty; @@ -829,107 +1401,131 @@ async function getPartnerMatchSummaries({ PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, ); - // These three reads are independent — only the per-item vector scoring further - // down needs the recent-content ids — so issue them as one parallel wave rather + // These three reads are independent, so issue them as one parallel wave rather // than three serial round trips. (Prisma promises are lazy and don't execute // until awaited, so they fan out together under the Promise.all below.) - const contentCountsPromise = prisma.partnerContentItem.groupBy({ - by: ["partnerId"], - where: { - partnerId: { - in: partnerIds, + const contentCountsPromise = prisma.partnerContentItem + .groupBy({ + by: ["partnerId"], + where: { + partnerId: { + in: partnerIds, + }, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + embeddedChunkCount: { + gt: 0, + }, + ...(platforms?.length && { + partnerPlatform: { + type: { in: platforms }, + }, + }), }, - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - embeddedChunkCount: { - gt: 0, + _count: { + _all: true, }, - ...(platforms?.length && { - partnerPlatform: { - type: { in: platforms }, - }, - }), - }, - _count: { - _all: true, - }, - _min: { - publishedAt: true, - }, - _max: { - publishedAt: true, - }, - }); + _min: { + publishedAt: true, + }, + _max: { + publishedAt: true, + }, + }) + .then((contentCounts) => { + logTiming?.("match-summary-content-counts-complete", { + contentCountRows: contentCounts.length, + }); + return contentCounts; + }); - const recentContentRowsPromise = prisma.$queryRaw< - PartnerRecentContentBarRow[] - >( - Prisma.sql` - SELECT - partnerId, - partnerContentItemId, - platformType, - platformContentId, - contentType, - contentTitle, - contentDescription, - contentUrl, - contentDurationMs, - transcriptFetchStatus, - publishedAt, - viewCount, - rowNumber - FROM ( + const recentContentRowsPromise = prisma + .$queryRaw( + Prisma.sql` SELECT - pci.partnerId, - pci.id AS partnerContentItemId, - pp.type AS platformType, - pci.platformContentId, - pci.contentType, - pci.title AS contentTitle, - pci.description AS contentDescription, - pci.url AS contentUrl, - pci.durationMs AS contentDurationMs, - pci.transcriptFetchStatus, - pci.publishedAt, - pci.viewCount, - ROW_NUMBER() OVER ( - PARTITION BY pci.partnerId - ORDER BY COALESCE(pci.publishedAt, pci.createdAt) DESC, pci.id ASC - ) AS rowNumber - FROM PartnerContentItem pci - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - WHERE pci.partnerId IN (${Prisma.join(partnerIds)}) - ${platformFilter} - AND COALESCE(pci.publishedAt, pci.createdAt) >= ${recencyCutoff} - AND EXISTS ( - SELECT 1 - FROM PartnerContentChunk c - WHERE c.partnerContentItemId = pci.id - AND c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - ) - ) recentContent - WHERE rowNumber <= ${PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner} - ORDER BY partnerId ASC, rowNumber ASC - `, - ); + partnerId, + partnerContentItemId, + platformType, + platformContentId, + contentType, + contentTitle, + contentDescription, + contentUrl, + contentDurationMs, + transcriptFetchStatus, + publishedAt, + viewCount, + likeCount, + commentCount, + shareCount, + saveCount, + rowNumber + FROM ( + SELECT + pci.partnerId, + pci.id AS partnerContentItemId, + pp.type AS platformType, + pci.platformContentId, + pci.contentType, + pci.title AS contentTitle, + pci.description AS contentDescription, + pci.url AS contentUrl, + pci.durationMs AS contentDurationMs, + pci.transcriptFetchStatus, + pci.publishedAt, + pci.viewCount, + pci.likeCount, + pci.commentCount, + pci.shareCount, + pci.saveCount, + ROW_NUMBER() OVER ( + PARTITION BY pci.partnerId + ORDER BY pci.publishedAt DESC, pci.createdAt DESC, pci.id ASC + ) AS rowNumber + FROM PartnerContentItem pci + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + WHERE pci.partnerId IN (${Prisma.join(partnerIds)}) + ${platformFilter} + AND ( + pci.publishedAt >= ${recencyCutoff} + OR (pci.publishedAt IS NULL AND pci.createdAt >= ${recencyCutoff}) + ) + AND pci.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + AND pci.embeddedChunkCount > 0 + ) recentContent + WHERE rowNumber <= ${PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner} + ORDER BY partnerId ASC, rowNumber ASC + `, + ) + .then((recentContentRows) => { + logTiming?.("match-summary-recent-content-complete", { + recentContentRows: recentContentRows.length, + }); + return recentContentRows; + }); // Reach: total followers across the partner's platforms (or just the filtered // platform). The single signal brands most want and the card wasn't showing. - const followerRowsPromise = prisma.partnerPlatform.groupBy({ - by: ["partnerId"], - where: { - partnerId: { - in: partnerIds, + const followerRowsPromise = prisma.partnerPlatform + .groupBy({ + by: ["partnerId"], + where: { + partnerId: { + in: partnerIds, + }, + ...(platforms?.length && { + type: { in: platforms }, + }), }, - ...(platforms?.length && { - type: { in: platforms }, - }), - }, - _sum: { - subscribers: true, - }, - }); + _sum: { + subscribers: true, + }, + }) + .then((followerRows) => { + logTiming?.("match-summary-followers-complete", { + followerRows: followerRows.length, + }); + return followerRows; + }); logTiming?.("match-summary-base-queries-start", { partnerCount: partnerIds.length, @@ -954,143 +1550,27 @@ async function getPartnerMatchSummaries({ const countByPartnerId = new Map( contentCounts.map((row) => [row.partnerId, row]), ); - const rowsByPartnerId = new Map(); - const recentRowsByPartnerId = new Map(); - - for (const row of rows) { - const partnerRows = rowsByPartnerId.get(row.partnerId) ?? []; - partnerRows.push(row); - rowsByPartnerId.set(row.partnerId, partnerRows); - } - - for (const row of recentContentRows) { - const partnerRows = recentRowsByPartnerId.get(row.partnerId) ?? []; - partnerRows.push(row); - recentRowsByPartnerId.set(row.partnerId, partnerRows); - } - - // Query mode only: score every shown partner's recent videos exactly against the - // query, bounded by their content-item ids (a small, PK-filtered set, not the - // whole corpus). This makes "matched" coverage independent of which raw chunk - // happened to survive the global candidate cap. - const recentItemSourceBestDistance = new Map< - string, - Map - >(); - const recentItemSourceExactMentions = new Map< - string, - Map - >(); - if (queryVector) { - const recentItemIds = recentContentRows.map( - ({ partnerContentItemId }) => partnerContentItemId, - ); - if (recentItemIds.length > 0) { - logTiming?.("match-summary-item-vector-start", { - recentItemCount: recentItemIds.length, - }); - const itemScoresPromise = prisma - .$queryRaw< - { - partnerContentItemId: string; - source: string; - bestDistance: number | string; - }[] - >( - Prisma.sql` - SELECT - c.partnerContentItemId, - c.source, - MIN( - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') - ) AS bestDistance - FROM PartnerContentChunk c - WHERE c.partnerContentItemId IN (${Prisma.join(recentItemIds)}) - AND c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - GROUP BY c.partnerContentItemId, c.source - `, - ) - .then((itemScores) => { - logTiming?.("match-summary-item-vector-complete", { - itemScoreRows: itemScores.length, - }); - return itemScores; - }); - - const exactMentionRowsPromise = exactMentionPattern - ? (() => { - logTiming?.("match-summary-exact-mentions-start", { - recentItemCount: recentItemIds.length, - queryIntent: querySignals.intent, - }); - return prisma - .$queryRaw< - { - partnerContentItemId: string; - source: string; - exactMentionCount: bigint | number; - }[] - >( - Prisma.sql` - SELECT - c.partnerContentItemId, - c.source, - COUNT(*) AS exactMentionCount - FROM PartnerContentChunk c - WHERE c.partnerContentItemId IN (${Prisma.join(recentItemIds)}) - AND c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - AND LOWER(c.chunkText) LIKE ${exactMentionPattern} - GROUP BY c.partnerContentItemId, c.source - `, - ) - .then((exactMentionRows) => { - logTiming?.("match-summary-exact-mentions-complete", { - exactMentionRows: exactMentionRows.length, - }); - return exactMentionRows; - }); - })() - : Promise.resolve([]); - - if (!exactMentionPattern) { - logTiming?.("match-summary-exact-mentions-skipped", { - queryIntent: querySignals.intent, - }); - } - - const [itemScores, exactMentionRows] = await Promise.all([ - itemScoresPromise, - exactMentionRowsPromise, - ]); - - for (const row of itemScores) { - setSourceDistance( - recentItemSourceBestDistance, - row.partnerContentItemId, - getEvidenceSource(row.source), - Number(row.bestDistance), - ); - } + const rowsByPartnerId = groupRowsBy(rows, ({ partnerId }) => partnerId); + const recentRowsByPartnerId = groupRowsBy( + recentContentRows, + ({ partnerId }) => partnerId, + ); - for (const row of exactMentionRows) { - if (Number(row.exactMentionCount) === 0) continue; - setSourceExactMention( - recentItemSourceExactMentions, - row.partnerContentItemId, - getEvidenceSource(row.source), - ); - } - } - } + // Query mode only: gate which of each shown partner's recent videos count as + // "matched". A recent video is matched when its best chunk is at least as + // relevant as the weakest candidate (cutoffDistance). Because cutoffDistance is + // itself a candidate-pool item's distance, every recent video that could clear + // it is already in itemSourceBestDistance — so we reuse that map instead of + // re-running a per-item DISTANCE pass over the recent set. A recent video absent + // from the map is provably beyond the cutoff (not matched). + const recentItemSourceBestDistance = + queryVector && itemSourceBestDistance + ? itemSourceBestDistance + : new Map>(); // Calibrated reranker score per item + source (from the candidate pool), used // to gate which recent posts count as on-topic. - const rerankByItemSource = new Map< - string, - Map - >(); + const rerankByItemSource: SourceScoreByContentItemId = new Map(); for (const row of rows) { if (row.rerankScore != null) { setSourceScore( @@ -1102,6 +1582,15 @@ async function getPartnerMatchSummaries({ } } + const queryMatchContext: QueryContentMatchContext | null = queryVector + ? { + querySignals, + cutoffDistance, + recentItemSourceBestDistance, + rerankByItemSource, + } + : null; + logTiming?.("match-summary-aggregation-start", { partnerCount: partnerIds.length, recentContentRows: recentContentRows.length, @@ -1112,177 +1601,26 @@ async function getPartnerMatchSummaries({ const recentRows = recentRowsByPartnerId.get(partnerId) ?? []; const countRow = countByPartnerId.get(partnerId); const totalContentCount = countRow ? countRow._count._all : 0; - const bestMatchByContentItemId = new Map< - string, - { - contentType: string; - transcriptScore: number | null; - creatorTextScore: number | null; - } - >(); - - for (const row of partnerRows) { - // Use the effective score (rerank when present) so the content-match - // bars stay consistent with the reranked partner ordering. - const score = row.rerankScore ?? toScore(Number(row.distance)); - const evidenceSource = getEvidenceSource(row.chunkSource); - const existing = bestMatchByContentItemId.get(row.partnerContentItemId); - const next = existing ?? { - contentType: row.contentType, - transcriptScore: null, - creatorTextScore: null, - }; - - if (evidenceSource === "transcript") { - next.transcriptScore = - next.transcriptScore == null - ? score - : Math.max(next.transcriptScore, score); - } else { - next.creatorTextScore = - next.creatorTextScore == null - ? score - : Math.max(next.creatorTextScore, score); - } - - bestMatchByContentItemId.set(row.partnerContentItemId, next); - } - - const contentBars = recentRows.map((row) => { - let matchEvidence: PartnerContentMatchEvidence; - let matchScore: number | null; - - if (queryVector) { - // On-topic gate: prefer the reranker's calibrated relevance, which - // cleanly separates on- from off-topic. Fall back to the cosine cutoff - // only for items the reranker didn't score (rerank disabled/failed, or - // the item never reached the candidate pool). - const transcriptScore = getEntityTranscriptScore({ - currentScore: getMatchedSourceScore({ - rerankScore: getSourceScore( - rerankByItemSource, - row.partnerContentItemId, - "transcript", - ), - bestDistance: getSourceScore( - recentItemSourceBestDistance, - row.partnerContentItemId, - "transcript", - ), - cutoffDistance, - }), - hasExactQueryMention: getSourceExactMention( - recentItemSourceExactMentions, - row.partnerContentItemId, - "transcript", - ), - queryIntent: querySignals.intent, - }); - const vectorCreatorTextScore = getMatchedSourceScore({ - rerankScore: getSourceScore( - rerankByItemSource, - row.partnerContentItemId, - "creatorText", - ), - bestDistance: getSourceScore( - recentItemSourceBestDistance, - row.partnerContentItemId, - "creatorText", - ), - cutoffDistance, - }); - const titleHasExactQueryMention = hasExactQueryMention( - row.contentTitle, - querySignals, - ); - const descriptionHasExactQueryMention = hasExactQueryMention( - row.contentDescription, - querySignals, - ); - const creatorTextBoost = getEntityCreatorTextBoost({ - platformType: row.platformType, - contentType: row.contentType, - transcriptFetchStatus: row.transcriptFetchStatus, - titleHasExactQueryMention, - descriptionHasExactQueryMention, - chunkHasExactQueryMention: getSourceExactMention( - recentItemSourceExactMentions, - row.partnerContentItemId, - "creatorText", - ), - transcriptScore, - queryIntent: querySignals.intent, - }); - const creatorTextScore = maxNullableScore( - vectorCreatorTextScore, - creatorTextBoost?.score, - ); - - matchEvidence = createContentMatchEvidence({ - contentType: row.contentType, - transcriptScore, - creatorTextScore, - creatorTextWeightOverride: creatorTextBoost?.weight, - }); - matchScore = getEvidenceMatchScore(matchEvidence); - } else { - const match = bestMatchByContentItemId.get(row.partnerContentItemId); - matchEvidence = createContentMatchEvidence({ - contentType: row.contentType, - transcriptScore: match?.transcriptScore ?? null, - creatorTextScore: match?.creatorTextScore ?? null, - }); - matchScore = getEvidenceMatchScore(matchEvidence); - } - const matched = matchEvidence.sources.length > 0; - - return { - partnerContentItemId: row.partnerContentItemId, - platform: row.platformType, - platformContentId: row.platformContentId, - title: row.contentTitle, - url: row.contentUrl, - durationMs: - row.contentDurationMs != null - ? Number(row.contentDurationMs) - : null, - publishedAt: row.publishedAt?.toISOString() ?? null, - viewCount: row.viewCount != null ? Number(row.viewCount) : null, - matched, - matchScore, - matchEvidence, - }; - }); - const matchedContentCount = contentBars.filter( - ({ matched }) => matched, - ).length; - const transcriptMatchedContentCount = contentBars.filter( - ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), - ).length; - const creatorTextMatchedContentCount = contentBars.filter( - ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), - ).length; - const creatorTextOnlyContentCount = contentBars.filter( - ({ matchEvidence }) => - matchEvidence.primarySource === "creatorText" && - matchEvidence.sources.length === 1, - ).length; - const weightedMatchedContentCount = Number( - contentBars - .reduce((total, bar) => total + bar.matchEvidence.weight, 0) - .toFixed(3), - ); - const weightedMatchedContentScore = Number( - contentBars - .reduce( - (total, bar) => - total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), - 0, - ) - .toFixed(3), + const bestMatchByContentItemId = getBestMatchByContentItemId(partnerRows); + const contentMatchContext: ContentMatchContext = queryMatchContext ?? { + bestMatchByContentItemId, + }; + const contentBars = recentRows.map((row) => + toContentMatchBar({ + row, + context: contentMatchContext, + }), ); + const { + matchedBars, + matchedContentCount, + transcriptMatchedContentCount, + creatorTextMatchedContentCount, + creatorTextOnlyContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + } = getContentBarMatchStats(contentBars); const recentContentCount = contentBars.length; - const matchedBars = contentBars.filter((bar) => bar.matched); const { topicFit, band } = deriveTopicFit({ matchedContentCount, weightedMatchedContentCount, @@ -1393,6 +1731,18 @@ function toChunkResult(row: PartnerContentSearchRow, distance: number) { thumbnailUrl: row.contentThumbnailUrl, publishedAt: row.contentPublishedAt?.toISOString() ?? null, durationMs: row.contentDurationMs, + viewCount: + row.contentViewCount != null ? Number(row.contentViewCount) : null, + likeCount: + row.contentLikeCount != null ? Number(row.contentLikeCount) : null, + commentCount: + row.contentCommentCount != null + ? Number(row.contentCommentCount) + : null, + shareCount: + row.contentShareCount != null ? Number(row.contentShareCount) : null, + saveCount: + row.contentSaveCount != null ? Number(row.contentSaveCount) : null, }, chunk: { source: row.chunkSource, diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 61a7a162563..3b39269c695 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -92,7 +92,7 @@ export function NetworkPartnerDetailContent({ fetcher, { revalidateOnFocus: false }, ); - const partner = partners?.[0]; + const loadedPartner = partners?.[0]; // The results page already ran the (global) Voyage search and handed us this // partner's match data on click, so we render straight from it — opening the @@ -104,10 +104,10 @@ export function NetworkPartnerDetailContent({ // tell-tale "65% vs 95%" banding). A per-partner rerank covers all of this // creator's items, so every row shares the reranker scale. // - // This is non-blocking: the cached summary paints immediately and the unified - // scores swap in when the scoped run resolves. Topic Fit / bars / counts stay - // sourced from the cached summary (re-running scoped would recompute the cutoff - // against a one-partner set and drift them), so the headline never shifts. + // This is non-blocking: cached summary data can paint the stable Topic Fit + // headline immediately, while content rows stay skeletoned until the scoped + // run resolves. That avoids showing global transcript snippets or row order and + // then swapping them for the single-partner reranked result. const shouldFetchSearch = hasContentSearch; const { data: searchResults, @@ -126,6 +126,8 @@ export function NetworkPartnerDetailContent({ }); const fetchedPartner = searchResults?.partners?.[0]; + const partner = + loadedPartner ?? initialSearchPartner?.partner ?? fetchedPartner?.partner; // Chunks (snippet text) prefer the fuller fetched set once it arrives; the // summary always prefers the cached one so scores never drift on click. const searchPartner = fetchedPartner ?? initialSearchPartner; @@ -335,12 +337,20 @@ function ContentMatchBars({ }: { summary: PartnerContentSearchPartner["matchSummary"] | undefined; }) { + // One open tooltip at a time: each bar is its own tooltip root that only + // closes on its own pointerleave, so a fast cursor flick can leave several + // open. Driving every bar's open state from one value prevents that. + const [openBarId, setOpenBarId] = useState(null); + const allBars = summary?.contentBars ?? []; if (!allBars.length) return null; const bars = allBars.slice(0, MAX_VISIBLE_CONTENT_BARS); return ( -
+
setOpenBarId(null)} + > {bars.map((bar) => { const score = bar.matched && bar.matchScore != null @@ -379,12 +389,22 @@ function ContentMatchBars({ } - // Snappy bar-to-bar hover: open instantly, drop Radix's hoverable - // "safe polygon" grace area (so it switches the moment you cross to - // the next bar instead of lingering), and trim the 400ms slide-fade. + // Snappy bar-to-bar hover: open instantly, drop the hoverable grace + // area, and skip the animation so each tooltip closes immediately + // rather than lingering as you sweep across. delayDuration={0} disableHoverableContent - style={{ animationDuration: "100ms" }} + disableAnimation + open={openBarId === bar.partnerContentItemId} + onOpenChange={(nextOpen) => + setOpenBarId((current) => + nextOpen + ? bar.partnerContentItemId + : current === bar.partnerContentItemId + ? null + : current, + ) + } > {bar.url ? ( ; + } + + const isLoadingRows = isLoading || isRefining; + // Per-item relevance on a single scale, from the scoped reranked summary. const unifiedRelevanceByItemId = buildUnifiedRelevanceMap(relevanceSummary); - // Both lists come from the cached summary's full matched set (instant, complete); - // loaded chunks only enrich a row with its snippet/thumbnail when available, and - // the unified-relevance map upgrades each row's score in place once available. + // Keep cached summary data for the headline, but hold row rendering until the + // scoped run finishes so transcript snippets and row ordering do not swap in. const items = buildMatchedContentItems( summary, - searchPartner?.chunks ?? [], + isLoadingRows ? [] : (searchPartner?.chunks ?? []), unifiedRelevanceByItemId, ); @@ -489,6 +513,7 @@ function SearchFitPanel({ // A separate "All content" list only earns its place when it adds rows beyond // the top set; with ≤ topContentCount matches the top list already shows them all. const showAllSection = + !isLoadingRows && allContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; const band = summary?.band ?? "none"; @@ -596,8 +621,10 @@ function SearchFitPanel({
Failed to load search matches
- ) : isLoading ? ( - + ) : isLoadingRows ? ( + ) : topContent.length ? ( topContent.map((item) => ( @@ -652,16 +679,67 @@ function SearchFitPanel({ ); } +function SearchFitPanelSkeleton() { + return ( +
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+ {[...Array(18)].map((_, index) => ( +
+ ))} +
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+ ); +} + function ContentMatchSkeletons({ count }: { count: number }) { return ( <> {[...Array(count)].map((_, index) => (
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
))} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index cd3d131e28c..f8e6cca4306 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -1,7 +1,9 @@ "use client"; import type { PartnerContentTopicFitBand } from "@/lib/partner-content-search/constants"; +import { getPartnerContentThumbnailUrl } from "@/lib/partner-content-search/thumbnail-url"; import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { Tooltip } from "@dub/ui"; import { Globe, Instagram, @@ -13,6 +15,7 @@ import { } from "@dub/ui/icons"; import { cn, nFormatter } from "@dub/utils"; import type { PlatformType } from "@prisma/client"; +import { useEffect, useState } from "react"; import { NetworkPartnerCard } from "./network-partner-card"; const PLATFORM_LABELS: Partial> = { @@ -29,6 +32,17 @@ function contentLabel(platform?: PlatformType) { return platform ? platformLabel(platform) : "indexed"; } +const TOP_CONTENT_PREVIEW_COUNT = 2; + +type ContentSearchChunk = PartnerContentSearchPartner["chunks"][number]; +type ContentSearchBar = NonNullable< + PartnerContentSearchPartner["matchSummary"] +>["contentBars"][number]; +type ContentPreviewItem = { + chunk: ContentSearchChunk; + bar?: ContentSearchBar; +}; + export function NetworkContentSearchResults({ error, hasQuery, @@ -74,16 +88,24 @@ export function NetworkContentSearchResults({
-
-
- {[...Array(12)].map((_, index) => ( -
- ))} +
+
+
+
+
+
+
+
+ {[...Array(TOP_CONTENT_PREVIEW_COUNT)].map((_, index) => ( +
+ ))} +
+
+
))} @@ -175,31 +197,40 @@ function NetworkPartnerContentMatch({ const medianViews = summary?.medianViews ?? null; const matchLabel = formatMatchEvidenceLabel(summary); const lastOnTopic = lastPublishedLabel(summary?.lastOnTopicAt); + const previewItems = getContentPreviewItems(partner); return (
- - Topic fit - -
- - {summary?.topicFit ?? 0} - - - {BAND_LABELS[band]} +
+ + Topic fit + {previewItems.length > 0 && ( + + Top matches + + )} +
+ + {summary?.topicFit ?? 0} + + + {BAND_LABELS[band]} + +
+ +
- {(followers || medianViews) && (
{followers ? ( @@ -231,31 +262,338 @@ function NetworkPartnerContentMatch({ ); } -function ContentMatchScoreDebug({ - partner, +function getContentPreviewItems( + partner: PartnerContentSearchPartner, +): ContentPreviewItem[] { + const barsByContentItemId = new Map( + partner.matchSummary?.contentBars.map((bar) => [ + bar.partnerContentItemId, + bar, + ]) ?? [], + ); + const seenContentItems = new Set(); + const items: ContentPreviewItem[] = []; + + for (const chunk of partner.chunks) { + if (seenContentItems.has(chunk.partnerContentItemId)) continue; + + seenContentItems.add(chunk.partnerContentItemId); + items.push({ + chunk, + bar: barsByContentItemId.get(chunk.partnerContentItemId), + }); + + if (items.length === TOP_CONTENT_PREVIEW_COUNT) break; + } + + return items; +} + +function ContentPreviewTiles({ items }: { items: ContentPreviewItem[] }) { + if (items.length === 0) return null; + + return ( +
+ {items.map((item) => ( + + ))} +
+ ); +} + +function ContentPreviewTile({ item }: { item: ContentPreviewItem }) { + const title = getContentTitle(item.chunk); + + return ( + } + delayDuration={100} + > + event.stopPropagation()} + className="group block size-11 overflow-hidden rounded-lg border border-neutral-200 bg-neutral-100 shadow-sm transition duration-150 hover:-translate-y-0.5 hover:border-neutral-300 hover:shadow-md focus:outline-none focus-visible:ring-2 focus-visible:ring-neutral-900/20" + aria-label={"Open " + title} + > + + + + ); +} + +function ContentPreviewImage({ + chunk, + className, + fallbackIconClassName = "size-5", + imageClassName, + showPlatformBadge = false, }: { - partner: PartnerContentSearchPartner; + chunk: ContentSearchChunk; + className?: string; + fallbackIconClassName?: string; + imageClassName?: string; + showPlatformBadge?: boolean; }) { - const { cosineScore, rerankScore } = partner; + const thumbnail = getContentThumbnail(chunk); + const [hasImageError, setHasImageError] = useState(false); + const thumbnailSrc = hasImageError ? null : thumbnail; - if (cosineScore == null && rerankScore == null) return null; + useEffect(() => { + setHasImageError(false); + }, [thumbnail]); return ( -
- - cos {cosineScore == null ? "-" : formatMatchPercent(cosineScore)} - - {"->"} - - rerank {rerankScore == null ? "-" : formatMatchPercent(rerankScore)} - +
+ {thumbnailSrc ? ( + setHasImageError(true)} + /> + ) : ( +
+ +
+ )} + {showPlatformBadge && ( + + + + )} +
+ ); +} + +function ContentPreviewTooltip({ item }: { item: ContentPreviewItem }) { + const { chunk, bar } = item; + const title = getContentTitle(chunk); + const metadata = [ + formatDuration(chunk.content.durationMs), + formatPublishedDate(chunk.content.publishedAt), + ] + .filter(Boolean) + .join(" · "); + const engagementMetrics = getContentEngagementMetrics(chunk, bar); + const matchLocation = formatChunkMatchLocation(chunk); + const matchScore = formatMatchPercent(chunk.rerankScore ?? chunk.score); + + return ( +
+
+ +
+
+ {title} +
+ {metadata && ( +
+ {metadata} +
+ )} +
+
+ +
+ + {matchScore} relevance + + {matchLocation && ( + + {matchLocation} + + )} +
+ + {engagementMetrics.length > 0 && ( +
+ {engagementMetrics.map((metric) => ( +
+ + {metric.value} + + + {metric.label} + +
+ ))} +
+ )}
); } +function PlatformIcon({ + platform, + className, +}: { + platform: string; + className?: string; +}) { + const Icon = PLATFORM_ICONS[platform as PlatformType] ?? User; + + return ; +} + +function getContentThumbnail(chunk: ContentSearchChunk) { + if (chunk.content.thumbnailUrl) { + return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); + } + + if (chunk.platform.type === "youtube") { + return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; + } + + return null; +} + +function getContentTitle(chunk: ContentSearchChunk) { + return ( + chunk.content.title?.trim() || + chunk.content.description?.trim().split(/\r?\n/)[0] || + "Untitled content" + ); +} + +function getContentHref(chunk: ContentSearchChunk) { + if (chunk.platform.type === "instagram") { + return getInstagramContentHref(chunk); + } + + if (chunk.platform.type !== "youtube" || chunk.chunk.startMs === null) { + return chunk.content.url; + } + + try { + const url = new URL(chunk.content.url); + url.searchParams.set("t", `${Math.floor(chunk.chunk.startMs / 1000)}s`); + return url.toString(); + } catch { + return chunk.content.url; + } +} + +function getInstagramContentHref(chunk: ContentSearchChunk) { + const shortcode = + extractInstagramShortcode(chunk.content.url) || chunk.content.platformContentId; + + return `https://www.instagram.com/${chunk.content.type === "reel" ? "reel" : "p"}/${shortcode}/`; +} + +function extractInstagramShortcode(url: string) { + try { + return ( + new URL(url).pathname.match( + /^\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1] ?? null + ); + } catch { + return ( + url.match( + /instagram\.com\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1] ?? null + ); + } +} + +function getContentEngagementMetrics( + chunk: ContentSearchChunk, + bar: ContentSearchBar | undefined, +) { + return [ + { label: "Views", value: chunk.content.viewCount ?? bar?.viewCount }, + { label: "Likes", value: chunk.content.likeCount ?? bar?.likeCount }, + { + label: "Comments", + value: chunk.content.commentCount ?? bar?.commentCount, + }, + { label: "Shares", value: chunk.content.shareCount ?? bar?.shareCount }, + { label: "Saves", value: chunk.content.saveCount ?? bar?.saveCount }, + ] + .filter((metric): metric is { label: string; value: number } => { + return metric.value != null && metric.value > 0; + }) + .map((metric) => ({ + label: metric.label, + value: nFormatter(metric.value), + })); +} + +function formatChunkMatchLocation(chunk: ContentSearchChunk) { + if (chunk.chunk.source === "metadata") return "Matched creator text"; + if (chunk.chunk.startMs === null && chunk.chunk.endMs === null) { + return "Matched transcript"; + } + if (chunk.chunk.startMs !== null && chunk.chunk.endMs !== null) { + return `Matched transcript ${formatTimestamp( + chunk.chunk.startMs, + )} - ${formatTimestamp(chunk.chunk.endMs)}`; + } + return `Matched transcript ${formatTimestamp( + chunk.chunk.startMs ?? chunk.chunk.endMs ?? 0, + )}`; +} + +function formatTimestamp(ms: number) { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +function formatDuration(durationMs: number | null) { + if (!durationMs || durationMs <= 0) return null; + + const totalSeconds = Math.floor(durationMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds + .toString() + .padStart(2, "0")}`; + } + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +function formatPublishedDate(publishedAt: string | null) { + if (!publishedAt) return null; + + const date = new Date(publishedAt); + if (Number.isNaN(date.getTime())) return null; + + return new Intl.DateTimeFormat(undefined, { + month: "short", + day: "numeric", + year: "numeric", + }).format(date); +} + const BAND_LABELS: Record = { consistent: "Consistent", frequent: "Frequent", diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx deleted file mode 100644 index ab18e271bf3..00000000000 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-country-filter.tsx +++ /dev/null @@ -1,136 +0,0 @@ -import { Popover } from "@dub/ui"; -import { FlagWavy } from "@dub/ui/icons"; -import { cn } from "@dub/utils"; -import { Check, ChevronDown } from "lucide-react"; -import { ReactNode, useMemo, useState } from "react"; - -type CountryOption = { value: string; label: string; right?: ReactNode }; - -export function NetworkCountryFilter({ - options, - getOptionIcon, - selectedValue, - onSelect, - onClear, - className, -}: { - options: CountryOption[]; - getOptionIcon?: (value: string) => ReactNode; - selectedValue?: string; - onSelect: (value: string) => void; - onClear: () => void; - className?: string; -}) { - const [openPopover, setOpenPopover] = useState(false); - const [search, setSearch] = useState(""); - - const selectedOption = options.find((o) => o.value === selectedValue); - - const filtered = useMemo(() => { - const q = search.trim().toLowerCase(); - if (!q) return options; - return options.filter((o) => o.label.toLowerCase().includes(q)); - }, [options, search]); - - return ( - -
- setSearch(e.target.value)} - placeholder="Search countries…" - className="h-8 w-full rounded-md border-none bg-transparent px-2 text-sm outline-none placeholder:text-neutral-400" - /> -
-
- {selectedValue && ( - - )} - {filtered.length === 0 ? ( -
- No countries found -
- ) : ( - filtered.map((option) => { - const isSelected = option.value === selectedValue; - - return ( - - ); - }) - )} -
-
- } - openPopover={openPopover} - setOpenPopover={setOpenPopover} - align="start" - > - - - ); -} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx index 015b7c3d753..f215288b4fe 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx @@ -1,196 +1,176 @@ -import { Popover } from "@dub/ui"; -import { - Globe, - Instagram, - LinkedIn, - TikTok, - Twitter, - User, - YouTube, -} from "@dub/ui/icons"; -import { cn } from "@dub/utils"; -import { PlatformType } from "@prisma/client"; -import { ChevronDown } from "lucide-react"; -import { useState } from "react"; -import { - isAllPlatformsSelected, - NETWORK_FILTER_PLATFORMS, -} from "./platform-filter-utils"; - -const PLATFORM_META: Record< - (typeof NETWORK_FILTER_PLATFORMS)[number], - { label: string; icon: typeof User } -> = { - website: { label: "Website", icon: Globe }, - youtube: { label: "YouTube", icon: YouTube }, - twitter: { label: "X", icon: Twitter }, - linkedin: { label: "LinkedIn", icon: LinkedIn }, - instagram: { label: "Instagram", icon: Instagram }, - tiktok: { label: "TikTok", icon: TikTok }, -}; - -export function NetworkPlatformFilter({ - selectedPlatforms, - onChange, - className, -}: { - selectedPlatforms: PlatformType[]; - onChange: (platforms: PlatformType[]) => void; - className?: string; -}) { - const [openPopover, setOpenPopover] = useState(false); - - const selectedSet = new Set(selectedPlatforms); - const isFiltered = !isAllPlatformsSelected(selectedPlatforms); - - const toggle = (platform: PlatformType) => { - if (selectedSet.has(platform)) { - // Never let the user clear the last platform — an empty selection would - // show nothing. Deselecting the last one is a no-op. - if (selectedSet.size === 1) return; - // Remove from the CURRENT selection, not the full list — otherwise a second - // deselect would silently re-add the first. - onChange( - NETWORK_FILTER_PLATFORMS.filter( - (p) => selectedSet.has(p) && p !== platform, - ), - ); - } else { - onChange( - NETWORK_FILTER_PLATFORMS.filter( - (p) => selectedSet.has(p) || p === platform, - ), - ); - } - }; - - return ( - - {NETWORK_FILTER_PLATFORMS.map((platform) => { - const { label, icon: Icon } = PLATFORM_META[platform]; - const isSelected = selectedSet.has(platform); - - return ( - - ); - })} -
- -
- } - openPopover={openPopover} - setOpenPopover={setOpenPopover} - align="end" - > - - - ); -} +import { Popover } from "@dub/ui"; +import { + Globe, + Instagram, + LinkedIn, + TikTok, + Twitter, + User, + YouTube, +} from "@dub/ui/icons"; +import { cn } from "@dub/utils"; +import { PlatformType } from "@prisma/client"; +import { ChevronDown } from "lucide-react"; +import { useState } from "react"; +import { + isAllPlatformsSelected, + NETWORK_FILTER_PLATFORMS, +} from "./platform-filter-utils"; + +const PLATFORM_META: Record< + (typeof NETWORK_FILTER_PLATFORMS)[number], + { label: string; icon: typeof User } +> = { + website: { label: "Website", icon: Globe }, + youtube: { label: "YouTube", icon: YouTube }, + twitter: { label: "X", icon: Twitter }, + linkedin: { label: "LinkedIn", icon: LinkedIn }, + instagram: { label: "Instagram", icon: Instagram }, + tiktok: { label: "TikTok", icon: TikTok }, +}; + +export function NetworkPlatformFilter({ + selectedPlatforms, + onChange, + className, +}: { + selectedPlatforms: PlatformType[]; + onChange: (platforms: PlatformType[]) => void; + className?: string; +}) { + const [openPopover, setOpenPopover] = useState(false); + + const selectedSet = new Set(selectedPlatforms); + const isFiltered = !isAllPlatformsSelected(selectedPlatforms); + + const toggle = (platform: PlatformType) => { + if (selectedSet.has(platform)) { + // Never let the user clear the last platform — an empty selection would + // show nothing. Deselecting the last one is a no-op. + if (selectedSet.size === 1) return; + // Remove from the CURRENT selection, not the full list — otherwise a second + // deselect would silently re-add the first. + onChange( + NETWORK_FILTER_PLATFORMS.filter( + (p) => selectedSet.has(p) && p !== platform, + ), + ); + } else { + onChange( + NETWORK_FILTER_PLATFORMS.filter( + (p) => selectedSet.has(p) || p === platform, + ), + ); + } + }; + + return ( + + {NETWORK_FILTER_PLATFORMS.map((platform) => { + const { label, icon: Icon } = PLATFORM_META[platform]; + const isSelected = selectedSet.has(platform); + + return ( + + ); + })} +
+ +
+ } + openPopover={openPopover} + setOpenPopover={setOpenPopover} + align="end" + > + + + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx deleted file mode 100644 index 60647ecae28..00000000000 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-reach-filter.tsx +++ /dev/null @@ -1,129 +0,0 @@ -import { - REACH_TIER_KEYS, - REACH_TIERS, - type ReachTier, -} from "@/lib/api/network/reach-tiers"; -import { Popover } from "@dub/ui"; -import { cn } from "@dub/utils"; -import { ChevronDown, Users } from "lucide-react"; -import { useState } from "react"; - -export function NetworkReachFilter({ - selectedTiers, - onChange, - className, -}: { - selectedTiers: ReachTier[]; - onChange: (tiers: ReachTier[]) => void; - className?: string; -}) { - const [openPopover, setOpenPopover] = useState(false); - - const selectedSet = new Set(selectedTiers); - const isFiltered = selectedTiers.length > 0; - - const toggle = (tier: ReachTier) => - onChange( - selectedSet.has(tier) - ? REACH_TIER_KEYS.filter((t) => t !== tier && selectedSet.has(t)) - : REACH_TIER_KEYS.filter((t) => selectedSet.has(t) || t === tier), - ); - - const label = !isFiltered - ? "Audience" - : selectedTiers.length === 1 - ? REACH_TIERS[selectedTiers[0]].range - : `Audience · ${selectedTiers.length}`; - - return ( - - {REACH_TIER_KEYS.map((tier) => { - const { range, descriptor } = REACH_TIERS[tier]; - const isSelected = selectedSet.has(tier); - - return ( - - ); - })} - {isFiltered && ( - <> -
- - - )} -
- } - openPopover={openPopover} - setOpenPopover={setOpenPopover} - align="start" - > - - - ); -} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index a728acd9646..eb7027bf3ea 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -1,10 +1,7 @@ "use client"; import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner"; -import { - parseReachTiers, - type ReachTier, -} from "@/lib/api/network/reach-tiers"; +import { parseReachTiers } from "@/lib/api/network/reach-tiers"; import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; import usePartnerContentSearch from "@/lib/swr/use-partner-content-search"; import useWorkspace from "@/lib/swr/use-workspace"; @@ -12,7 +9,9 @@ import { NetworkPartnerProps } from "@/lib/types"; import { PARTNER_NETWORK_MAX_PAGE_SIZE } from "@/lib/zod/schemas/partner-network"; import { SearchBoxPersisted } from "@/ui/shared/search-box"; import { + AnimatedSizeContainer, Button, + Filter, PaginationControls, usePagination, useRouterStuff, @@ -26,12 +25,10 @@ import { toast } from "sonner"; import { useDebounce } from "use-debounce"; import useSWR from "swr"; import { NetworkContentSearchResults } from "./network-content-search-results"; -import { NetworkCountryFilter } from "./network-country-filter"; import { NetworkEmptyState } from "./network-empty-state"; import { NetworkPartnerCard } from "./network-partner-card"; import { NetworkPartnerDetailSheet } from "./network-partner-detail-sheet"; import { NetworkPlatformFilter } from "./network-platform-filter"; -import { NetworkReachFilter } from "./network-reach-filter"; import { getContentSearchPlatforms, isAllPlatformsSelected, @@ -117,13 +114,6 @@ export function ProgramPartnerNetworkPageClient({ : { set: { platform: platforms.join(",") }, del: "page" }, ); - const onReachChange = (tiers: ReachTier[]) => - updateSearchParams( - tiers.length - ? { set: { reach: tiers.join(",") }, del: "page" } - : { del: ["reach", "page"] }, - ); - const { data: partnerCounts, error: countError } = useNetworkPartnersCount(); const { @@ -180,26 +170,34 @@ export function ProgramPartnerNetworkPageClient({ PARTNER_NETWORK_MAX_PAGE_SIZE, ); - const { filters, isFiltered, onSelect, onRemove, onRemoveAll } = - usePartnerNetworkFilters({ status }); - - const countryFilter = filters.find((f) => f.key === "country"); + const { + filters, + activeFilters, + isFiltered, + onSelect, + onRemove, + onRemoveAll, + } = usePartnerNetworkFilters({ status }); const isStarred = searchParams.get("starred") === "true"; + const selectedPartnerId = searchParams.get("partnerId"); const [detailsSheetState, setDetailsSheetState] = useState< | { open: false; partnerId: string | null } | { open: true; partnerId: string } - >({ open: false, partnerId: null }); + >( + selectedPartnerId + ? { open: true, partnerId: selectedPartnerId } + : { open: false, partnerId: null }, + ); useEffect(() => { - const partnerId = searchParams.get("partnerId"); - if (partnerId) { - setDetailsSheetState({ open: true, partnerId }); + if (selectedPartnerId) { + setDetailsSheetState({ open: true, partnerId: selectedPartnerId }); } else { setDetailsSheetState({ open: false, partnerId: null }); } - }, [searchParams]); + }, [selectedPartnerId]); const sheetPartnerIds = useMemo( () => @@ -305,12 +303,12 @@ export function ProgramPartnerNetworkPageClient({
- onSelect("country", value)} - onClear={() => country && onRemove("country", country)} +
+ + {activeFilters.length > 0 && ( +
+ +
+ )} +
)} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts index 9c9ab94b372..e0af82a1936 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts @@ -1,53 +1,53 @@ -import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; -import type { PlatformType } from "@prisma/client"; - -// The platforms a partner can be filtered by, in display order. This is the -// inclusion set: with all of them selected the filter is a no-op (we omit the -// `platform` param entirely), and deselecting narrows results to partners -// present on any of the still-selected platforms. -export const NETWORK_FILTER_PLATFORMS = [ - "youtube", - "instagram", - "tiktok", - "website", - "twitter", - "linkedin", -] as const satisfies readonly PlatformType[]; - -const PLATFORM_SET = new Set(NETWORK_FILTER_PLATFORMS); - -// Parse the comma-separated `platform` query param. Absent/empty means "all -// platforms" (no filter), which we represent as the full list so callers always -// get a concrete selection to render. -export function parseSelectedPlatforms( - param: string | null | undefined, -): PlatformType[] { - if (!param) return [...NETWORK_FILTER_PLATFORMS]; - - const selected = param - .split(",") - .map((value) => value.trim()) - .filter((value): value is PlatformType => PLATFORM_SET.has(value)); - - return selected.length ? selected : [...NETWORK_FILTER_PLATFORMS]; -} - -export function isAllPlatformsSelected(selected: PlatformType[]): boolean { - return selected.length >= NETWORK_FILTER_PLATFORMS.length; -} - -// What to actually send to the API: `undefined` when everything is selected -// (keeps the default ranking path and clean URLs), otherwise the selection. -export function platformFilterParam( - selected: PlatformType[], -): PlatformType[] | undefined { - return isAllPlatformsSelected(selected) ? undefined : selected; -} - -// Semantic content search only covers a subset of platforms; this intersection -// decides whether content search runs at all for the current selection. -export function getContentSearchPlatforms( - selected: PlatformType[], -): PlatformType[] { - return selected.filter(isPartnerContentSearchPlatform); -} +import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; +import type { PlatformType } from "@prisma/client"; + +// The platforms a partner can be filtered by, in display order. This is the +// inclusion set: with all of them selected the filter is a no-op (we omit the +// `platform` param entirely), and deselecting narrows results to partners +// present on any of the still-selected platforms. +export const NETWORK_FILTER_PLATFORMS = [ + "youtube", + "instagram", + "tiktok", + "website", + "twitter", + "linkedin", +] as const satisfies readonly PlatformType[]; + +const PLATFORM_SET = new Set(NETWORK_FILTER_PLATFORMS); + +// Parse the comma-separated `platform` query param. Absent/empty means "all +// platforms" (no filter), which we represent as the full list so callers always +// get a concrete selection to render. +export function parseSelectedPlatforms( + param: string | null | undefined, +): PlatformType[] { + if (!param) return [...NETWORK_FILTER_PLATFORMS]; + + const selected = param + .split(",") + .map((value) => value.trim()) + .filter((value): value is PlatformType => PLATFORM_SET.has(value)); + + return selected.length ? selected : [...NETWORK_FILTER_PLATFORMS]; +} + +export function isAllPlatformsSelected(selected: PlatformType[]): boolean { + return selected.length >= NETWORK_FILTER_PLATFORMS.length; +} + +// What to actually send to the API: `undefined` when everything is selected +// (keeps the default ranking path and clean URLs), otherwise the selection. +export function platformFilterParam( + selected: PlatformType[], +): PlatformType[] | undefined { + return isAllPlatformsSelected(selected) ? undefined : selected; +} + +// Semantic content search only covers a subset of platforms; this intersection +// decides whether content search runs at all for the current selection. +export function getContentSearchPlatforms( + selected: PlatformType[], +): PlatformType[] { + return selected.filter(isPartnerContentSearchPlatform); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx index b4cb8b14933..3dd60bc77db 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-partner-network-filters.tsx @@ -1,135 +1,155 @@ -import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; -import { CountryFlag } from "@/ui/shared/country-flag"; -import { useRouterStuff } from "@dub/ui"; -import { FlagWavy } from "@dub/ui/icons"; -import { COUNTRIES, nFormatter } from "@dub/utils"; -import { useCallback, useMemo } from "react"; - -export function usePartnerNetworkFilters({ - status, -}: { - status: "discover" | "invited" | "recruited" | "ignored"; -}) { - const { searchParamsObj, queryParams } = useRouterStuff(); - - // Apply filter changes via the History API (instant) rather than router.push, - // which would trigger a full RSC navigation per click. See page-client for the - // rationale — all data here is client-side SWR keyed off useSearchParams. - const updateSearchParams = useCallback( - (opts: { set?: Record; del?: string | string[] }) => { - const newPath = queryParams({ ...opts, getNewPath: true }) as string; - window.history.pushState(null, "", newPath); - }, - [queryParams], - ); - - const { data: countriesCount } = useNetworkPartnersCount< - | { - country: string; - _count: number; - }[] - | undefined - >({ - query: { - status, - groupBy: "country", - }, - excludeParams: ["country"], - }); - - const filters = useMemo( - () => [ - { - key: "country", - icon: FlagWavy, - label: "Partner country", - getOptionIcon: (value) => ( - - ), - getOptionLabel: (value) => COUNTRIES[value], - options: - countriesCount - ?.filter(({ country }) => COUNTRIES[country]) - .map(({ country, _count }) => ({ - value: country, - label: COUNTRIES[country], - right: nFormatter(_count, { full: true }), - })) ?? [], - }, - ], - [countriesCount], - ); - - const multiFilters = useMemo(() => ({}), []) as Record; - - const activeFilters = useMemo(() => { - const { country } = searchParamsObj; - - return [ - ...Object.entries(multiFilters) - .map(([key, value]) => ({ key, value })) - .filter(({ value }) => value.length > 0), - - ...(country ? [{ key: "country", value: country }] : []), - ]; - }, [searchParamsObj, multiFilters]); - - const onSelect = useCallback( - (key: string, value: any) => - updateSearchParams({ - set: Object.keys(multiFilters).includes(key) - ? { - [key]: multiFilters[key].concat(value).join(","), - } - : { - [key]: value, - }, - del: "page", - }), - [updateSearchParams, multiFilters], - ); - - const onRemove = useCallback( - (key: string, value: any) => { - if ( - Object.keys(multiFilters).includes(key) && - !(multiFilters[key].length === 1 && multiFilters[key][0] === value) - ) { - updateSearchParams({ - set: { - [key]: multiFilters[key].filter((id) => id !== value).join(","), - }, - del: "page", - }); - } else { - updateSearchParams({ - del: [key, "page"], - }); - } - }, - [updateSearchParams, multiFilters], - ); - - const onRemoveAll = useCallback( - () => - updateSearchParams({ - del: ["country", "starred", "platform", "reach"], - }), - [updateSearchParams], - ); - - const isFiltered = - activeFilters.length > 0 || - !!searchParamsObj.platform || - !!searchParamsObj.reach || - !!searchParamsObj.search; - - return { - filters, - activeFilters, - onSelect, - onRemove, - onRemoveAll, - isFiltered, - }; -} +import { REACH_TIER_KEYS, REACH_TIERS } from "@/lib/api/network/reach-tiers"; +import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; +import { CountryFlag } from "@/ui/shared/country-flag"; +import { useRouterStuff } from "@dub/ui"; +import { FlagWavy, User } from "@dub/ui/icons"; +import { COUNTRIES, nFormatter } from "@dub/utils"; +import { useCallback, useMemo } from "react"; + +export function usePartnerNetworkFilters({ + status, +}: { + status: "discover" | "invited" | "recruited" | "ignored"; +}) { + const { searchParamsObj, queryParams } = useRouterStuff(); + + // Apply filter changes via the History API (instant) rather than router.push, + // which would trigger a full RSC navigation per click. See page-client for the + // rationale — all data here is client-side SWR keyed off useSearchParams. + const updateSearchParams = useCallback( + (opts: { set?: Record; del?: string | string[] }) => { + const newPath = queryParams({ ...opts, getNewPath: true }) as string; + window.history.pushState(null, "", newPath); + }, + [queryParams], + ); + + const { data: countriesCount } = useNetworkPartnersCount< + | { + country: string; + _count: number; + }[] + | undefined + >({ + query: { + status, + groupBy: "country", + }, + excludeParams: ["country"], + }); + + const filters = useMemo( + () => [ + { + key: "country", + icon: FlagWavy, + label: "Partner country", + getOptionIcon: (value) => ( + + ), + getOptionLabel: (value) => COUNTRIES[value], + options: + countriesCount + ?.filter(({ country }) => COUNTRIES[country]) + .map(({ country, _count }) => ({ + value: country, + label: COUNTRIES[country], + right: nFormatter(_count, { full: true }), + })) ?? [], + }, + { + key: "reach", + icon: User, + label: "Creator size", + multiple: true, + // Range leads (objective); the muted descriptor sits on the right. + options: REACH_TIER_KEYS.map((tier) => ({ + value: tier, + label: REACH_TIERS[tier].range, + right: REACH_TIERS[tier].descriptor, + })), + }, + ], + [countriesCount], + ); + + const multiFilters = useMemo( + () => ({ + reach: searchParamsObj.reach?.split(",").filter(Boolean) ?? [], + }), + [searchParamsObj.reach], + ) as Record; + + const activeFilters = useMemo(() => { + const { country } = searchParamsObj; + + return [ + // Multi-select filters use the plural `values` shape so the Filter component + // reflects each option's checked state and renders per-value labels/icons. + ...Object.entries(multiFilters) + .map(([key, values]) => ({ key, values })) + .filter(({ values }) => values.length > 0), + + ...(country ? [{ key: "country", value: country }] : []), + ]; + }, [searchParamsObj, multiFilters]); + + const onSelect = useCallback( + (key: string, value: any) => + updateSearchParams({ + set: Object.keys(multiFilters).includes(key) + ? { + [key]: multiFilters[key].concat(value).join(","), + } + : { + [key]: value, + }, + del: "page", + }), + [updateSearchParams, multiFilters], + ); + + const onRemove = useCallback( + (key: string, value: any) => { + if ( + Object.keys(multiFilters).includes(key) && + !(multiFilters[key].length === 1 && multiFilters[key][0] === value) + ) { + updateSearchParams({ + set: { + [key]: multiFilters[key].filter((id) => id !== value).join(","), + }, + del: "page", + }); + } else { + updateSearchParams({ + del: [key, "page"], + }); + } + }, + [updateSearchParams, multiFilters], + ); + + const onRemoveAll = useCallback( + () => + updateSearchParams({ + del: ["country", "starred", "platform", "reach"], + }), + [updateSearchParams], + ); + + const isFiltered = + activeFilters.length > 0 || + !!searchParamsObj.platform || + !!searchParamsObj.reach || + !!searchParamsObj.search; + + return { + filters, + activeFilters, + onSelect, + onRemove, + onRemoveAll, + isFiltered, + }; +} diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts index 72313c8b849..012ef1e48b7 100644 --- a/apps/web/lib/partner-content-search/constants.ts +++ b/apps/web/lib/partner-content-search/constants.ts @@ -6,6 +6,17 @@ export const PARTNER_CONTENT_SEARCH_ENV_VARS = { voyageApiKey: "VOYAGE_API_KEY", } as const; +// Name of the manually-created PlanetScale VECTOR index on +// PartnerContentChunk.embedding, and the distance metric it's built with. These +// are the single source of truth: the search routes feed them into the FORCE +// INDEX hint and the DISTANCE() call, and they must match the @@index(map:) +// files that can't import this constant use a drift check test in +// tests/partner-content-search/vector-index-sync.test.ts. +export const PARTNER_CONTENT_CHUNK_VECTOR_INDEX = + "partner_content_chunk_embedding_cosine_idx"; + +export const PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE = "cosine"; + export const PARTNER_CONTENT_SEARCH_MODELS = { embedding: { provider: "voyage", @@ -34,12 +45,11 @@ export const PARTNER_CONTENT_SEARCH_LIMITS = { // per-partner item set so the exact per-item scoring query stays cheap for // extremely prolific creators. recentContentMaxPerPartner: 200, - // Pre-dedup ANN over-fetch for retrieval. We pull this many raw chunks via the - // vector index (flat `ORDER BY DISTANCE ... LIMIT`, which stays index-friendly), - // then collapse to the best chunk per content item in app code. Larger = better - // video/partner diversity when one high-volume creator dominates a query; - // bounded so the ANN fetch stays fast. - vectorSearchChunkPoolSize: 1000, + // Pre-dedup ANN over-fetch for retrieval. We pull limit * multiplier raw + // chunk candidates via the vector index, capped so the user-facing path does + // not request a large slice of the corpus as the chunk table grows. + vectorSearchChunkPoolMultiplier: 3, + vectorSearchChunkPoolMaxSize: 600, rerankerCandidateCount: 150, // Cap each document sent to the reranker. Chunks are <=800 tokens; trimming the // tail keeps the rerank payload (and thus latency) down with little quality loss. diff --git a/apps/web/lib/partner-content-search/search-utils.ts b/apps/web/lib/partner-content-search/search-utils.ts index aa0d0ffcf11..3d0865e71e2 100644 --- a/apps/web/lib/partner-content-search/search-utils.ts +++ b/apps/web/lib/partner-content-search/search-utils.ts @@ -30,6 +30,11 @@ export type PartnerContentSearchRow = { contentThumbnailUrl: string | null; contentPublishedAt: Date | null; contentDurationMs: number | null; + contentViewCount?: bigint | number | null; + contentLikeCount?: bigint | number | null; + contentCommentCount?: bigint | number | null; + contentShareCount?: bigint | number | null; + contentSaveCount?: bigint | number | null; chunkSource: string; // Only the admin query selects chunkIndex; the network query omits it. chunkIndex?: number; @@ -57,9 +62,15 @@ export function effectiveRowScore(row: PartnerContentSearchRow) { // content item is its best chunk. This turns a top-k *chunk* pool into a per-video // candidate set in app code, avoiding a SQL window-function dedup that would force // a full distance scan and defeat the ANN vector index. -export function dedupeBestChunkPerContentItem(rows: PartnerContentSearchRow[]) { +// +// Generic over the row shape so callers can dedup lightweight candidate rows +// (chunkId + ids + distance) before the hydration join as well as fully-hydrated +// rows. +export function dedupeBestChunkPerContentItem< + T extends { partnerContentItemId: string }, +>(rows: T[]) { const seen = new Set(); - const deduped: PartnerContentSearchRow[] = []; + const deduped: T[] = []; for (const row of rows) { if (seen.has(row.partnerContentItemId)) continue; @@ -70,11 +81,11 @@ export function dedupeBestChunkPerContentItem(rows: PartnerContentSearchRow[]) { return deduped; } -export function dedupeBestChunkPerContentItemSource( - rows: PartnerContentSearchRow[], -) { +export function dedupeBestChunkPerContentItemSource< + T extends { partnerContentItemId: string; chunkSource: string }, +>(rows: T[]) { const seen = new Set(); - const deduped: PartnerContentSearchRow[] = []; + const deduped: T[] = []; for (const row of rows) { const key = `${row.partnerContentItemId}:${row.chunkSource}`; diff --git a/apps/web/lib/partner-content-search/thumbnail-url.ts b/apps/web/lib/partner-content-search/thumbnail-url.ts index ef527b6709d..620d55a36a8 100644 --- a/apps/web/lib/partner-content-search/thumbnail-url.ts +++ b/apps/web/lib/partner-content-search/thumbnail-url.ts @@ -12,7 +12,9 @@ export function isInstagramCdnUrl(url: string) { try { const { hostname } = new URL(url); return ( - hostname === "cdninstagram.com" || hostname.endsWith(".cdninstagram.com") + hostname === "cdninstagram.com" || + hostname.endsWith(".cdninstagram.com") || + (hostname.endsWith(".fbcdn.net") && hostname.startsWith("instagram.")) ); } catch { return false; diff --git a/apps/web/lib/swr/use-partner-content-search.ts b/apps/web/lib/swr/use-partner-content-search.ts index 120d0790e39..638fad9ac32 100644 --- a/apps/web/lib/swr/use-partner-content-search.ts +++ b/apps/web/lib/swr/use-partner-content-search.ts @@ -48,6 +48,11 @@ export type PartnerContentSearchPartner = { thumbnailUrl: string | null; publishedAt: string | null; durationMs: number | null; + viewCount: number | null; + likeCount: number | null; + commentCount: number | null; + shareCount: number | null; + saveCount: number | null; }; chunk: { source: "metadata" | "transcript" | string; @@ -91,6 +96,10 @@ export type PartnerContentSearchPartner = { durationMs: number | null; publishedAt: string | null; viewCount: number | null; + likeCount: number | null; + commentCount: number | null; + shareCount: number | null; + saveCount: number | null; matched: boolean; matchScore: number | null; matchEvidence: PartnerContentMatchEvidence; diff --git a/apps/web/prisma/manual/partner-content-search-vector-index.sql b/apps/web/prisma/manual/partner-content-search-vector-index.sql index fdd61f6a671..00fc127e704 100644 --- a/apps/web/prisma/manual/partner-content-search-vector-index.sql +++ b/apps/web/prisma/manual/partner-content-search-vector-index.sql @@ -6,12 +6,12 @@ -- The schema includes a same-named @@index only so Prisma db push preserves this -- manual index; Prisma cannot define the VECTOR INDEX distance/options below. -- --- Keep the index distance in sync with the search query in: --- apps/web/app/(ee)/api/admin/partner-content/search/route.ts and --- apps/web/app/(ee)/api/network/partners/content-search/route.ts. --- That query currently uses DISTANCE(..., 'cosine'); if these metrics differ, --- PlanetScale cannot use the vector index and will fall back to a full scan. - +-- The index name and distance metric below are the same values the search routes +-- feed into their FORCE INDEX hint and DISTANCE() call (if they diverge, the +-- planner falls back to a full scan). They mirror PARTNER_CONTENT_CHUNK_VECTOR_INDEX +-- and PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE in +-- apps/web/lib/partner-content-search/constants.ts; the match is enforced by +-- apps/web/tests/partner-content-search/vector-index-sync.test.ts. CREATE /*vt+ QUERY_TIMEOUT_MS=0 */ VECTOR INDEX partner_content_chunk_embedding_cosine_idx ON PartnerContentChunk(embedding) diff --git a/apps/web/prisma/schema/partner-content-search.prisma b/apps/web/prisma/schema/partner-content-search.prisma index abca05ec2b0..61276392eb0 100644 --- a/apps/web/prisma/schema/partner-content-search.prisma +++ b/apps/web/prisma/schema/partner-content-search.prisma @@ -56,6 +56,7 @@ model PartnerContentItem { @@unique([partnerPlatformId, platformContentId]) @@index([partnerId, partnerPlatformId]) + @@index([partnerId, embeddingModel, publishedAt]) @@index(publishedAt) } @@ -71,6 +72,12 @@ model PartnerContentChunk { textHash String embedding Unsupported("vector(1024)")? embeddingModel String + + // Denormalized from Partner / PartnerPlatform so the vector ANN can pre-filter + // inline (no joins) on the index fast path; both fields are effectively static. + country String? // denormalized from Partner.country + platformType PlatformType? // denormalized from PartnerPlatform.type + createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -86,5 +93,8 @@ model PartnerContentChunk { @@index([partnerContentItemId, embeddingModel]) // Drift guard only: Prisma cannot express PlanetScale VECTOR INDEX options. // Create/recreate the real cosine ANN index with prisma/manual/partner-content-search-vector-index.sql. + // The map name mirrors PARTNER_CONTENT_CHUNK_VECTOR_INDEX in + // apps/web/lib/partner-content-search/constants.ts (the FORCE INDEX hint); the + // match is enforced by tests/partner-content-search/vector-index-sync.test.ts. @@index([embedding], map: "partner_content_chunk_embedding_cosine_idx") } diff --git a/apps/web/tests/partner-content-search/vector-index-sync.test.ts b/apps/web/tests/partner-content-search/vector-index-sync.test.ts new file mode 100644 index 00000000000..c3e15efae70 --- /dev/null +++ b/apps/web/tests/partner-content-search/vector-index-sync.test.ts @@ -0,0 +1,32 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { + PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, + PARTNER_CONTENT_CHUNK_VECTOR_INDEX, +} from "@/lib/partner-content-search/constants"; + +// The manual PlanetScale VECTOR index DDL and the Prisma @@index drift guard both +// hardcode the index name + distance metric because neither a .sql file nor the +// Prisma schema can import a TS constant. These assertions are the enforcement +// that replaces the old "keep in sync" comments: if the constant and either file +// disagree, this fails instead of silently shipping an unusable index. +describe("partner content search vector index", () => { + const read = (relativePath: string) => + readFileSync(join(process.cwd(), relativePath), "utf8"); + + test("manual DDL matches the index name and distance constants", () => { + const ddl = read( + "prisma/manual/partner-content-search-vector-index.sql", + ); + + expect(ddl).toContain(`VECTOR INDEX ${PARTNER_CONTENT_CHUNK_VECTOR_INDEX}`); + expect(ddl).toContain(`"distance":"${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}"`); + }); + + test("prisma @@index map matches the index name constant", () => { + const schema = read("prisma/schema/partner-content-search.prisma"); + + expect(schema).toContain(`map: "${PARTNER_CONTENT_CHUNK_VECTOR_INDEX}"`); + }); +}); diff --git a/packages/ui/src/tooltip.tsx b/packages/ui/src/tooltip.tsx index d7b3a327b9e..cbc578af191 100644 --- a/packages/ui/src/tooltip.tsx +++ b/packages/ui/src/tooltip.tsx @@ -72,6 +72,14 @@ export interface TooltipProps disabled?: boolean; disableHoverableContent?: TooltipPrimitive.TooltipProps["disableHoverableContent"]; delayDuration?: TooltipPrimitive.TooltipProps["delayDuration"]; + /** + * Controlled open state. Pass `open` and `onOpenChange` together to control + * it from the parent; omit both to let the tooltip manage its own. + */ + open?: boolean; + onOpenChange?: (open: boolean) => void; + /** Drop the entrance animation so the tooltip closes instantly without lingering. */ + disableAnimation?: boolean; } export function Tooltip({ @@ -82,9 +90,16 @@ export function Tooltip({ side = "top", disableHoverableContent, delayDuration = 0, + open: openProp, + onOpenChange, + disableAnimation, ...rest }: TooltipProps) { - const [open, setOpen] = useState(false); + const [uncontrolledOpen, setUncontrolledOpen] = useState(false); + const isControlled = openProp !== undefined; + const open = isControlled ? openProp : uncontrolledOpen; + const setOpen = (next: boolean) => + isControlled ? onOpenChange?.(next) : setUncontrolledOpen(next); return ( From 7106cf298ad825949a8f39dd6af640012ae2d6b5 Mon Sep 17 00:00:00 2001 From: David Clark Date: Mon, 22 Jun 2026 16:57:26 -0400 Subject: [PATCH 13/25] drop redundant indexes --- apps/web/prisma/schema/partner-content-search.prisma | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/web/prisma/schema/partner-content-search.prisma b/apps/web/prisma/schema/partner-content-search.prisma index 61276392eb0..291864cfd1b 100644 --- a/apps/web/prisma/schema/partner-content-search.prisma +++ b/apps/web/prisma/schema/partner-content-search.prisma @@ -86,8 +86,6 @@ model PartnerContentChunk { @@unique([partnerContentItemId, source, chunkIndex, embeddingModel]) @@index(partnerId) - @@index(partnerContentItemId) - @@index([partnerContentItemId, source]) // Covers the empty-query "recent content" list path, which filters on // embeddingModel while joining chunks by partnerContentItemId. @@index([partnerContentItemId, embeddingModel]) From 1faf70f59b0b6874d1589b3508a134678766a9a9 Mon Sep 17 00:00:00 2001 From: David Clark Date: Mon, 22 Jun 2026 17:10:46 -0400 Subject: [PATCH 14/25] refactor partner content search route --- .../network/partners/content-search/route.ts | 1493 +---------------- .../web/lib/partner-content-search/listing.ts | 179 ++ .../partner-content-search/match-summaries.ts | 673 ++++++++ .../network-partners.ts | 40 + .../web/lib/partner-content-search/ranking.ts | 8 + .../lib/partner-content-search/retrieval.ts | 540 ++++++ apps/web/lib/partner-content-search/timing.ts | 59 + apps/web/lib/zod/schemas/partner-network.ts | 39 + 8 files changed, 1544 insertions(+), 1487 deletions(-) create mode 100644 apps/web/lib/partner-content-search/listing.ts create mode 100644 apps/web/lib/partner-content-search/match-summaries.ts create mode 100644 apps/web/lib/partner-content-search/network-partners.ts create mode 100644 apps/web/lib/partner-content-search/retrieval.ts create mode 100644 apps/web/lib/partner-content-search/timing.ts diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index a1a9c8f1af6..91450a0faef 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -1,133 +1,34 @@ import { DubApiError } from "@/lib/api/errors"; -import { calculatePartnerRanking } from "@/lib/api/network/calculate-partner-ranking"; -import { parseRankedNetworkPartners } from "@/lib/api/network/normalize-ranked-network-partner"; import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw"; import { parseRequestBody } from "@/lib/api/utils"; import { withWorkspace } from "@/lib/auth"; import { - PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, - PARTNER_CONTENT_CHUNK_VECTOR_INDEX, - PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_LIMITS, - PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_MODELS, PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, - PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, } from "@/lib/partner-content-search/constants"; +import { listPartnerNetworkContent } from "@/lib/partner-content-search/listing"; +import { getPartnerMatchSummaries } from "@/lib/partner-content-search/match-summaries"; +import { getNetworkPartnersById } from "@/lib/partner-content-search/network-partners"; import { createContentMatchEvidence, - deriveTopicFit, - getEntityCreatorTextBoost, - getEntityTranscriptScore, - getEvidenceMatchScore, - getEvidenceTopicScore, getEvidenceSource, - getMatchedSourceScore, - getPartnerContentSearchQuerySignals, getRowRelevanceScore, - getSourceScore, - hasExactQueryMention, - maxNullableScore, - setSourceDistance, - setSourceScore, - sortRowsByRelevanceScore, - type PartnerContentMatchEvidence, - type PartnerContentMatchSource, - type PartnerContentSearchQuerySignals, } from "@/lib/partner-content-search/ranking"; +import { searchPartnerNetworkContent } from "@/lib/partner-content-search/retrieval"; import { - dedupeBestChunkPerContentItem, - dedupeBestChunkPerContentItemSource, groupPartnerSearchResults, - rerankPartnerSearchRows, toScore, type PartnerContentSearchRow, } from "@/lib/partner-content-search/search-utils"; -import { - embedPartnerContentTexts, - serializeEmbeddingForVector, - VoyageTimeoutError, -} from "@/lib/partner-content-search/voyage"; -import { REACH_TIER_KEYS, type ReachTier } from "@/lib/api/network/reach-tiers"; +import { createPartnerContentSearchTimingLogger } from "@/lib/partner-content-search/timing"; import { prisma } from "@/lib/prisma"; -import { PlatformType, Prisma } from "@prisma/client"; +import { partnerNetworkContentSearchSchema } from "@/lib/zod/schemas/partner-network"; import { NextResponse } from "next/server"; -import * as z from "zod/v4"; export const dynamic = "force-dynamic"; export const maxDuration = 30; -const DEFAULT_PARTNER_LIMIT = 20; -const MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS = 5; -const PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES = new Set([ - "query-embedding-complete", - "partner-eligibility-resolved", - "vector-candidate-search-complete", - "vector-candidate-hydration-complete", - "vector-search-pool-expanded", - "vector-search-complete", - "candidate-dedupe-complete", - "chunk-text-hydration-complete", - "rerank-complete", - "partner-candidates-grouped", - "match-summary-base-queries-complete", - "match-summary-content-counts-complete", - "match-summary-recent-content-complete", - "match-summary-followers-complete", - "match-summary-aggregation-complete", - "partner-hydration-complete", - "response-ready", -]); - -type PartnerContentSearchTimingLogger = ( - stage: string, - metadata?: Record, -) => void; - -type SourceScoreByContentItemId = Map< - string, - Map ->; - -type PartnerContentSearchCandidateRow = Pick< - PartnerContentSearchRow, - "chunkId" | "partnerContentItemId" | "chunkSource" | "distance" ->; - -type PartnerContentSearchHydrationRow = Omit< - PartnerContentSearchRow, - "distance" ->; - -const partnerNetworkContentSearchSchema = z.object({ - query: z.string().trim().max(500).optional(), - platforms: z.array(z.enum(PlatformType)).min(1).optional(), - reach: z.array(z.enum(REACH_TIER_KEYS)).min(1).optional(), - country: z.string().trim().min(1).optional(), - partnerIds: z.array(z.string()).min(1).max(100).optional(), - starred: z.boolean().optional(), - limit: z - .number() - .int() - .positive() - .max(PARTNER_CONTENT_SEARCH_PARTNER_LIMIT) - .default(DEFAULT_PARTNER_LIMIT), - chunksPerPartner: z - .number() - .int() - .positive() - .max(PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER) - .default(PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER), - candidateChunkCount: z - .number() - .int() - .positive() - .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) - .optional(), - // Second-stage reranking is on by default; pass `false` for diagnostics. - rerank: z.boolean().default(true), -}); - // POST /api/network/partners/content-search - semantic search over indexed partner content export const POST = withWorkspace( async ({ workspace, req }) => { @@ -295,54 +196,10 @@ export const POST = withWorkspace( }, ); -function createPartnerContentSearchTimingLogger( - context: Record, -): PartnerContentSearchTimingLogger { - const startedAt = Date.now(); - let previousAt = startedAt; - - return (stage, metadata = {}) => { - const now = Date.now(); - const elapsedMs = now - startedAt; - const deltaMs = now - previousAt; - previousAt = now; - - if ( - deltaMs < MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS && - !PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES.has(stage) - ) { - return; - } - - console.info("[partner-content-search:timing]", { - stage, - elapsedMs, - deltaMs, - ...context, - ...metadata, - }); - }; -} - function isNonNull(value: T | null): value is T { return value !== null; } -function getVectorSearchChunkPoolSize(limit: number) { - return Math.max( - limit, - Math.min( - PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize, - limit * PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMultiplier, - ), - ); -} - -function countDistinctContentItems(rows: { partnerContentItemId: string }[]) { - return new Set(rows.map(({ partnerContentItemId }) => partnerContentItemId)) - .size; -} - function sortPartnersByTopicFit< T extends { score: number; @@ -376,1344 +233,6 @@ function sortPartnersByTopicFit< }); } -async function getNetworkPartnersById({ - programId, - partnerIds, - platforms, - reach, - country, - starred, -}: { - programId: string; - partnerIds: string[]; - platforms?: PlatformType[]; - reach?: ReachTier[]; - country?: string; - starred?: boolean; -}) { - if (partnerIds.length === 0) return new Map(); - - const rankedPartners = await calculatePartnerRanking({ - programId, - partnerIds, - status: "discover", - page: 1, - pageSize: partnerIds.length, - country, - starred: starred ?? undefined, - platform: platforms, - reach, - }); - const partners = parseRankedNetworkPartners(rankedPartners); - - return new Map(partners.map((partner) => [partner.id, partner])); -} - -// Resolves a program's partner-eligibility sets for inline ANN pre-filtering: -// the deny-set (enrolled ∪ ignored partners) excluded from every query, plus the -// starred set the starred filter includes/excludes. All are per-program and bounded -// by the program's roster, so they ride into the flat ANN as id-set predicates -// without dragging the relational joins onto the vector-index traversal path. -async function resolveProgramPartnerEligibility({ - programId, - starred, - logTiming, -}: { - programId: string; - starred?: boolean; - logTiming?: PartnerContentSearchTimingLogger; -}) { - const [enrolledRows, ignoredRows, starredRows] = await Promise.all([ - prisma.programEnrollment.findMany({ - where: { programId }, - select: { partnerId: true }, - }), - prisma.discoveredPartner.findMany({ - where: { programId, ignoredAt: { not: null } }, - select: { partnerId: true }, - }), - // Only needed when the starred filter is active. - starred !== undefined - ? prisma.discoveredPartner.findMany({ - where: { programId, starredAt: { not: null } }, - select: { partnerId: true }, - }) - : Promise.resolve<{ partnerId: string }[]>([]), - ]); - - const excludedPartnerIds = [ - ...new Set([ - ...enrolledRows.map(({ partnerId }) => partnerId), - ...ignoredRows.map(({ partnerId }) => partnerId), - ]), - ]; - const starredPartnerIds = [ - ...new Set(starredRows.map(({ partnerId }) => partnerId)), - ]; - - logTiming?.("partner-eligibility-resolved", { - enrolledCount: enrolledRows.length, - ignoredCount: ignoredRows.length, - excludedPartnerCount: excludedPartnerIds.length, - starredPartnerCount: starredPartnerIds.length, - }); - - return { excludedPartnerIds, starredPartnerIds }; -} - -async function searchPartnerNetworkContent({ - programId, - query, - platforms, - country, - partnerIds, - starred, - limit, - rerank, - logTiming, -}: { - programId: string; - query: string; - platforms?: PlatformType[]; - country?: string; - partnerIds?: string[]; - starred?: boolean; - limit: number; - rerank: boolean; - logTiming?: PartnerContentSearchTimingLogger; -}) { - // Resolve the program's partner-eligibility sets in parallel with the query - // embedding — they need only programId, so they run under the Voyage round-trip - // and add little to no incremental time delay. (PrismaPromise.all kicks the queries off - // synchronously here, before we await the embedding below.) - const eligibilityPromise = resolveProgramPartnerEligibility({ - programId, - starred, - logTiming, - }); - - let queryEmbedding: number[]; - logTiming?.("query-embedding-start"); - try { - [queryEmbedding] = await embedPartnerContentTexts({ - input: [query], - inputType: "query", - timeoutMs: PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, - }); - } catch (error) { - if (error instanceof VoyageTimeoutError) { - throw new DubApiError({ - code: "internal_server_error", - message: "Partner content search timed out. Please try again.", - }); - } - throw error; - } - logTiming?.("query-embedding-complete", { - embeddingDimensions: queryEmbedding.length, - }); - const queryVector = serializeEmbeddingForVector(queryEmbedding); - const eligibility = await eligibilityPromise; - - // Pre-filter eligibility + user filters INLINE on the flat ANN (no joins): exclude - // the program's enrolled + ignored partners, honor explicit partnerIds, apply the - // starred filter, and apply the country/platform user filters via the denormalized - // c.country / c.platformType columns. Pre-filtering (vs. the old post-hydration - // filter) keeps these from biasing/starving the candidate pool — enrolled partners - // are exactly a program's strongest matches, so post-filtering them silently - // dropped the best results as a program matured. networkStatus stays a post-filter - // (non-selective: ingestion only embeds approved/trusted partners). - const annFilters: Prisma.Sql[] = []; - if (eligibility.excludedPartnerIds.length > 0) { - annFilters.push( - Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.excludedPartnerIds)})`, - ); - } - if (partnerIds?.length) { - annFilters.push(Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})`); - } - if (starred === true) { - // starred filter on with no starred partners → no eligible candidates. - annFilters.push( - eligibility.starredPartnerIds.length > 0 - ? Prisma.sql`AND c.partnerId IN (${Prisma.join(eligibility.starredPartnerIds)})` - : Prisma.sql`AND 1 = 0`, - ); - } else if (starred === false && eligibility.starredPartnerIds.length > 0) { - annFilters.push( - Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.starredPartnerIds)})`, - ); - } - if (country) { - annFilters.push(Prisma.sql`AND c.country = ${country}`); - } - if (platforms?.length) { - annFilters.push( - Prisma.sql`AND c.platformType IN (${Prisma.join(platforms)})`, - ); - } - const annFilter = - annFilters.length > 0 ? Prisma.join(annFilters, " ") : Prisma.empty; - - // Retrieval is split into two bounded phases. First, keep the ANN query flat so - // PlanetScale can use the vector index for a small chunk-candidate pool. Then - // hydrate/filter only those chunk ids through the relational partner/platform - // joins below. This keeps the expensive joins off the vector traversal path. - // The cheap phase also returns the content-item id + source so we can dedup to - // the best chunk per item+source BEFORE the join (see retrievePool). - const fetchCandidateChunks = (poolSize: number) => - prisma.$queryRaw(Prisma.sql` - SELECT - c.id AS chunkId, - c.partnerContentItemId, - c.source AS chunkSource, - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance - FROM PartnerContentChunk c FORCE INDEX (${Prisma.raw(PARTNER_CONTENT_CHUNK_VECTOR_INDEX)}) - WHERE c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - ${annFilter} - ORDER BY distance ASC - LIMIT ${poolSize} - `); - - // Dedup to the best chunk per content-item + source BEFORE the relational - // hydration join. The post-ANN filters are all per-item/per-partner, so the - // surviving items are identical whether we filter-then-dedup or dedup-then- - // filter; this just keeps chunk-heavy items from sending redundant rows through - // the 5-way join. itemRows (cutoff) and rows (rerank) are both derived from the - // per-item+source set below, so nothing downstream changes. - const retrievePool = async (poolSize: number) => { - const candidateRows = await fetchCandidateChunks(poolSize); - logTiming?.("vector-candidate-search-complete", { - candidateRowCount: candidateRows.length, - poolSize, - vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, - }); - const dedupedCandidates = - dedupeBestChunkPerContentItemSource(candidateRows); - const poolRows = await hydratePartnerContentSearchRows({ - candidateRows: dedupedCandidates, - logTiming, - }); - return { candidateRowCount: candidateRows.length, poolRows }; - }; - - const initialPoolSize = getVectorSearchChunkPoolSize(limit); - const maxPoolSize = PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize; - logTiming?.("vector-search-start", { - poolSize: initialPoolSize, - maxPoolSize, - retrievalShape: "two-phase", - }); - - let poolSize = initialPoolSize; - let { candidateRowCount, poolRows } = await retrievePool(poolSize); - - // Eligibility + user filters run INLINE in the ANN, so the pool comes back - // already eligible; networkStatus is the only remaining post-ANN filter and it's - // non-selective (ingestion gates on approved/trusted), so the hydrated pool rarely - // shrinks. Keep a one-shot expansion as a safety net for the rare case it does - // (e.g. a cluster of demoted partners): if we under-filled and the ANN hadn't - // exhausted the index (it returned a full pool), widen to the cap once and retry. - let distinctItemCount = countDistinctContentItems(poolRows); - if ( - poolSize < maxPoolSize && - candidateRowCount === poolSize && - distinctItemCount < limit - ) { - poolSize = maxPoolSize; - logTiming?.("vector-search-pool-expanded", { - previousPoolSize: initialPoolSize, - poolSize, - distinctItemCount, - limit, - }); - ({ candidateRowCount, poolRows } = await retrievePool(poolSize)); - distinctItemCount = countDistinctContentItems(poolRows); - } - logTiming?.("vector-search-complete", { - candidateRowCount, - poolRowCount: poolRows.length, - distinctItemCount, - poolSize, - expanded: poolSize !== initialPoolSize, - vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, - retrievalShape: "two-phase", - }); - - // Best cosine distance per content-item + evidence source across the candidate - // pool. getPartnerMatchSummaries reuses this to gate which recent posts count as - // "matched" instead of recomputing DISTANCE per item in SQL: cutoffDistance is - // itself a pool item's distance, so any item that could clear it is already in - // this map (an item missing here is provably beyond the cutoff — not matched). - const itemSourceBestDistance: SourceScoreByContentItemId = new Map(); - for (const row of poolRows) { - setSourceDistance( - itemSourceBestDistance, - row.partnerContentItemId, - getEvidenceSource(row.chunkSource), - Number(row.distance), - ); - } - - // Collapse to one best chunk per content item for the item-level cutoff, and - // one best chunk per content item + source for reranking/source-aware evidence. - // This keeps metadata and transcript evidence separable without letting one - // chunk-heavy video flood the candidate pool. - const itemRows = dedupeBestChunkPerContentItem(poolRows).slice(0, limit); - const rows = dedupeBestChunkPerContentItemSource(poolRows).slice( - 0, - Math.min(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, limit * 2), - ); - - // Cosine-distance cutoff = the least-relevant candidate kept. getPartnerMatchSummaries - // uses this to decide which of a partner's recent videos count as "matched" (i.e. - // at least as relevant as the weakest video in the candidate pool), computed - // exactly per item, independent of which raw chunk survived the global cap. - const cutoffDistance = itemRows.length - ? Number(itemRows[itemRows.length - 1].distance) - : null; - logTiming?.("candidate-dedupe-complete", { - itemRowCount: itemRows.length, - sourceRowCount: rows.length, - cutoffDistance, - }); - - const rowsWithChunkText = await hydratePartnerContentChunkText({ - rows, - maxRows: rerank - ? PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount - : rows.length, - logTiming, - }); - - if (!rerank) { - return { - rows: sortRowsByRelevanceScore(rowsWithChunkText), - reranked: false, - queryVector, - cutoffDistance, - itemSourceBestDistance, - }; - } - - logTiming?.("rerank-start", { - rowCount: rowsWithChunkText.length, - rerankerCandidateCount: Math.min( - rowsWithChunkText.length, - PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, - ), - }); - const rerankResult = await rerankPartnerSearchRows({ - query, - rows: rowsWithChunkText, - }); - logTiming?.("rerank-complete", { - reranked: rerankResult.reranked, - rowCount: rerankResult.rows.length, - rerankedRowCount: rerankResult.rows.filter( - ({ rerankScore }) => rerankScore != null, - ).length, - }); - return { - ...rerankResult, - rows: sortRowsByRelevanceScore(rerankResult.rows), - queryVector, - cutoffDistance, - itemSourceBestDistance, - }; -} - -async function hydratePartnerContentSearchRows({ - candidateRows, - logTiming, -}: { - candidateRows: PartnerContentSearchCandidateRow[]; - logTiming?: PartnerContentSearchTimingLogger; -}) { - if (candidateRows.length === 0) return []; - - const chunkIds = candidateRows.map(({ chunkId }) => chunkId); - // Eligibility + user filters (enrolled/ignored/starred/country/platform) are all - // pre-filtered in the ANN now, so hydration is a plain metadata fetch over - // already-eligible chunk ids. networkStatus is the one remaining post-filter - // (cheap and non-selective — see the search query note); the partner/platform - // joins stay only to supply display columns and that networkStatus check. - - // Plain relational hydration over an already-eligible chunk-id set: the inner - // joins in the prior raw form were pure filters (all relations are required - // FKs), and the lone post-ANN predicate, networkStatus, maps to the partner - // relation filter below. `chunkText` is deliberately not selected here (it's - // hydrated separately in hydratePartnerContentChunkText); we backfill the "" - // placeholder when flattening to the row shape. - const hydratedRows = await prisma.partnerContentChunk.findMany({ - where: { - id: { in: chunkIds }, - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - partner: { - networkStatus: { in: ["approved", "trusted"] }, - }, - }, - select: { - id: true, - partnerContentItemId: true, - partnerId: true, - source: true, - startMs: true, - endMs: true, - partner: { - select: { - name: true, - username: true, - image: true, - description: true, - }, - }, - partnerContentItem: { - select: { - platformContentId: true, - url: true, - contentType: true, - title: true, - description: true, - thumbnailUrl: true, - publishedAt: true, - durationMs: true, - viewCount: true, - likeCount: true, - commentCount: true, - shareCount: true, - saveCount: true, - partnerPlatform: { - select: { - type: true, - identifier: true, - }, - }, - }, - }, - }, - }); - const hydratedByChunkId = new Map( - hydratedRows.map(({ partner, partnerContentItem: item, ...chunk }): [ - string, - PartnerContentSearchHydrationRow, - ] => [ - chunk.id, - { - chunkId: chunk.id, - partnerContentItemId: chunk.partnerContentItemId, - partnerId: chunk.partnerId, - partnerName: partner.name, - partnerUsername: partner.username, - partnerImage: partner.image, - partnerDescription: partner.description, - platformType: item.partnerPlatform.type, - platformIdentifier: item.partnerPlatform.identifier, - platformContentId: item.platformContentId, - contentUrl: item.url, - contentType: item.contentType, - contentTitle: item.title, - contentDescription: item.description, - contentThumbnailUrl: item.thumbnailUrl, - contentPublishedAt: item.publishedAt, - contentDurationMs: item.durationMs, - contentViewCount: item.viewCount, - contentLikeCount: item.likeCount, - contentCommentCount: item.commentCount, - contentShareCount: item.shareCount, - contentSaveCount: item.saveCount, - chunkSource: chunk.source, - chunkText: "", - startMs: chunk.startMs, - endMs: chunk.endMs, - }, - ]), - ); - const rows = candidateRows.flatMap((candidate) => { - const hydrated = hydratedByChunkId.get(candidate.chunkId); - if (!hydrated) return []; - - return [ - { - ...hydrated, - distance: candidate.distance, - }, - ]; - }); - logTiming?.("vector-candidate-hydration-complete", { - candidateRowCount: candidateRows.length, - hydratedRowCount: hydratedRows.length, - filteredOutRowCount: candidateRows.length - rows.length, - }); - - return rows; -} - -async function hydratePartnerContentChunkText({ - rows, - maxRows, - logTiming, -}: { - rows: PartnerContentSearchRow[]; - maxRows: number; - logTiming?: PartnerContentSearchTimingLogger; -}) { - const rowsToHydrate = rows.slice(0, maxRows); - const chunkIds = rowsToHydrate.map(({ chunkId }) => chunkId); - - if (chunkIds.length === 0) return rows; - - logTiming?.("chunk-text-hydration-start", { - requestedRowCount: rowsToHydrate.length, - totalRowCount: rows.length, - }); - const chunkTextRows = await prisma.partnerContentChunk.findMany({ - where: { - id: { - in: chunkIds, - }, - }, - select: { - id: true, - chunkText: true, - }, - }); - const chunkTextById = new Map( - chunkTextRows.map((row) => [row.id, row.chunkText]), - ); - logTiming?.("chunk-text-hydration-complete", { - hydratedRowCount: chunkTextById.size, - }); - - return rows.map((row, index) => - index < maxRows - ? { - ...row, - chunkText: chunkTextById.get(row.chunkId) ?? row.chunkText, - } - : row, - ); -} - -async function listPartnerNetworkContent({ - programId, - platforms, - country, - partnerIds, - starred, - limit, -}: { - programId: string; - platforms?: PlatformType[]; - country?: string; - partnerIds?: string[]; - starred?: boolean; - limit: number; -}) { - const discoveredFilters: Prisma.PartnerWhereInput[] = [ - { - discoveredByPrograms: { - none: { - programId, - ignoredAt: { - not: null, - }, - }, - }, - }, - ]; - - if (starred === true) { - discoveredFilters.push({ - discoveredByPrograms: { - some: { - programId, - starredAt: { - not: null, - }, - }, - }, - }); - } else if (starred === false) { - discoveredFilters.push({ - discoveredByPrograms: { - none: { - programId, - starredAt: { - not: null, - }, - }, - }, - }); - } - - const contentItems = await prisma.partnerContentItem.findMany({ - where: { - ...(partnerIds?.length && { - partnerId: { - in: partnerIds, - }, - }), - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - embeddedChunkCount: { - gt: 0, - }, - ...(platforms?.length && { - partnerPlatform: { - type: { in: platforms }, - }, - }), - partner: { - networkStatus: { - in: ["approved", "trusted"], - }, - ...(country && { country }), - programs: { - none: { - programId, - }, - }, - AND: discoveredFilters, - }, - }, - select: { - id: true, - partnerId: true, - platformContentId: true, - url: true, - contentType: true, - title: true, - description: true, - thumbnailUrl: true, - publishedAt: true, - durationMs: true, - viewCount: true, - likeCount: true, - commentCount: true, - shareCount: true, - saveCount: true, - partner: { - select: { - name: true, - username: true, - image: true, - description: true, - }, - }, - partnerPlatform: { - select: { - type: true, - identifier: true, - }, - }, - chunks: { - where: { - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - }, - select: { - id: true, - source: true, - }, - orderBy: { - id: "asc", - }, - take: 1, - }, - }, - orderBy: [ - { - publishedAt: "desc", - }, - { - id: "asc", - }, - ], - take: limit, - }); - - return contentItems.flatMap((contentItem) => { - const chunk = contentItem.chunks[0]; - if (!chunk) return []; - - return { - chunkId: chunk.id, - partnerContentItemId: contentItem.id, - partnerId: contentItem.partnerId, - partnerName: contentItem.partner.name, - partnerUsername: contentItem.partner.username, - partnerImage: contentItem.partner.image, - partnerDescription: contentItem.partner.description, - platformType: contentItem.partnerPlatform.type, - platformIdentifier: contentItem.partnerPlatform.identifier, - platformContentId: contentItem.platformContentId, - contentUrl: contentItem.url, - contentType: contentItem.contentType, - contentTitle: contentItem.title, - contentDescription: contentItem.description, - contentThumbnailUrl: contentItem.thumbnailUrl, - contentPublishedAt: contentItem.publishedAt, - contentDurationMs: contentItem.durationMs, - contentViewCount: contentItem.viewCount, - contentLikeCount: contentItem.likeCount, - contentCommentCount: contentItem.commentCount, - contentShareCount: contentItem.shareCount, - contentSaveCount: contentItem.saveCount, - chunkSource: chunk.source, - chunkText: "", - startMs: null, - endMs: null, - distance: 0, - }; - }); -} - -type PartnerRecentContentBarRow = { - partnerId: string; - partnerContentItemId: string; - platformType: string; - platformContentId: string; - contentType: string; - contentTitle: string | null; - contentDescription: string | null; - contentUrl: string | null; - contentDurationMs: number | null; - transcriptFetchStatus: string | null; - publishedAt: Date | null; - viewCount: bigint | number | null; - likeCount: bigint | number | null; - commentCount: bigint | number | null; - shareCount: bigint | number | null; - saveCount: bigint | number | null; - rowNumber: bigint | number; -}; - -type PartnerContentItemBestMatch = { - transcriptScore: number | null; - creatorTextScore: number | null; -}; - -type PartnerContentMatchBar = { - partnerContentItemId: string; - platform: string; - platformContentId: string; - title: string | null; - url: string | null; - durationMs: number | null; - publishedAt: string | null; - viewCount: number | null; - likeCount: number | null; - commentCount: number | null; - shareCount: number | null; - saveCount: number | null; - matched: boolean; - matchScore: number | null; - matchEvidence: PartnerContentMatchEvidence; -}; - -type QueryContentMatchContext = { - querySignals: PartnerContentSearchQuerySignals; - cutoffDistance?: number | null; - recentItemSourceBestDistance: SourceScoreByContentItemId; - rerankByItemSource: SourceScoreByContentItemId; -}; - -type ListContentMatchContext = { - bestMatchByContentItemId: Map; -}; - -type ContentMatchContext = QueryContentMatchContext | ListContentMatchContext; - -// Median of a numeric list (robust to viral outliers vs. a mean). Returns null -// for an empty list so callers can omit the signal when there's no data. -function median(values: number[]): number | null { - if (values.length === 0) return null; - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 - ? Math.round((sorted[mid - 1] + sorted[mid]) / 2) - : sorted[mid]; -} - -function groupRowsBy(rows: T[], getKey: (row: T) => K) { - const rowsByKey = new Map(); - - for (const row of rows) { - const key = getKey(row); - const group = rowsByKey.get(key) ?? []; - group.push(row); - rowsByKey.set(key, group); - } - - return rowsByKey; -} - -function getBestMatchByContentItemId(rows: PartnerContentSearchRow[]) { - const bestMatchByContentItemId = new Map< - string, - PartnerContentItemBestMatch - >(); - - for (const row of rows) { - // Use the effective score (rerank when present) so the content-match bars - // stay consistent with the reranked partner ordering. - const score = row.rerankScore ?? toScore(Number(row.distance)); - const evidenceSource = getEvidenceSource(row.chunkSource); - const existing = bestMatchByContentItemId.get(row.partnerContentItemId); - const next = existing ?? { - transcriptScore: null, - creatorTextScore: null, - }; - - if (evidenceSource === "transcript") { - next.transcriptScore = - next.transcriptScore == null - ? score - : Math.max(next.transcriptScore, score); - } else { - next.creatorTextScore = - next.creatorTextScore == null - ? score - : Math.max(next.creatorTextScore, score); - } - - bestMatchByContentItemId.set(row.partnerContentItemId, next); - } - - return bestMatchByContentItemId; -} - -function getQueryContentMatch({ - row, - context, -}: { - row: PartnerRecentContentBarRow; - context: QueryContentMatchContext; -}) { - // On-topic gate: prefer the reranker's calibrated relevance, which cleanly - // separates on- from off-topic. Fall back to the cosine cutoff only for items - // the reranker didn't score (rerank disabled/failed, or the item never reached - // the candidate pool). - const transcriptScore = getEntityTranscriptScore({ - currentScore: getMatchedSourceScore({ - rerankScore: getSourceScore( - context.rerankByItemSource, - row.partnerContentItemId, - "transcript", - ), - bestDistance: getSourceScore( - context.recentItemSourceBestDistance, - row.partnerContentItemId, - "transcript", - ), - cutoffDistance: context.cutoffDistance, - }), - // Transcript-level substring matching was removed from the search path; only - // title/description exact mentions (computed in memory below) feed scoring now. - hasExactQueryMention: false, - queryIntent: context.querySignals.intent, - }); - const vectorCreatorTextScore = getMatchedSourceScore({ - rerankScore: getSourceScore( - context.rerankByItemSource, - row.partnerContentItemId, - "creatorText", - ), - bestDistance: getSourceScore( - context.recentItemSourceBestDistance, - row.partnerContentItemId, - "creatorText", - ), - cutoffDistance: context.cutoffDistance, - }); - const titleHasExactQueryMention = hasExactQueryMention( - row.contentTitle, - context.querySignals, - ); - const descriptionHasExactQueryMention = hasExactQueryMention( - row.contentDescription, - context.querySignals, - ); - const creatorTextBoost = getEntityCreatorTextBoost({ - platformType: row.platformType, - contentType: row.contentType, - transcriptFetchStatus: row.transcriptFetchStatus, - titleHasExactQueryMention, - descriptionHasExactQueryMention, - // Chunk-level substring matching was removed from the search path (see above). - chunkHasExactQueryMention: false, - transcriptScore, - queryIntent: context.querySignals.intent, - }); - const creatorTextScore = maxNullableScore( - vectorCreatorTextScore, - creatorTextBoost?.score, - ); - - const matchEvidence = createContentMatchEvidence({ - contentType: row.contentType, - transcriptScore, - creatorTextScore, - creatorTextWeightOverride: creatorTextBoost?.weight, - }); - - return { - matchEvidence, - matchScore: getEvidenceMatchScore(matchEvidence), - }; -} - -function getListContentMatch({ - row, - context, -}: { - row: PartnerRecentContentBarRow; - context: ListContentMatchContext; -}) { - const match = context.bestMatchByContentItemId.get(row.partnerContentItemId); - const matchEvidence = createContentMatchEvidence({ - contentType: row.contentType, - transcriptScore: match?.transcriptScore ?? null, - creatorTextScore: match?.creatorTextScore ?? null, - }); - - return { - matchEvidence, - matchScore: getEvidenceMatchScore(matchEvidence), - }; -} - -function toContentMatchBar({ - row, - context, -}: { - row: PartnerRecentContentBarRow; - context: ContentMatchContext; -}): PartnerContentMatchBar { - const { matchEvidence, matchScore } = - "bestMatchByContentItemId" in context - ? getListContentMatch({ row, context }) - : getQueryContentMatch({ row, context }); - const matched = matchEvidence.sources.length > 0; - - return { - partnerContentItemId: row.partnerContentItemId, - platform: row.platformType, - platformContentId: row.platformContentId, - title: row.contentTitle, - url: row.contentUrl, - durationMs: - row.contentDurationMs != null ? Number(row.contentDurationMs) : null, - publishedAt: row.publishedAt?.toISOString() ?? null, - viewCount: row.viewCount != null ? Number(row.viewCount) : null, - likeCount: row.likeCount != null ? Number(row.likeCount) : null, - commentCount: row.commentCount != null ? Number(row.commentCount) : null, - shareCount: row.shareCount != null ? Number(row.shareCount) : null, - saveCount: row.saveCount != null ? Number(row.saveCount) : null, - matched, - matchScore, - matchEvidence, - }; -} - -function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { - const matchedBars = contentBars.filter((bar) => bar.matched); - const matchedContentCount = matchedBars.length; - const transcriptMatchedContentCount = contentBars.filter( - ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), - ).length; - const creatorTextMatchedContentCount = contentBars.filter( - ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), - ).length; - const creatorTextOnlyContentCount = contentBars.filter( - ({ matchEvidence }) => - matchEvidence.primarySource === "creatorText" && - matchEvidence.sources.length === 1, - ).length; - const weightedMatchedContentCount = Number( - contentBars - .reduce((total, bar) => total + bar.matchEvidence.weight, 0) - .toFixed(3), - ); - const weightedMatchedContentScore = Number( - contentBars - .reduce( - (total, bar) => total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), - 0, - ) - .toFixed(3), - ); - - return { - matchedBars, - matchedContentCount, - transcriptMatchedContentCount, - creatorTextMatchedContentCount, - creatorTextOnlyContentCount, - weightedMatchedContentCount, - weightedMatchedContentScore, - }; -} - -async function getPartnerMatchSummaries({ - rows, - partnerIds, - platforms, - query, - queryVector, - cutoffDistance, - itemSourceBestDistance, - logTiming, -}: { - rows: PartnerContentSearchRow[]; - partnerIds: string[]; - platforms?: PlatformType[]; - query?: string | null; - // Present only in query mode. When set, each shown partner's recent videos are - // counted as matched when at least as relevant as the weakest candidate - // (cutoffDistance). When null (list mode) we fall back to the retrieval rows for - // the match determination. - queryVector?: string | null; - cutoffDistance?: number | null; - // Best cosine distance per content-item + source across the candidate pool, - // computed once during retrieval. Reused here to gate matched recent content - // without a second per-item DISTANCE pass (any item that clears the cutoff is - // already in this map — see the producer in searchPartnerNetworkContent). - itemSourceBestDistance?: SourceScoreByContentItemId; - logTiming?: PartnerContentSearchTimingLogger; -}) { - if (partnerIds.length === 0) return new Map(); - - const querySignals = getPartnerContentSearchQuerySignals(query); - const platformFilter = platforms?.length - ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` - : Prisma.empty; - - // "Recent" is a time window (last recencyWindowMonths), not a fixed post count. - // A count window means wildly different time spans per creator (a daily poster's - // 28 posts span a month; a monthly poster's span 2+ years), so coverage over it - // conflates active and dormant creators. A time window normalizes that and keeps - // every platform with content in the period (incl. ones the creator has since - // moved on from; those still count toward topical authority). - const recencyCutoff = new Date(); - recencyCutoff.setMonth( - recencyCutoff.getMonth() - - PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, - ); - - // These three reads are independent, so issue them as one parallel wave rather - // than three serial round trips. (Prisma promises are lazy and don't execute - // until awaited, so they fan out together under the Promise.all below.) - const contentCountsPromise = prisma.partnerContentItem - .groupBy({ - by: ["partnerId"], - where: { - partnerId: { - in: partnerIds, - }, - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - embeddedChunkCount: { - gt: 0, - }, - ...(platforms?.length && { - partnerPlatform: { - type: { in: platforms }, - }, - }), - }, - _count: { - _all: true, - }, - _min: { - publishedAt: true, - }, - _max: { - publishedAt: true, - }, - }) - .then((contentCounts) => { - logTiming?.("match-summary-content-counts-complete", { - contentCountRows: contentCounts.length, - }); - return contentCounts; - }); - - const recentContentRowsPromise = prisma - .$queryRaw( - Prisma.sql` - SELECT - partnerId, - partnerContentItemId, - platformType, - platformContentId, - contentType, - contentTitle, - contentDescription, - contentUrl, - contentDurationMs, - transcriptFetchStatus, - publishedAt, - viewCount, - likeCount, - commentCount, - shareCount, - saveCount, - rowNumber - FROM ( - SELECT - pci.partnerId, - pci.id AS partnerContentItemId, - pp.type AS platformType, - pci.platformContentId, - pci.contentType, - pci.title AS contentTitle, - pci.description AS contentDescription, - pci.url AS contentUrl, - pci.durationMs AS contentDurationMs, - pci.transcriptFetchStatus, - pci.publishedAt, - pci.viewCount, - pci.likeCount, - pci.commentCount, - pci.shareCount, - pci.saveCount, - ROW_NUMBER() OVER ( - PARTITION BY pci.partnerId - ORDER BY pci.publishedAt DESC, pci.createdAt DESC, pci.id ASC - ) AS rowNumber - FROM PartnerContentItem pci - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - WHERE pci.partnerId IN (${Prisma.join(partnerIds)}) - ${platformFilter} - AND ( - pci.publishedAt >= ${recencyCutoff} - OR (pci.publishedAt IS NULL AND pci.createdAt >= ${recencyCutoff}) - ) - AND pci.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - AND pci.embeddedChunkCount > 0 - ) recentContent - WHERE rowNumber <= ${PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner} - ORDER BY partnerId ASC, rowNumber ASC - `, - ) - .then((recentContentRows) => { - logTiming?.("match-summary-recent-content-complete", { - recentContentRows: recentContentRows.length, - }); - return recentContentRows; - }); - - // Reach: total followers across the partner's platforms (or just the filtered - // platform). The single signal brands most want and the card wasn't showing. - const followerRowsPromise = prisma.partnerPlatform - .groupBy({ - by: ["partnerId"], - where: { - partnerId: { - in: partnerIds, - }, - ...(platforms?.length && { - type: { in: platforms }, - }), - }, - _sum: { - subscribers: true, - }, - }) - .then((followerRows) => { - logTiming?.("match-summary-followers-complete", { - followerRows: followerRows.length, - }); - return followerRows; - }); - - logTiming?.("match-summary-base-queries-start", { - partnerCount: partnerIds.length, - }); - const [contentCounts, recentContentRows, followerRows] = await Promise.all([ - contentCountsPromise, - recentContentRowsPromise, - followerRowsPromise, - ]); - logTiming?.("match-summary-base-queries-complete", { - contentCountRows: contentCounts.length, - recentContentRows: recentContentRows.length, - followerRows: followerRows.length, - }); - const followersByPartner = new Map( - followerRows.map((row) => [ - row.partnerId, - row._sum.subscribers != null ? Number(row._sum.subscribers) : 0, - ]), - ); - - const countByPartnerId = new Map( - contentCounts.map((row) => [row.partnerId, row]), - ); - const rowsByPartnerId = groupRowsBy(rows, ({ partnerId }) => partnerId); - const recentRowsByPartnerId = groupRowsBy( - recentContentRows, - ({ partnerId }) => partnerId, - ); - - // Query mode only: gate which of each shown partner's recent videos count as - // "matched". A recent video is matched when its best chunk is at least as - // relevant as the weakest candidate (cutoffDistance). Because cutoffDistance is - // itself a candidate-pool item's distance, every recent video that could clear - // it is already in itemSourceBestDistance — so we reuse that map instead of - // re-running a per-item DISTANCE pass over the recent set. A recent video absent - // from the map is provably beyond the cutoff (not matched). - const recentItemSourceBestDistance = - queryVector && itemSourceBestDistance - ? itemSourceBestDistance - : new Map>(); - - // Calibrated reranker score per item + source (from the candidate pool), used - // to gate which recent posts count as on-topic. - const rerankByItemSource: SourceScoreByContentItemId = new Map(); - for (const row of rows) { - if (row.rerankScore != null) { - setSourceScore( - rerankByItemSource, - row.partnerContentItemId, - getEvidenceSource(row.chunkSource), - row.rerankScore, - ); - } - } - - const queryMatchContext: QueryContentMatchContext | null = queryVector - ? { - querySignals, - cutoffDistance, - recentItemSourceBestDistance, - rerankByItemSource, - } - : null; - - logTiming?.("match-summary-aggregation-start", { - partnerCount: partnerIds.length, - recentContentRows: recentContentRows.length, - }); - const summaries = new Map( - partnerIds.map((partnerId) => { - const partnerRows = rowsByPartnerId.get(partnerId) ?? []; - const recentRows = recentRowsByPartnerId.get(partnerId) ?? []; - const countRow = countByPartnerId.get(partnerId); - const totalContentCount = countRow ? countRow._count._all : 0; - const bestMatchByContentItemId = getBestMatchByContentItemId(partnerRows); - const contentMatchContext: ContentMatchContext = queryMatchContext ?? { - bestMatchByContentItemId, - }; - const contentBars = recentRows.map((row) => - toContentMatchBar({ - row, - context: contentMatchContext, - }), - ); - const { - matchedBars, - matchedContentCount, - transcriptMatchedContentCount, - creatorTextMatchedContentCount, - creatorTextOnlyContentCount, - weightedMatchedContentCount, - weightedMatchedContentScore, - } = getContentBarMatchStats(contentBars); - const recentContentCount = contentBars.length; - const { topicFit, band } = deriveTopicFit({ - matchedContentCount, - weightedMatchedContentCount, - weightedMatchedContentScore, - recentContentCount, - }); - // Brand-facing signals over the on-topic posts: typical reach (median views, - // robust to a viral outlier) and how recently they last posted on topic. - const medianViews = median( - matchedBars - .map((bar) => bar.viewCount) - .filter((views): views is number => views != null && views > 0), - ); - const matchedTimestamps = matchedBars - .map((bar) => bar.publishedAt) - .filter((publishedAt): publishedAt is string => Boolean(publishedAt)) - .map((publishedAt) => new Date(publishedAt).getTime()); - const lastOnTopicAt = matchedTimestamps.length - ? new Date(Math.max(...matchedTimestamps)).toISOString() - : null; - const followers = followersByPartner.get(partnerId) ?? null; - // Top platforms ranked by matched-post frequency, falling back to all recent - // posts when nothing matched (so the card can still show a platform). - const platformFrequency = new Map(); - for (const bar of matchedBars.length ? matchedBars : contentBars) { - platformFrequency.set( - bar.platform, - (platformFrequency.get(bar.platform) ?? 0) + 1, - ); - } - const topPlatforms = Array.from(platformFrequency.entries()) - .sort((a, b) => b[1] - a[1]) - .map(([platform]) => platform); - const publishedDates = contentBars - .map(({ publishedAt }) => (publishedAt ? new Date(publishedAt) : null)) - .filter((date): date is Date => Boolean(date)); - const oldestPublishedAt = publishedDates.length - ? new Date( - Math.min(...publishedDates.map((date) => date.getTime())), - ).toISOString() - : countRow?._min.publishedAt?.toISOString() ?? null; - const newestPublishedAt = publishedDates.length - ? new Date( - Math.max(...publishedDates.map((date) => date.getTime())), - ).toISOString() - : countRow?._max.publishedAt?.toISOString() ?? null; - - return [ - partnerId, - { - matchedContentCount, - transcriptMatchedContentCount, - creatorTextMatchedContentCount, - creatorTextOnlyContentCount, - weightedMatchedContentCount, - weightedMatchedContentScore, - recentContentCount, - totalContentCount, - topicFit, - band, - followers, - medianViews, - lastOnTopicAt, - topPlatforms, - platforms: Array.from( - new Set( - (recentRows.map(({ platformType }) => platformType).length - ? recentRows.map(({ platformType }) => platformType) - : partnerRows.map(({ platformType }) => platformType) - ).filter(Boolean), - ), - ).sort(), - sources: Array.from( - new Set( - partnerRows.map(({ chunkSource }) => - getEvidenceSource(chunkSource), - ), - ), - ).sort(), - oldestPublishedAt, - newestPublishedAt, - contentBars, - }, - ] as const; - }), - ); - logTiming?.("match-summary-aggregation-complete", { - summaryCount: summaries.size, - }); - - return summaries; -} - function toChunkResult(row: PartnerContentSearchRow, distance: number) { return { chunkId: row.chunkId, diff --git a/apps/web/lib/partner-content-search/listing.ts b/apps/web/lib/partner-content-search/listing.ts new file mode 100644 index 00000000000..61a73e8989e --- /dev/null +++ b/apps/web/lib/partner-content-search/listing.ts @@ -0,0 +1,179 @@ +import { prisma } from "@/lib/prisma"; +import { PlatformType, Prisma } from "@prisma/client"; +import { PARTNER_CONTENT_SEARCH_MODELS } from "./constants"; +import type { PartnerContentSearchRow } from "./search-utils"; + +// Empty-query path: the most recent embedded content across the eligible network, +// shaped into the same per-chunk row as the vector search so the downstream +// grouping/match-summary code is identical for both modes. +export async function listPartnerNetworkContent({ + programId, + platforms, + country, + partnerIds, + starred, + limit, +}: { + programId: string; + platforms?: PlatformType[]; + country?: string; + partnerIds?: string[]; + starred?: boolean; + limit: number; +}) { + const discoveredFilters: Prisma.PartnerWhereInput[] = [ + { + discoveredByPrograms: { + none: { + programId, + ignoredAt: { + not: null, + }, + }, + }, + }, + ]; + + if (starred === true) { + discoveredFilters.push({ + discoveredByPrograms: { + some: { + programId, + starredAt: { + not: null, + }, + }, + }, + }); + } else if (starred === false) { + discoveredFilters.push({ + discoveredByPrograms: { + none: { + programId, + starredAt: { + not: null, + }, + }, + }, + }); + } + + const contentItems = await prisma.partnerContentItem.findMany({ + where: { + ...(partnerIds?.length && { + partnerId: { + in: partnerIds, + }, + }), + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + embeddedChunkCount: { + gt: 0, + }, + ...(platforms?.length && { + partnerPlatform: { + type: { in: platforms }, + }, + }), + partner: { + networkStatus: { + in: ["approved", "trusted"], + }, + ...(country && { country }), + programs: { + none: { + programId, + }, + }, + AND: discoveredFilters, + }, + }, + select: { + id: true, + partnerId: true, + platformContentId: true, + url: true, + contentType: true, + title: true, + description: true, + thumbnailUrl: true, + publishedAt: true, + durationMs: true, + viewCount: true, + likeCount: true, + commentCount: true, + shareCount: true, + saveCount: true, + partner: { + select: { + name: true, + username: true, + image: true, + description: true, + }, + }, + partnerPlatform: { + select: { + type: true, + identifier: true, + }, + }, + chunks: { + where: { + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + }, + select: { + id: true, + source: true, + }, + orderBy: { + id: "asc", + }, + take: 1, + }, + }, + orderBy: [ + { + publishedAt: "desc", + }, + { + id: "asc", + }, + ], + take: limit, + }); + + return contentItems.flatMap((contentItem) => { + const chunk = contentItem.chunks[0]; + if (!chunk) return []; + + return { + chunkId: chunk.id, + partnerContentItemId: contentItem.id, + partnerId: contentItem.partnerId, + partnerName: contentItem.partner.name, + partnerUsername: contentItem.partner.username, + partnerImage: contentItem.partner.image, + partnerDescription: contentItem.partner.description, + platformType: contentItem.partnerPlatform.type, + platformIdentifier: contentItem.partnerPlatform.identifier, + platformContentId: contentItem.platformContentId, + contentUrl: contentItem.url, + contentType: contentItem.contentType, + contentTitle: contentItem.title, + contentDescription: contentItem.description, + contentThumbnailUrl: contentItem.thumbnailUrl, + contentPublishedAt: contentItem.publishedAt, + contentDurationMs: contentItem.durationMs, + contentViewCount: contentItem.viewCount, + contentLikeCount: contentItem.likeCount, + contentCommentCount: contentItem.commentCount, + contentShareCount: contentItem.shareCount, + contentSaveCount: contentItem.saveCount, + chunkSource: chunk.source, + chunkText: "", + startMs: null, + endMs: null, + distance: 0, + }; + }); +} diff --git a/apps/web/lib/partner-content-search/match-summaries.ts b/apps/web/lib/partner-content-search/match-summaries.ts new file mode 100644 index 00000000000..13abaa38ec9 --- /dev/null +++ b/apps/web/lib/partner-content-search/match-summaries.ts @@ -0,0 +1,673 @@ +import { prisma } from "@/lib/prisma"; +import { PlatformType, Prisma } from "@prisma/client"; +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MODELS, +} from "./constants"; +import { + createContentMatchEvidence, + deriveTopicFit, + getEntityCreatorTextBoost, + getEntityTranscriptScore, + getEvidenceMatchScore, + getEvidenceSource, + getEvidenceTopicScore, + getMatchedSourceScore, + getPartnerContentSearchQuerySignals, + getSourceScore, + hasExactQueryMention, + maxNullableScore, + setSourceScore, + type PartnerContentMatchEvidence, + type PartnerContentMatchSource, + type PartnerContentSearchQuerySignals, + type SourceScoreByContentItemId, +} from "./ranking"; +import { toScore, type PartnerContentSearchRow } from "./search-utils"; +import type { PartnerContentSearchTimingLogger } from "./timing"; + +type PartnerRecentContentBarRow = { + partnerId: string; + partnerContentItemId: string; + platformType: string; + platformContentId: string; + contentType: string; + contentTitle: string | null; + contentDescription: string | null; + contentUrl: string | null; + contentDurationMs: number | null; + transcriptFetchStatus: string | null; + publishedAt: Date | null; + viewCount: bigint | number | null; + likeCount: bigint | number | null; + commentCount: bigint | number | null; + shareCount: bigint | number | null; + saveCount: bigint | number | null; + rowNumber: bigint | number; +}; + +type PartnerContentItemBestMatch = { + transcriptScore: number | null; + creatorTextScore: number | null; +}; + +type PartnerContentMatchBar = { + partnerContentItemId: string; + platform: string; + platformContentId: string; + title: string | null; + url: string | null; + durationMs: number | null; + publishedAt: string | null; + viewCount: number | null; + likeCount: number | null; + commentCount: number | null; + shareCount: number | null; + saveCount: number | null; + matched: boolean; + matchScore: number | null; + matchEvidence: PartnerContentMatchEvidence; +}; + +type QueryContentMatchContext = { + querySignals: PartnerContentSearchQuerySignals; + cutoffDistance?: number | null; + recentItemSourceBestDistance: SourceScoreByContentItemId; + rerankByItemSource: SourceScoreByContentItemId; +}; + +type ListContentMatchContext = { + bestMatchByContentItemId: Map; +}; + +type ContentMatchContext = QueryContentMatchContext | ListContentMatchContext; + +// Median of a numeric list (robust to viral outliers vs. a mean). Returns null +// for an empty list so callers can omit the signal when there's no data. +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? Math.round((sorted[mid - 1] + sorted[mid]) / 2) + : sorted[mid]; +} + +function groupRowsBy(rows: T[], getKey: (row: T) => K) { + const rowsByKey = new Map(); + + for (const row of rows) { + const key = getKey(row); + const group = rowsByKey.get(key) ?? []; + group.push(row); + rowsByKey.set(key, group); + } + + return rowsByKey; +} + +function getBestMatchByContentItemId(rows: PartnerContentSearchRow[]) { + const bestMatchByContentItemId = new Map< + string, + PartnerContentItemBestMatch + >(); + + for (const row of rows) { + // Use the effective score (rerank when present) so the content-match bars + // stay consistent with the reranked partner ordering. + const score = row.rerankScore ?? toScore(Number(row.distance)); + const evidenceSource = getEvidenceSource(row.chunkSource); + const existing = bestMatchByContentItemId.get(row.partnerContentItemId); + const next = existing ?? { + transcriptScore: null, + creatorTextScore: null, + }; + + if (evidenceSource === "transcript") { + next.transcriptScore = + next.transcriptScore == null + ? score + : Math.max(next.transcriptScore, score); + } else { + next.creatorTextScore = + next.creatorTextScore == null + ? score + : Math.max(next.creatorTextScore, score); + } + + bestMatchByContentItemId.set(row.partnerContentItemId, next); + } + + return bestMatchByContentItemId; +} + +function getQueryContentMatch({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: QueryContentMatchContext; +}) { + // On-topic gate: prefer the reranker's calibrated relevance, which cleanly + // separates on- from off-topic. Fall back to the cosine cutoff only for items + // the reranker didn't score (rerank disabled/failed, or the item never reached + // the candidate pool). + const transcriptScore = getEntityTranscriptScore({ + currentScore: getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "transcript", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "transcript", + ), + cutoffDistance: context.cutoffDistance, + }), + // Transcript-level substring matching was removed from the search path; only + // title/description exact mentions (computed in memory below) feed scoring now. + hasExactQueryMention: false, + queryIntent: context.querySignals.intent, + }); + const vectorCreatorTextScore = getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "creatorText", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "creatorText", + ), + cutoffDistance: context.cutoffDistance, + }); + const titleHasExactQueryMention = hasExactQueryMention( + row.contentTitle, + context.querySignals, + ); + const descriptionHasExactQueryMention = hasExactQueryMention( + row.contentDescription, + context.querySignals, + ); + const creatorTextBoost = getEntityCreatorTextBoost({ + platformType: row.platformType, + contentType: row.contentType, + transcriptFetchStatus: row.transcriptFetchStatus, + titleHasExactQueryMention, + descriptionHasExactQueryMention, + // Chunk-level substring matching was removed from the search path (see above). + chunkHasExactQueryMention: false, + transcriptScore, + queryIntent: context.querySignals.intent, + }); + const creatorTextScore = maxNullableScore( + vectorCreatorTextScore, + creatorTextBoost?.score, + ); + + const matchEvidence = createContentMatchEvidence({ + contentType: row.contentType, + transcriptScore, + creatorTextScore, + creatorTextWeightOverride: creatorTextBoost?.weight, + }); + + return { + matchEvidence, + matchScore: getEvidenceMatchScore(matchEvidence), + }; +} + +function getListContentMatch({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: ListContentMatchContext; +}) { + const match = context.bestMatchByContentItemId.get(row.partnerContentItemId); + const matchEvidence = createContentMatchEvidence({ + contentType: row.contentType, + transcriptScore: match?.transcriptScore ?? null, + creatorTextScore: match?.creatorTextScore ?? null, + }); + + return { + matchEvidence, + matchScore: getEvidenceMatchScore(matchEvidence), + }; +} + +function toContentMatchBar({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: ContentMatchContext; +}): PartnerContentMatchBar { + const { matchEvidence, matchScore } = + "bestMatchByContentItemId" in context + ? getListContentMatch({ row, context }) + : getQueryContentMatch({ row, context }); + const matched = matchEvidence.sources.length > 0; + + return { + partnerContentItemId: row.partnerContentItemId, + platform: row.platformType, + platformContentId: row.platformContentId, + title: row.contentTitle, + url: row.contentUrl, + durationMs: + row.contentDurationMs != null ? Number(row.contentDurationMs) : null, + publishedAt: row.publishedAt?.toISOString() ?? null, + viewCount: row.viewCount != null ? Number(row.viewCount) : null, + likeCount: row.likeCount != null ? Number(row.likeCount) : null, + commentCount: row.commentCount != null ? Number(row.commentCount) : null, + shareCount: row.shareCount != null ? Number(row.shareCount) : null, + saveCount: row.saveCount != null ? Number(row.saveCount) : null, + matched, + matchScore, + matchEvidence, + }; +} + +function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { + const matchedBars = contentBars.filter((bar) => bar.matched); + const matchedContentCount = matchedBars.length; + const transcriptMatchedContentCount = contentBars.filter( + ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), + ).length; + const creatorTextMatchedContentCount = contentBars.filter( + ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), + ).length; + const creatorTextOnlyContentCount = contentBars.filter( + ({ matchEvidence }) => + matchEvidence.primarySource === "creatorText" && + matchEvidence.sources.length === 1, + ).length; + const weightedMatchedContentCount = Number( + contentBars + .reduce((total, bar) => total + bar.matchEvidence.weight, 0) + .toFixed(3), + ); + const weightedMatchedContentScore = Number( + contentBars + .reduce( + (total, bar) => total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), + 0, + ) + .toFixed(3), + ); + + return { + matchedBars, + matchedContentCount, + transcriptMatchedContentCount, + creatorTextMatchedContentCount, + creatorTextOnlyContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + }; +} + +export async function getPartnerMatchSummaries({ + rows, + partnerIds, + platforms, + query, + queryVector, + cutoffDistance, + itemSourceBestDistance, + logTiming, +}: { + rows: PartnerContentSearchRow[]; + partnerIds: string[]; + platforms?: PlatformType[]; + query?: string | null; + // Present only in query mode. When set, each shown partner's recent videos are + // counted as matched when at least as relevant as the weakest candidate + // (cutoffDistance). When null (list mode) we fall back to the retrieval rows for + // the match determination. + queryVector?: string | null; + cutoffDistance?: number | null; + // Best cosine distance per content-item + source across the candidate pool, + // computed once during retrieval. Reused here to gate matched recent content + // without a second per-item DISTANCE pass (any item that clears the cutoff is + // already in this map — see the producer in searchPartnerNetworkContent). + itemSourceBestDistance?: SourceScoreByContentItemId; + logTiming?: PartnerContentSearchTimingLogger; +}) { + if (partnerIds.length === 0) return new Map(); + + const querySignals = getPartnerContentSearchQuerySignals(query); + const platformFilter = platforms?.length + ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` + : Prisma.empty; + + // "Recent" is a time window (last recencyWindowMonths), not a fixed post count. + // A count window means wildly different time spans per creator (a daily poster's + // 28 posts span a month; a monthly poster's span 2+ years), so coverage over it + // conflates active and dormant creators. A time window normalizes that and keeps + // every platform with content in the period (incl. ones the creator has since + // moved on from; those still count toward topical authority). + const recencyCutoff = new Date(); + recencyCutoff.setMonth( + recencyCutoff.getMonth() - + PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, + ); + + // These three reads are independent, so issue them as one parallel wave rather + // than three serial round trips. (Prisma promises are lazy and don't execute + // until awaited, so they fan out together under the Promise.all below.) + const contentCountsPromise = prisma.partnerContentItem + .groupBy({ + by: ["partnerId"], + where: { + partnerId: { + in: partnerIds, + }, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + embeddedChunkCount: { + gt: 0, + }, + ...(platforms?.length && { + partnerPlatform: { + type: { in: platforms }, + }, + }), + }, + _count: { + _all: true, + }, + _min: { + publishedAt: true, + }, + _max: { + publishedAt: true, + }, + }) + .then((contentCounts) => { + logTiming?.("match-summary-content-counts-complete", { + contentCountRows: contentCounts.length, + }); + return contentCounts; + }); + + const recentContentRowsPromise = prisma + .$queryRaw( + Prisma.sql` + SELECT + partnerId, + partnerContentItemId, + platformType, + platformContentId, + contentType, + contentTitle, + contentDescription, + contentUrl, + contentDurationMs, + transcriptFetchStatus, + publishedAt, + viewCount, + likeCount, + commentCount, + shareCount, + saveCount, + rowNumber + FROM ( + SELECT + pci.partnerId, + pci.id AS partnerContentItemId, + pp.type AS platformType, + pci.platformContentId, + pci.contentType, + pci.title AS contentTitle, + pci.description AS contentDescription, + pci.url AS contentUrl, + pci.durationMs AS contentDurationMs, + pci.transcriptFetchStatus, + pci.publishedAt, + pci.viewCount, + pci.likeCount, + pci.commentCount, + pci.shareCount, + pci.saveCount, + ROW_NUMBER() OVER ( + PARTITION BY pci.partnerId + ORDER BY pci.publishedAt DESC, pci.createdAt DESC, pci.id ASC + ) AS rowNumber + FROM PartnerContentItem pci + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + WHERE pci.partnerId IN (${Prisma.join(partnerIds)}) + ${platformFilter} + AND ( + pci.publishedAt >= ${recencyCutoff} + OR (pci.publishedAt IS NULL AND pci.createdAt >= ${recencyCutoff}) + ) + AND pci.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + AND pci.embeddedChunkCount > 0 + ) recentContent + WHERE rowNumber <= ${PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner} + ORDER BY partnerId ASC, rowNumber ASC + `, + ) + .then((recentContentRows) => { + logTiming?.("match-summary-recent-content-complete", { + recentContentRows: recentContentRows.length, + }); + return recentContentRows; + }); + + // Reach: total followers across the partner's platforms (or just the filtered + // platform). The single signal brands most want and the card wasn't showing. + const followerRowsPromise = prisma.partnerPlatform + .groupBy({ + by: ["partnerId"], + where: { + partnerId: { + in: partnerIds, + }, + ...(platforms?.length && { + type: { in: platforms }, + }), + }, + _sum: { + subscribers: true, + }, + }) + .then((followerRows) => { + logTiming?.("match-summary-followers-complete", { + followerRows: followerRows.length, + }); + return followerRows; + }); + + logTiming?.("match-summary-base-queries-start", { + partnerCount: partnerIds.length, + }); + const [contentCounts, recentContentRows, followerRows] = await Promise.all([ + contentCountsPromise, + recentContentRowsPromise, + followerRowsPromise, + ]); + logTiming?.("match-summary-base-queries-complete", { + contentCountRows: contentCounts.length, + recentContentRows: recentContentRows.length, + followerRows: followerRows.length, + }); + const followersByPartner = new Map( + followerRows.map((row) => [ + row.partnerId, + row._sum.subscribers != null ? Number(row._sum.subscribers) : 0, + ]), + ); + + const countByPartnerId = new Map( + contentCounts.map((row) => [row.partnerId, row]), + ); + const rowsByPartnerId = groupRowsBy(rows, ({ partnerId }) => partnerId); + const recentRowsByPartnerId = groupRowsBy( + recentContentRows, + ({ partnerId }) => partnerId, + ); + + // Query mode only: gate which of each shown partner's recent videos count as + // "matched". A recent video is matched when its best chunk is at least as + // relevant as the weakest candidate (cutoffDistance). Because cutoffDistance is + // itself a candidate-pool item's distance, every recent video that could clear + // it is already in itemSourceBestDistance — so we reuse that map instead of + // re-running a per-item DISTANCE pass over the recent set. A recent video absent + // from the map is provably beyond the cutoff (not matched). + const recentItemSourceBestDistance = + queryVector && itemSourceBestDistance + ? itemSourceBestDistance + : new Map>(); + + // Calibrated reranker score per item + source (from the candidate pool), used + // to gate which recent posts count as on-topic. + const rerankByItemSource: SourceScoreByContentItemId = new Map(); + for (const row of rows) { + if (row.rerankScore != null) { + setSourceScore( + rerankByItemSource, + row.partnerContentItemId, + getEvidenceSource(row.chunkSource), + row.rerankScore, + ); + } + } + + const queryMatchContext: QueryContentMatchContext | null = queryVector + ? { + querySignals, + cutoffDistance, + recentItemSourceBestDistance, + rerankByItemSource, + } + : null; + + logTiming?.("match-summary-aggregation-start", { + partnerCount: partnerIds.length, + recentContentRows: recentContentRows.length, + }); + const summaries = new Map( + partnerIds.map((partnerId) => { + const partnerRows = rowsByPartnerId.get(partnerId) ?? []; + const recentRows = recentRowsByPartnerId.get(partnerId) ?? []; + const countRow = countByPartnerId.get(partnerId); + const totalContentCount = countRow ? countRow._count._all : 0; + const bestMatchByContentItemId = getBestMatchByContentItemId(partnerRows); + const contentMatchContext: ContentMatchContext = queryMatchContext ?? { + bestMatchByContentItemId, + }; + const contentBars = recentRows.map((row) => + toContentMatchBar({ + row, + context: contentMatchContext, + }), + ); + const { + matchedBars, + matchedContentCount, + transcriptMatchedContentCount, + creatorTextMatchedContentCount, + creatorTextOnlyContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + } = getContentBarMatchStats(contentBars); + const recentContentCount = contentBars.length; + const { topicFit, band } = deriveTopicFit({ + matchedContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + recentContentCount, + }); + // Brand-facing signals over the on-topic posts: typical reach (median views, + // robust to a viral outlier) and how recently they last posted on topic. + const medianViews = median( + matchedBars + .map((bar) => bar.viewCount) + .filter((views): views is number => views != null && views > 0), + ); + const matchedTimestamps = matchedBars + .map((bar) => bar.publishedAt) + .filter((publishedAt): publishedAt is string => Boolean(publishedAt)) + .map((publishedAt) => new Date(publishedAt).getTime()); + const lastOnTopicAt = matchedTimestamps.length + ? new Date(Math.max(...matchedTimestamps)).toISOString() + : null; + const followers = followersByPartner.get(partnerId) ?? null; + // Top platforms ranked by matched-post frequency, falling back to all recent + // posts when nothing matched (so the card can still show a platform). + const platformFrequency = new Map(); + for (const bar of matchedBars.length ? matchedBars : contentBars) { + platformFrequency.set( + bar.platform, + (platformFrequency.get(bar.platform) ?? 0) + 1, + ); + } + const topPlatforms = Array.from(platformFrequency.entries()) + .sort((a, b) => b[1] - a[1]) + .map(([platform]) => platform); + const publishedDates = contentBars + .map(({ publishedAt }) => (publishedAt ? new Date(publishedAt) : null)) + .filter((date): date is Date => Boolean(date)); + const oldestPublishedAt = publishedDates.length + ? new Date( + Math.min(...publishedDates.map((date) => date.getTime())), + ).toISOString() + : countRow?._min.publishedAt?.toISOString() ?? null; + const newestPublishedAt = publishedDates.length + ? new Date( + Math.max(...publishedDates.map((date) => date.getTime())), + ).toISOString() + : countRow?._max.publishedAt?.toISOString() ?? null; + + return [ + partnerId, + { + matchedContentCount, + transcriptMatchedContentCount, + creatorTextMatchedContentCount, + creatorTextOnlyContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + recentContentCount, + totalContentCount, + topicFit, + band, + followers, + medianViews, + lastOnTopicAt, + topPlatforms, + platforms: Array.from( + new Set( + (recentRows.map(({ platformType }) => platformType).length + ? recentRows.map(({ platformType }) => platformType) + : partnerRows.map(({ platformType }) => platformType) + ).filter(Boolean), + ), + ).sort(), + sources: Array.from( + new Set( + partnerRows.map(({ chunkSource }) => + getEvidenceSource(chunkSource), + ), + ), + ).sort(), + oldestPublishedAt, + newestPublishedAt, + contentBars, + }, + ] as const; + }), + ); + logTiming?.("match-summary-aggregation-complete", { + summaryCount: summaries.size, + }); + + return summaries; +} diff --git a/apps/web/lib/partner-content-search/network-partners.ts b/apps/web/lib/partner-content-search/network-partners.ts new file mode 100644 index 00000000000..4a16810621f --- /dev/null +++ b/apps/web/lib/partner-content-search/network-partners.ts @@ -0,0 +1,40 @@ +import { calculatePartnerRanking } from "@/lib/api/network/calculate-partner-ranking"; +import { parseRankedNetworkPartners } from "@/lib/api/network/normalize-ranked-network-partner"; +import type { ReachTier } from "@/lib/api/network/reach-tiers"; +import type { PlatformType } from "@prisma/client"; + +// Hydrate the shown partner candidates into full network-partner cards, applying +// the same ranking + post-retrieval user filters (reach/country/starred/platform) +// the listing endpoints use, keyed by id for the caller to merge back in. +export async function getNetworkPartnersById({ + programId, + partnerIds, + platforms, + reach, + country, + starred, +}: { + programId: string; + partnerIds: string[]; + platforms?: PlatformType[]; + reach?: ReachTier[]; + country?: string; + starred?: boolean; +}) { + if (partnerIds.length === 0) return new Map(); + + const rankedPartners = await calculatePartnerRanking({ + programId, + partnerIds, + status: "discover", + page: 1, + pageSize: partnerIds.length, + country, + starred: starred ?? undefined, + platform: platforms, + reach, + }); + const partners = parseRankedNetworkPartners(rankedPartners); + + return new Map(partners.map((partner) => [partner.id, partner])); +} diff --git a/apps/web/lib/partner-content-search/ranking.ts b/apps/web/lib/partner-content-search/ranking.ts index 7c6e45d53f5..db88fcd7ff4 100644 --- a/apps/web/lib/partner-content-search/ranking.ts +++ b/apps/web/lib/partner-content-search/ranking.ts @@ -4,6 +4,14 @@ import { toScore, type PartnerContentSearchRow } from "./search-utils"; export type PartnerContentMatchSource = "transcript" | "creatorText"; +// Best score (rerank or distance) per content item, split by evidence source. +// Produced during retrieval and reused by the match-summary pass; the source-map +// helpers below (get/set) all operate on exactly this shape. +export type SourceScoreByContentItemId = Map< + string, + Map +>; + export type PartnerContentSearchQueryIntent = "entity" | "semantic"; export type PartnerContentSearchQuerySignals = { diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts new file mode 100644 index 00000000000..b9f0d762dae --- /dev/null +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -0,0 +1,540 @@ +import { DubApiError } from "@/lib/api/errors"; +import { prisma } from "@/lib/prisma"; +import { PlatformType, Prisma } from "@prisma/client"; +import { + PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, + PARTNER_CONTENT_CHUNK_VECTOR_INDEX, + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MODELS, + PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, +} from "./constants"; +import { + getEvidenceSource, + setSourceDistance, + sortRowsByRelevanceScore, + type SourceScoreByContentItemId, +} from "./ranking"; +import { + dedupeBestChunkPerContentItem, + dedupeBestChunkPerContentItemSource, + rerankPartnerSearchRows, + type PartnerContentSearchRow, +} from "./search-utils"; +import type { PartnerContentSearchTimingLogger } from "./timing"; +import { + embedPartnerContentTexts, + serializeEmbeddingForVector, + VoyageTimeoutError, +} from "./voyage"; + +type PartnerContentSearchCandidateRow = Pick< + PartnerContentSearchRow, + "chunkId" | "partnerContentItemId" | "chunkSource" | "distance" +>; + +type PartnerContentSearchHydrationRow = Omit< + PartnerContentSearchRow, + "distance" +>; + +function getVectorSearchChunkPoolSize(limit: number) { + return Math.max( + limit, + Math.min( + PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize, + limit * PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMultiplier, + ), + ); +} + +function countDistinctContentItems(rows: { partnerContentItemId: string }[]) { + return new Set(rows.map(({ partnerContentItemId }) => partnerContentItemId)) + .size; +} + +// Resolves a program's partner-eligibility sets for inline ANN pre-filtering: +// the deny-set (enrolled ∪ ignored partners) excluded from every query, plus the +// starred set the starred filter includes/excludes. All are per-program and bounded +// by the program's roster, so they ride into the flat ANN as id-set predicates +// without dragging the relational joins onto the vector-index traversal path. +async function resolveProgramPartnerEligibility({ + programId, + starred, + logTiming, +}: { + programId: string; + starred?: boolean; + logTiming?: PartnerContentSearchTimingLogger; +}) { + const [enrolledRows, ignoredRows, starredRows] = await Promise.all([ + prisma.programEnrollment.findMany({ + where: { programId }, + select: { partnerId: true }, + }), + prisma.discoveredPartner.findMany({ + where: { programId, ignoredAt: { not: null } }, + select: { partnerId: true }, + }), + // Only needed when the starred filter is active. + starred !== undefined + ? prisma.discoveredPartner.findMany({ + where: { programId, starredAt: { not: null } }, + select: { partnerId: true }, + }) + : Promise.resolve<{ partnerId: string }[]>([]), + ]); + + const excludedPartnerIds = [ + ...new Set([ + ...enrolledRows.map(({ partnerId }) => partnerId), + ...ignoredRows.map(({ partnerId }) => partnerId), + ]), + ]; + const starredPartnerIds = [ + ...new Set(starredRows.map(({ partnerId }) => partnerId)), + ]; + + logTiming?.("partner-eligibility-resolved", { + enrolledCount: enrolledRows.length, + ignoredCount: ignoredRows.length, + excludedPartnerCount: excludedPartnerIds.length, + starredPartnerCount: starredPartnerIds.length, + }); + + return { excludedPartnerIds, starredPartnerIds }; +} + +export async function searchPartnerNetworkContent({ + programId, + query, + platforms, + country, + partnerIds, + starred, + limit, + rerank, + logTiming, +}: { + programId: string; + query: string; + platforms?: PlatformType[]; + country?: string; + partnerIds?: string[]; + starred?: boolean; + limit: number; + rerank: boolean; + logTiming?: PartnerContentSearchTimingLogger; +}) { + // Resolve the program's partner-eligibility sets in parallel with the query + // embedding — they need only programId, so they run under the Voyage round-trip + // and add little to no incremental time delay. (PrismaPromise.all kicks the queries off + // synchronously here, before we await the embedding below.) + const eligibilityPromise = resolveProgramPartnerEligibility({ + programId, + starred, + logTiming, + }); + + let queryEmbedding: number[]; + logTiming?.("query-embedding-start"); + try { + [queryEmbedding] = await embedPartnerContentTexts({ + input: [query], + inputType: "query", + timeoutMs: PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, + }); + } catch (error) { + if (error instanceof VoyageTimeoutError) { + throw new DubApiError({ + code: "internal_server_error", + message: "Partner content search timed out. Please try again.", + }); + } + throw error; + } + logTiming?.("query-embedding-complete", { + embeddingDimensions: queryEmbedding.length, + }); + const queryVector = serializeEmbeddingForVector(queryEmbedding); + const eligibility = await eligibilityPromise; + + // Pre-filter eligibility + user filters INLINE on the flat ANN (no joins): exclude + // the program's enrolled + ignored partners, honor explicit partnerIds, apply the + // starred filter, and apply the country/platform user filters via the denormalized + // c.country / c.platformType columns. Pre-filtering (vs. the old post-hydration + // filter) keeps these from biasing/starving the candidate pool — enrolled partners + // are exactly a program's strongest matches, so post-filtering them silently + // dropped the best results as a program matured. networkStatus stays a post-filter + // (non-selective: ingestion only embeds approved/trusted partners). + const annFilters: Prisma.Sql[] = []; + if (eligibility.excludedPartnerIds.length > 0) { + annFilters.push( + Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.excludedPartnerIds)})`, + ); + } + if (partnerIds?.length) { + annFilters.push(Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})`); + } + if (starred === true) { + // starred filter on with no starred partners → no eligible candidates. + annFilters.push( + eligibility.starredPartnerIds.length > 0 + ? Prisma.sql`AND c.partnerId IN (${Prisma.join(eligibility.starredPartnerIds)})` + : Prisma.sql`AND 1 = 0`, + ); + } else if (starred === false && eligibility.starredPartnerIds.length > 0) { + annFilters.push( + Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.starredPartnerIds)})`, + ); + } + if (country) { + annFilters.push(Prisma.sql`AND c.country = ${country}`); + } + if (platforms?.length) { + annFilters.push( + Prisma.sql`AND c.platformType IN (${Prisma.join(platforms)})`, + ); + } + const annFilter = + annFilters.length > 0 ? Prisma.join(annFilters, " ") : Prisma.empty; + + // Retrieval is split into two bounded phases. First, keep the ANN query flat so + // PlanetScale can use the vector index for a small chunk-candidate pool. Then + // hydrate/filter only those chunk ids through the relational partner/platform + // joins below. This keeps the expensive joins off the vector traversal path. + // The cheap phase also returns the content-item id + source so we can dedup to + // the best chunk per item+source BEFORE the join (see retrievePool). + const fetchCandidateChunks = (poolSize: number) => + prisma.$queryRaw(Prisma.sql` + SELECT + c.id AS chunkId, + c.partnerContentItemId, + c.source AS chunkSource, + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance + FROM PartnerContentChunk c FORCE INDEX (${Prisma.raw(PARTNER_CONTENT_CHUNK_VECTOR_INDEX)}) + WHERE c.embedding IS NOT NULL + AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + ${annFilter} + ORDER BY distance ASC + LIMIT ${poolSize} + `); + + // Dedup to the best chunk per content-item + source BEFORE the relational + // hydration join. The post-ANN filters are all per-item/per-partner, so the + // surviving items are identical whether we filter-then-dedup or dedup-then- + // filter; this just keeps chunk-heavy items from sending redundant rows through + // the 5-way join. itemRows (cutoff) and rows (rerank) are both derived from the + // per-item+source set below, so nothing downstream changes. + const retrievePool = async (poolSize: number) => { + const candidateRows = await fetchCandidateChunks(poolSize); + logTiming?.("vector-candidate-search-complete", { + candidateRowCount: candidateRows.length, + poolSize, + vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, + }); + const dedupedCandidates = + dedupeBestChunkPerContentItemSource(candidateRows); + const poolRows = await hydratePartnerContentSearchRows({ + candidateRows: dedupedCandidates, + logTiming, + }); + return { candidateRowCount: candidateRows.length, poolRows }; + }; + + const initialPoolSize = getVectorSearchChunkPoolSize(limit); + const maxPoolSize = PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize; + logTiming?.("vector-search-start", { + poolSize: initialPoolSize, + maxPoolSize, + retrievalShape: "two-phase", + }); + + let poolSize = initialPoolSize; + let { candidateRowCount, poolRows } = await retrievePool(poolSize); + + // Eligibility + user filters run INLINE in the ANN, so the pool comes back + // already eligible; networkStatus is the only remaining post-ANN filter and it's + // non-selective (ingestion gates on approved/trusted), so the hydrated pool rarely + // shrinks. Keep a one-shot expansion as a safety net for the rare case it does + // (e.g. a cluster of demoted partners): if we under-filled and the ANN hadn't + // exhausted the index (it returned a full pool), widen to the cap once and retry. + let distinctItemCount = countDistinctContentItems(poolRows); + if ( + poolSize < maxPoolSize && + candidateRowCount === poolSize && + distinctItemCount < limit + ) { + poolSize = maxPoolSize; + logTiming?.("vector-search-pool-expanded", { + previousPoolSize: initialPoolSize, + poolSize, + distinctItemCount, + limit, + }); + ({ candidateRowCount, poolRows } = await retrievePool(poolSize)); + distinctItemCount = countDistinctContentItems(poolRows); + } + logTiming?.("vector-search-complete", { + candidateRowCount, + poolRowCount: poolRows.length, + distinctItemCount, + poolSize, + expanded: poolSize !== initialPoolSize, + vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, + retrievalShape: "two-phase", + }); + + // Best cosine distance per content-item + evidence source across the candidate + // pool. getPartnerMatchSummaries reuses this to gate which recent posts count as + // "matched" instead of recomputing DISTANCE per item in SQL: cutoffDistance is + // itself a pool item's distance, so any item that could clear it is already in + // this map (an item missing here is provably beyond the cutoff — not matched). + const itemSourceBestDistance: SourceScoreByContentItemId = new Map(); + for (const row of poolRows) { + setSourceDistance( + itemSourceBestDistance, + row.partnerContentItemId, + getEvidenceSource(row.chunkSource), + Number(row.distance), + ); + } + + // Collapse to one best chunk per content item for the item-level cutoff, and + // one best chunk per content item + source for reranking/source-aware evidence. + // This keeps metadata and transcript evidence separable without letting one + // chunk-heavy video flood the candidate pool. + const itemRows = dedupeBestChunkPerContentItem(poolRows).slice(0, limit); + const rows = dedupeBestChunkPerContentItemSource(poolRows).slice( + 0, + Math.min(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, limit * 2), + ); + + // Cosine-distance cutoff = the least-relevant candidate kept. getPartnerMatchSummaries + // uses this to decide which of a partner's recent videos count as "matched" (i.e. + // at least as relevant as the weakest video in the candidate pool), computed + // exactly per item, independent of which raw chunk survived the global cap. + const cutoffDistance = itemRows.length + ? Number(itemRows[itemRows.length - 1].distance) + : null; + logTiming?.("candidate-dedupe-complete", { + itemRowCount: itemRows.length, + sourceRowCount: rows.length, + cutoffDistance, + }); + + const rowsWithChunkText = await hydratePartnerContentChunkText({ + rows, + maxRows: rerank + ? PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount + : rows.length, + logTiming, + }); + + if (!rerank) { + return { + rows: sortRowsByRelevanceScore(rowsWithChunkText), + reranked: false, + queryVector, + cutoffDistance, + itemSourceBestDistance, + }; + } + + logTiming?.("rerank-start", { + rowCount: rowsWithChunkText.length, + rerankerCandidateCount: Math.min( + rowsWithChunkText.length, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, + ), + }); + const rerankResult = await rerankPartnerSearchRows({ + query, + rows: rowsWithChunkText, + }); + logTiming?.("rerank-complete", { + reranked: rerankResult.reranked, + rowCount: rerankResult.rows.length, + rerankedRowCount: rerankResult.rows.filter( + ({ rerankScore }) => rerankScore != null, + ).length, + }); + return { + ...rerankResult, + rows: sortRowsByRelevanceScore(rerankResult.rows), + queryVector, + cutoffDistance, + itemSourceBestDistance, + }; +} + +async function hydratePartnerContentSearchRows({ + candidateRows, + logTiming, +}: { + candidateRows: PartnerContentSearchCandidateRow[]; + logTiming?: PartnerContentSearchTimingLogger; +}) { + if (candidateRows.length === 0) return []; + + const chunkIds = candidateRows.map(({ chunkId }) => chunkId); + // Eligibility + user filters (enrolled/ignored/starred/country/platform) are all + // pre-filtered in the ANN now, so hydration is a plain metadata fetch over + // already-eligible chunk ids. networkStatus is the one remaining post-filter + // (cheap and non-selective — see the search query note); the partner/platform + // joins stay only to supply display columns and that networkStatus check. + + // Plain relational hydration over an already-eligible chunk-id set: the inner + // joins in the prior raw form were pure filters (all relations are required + // FKs), and the lone post-ANN predicate, networkStatus, maps to the partner + // relation filter below. `chunkText` is deliberately not selected here (it's + // hydrated separately in hydratePartnerContentChunkText); we backfill the "" + // placeholder when flattening to the row shape. + const hydratedRows = await prisma.partnerContentChunk.findMany({ + where: { + id: { in: chunkIds }, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + partner: { + networkStatus: { in: ["approved", "trusted"] }, + }, + }, + select: { + id: true, + partnerContentItemId: true, + partnerId: true, + source: true, + startMs: true, + endMs: true, + partner: { + select: { + name: true, + username: true, + image: true, + description: true, + }, + }, + partnerContentItem: { + select: { + platformContentId: true, + url: true, + contentType: true, + title: true, + description: true, + thumbnailUrl: true, + publishedAt: true, + durationMs: true, + viewCount: true, + likeCount: true, + commentCount: true, + shareCount: true, + saveCount: true, + partnerPlatform: { + select: { + type: true, + identifier: true, + }, + }, + }, + }, + }, + }); + const hydratedByChunkId = new Map( + hydratedRows.map(({ partner, partnerContentItem: item, ...chunk }): [ + string, + PartnerContentSearchHydrationRow, + ] => [ + chunk.id, + { + chunkId: chunk.id, + partnerContentItemId: chunk.partnerContentItemId, + partnerId: chunk.partnerId, + partnerName: partner.name, + partnerUsername: partner.username, + partnerImage: partner.image, + partnerDescription: partner.description, + platformType: item.partnerPlatform.type, + platformIdentifier: item.partnerPlatform.identifier, + platformContentId: item.platformContentId, + contentUrl: item.url, + contentType: item.contentType, + contentTitle: item.title, + contentDescription: item.description, + contentThumbnailUrl: item.thumbnailUrl, + contentPublishedAt: item.publishedAt, + contentDurationMs: item.durationMs, + contentViewCount: item.viewCount, + contentLikeCount: item.likeCount, + contentCommentCount: item.commentCount, + contentShareCount: item.shareCount, + contentSaveCount: item.saveCount, + chunkSource: chunk.source, + chunkText: "", + startMs: chunk.startMs, + endMs: chunk.endMs, + }, + ]), + ); + const rows = candidateRows.flatMap((candidate) => { + const hydrated = hydratedByChunkId.get(candidate.chunkId); + if (!hydrated) return []; + + return [ + { + ...hydrated, + distance: candidate.distance, + }, + ]; + }); + logTiming?.("vector-candidate-hydration-complete", { + candidateRowCount: candidateRows.length, + hydratedRowCount: hydratedRows.length, + filteredOutRowCount: candidateRows.length - rows.length, + }); + + return rows; +} + +async function hydratePartnerContentChunkText({ + rows, + maxRows, + logTiming, +}: { + rows: PartnerContentSearchRow[]; + maxRows: number; + logTiming?: PartnerContentSearchTimingLogger; +}) { + const rowsToHydrate = rows.slice(0, maxRows); + const chunkIds = rowsToHydrate.map(({ chunkId }) => chunkId); + + if (chunkIds.length === 0) return rows; + + logTiming?.("chunk-text-hydration-start", { + requestedRowCount: rowsToHydrate.length, + totalRowCount: rows.length, + }); + const chunkTextRows = await prisma.partnerContentChunk.findMany({ + where: { + id: { + in: chunkIds, + }, + }, + select: { + id: true, + chunkText: true, + }, + }); + const chunkTextById = new Map( + chunkTextRows.map((row) => [row.id, row.chunkText]), + ); + logTiming?.("chunk-text-hydration-complete", { + hydratedRowCount: chunkTextById.size, + }); + + return rows.map((row, index) => + index < maxRows + ? { + ...row, + chunkText: chunkTextById.get(row.chunkId) ?? row.chunkText, + } + : row, + ); +} diff --git a/apps/web/lib/partner-content-search/timing.ts b/apps/web/lib/partner-content-search/timing.ts new file mode 100644 index 00000000000..14afc8a0538 --- /dev/null +++ b/apps/web/lib/partner-content-search/timing.ts @@ -0,0 +1,59 @@ +// Stage timing logger for the partner content search route. Emits one structured +// console line per pipeline stage with the elapsed/delta timing plus a shared +// request context, throttling noisy sub-5ms stages unless they're on the +// always-log allowlist below. + +const MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS = 5; +const PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES = new Set([ + "query-embedding-complete", + "partner-eligibility-resolved", + "vector-candidate-search-complete", + "vector-candidate-hydration-complete", + "vector-search-pool-expanded", + "vector-search-complete", + "candidate-dedupe-complete", + "chunk-text-hydration-complete", + "rerank-complete", + "partner-candidates-grouped", + "match-summary-base-queries-complete", + "match-summary-content-counts-complete", + "match-summary-recent-content-complete", + "match-summary-followers-complete", + "match-summary-aggregation-complete", + "partner-hydration-complete", + "response-ready", +]); + +export type PartnerContentSearchTimingLogger = ( + stage: string, + metadata?: Record, +) => void; + +export function createPartnerContentSearchTimingLogger( + context: Record, +): PartnerContentSearchTimingLogger { + const startedAt = Date.now(); + let previousAt = startedAt; + + return (stage, metadata = {}) => { + const now = Date.now(); + const elapsedMs = now - startedAt; + const deltaMs = now - previousAt; + previousAt = now; + + if ( + deltaMs < MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS && + !PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES.has(stage) + ) { + return; + } + + console.info("[partner-content-search:timing]", { + stage, + elapsedMs, + deltaMs, + ...context, + ...metadata, + }); + }; +} diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index ab21daef080..0a1e294eac1 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -1,5 +1,11 @@ import { REACH_TIER_KEYS } from "@/lib/api/network/reach-tiers"; import { processKey } from "@/lib/api/links/utils"; +import { + PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER, + PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, +} from "@/lib/partner-content-search/constants"; import { PlatformType } from "@prisma/client"; import * as z from "zod/v4"; import { booleanQuerySchema, getPaginationQuerySchema } from "./misc"; @@ -123,3 +129,36 @@ export const invitePartnerFromNetworkSchema = z.object({ emailTitle: z.string().trim().max(255).optional(), emailBody: z.string().trim().max(3000).optional(), }); + +const PARTNER_NETWORK_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT = 20; + +// Request body for POST /api/network/partners/content-search (semantic search +// over indexed partner content). +export const partnerNetworkContentSearchSchema = z.object({ + query: z.string().trim().max(500).optional(), + platforms: z.array(z.enum(PlatformType)).min(1).optional(), + reach: z.array(z.enum(REACH_TIER_KEYS)).min(1).optional(), + country: z.string().trim().min(1).optional(), + partnerIds: z.array(z.string()).min(1).max(100).optional(), + starred: z.boolean().optional(), + limit: z + .number() + .int() + .positive() + .max(PARTNER_CONTENT_SEARCH_PARTNER_LIMIT) + .default(PARTNER_NETWORK_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT), + chunksPerPartner: z + .number() + .int() + .positive() + .max(PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER) + .default(PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER), + candidateChunkCount: z + .number() + .int() + .positive() + .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) + .optional(), + // Second-stage reranking is on by default; pass `false` for diagnostics. + rerank: z.boolean().default(true), +}); From 926162c3a6378b239fffde6567d2244db7a3eba0 Mon Sep 17 00:00:00 2001 From: David Clark Date: Mon, 22 Jun 2026 21:50:53 -0400 Subject: [PATCH 15/25] refactor partner content search logic; simplify ranking --- .../api/admin/partner-content/search/route.ts | 90 +----- .../api/cron/partner-content/embed/route.ts | 91 +----- .../cron/partner-content/enumerate/route.ts | 45 +-- .../cron/partner-content/transcript/route.ts | 177 +---------- .../network/partners/content-search/route.ts | 35 +-- .../network/[partnerId]/page-client.tsx | 241 +++------------ .../program/network/content-display-utils.ts | 120 ++++++++ .../network-content-search-results.tsx | 144 +-------- .../chunk-transcript.ts | 3 +- .../lib/partner-content-search/constants.ts | 90 +----- .../ingestion/chunk-counts.ts | 6 +- .../ingestion/enqueue.ts | 187 ++++++++++-- .../ingestion/write-fetched-content-items.ts | 12 +- .../ingestion/write-transcript-chunks.ts | 118 ++++++++ .../web/lib/partner-content-search/listing.ts | 5 +- .../partner-content-search/match-summaries.ts | 139 ++------- .../network-partners.ts | 5 +- .../web/lib/partner-content-search/ranking.ts | 286 ++++-------------- .../lib/partner-content-search/retrieval.ts | 142 +++++---- .../partner-content-search/search-utils.ts | 52 ++-- apps/web/lib/partner-content-search/timing.ts | 6 +- .../top-content-ranking.ts | 45 +-- apps/web/lib/partner-content-search/voyage.ts | 21 +- apps/web/lib/zod/schemas/partner-network.ts | 33 ++ .../partner-content-search/ranking.test.ts | 108 ++++--- 25 files changed, 833 insertions(+), 1368 deletions(-) create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts create mode 100644 apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts diff --git a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts index 6c9f1a24454..3734b90dccc 100644 --- a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts +++ b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts @@ -2,11 +2,11 @@ import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; import { parseRequestBody } from "@/lib/api/utils"; import { withAdmin } from "@/lib/auth"; import { - PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, PARTNER_CONTENT_SEARCH_LIMITS, PARTNER_CONTENT_SEARCH_MODELS, PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, } from "@/lib/partner-content-search/constants"; +import { searchAdminPartnerContentChunks } from "@/lib/partner-content-search/retrieval"; import { groupPartnerSearchResults, rerankPartnerSearchRows, @@ -18,44 +18,17 @@ import { serializeEmbeddingForVector, VoyageTimeoutError, } from "@/lib/partner-content-search/voyage"; -import { prisma } from "@/lib/prisma"; -import { PlatformType, Prisma } from "@prisma/client"; +import { partnerAdminContentSearchSchema } from "@/lib/zod/schemas/partner-network"; import { NextResponse } from "next/server"; -import * as z from "zod/v4"; export const dynamic = "force-dynamic"; export const maxDuration = 30; -const DEFAULT_PARTNER_LIMIT = 10; -const DEFAULT_CHUNKS_PER_PARTNER = 3; - -const partnerContentSearchSchema = z.object({ - query: z.string().trim().min(1).max(500), - limit: z.number().int().positive().max(50).default(DEFAULT_PARTNER_LIMIT), - chunksPerPartner: z - .number() - .int() - .positive() - .max(10) - .default(DEFAULT_CHUNKS_PER_PARTNER), - candidateChunkCount: z - .number() - .int() - .positive() - .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) - .optional(), - partnerIds: z.array(z.string()).min(1).max(100).optional(), - platform: z.enum(PlatformType).optional(), - // Second-stage reranking is on by default; pass `false` to inspect cosine-only - // ranking for comparison. - rerank: z.boolean().default(true), -}); - // POST /api/admin/partner-content/search export const POST = withAdmin( async ({ req }) => { try { - const body = partnerContentSearchSchema.parse( + const body = partnerAdminContentSearchSchema.parse( await parseRequestBody(req), ); const candidateChunkCount = @@ -83,7 +56,7 @@ export const POST = withAdmin( } const queryVector = serializeEmbeddingForVector(queryEmbedding); - const candidateRows = await searchPartnerContentChunks({ + const candidateRows = await searchAdminPartnerContentChunks({ queryVector, limit: candidateChunkCount, partnerIds: body.partnerIds, @@ -122,61 +95,6 @@ export const POST = withAdmin( }, ); -async function searchPartnerContentChunks({ - queryVector, - limit, - partnerIds, - platform, -}: { - queryVector: string; - limit: number; - partnerIds?: string[]; - platform?: PlatformType; -}) { - const partnerFilter = partnerIds?.length - ? Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})` - : Prisma.empty; - const platformFilter = platform - ? Prisma.sql`AND pp.type = ${platform}` - : Prisma.empty; - - return await prisma.$queryRaw(Prisma.sql` - SELECT - c.id AS chunkId, - c.partnerContentItemId, - c.partnerId, - p.name AS partnerName, - p.username AS partnerUsername, - p.image AS partnerImage, - p.description AS partnerDescription, - pp.type AS platformType, - pp.identifier AS platformIdentifier, - pci.platformContentId, - pci.url AS contentUrl, - pci.contentType, - pci.title AS contentTitle, - pci.thumbnailUrl AS contentThumbnailUrl, - pci.publishedAt AS contentPublishedAt, - pci.durationMs AS contentDurationMs, - c.source AS chunkSource, - c.chunkIndex, - c.chunkText, - c.startMs, - c.endMs, - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance - FROM PartnerContentChunk c - INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId - INNER JOIN Partner p ON p.id = c.partnerId - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - WHERE c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - ${partnerFilter} - ${platformFilter} - ORDER BY distance ASC - LIMIT ${limit} - `); -} - function toChunkResult(row: PartnerContentSearchRow, distance: number) { return { chunkId: row.chunkId, diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts index d30f2449766..33c079fc3ba 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -4,12 +4,12 @@ import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/cons import { refreshPartnerContentItemChunkCounts } from "@/lib/partner-content-search/ingestion/chunk-counts"; import { createPartnerContentDeduplicationId, + enqueueEmbedJobsForPartnerPlatform, getPartnerContentUrl, parsePartnerContentCronPayload, PARTNER_CONTENT_EMBED_FLOW_CONTROL, PARTNER_CONTENT_SEARCH_ROUTES, partnerContentEmbedPayloadSchema, - PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; import { embedPartnerContentTexts, @@ -52,10 +52,8 @@ export const POST = withCron(async ({ rawBody }) => { partnerId: true, totalChunkCount: true, embeddedChunkCount: true, - // Source values for the denormalized search pre-filter columns on - // PartnerContentChunk. The embed write is the single choke point that makes - // a chunk searchable, so we stamp these here to keep new chunks correctly - // filterable the moment they gain an embedding. + // Source values for the denormalized pre-filter columns (country, platformType), + // stamped on the chunk at embed time — the point it becomes searchable. partner: { select: { country: true, @@ -148,13 +146,8 @@ export const POST = withCron(async ({ rawBody }) => { ); } - // Denormalized search pre-filter values, stamped alongside the embedding so a - // chunk is correctly filterable as soon as it becomes searchable. All chunks in - // this job share one content item, hence one partner/platform, so these are - // constant across the batch. Both are effectively static (platformType immutable, - // country ~never changes); a periodic reconcile catches the rare country edit. - // networkStatus is intentionally not denormalized — it stays a post-filter in the - // search query (ingestion already gates content on approved/trusted partners). + // Denormalized pre-filter values stamped with the embedding (constant across the + // batch — one content item → one partner/platform). Both are ~static const country = contentItem.partner.country; const platformType = contentItem.partnerPlatform.type; @@ -207,80 +200,6 @@ export const POST = withCron(async ({ rawBody }) => { ); }); -async function enqueueEmbedJobsForPartnerPlatform({ - mode, - runStamp, - partnerId, - partnerPlatformId, - limitContentItems, - maxChunks, -}: { - mode: PartnerContentIngestionMode; - runStamp: string; - partnerId: string; - partnerPlatformId?: string; - limitContentItems: number; - maxChunks: number; -}) { - if (!partnerPlatformId) { - return logAndRespond( - `[PartnerContentSearch] Embed enqueue requires partnerPlatformId for ${mode} run ${runStamp}.`, - { status: 400, logLevel: "warn" }, - ); - } - - const contentItems = await prisma.partnerContentItem.findMany({ - where: { - partnerId, - partnerPlatformId, - totalChunkCount: { - gt: 0, - }, - }, - select: { - id: true, - totalChunkCount: true, - embeddedChunkCount: true, - }, - orderBy: { - id: "asc", - }, - take: limitContentItems, - }); - - const pendingContentItems = contentItems.filter( - ({ totalChunkCount, embeddedChunkCount }) => - embeddedChunkCount < totalChunkCount, - ); - - const messages = pendingContentItems.map((contentItem) => ({ - url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), - method: "POST" as const, - flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, - body: { - mode, - runStamp, - partnerId, - partnerContentItemId: contentItem.id, - maxChunks, - }, - deduplicationId: createPartnerContentDeduplicationId( - "partner-content-embed", - mode, - runStamp, - contentItem.id, - ), - })); - - if (messages.length > 0) { - await qstash.batchJSON(messages); - } - - return logAndRespond( - `[PartnerContentSearch] Enqueued ${messages.length} embed jobs (${pendingContentItems.length} pending of ${contentItems.length} inspected) for partner platform ${partnerPlatformId} on ${mode} run ${runStamp}.`, - ); -} - async function refreshEmbeddedChunkCount(partnerContentItemId: string) { const { embeddedChunkCount } = await refreshPartnerContentItemChunkCounts(partnerContentItemId); diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts index bfa0f09bd97..adefdb8b0f8 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts @@ -1,18 +1,15 @@ import { qstash } from "@/lib/cron"; import { withCron } from "@/lib/cron/with-cron"; import { - buildEligiblePartnerPlatformWhere, + buildEligiblePartnerWhere, createPartnerContentDeduplicationId, getPartnerContentUrl, PARTNER_CONTENT_ENUMERATE_PAGE_SIZE, PARTNER_CONTENT_SEARCH_ROUTES, parsePartnerContentCronPayload, partnerContentEnumeratePayloadSchema, - PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; -import { PartnerContentPlatform } from "@/lib/partner-content-search/types"; import { prisma } from "@/lib/prisma"; -import { Prisma } from "@prisma/client"; import { logAndRespond } from "../../utils"; export const dynamic = "force-dynamic"; @@ -26,9 +23,8 @@ export const POST = withCron(async ({ rawBody }) => { ); if (payload instanceof Response) return payload; - // Process a single page per invocation and hand the next cursor back to - // this same route, rather than draining the whole partner set in one - // request (keeps each run well under maxDuration regardless of scale). + // One page per invocation, handing the next cursor back to this route — keeps + // each run under maxDuration regardless of scale. const remainingPartners = payload.remainingPartners ?? payload.filter.limitPartners; @@ -92,8 +88,7 @@ export const POST = withCron(async ({ rawBody }) => { partnerIds, }, }, - // Self-continuation hop: re-enter this route from the last id instead of - // draining the whole partner set in a single invocation. + // Self-continuation hop: re-enter from the last id instead of draining in one go. ...(hasMore ? [ { @@ -132,35 +127,3 @@ export const POST = withCron(async ({ rawBody }) => { }${hasMore ? ` (continuing after ${lastPartnerId})` : " (final page)"}.`, ); }); - -function buildEligiblePartnerWhere({ - mode, - filter, -}: { - mode: PartnerContentIngestionMode; - filter: { - partnerId?: string; - partnerIds?: string[]; - platforms: PartnerContentPlatform[]; - }; -}): Prisma.PartnerWhereInput { - return { - networkStatus: { - in: ["approved", "trusted"], - }, - ...(filter.partnerId && { - id: filter.partnerId, - }), - ...(filter.partnerIds?.length && { - id: { - in: filter.partnerIds, - }, - }), - platforms: { - some: buildEligiblePartnerPlatformWhere({ - mode, - platforms: filter.platforms, - }), - }, - }; -} diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts index 3cf5218b298..da60ac65c42 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -1,23 +1,10 @@ -import { createId } from "@/lib/api/create-id"; -import { qstash } from "@/lib/cron"; import { withCron } from "@/lib/cron/with-cron"; import { - chunkTranscriptSegments, - hashTranscript, -} from "@/lib/partner-content-search/chunk-transcript"; -import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; -import { refreshPartnerContentItemChunkCounts } from "@/lib/partner-content-search/ingestion/chunk-counts"; -import { - createPartnerContentDeduplicationId, - getPartnerContentUrl, + enqueueEmbedJob, parsePartnerContentCronPayload, - PARTNER_CONTENT_EMBED_FLOW_CONTROL, - PARTNER_CONTENT_SEARCH_ROUTES, partnerContentTranscriptPayloadSchema, - type PartnerContentIngestionMode, } from "@/lib/partner-content-search/ingestion/enqueue"; -import { fetchPlatformTranscriptSegments } from "@/lib/partner-content-search/ingestion/platforms"; -import type { PartnerContentPlatform } from "@/lib/partner-content-search/types"; +import { writeTranscriptChunks } from "@/lib/partner-content-search/ingestion/write-transcript-chunks"; import { prisma } from "@/lib/prisma"; import { logAndRespond } from "../../utils"; @@ -64,10 +51,9 @@ export const POST = withCron(async ({ rawBody }) => { ); } - // QStash delivers at least once. If this transcript was already fetched and - // chunked, skip the (credit-burning) ScrapeCreators re-fetch — but still - // (re-)enqueue the embed job so a previously-failed enqueue is recovered on - // redelivery. Admin re-ingestion can force a fresh fetch via forceRefetch. + // QStash delivers at least once. If already fetched+chunked, skip the credit-burning + // re-fetch but still re-enqueue embed (recovers a previously-failed enqueue). + // forceRefetch forces a fresh fetch. if ( contentItem.transcriptFetchStatus === "fetched" && !payload.forceRefetch @@ -129,156 +115,3 @@ export const POST = withCron(async ({ rawBody }) => { `[PartnerContentSearch] Transcribed content item ${contentItem.id}: ${transcriptWriteResult.chunkCount} chunks (${transcriptWriteResult.chunksCreated} created), embed enqueued for ${payload.mode} run ${payload.runStamp}.`, ); }); - -async function writeTranscriptChunks(contentItem: { - id: string; - partnerId: string; - url: string; - platform: PartnerContentPlatform; -}) { - const transcriptSegments = await fetchPlatformTranscriptSegments({ - platform: contentItem.platform, - url: contentItem.url, - }); - const normalizedTranscript = transcriptSegments - .map(({ text }) => text.trim()) - .filter(Boolean) - .join(" "); - const transcriptHash = - transcriptSegments.length > 0 ? hashTranscript(transcriptSegments) : null; - - if (!transcriptHash) { - await prisma.partnerContentItem.update({ - where: { - id: contentItem.id, - }, - data: { - transcriptFetchStatus: "notAvailable", - transcriptLastAttemptedAt: new Date(), - normalizedTranscript: null, - transcriptHash: null, - transcriptHasTimestamps: false, - }, - }); - - await prisma.partnerContentChunk.deleteMany({ - where: { - partnerContentItemId: contentItem.id, - source: "transcript", - }, - }); - - await refreshPartnerContentItemChunkCounts(contentItem.id); - - return { - transcriptAvailable: false, - segmentCount: 0, - normalizedTranscriptLength: 0, - transcriptHash: null, - chunkCount: 0, - chunksCreated: 0, - }; - } - - const chunks = chunkTranscriptSegments(transcriptSegments); - const transcriptHasTimestamps = transcriptSegments.some( - ({ startMs, endMs }) => startMs !== null || endMs !== null, - ); - const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.id; - - const [, deletedChunks, createChunksResult] = await prisma.$transaction([ - prisma.partnerContentItem.update({ - where: { - id: contentItem.id, - }, - data: { - transcriptFetchStatus: "fetched", - transcriptLastAttemptedAt: new Date(), - normalizedTranscript, - transcriptHash, - transcriptHasTimestamps, - embeddingModel, - lastFetchedAt: new Date(), - }, - }), - prisma.partnerContentChunk.deleteMany({ - where: { - partnerContentItemId: contentItem.id, - source: "transcript", - }, - }), - prisma.partnerContentChunk.createMany({ - data: chunks.map((chunk) => ({ - id: createId({ prefix: "pcc_" }), - partnerContentItemId: contentItem.id, - partnerId: contentItem.partnerId, - source: "transcript", - chunkIndex: chunk.chunkIndex, - chunkText: chunk.text, - startMs: chunk.startMs, - endMs: chunk.endMs, - textHash: transcriptHash, - embeddingModel, - })), - }), - ]); - - await refreshPartnerContentItemChunkCounts(contentItem.id); - - return { - transcriptAvailable: true, - segmentCount: transcriptSegments.length, - normalizedTranscriptLength: normalizedTranscript.length, - transcriptHash, - chunkCount: chunks.length, - chunksDeleted: deletedChunks.count, - chunksCreated: createChunksResult.count, - }; -} - -async function enqueueEmbedJob({ - mode, - runStamp, - partnerId, - partnerContentItemId, -}: { - mode: PartnerContentIngestionMode; - runStamp: string; - partnerId: string; - partnerContentItemId: string; -}) { - try { - await qstash.publishJSON({ - url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), - method: "POST", - body: { - mode, - runStamp, - partnerId, - partnerContentItemId, - }, - flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, - deduplicationId: createPartnerContentDeduplicationId( - "partner-content-embed", - mode, - runStamp, - partnerContentItemId, - ), - }); - } catch (error) { - // Don't swallow this: the transcript chunks are already committed, so a - // dropped embed job leaves them permanently unsearchable. Rethrow so - // withCron returns 500 and QStash retries the whole transcript job — the - // forceRefetch=false guard above makes that retry skip the re-fetch and - // just re-enqueue here, and the embed deduplicationId keeps it idempotent. - console.error("[PartnerContentSearch] Failed to enqueue embed job", { - error, - mode, - runStamp, - partnerId, - partnerContentItemId, - }); - - throw error; - } -} diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index 91450a0faef..5face1e6740 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -14,6 +14,7 @@ import { createContentMatchEvidence, getEvidenceSource, getRowRelevanceScore, + sortPartnersByTopicFit, } from "@/lib/partner-content-search/ranking"; import { searchPartnerNetworkContent } from "@/lib/partner-content-search/retrieval"; import { @@ -134,7 +135,6 @@ export const POST = withWorkspace( rows, partnerIds: partnerCandidates.map(({ partnerId }) => partnerId), platforms: body.platforms, - query: body.query, queryVector, cutoffDistance, itemSourceBestDistance, @@ -200,39 +200,6 @@ function isNonNull(value: T | null): value is T { return value !== null; } -function sortPartnersByTopicFit< - T extends { - score: number; - matchSummary: { - topicFit: number; - weightedMatchedContentScore: number; - weightedMatchedContentCount: number; - transcriptMatchedContentCount: number; - matchedContentCount: number; - followers: number | null; - } | null; - }, ->(partners: T[]) { - return [...partners].sort((a, b) => { - const aSummary = a.matchSummary; - const bSummary = b.matchSummary; - - return ( - (bSummary?.topicFit ?? 0) - (aSummary?.topicFit ?? 0) || - (bSummary?.weightedMatchedContentScore ?? 0) - - (aSummary?.weightedMatchedContentScore ?? 0) || - (bSummary?.weightedMatchedContentCount ?? 0) - - (aSummary?.weightedMatchedContentCount ?? 0) || - (bSummary?.transcriptMatchedContentCount ?? 0) - - (aSummary?.transcriptMatchedContentCount ?? 0) || - (bSummary?.matchedContentCount ?? 0) - - (aSummary?.matchedContentCount ?? 0) || - b.score - a.score || - (bSummary?.followers ?? 0) - (aSummary?.followers ?? 0) - ); - }); -} - function toChunkResult(row: PartnerContentSearchRow, distance: number) { return { chunkId: row.chunkId, diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 3b39269c695..0e0edda9880 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -1,8 +1,8 @@ "use client"; import { + PARTNER_CONTENT_SEARCH_DETAIL_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_LIMITS, - PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_TOP_CONTENT, type PartnerContentTopicFitBand, } from "@/lib/partner-content-search/constants"; @@ -10,7 +10,6 @@ import { getBlendedTopContentScore, getViewBaseline, } from "@/lib/partner-content-search/top-content-ranking"; -import { getPartnerContentThumbnailUrl } from "@/lib/partner-content-search/thumbnail-url"; import { mutatePrefix } from "@/lib/swr/mutate"; import usePartnerContentSearch, { type PartnerContentMatchEvidence, @@ -28,6 +27,16 @@ import { cn, fetcher, nFormatter } from "@dub/utils"; import { EmailContent } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-email-preview"; import { InviteNetworkPartnerSheet } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-network-partner-sheet"; import Link from "next/link"; +import { + BAND_LABELS, + formatDuration, + formatMatchPercent, + formatPublishedDate, + formatTimestamp, + getContentHref, + getContentThumbnail, + getContentTitle, +} from "../content-display-utils"; import { getContentSearchPlatforms, parseSelectedPlatforms, @@ -94,20 +103,11 @@ export function NetworkPartnerDetailContent({ ); const loadedPartner = partners?.[0]; - // The results page already ran the (global) Voyage search and handed us this - // partner's match data on click, so we render straight from it — opening the - // detail is instant. In the background we also run a scoped, single-partner - // search whenever there's a content-search context, for one reason: to put the - // matched-content list on a SINGLE relevance scale. The global search only - // reranks items that made its top-N candidate pool, so a creator's other matched - // posts fall back to raw cosine — two incomparable scales mixed in one list (the - // tell-tale "65% vs 95%" banding). A per-partner rerank covers all of this - // creator's items, so every row shares the reranker scale. - // - // This is non-blocking: cached summary data can paint the stable Topic Fit - // headline immediately, while content rows stay skeletoned until the scoped - // run resolves. That avoids showing global transcript snippets or row order and - // then swapping them for the single-partner reranked result. + // The results page already ran the global search and handed us this partner's + // match data, so the detail opens instantly. In the background we run a scoped + // single-partner rerank so the matched list shares ONE relevance scale — the + // global search only reranks its top-N pool, mixing rerank + cosine otherwise. + // Non-blocking: the cached Topic Fit headline paints now; rows skeleton until it lands. const shouldFetchSearch = hasContentSearch; const { data: searchResults, @@ -121,25 +121,21 @@ export function NetworkPartnerDetailContent({ starred: false, partnerIds: [partnerId], limit: 1, - chunksPerPartner: PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER, + chunksPerPartner: PARTNER_CONTENT_SEARCH_DETAIL_CHUNKS_PER_PARTNER, candidateChunkCount: PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, }); const fetchedPartner = searchResults?.partners?.[0]; const partner = loadedPartner ?? initialSearchPartner?.partner ?? fetchedPartner?.partner; - // Chunks (snippet text) prefer the fuller fetched set once it arrives; the - // summary always prefers the cached one so scores never drift on click. + // Chunks prefer the fuller fetched set; the summary keeps the cached one so scores don't drift. const searchPartner = fetchedPartner ?? initialSearchPartner; const searchSummary = initialSearchPartner?.matchSummary ?? fetchedPartner?.matchSummary; - // The matched-content list's per-row relevance comes from the scoped, fully - // reranked summary (one scale) once it lands; null until then, so the list - // shows cached scores and quietly upgrades. Everything else stays on the - // cached summary above. + // Per-row relevance comes from the scoped reranked summary once it lands (one + // scale); null until then, so rows show cached scores and quietly upgrade. const relevanceSummary = fetchedPartner?.matchSummary ?? null; - // A deep-link has nothing cached → full skeleton. With cached data in hand the - // scoped run is a silent, non-blocking relevance refinement. + // Deep-link with nothing cached → full skeleton; otherwise a silent refinement. const isFallbackLoading = isLoadingSearch && !initialSearchPartner; const isRefiningRelevance = isLoadingSearch && Boolean(initialSearchPartner) && !relevanceSummary; @@ -279,14 +275,6 @@ function NetworkInviteControl({ ); } -const TOPIC_FIT_BAND_LABELS: Record = { - consistent: "Consistent", - frequent: "Frequent", - occasional: "Occasional", - "one-off": "One-off", - none: "No recent match", -}; - const TOPIC_FIT_BAND_STYLES: Record< PartnerContentTopicFitBand, { number: string; chip: string } @@ -312,10 +300,8 @@ function lastPostedLabel(iso: string | null) { return `${Math.floor(days / 365)}y ago`; } -// Per-post match bars: height encodes match magnitude, color hints at the -// platform. The palette is intentionally muted + evenly weighted (no full- -// saturation brand colors, no black) so platforms read as a calm spectrum -// rather than high-contrast — and so no platform looks like a "non-match". +// Per-post match bars: height = match magnitude, color = platform. Muted, evenly- +// weighted palette so platforms read as a calm spectrum (none looks like a non-match). const PLATFORM_BAR_COLORS: Record = { youtube: "bg-[#bd8488]", instagram: "bg-[#b083a2]", @@ -326,10 +312,8 @@ const PLATFORM_BAR_COLORS: Record = { website: "bg-[#bda77c]", }; -// The bars are a glanceable recent-activity strip, not the full archive — cap -// the visual to the most-recent N so columns stay wide enough to hover and the -// row doesn't turn into 200 hairlines. Topic Fit + the "X of Y on topic" counts -// still derive from the full server-side recent set, so scoring is unaffected. +// Glanceable recent-activity strip, capped to the most-recent N so columns stay +// hoverable. Topic Fit + the "X of Y" counts still use the full server-side set. const MAX_VISIBLE_CONTENT_BARS = 40; function ContentMatchBars({ @@ -337,9 +321,8 @@ function ContentMatchBars({ }: { summary: PartnerContentSearchPartner["matchSummary"] | undefined; }) { - // One open tooltip at a time: each bar is its own tooltip root that only - // closes on its own pointerleave, so a fast cursor flick can leave several - // open. Driving every bar's open state from one value prevents that. + // One open tooltip at a time — driving every bar's open state from one value + // prevents a fast cursor flick leaving several open. const [openBarId, setOpenBarId] = useState(null); const allBars = summary?.contentBars ?? []; @@ -361,10 +344,8 @@ function ContentMatchBars({ const isCreatorTextOnlyVideoMatch = bar.matchEvidence.primarySource === "creatorText" && bar.matchEvidence.weight < 1; - // The whole column (full height) is the hover/click target, not just the - // short bar — far easier to land on. On hover a gold wash fills the - // column and the bar itself turns gold, so the moused-over post is - // unmistakable. + // The whole column is the hover/click target (easier to land on); on hover + // a gold wash fills it and the bar turns gold. const columnClassName = cn( "group flex h-full min-w-[5px] flex-1 items-end rounded-[3px] transition-colors duration-75 hover:bg-amber-100/70", bar.url && "cursor-pointer", @@ -389,9 +370,7 @@ function ContentMatchBars({ } - // Snappy bar-to-bar hover: open instantly, drop the hoverable grace - // area, and skip the animation so each tooltip closes immediately - // rather than lingering as you sweep across. + // Snappy bar-to-bar hover: open instantly, no grace area, no close animation. delayDuration={0} disableHoverableContent disableAnimation @@ -426,8 +405,7 @@ function ContentMatchBars({ ); } -// Subtle hover card for a single content bar: platform, title, and the post's -// date · length · views — all already on the cached summary, no extra fetch. +// Hover card for a content bar: platform, title, date · length · views (all cached). function BarTooltip({ bar, }: { @@ -471,12 +449,10 @@ function SearchFitPanel({ }: { error: unknown; isLoading: boolean; - // The scoped single-partner rerank is in flight (cached scores still on screen); - // when it lands, every row's relevance moves onto one consistent scale. + // Scoped rerank in flight; when it lands, every row's relevance moves to one scale. isRefining?: boolean; summary?: PartnerContentSearchPartner["matchSummary"]; - // Scoped, fully reranked summary used only to put the list's per-row relevance - // on a single scale. Null until the background rerank resolves. + // Scoped reranked summary, only to put per-row relevance on one scale. Null until it resolves. relevanceSummary?: PartnerContentSearchPartner["matchSummary"] | null; searchPartner?: PartnerContentSearchPartner; }) { @@ -492,16 +468,14 @@ function SearchFitPanel({ const isLoadingRows = isLoading || isRefining; // Per-item relevance on a single scale, from the scoped reranked summary. const unifiedRelevanceByItemId = buildUnifiedRelevanceMap(relevanceSummary); - // Keep cached summary data for the headline, but hold row rendering until the - // scoped run finishes so transcript snippets and row ordering do not swap in. + // Hold row rendering until the scoped run finishes so snippets/order don't swap in. const items = buildMatchedContentItems( summary, isLoadingRows ? [] : (searchPartner?.chunks ?? []), unifiedRelevanceByItemId, ); - // Top content: the relevance-led blend of relevance + reach (robust to a single - // viral post). All content: the same matched set, simply newest-first. + // Top content: relevance + reach blend. All content: same set, newest-first. const topContent = [...items] .sort((a, b) => b.blendedScore - a.blendedScore) .slice(0, PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount); @@ -510,8 +484,7 @@ function SearchFitPanel({ ); const visibleAll = allContent.slice(0, visibleAllCount); const hiddenAllCount = Math.max(0, allContent.length - visibleAll.length); - // A separate "All content" list only earns its place when it adds rows beyond - // the top set; with ≤ topContentCount matches the top list already shows them all. + // "All content" only earns its place when it adds rows beyond the top set. const showAllSection = !isLoadingRows && allContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; @@ -550,7 +523,7 @@ function SearchFitPanel({ bandStyles.chip, )} > - {TOPIC_FIT_BAND_LABELS[band]} + {BAND_LABELS[band]}
@@ -747,9 +720,8 @@ function ContentMatchSkeletons({ count }: { count: number }) { ); } -// A matched post for the detail-pane lists. Built from the cached summary's -// content bars (complete + instant) and enriched with a loaded chunk (snippet, -// timed transcript, richer thumbnail) when one is available. +// A matched post for the detail lists: from the cached summary's bars, enriched +// with a loaded chunk (snippet, timed transcript, thumbnail) when available. type MatchedContentItem = { contentItemId: string; platform: string; @@ -767,10 +739,8 @@ type MatchedContentItem = { chunk?: PartnerContentSearchPartner["chunks"][number]; }; -// Per-item relevance from the scoped, fully reranked summary, keyed by content -// item. Used to upgrade each list row onto a single reranker scale (the cached -// global summary mixes reranker and cosine scores). Includes any item with -// evidence — not just `matched` ones — so boundary items still get unified. +// Per-item relevance from the scoped reranked summary (one scale; the cached global +// summary mixes rerank + cosine). Includes any item with evidence, not just matched. function buildUnifiedRelevanceMap( relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, ) { @@ -801,15 +771,13 @@ function buildMatchedContentItems( } } - // Per-creator engagement baseline: median views across all recent posts - // (matched + unmatched), so a viral hit can't skew the normalization. + // Per-creator engagement baseline: median recent views (matched + unmatched). const baselineViews = getViewBaseline(bars.map((bar) => bar.viewCount)); return bars .filter((bar) => bar.matched) .map((bar) => { - // Prefer the unified (single-scale) relevance when the scoped rerank has - // landed; otherwise fall back to the cached score so rows render instantly. + // Prefer the unified single-scale relevance once the rerank lands; else cached. const relevance = unifiedRelevanceByItemId?.get(bar.partnerContentItemId) ?? getEvidenceDisplayScore(bar.matchEvidence) ?? @@ -968,9 +936,8 @@ function getMatchTags( })); } -// The displayed snippet. Transcript chunks are real prose; creator-text chunks -// store the raw embedding input ("Content type: video Title: … Description: …"), -// so we surface just the creator-entered text for a cleaner preview. +// Displayed snippet. Transcript chunks are prose; creator-text chunks store the raw +// embedding input, so we surface just the creator-entered text. function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { const text = (chunk.chunk.text ?? "").trim(); if (!text) return null; @@ -1003,23 +970,9 @@ function PlatformIcon({ return Icon ? : null; } -function getPreviewThumbnail( - chunk: PartnerContentSearchPartner["chunks"][number], -) { - if (chunk.content.thumbnailUrl) { - return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); - } - if (chunk.platform.type === "youtube") { - return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; - } - return null; -} - -// Thumbnail for a matched item: the loaded chunk's preview when enriched, -// otherwise a YouTube thumbnail derived from the content id (the only platform -// with a stable URL pattern from the bar data alone). +// Item thumbnail: the loaded chunk's preview, else a YouTube thumbnail from the id. function getItemThumbnail(item: MatchedContentItem) { - if (item.chunk) return getPreviewThumbnail(item.chunk); + if (item.chunk) return getContentThumbnail(item.chunk); if (item.platform === "youtube" && item.platformContentId) { return `https://i.ytimg.com/vi/${item.platformContentId}/hqdefault.jpg`; } @@ -1038,10 +991,8 @@ function getItemHref(item: MatchedContentItem) { return item.url ?? "#"; } -// A noun phrase describing exactly what window the ranks are computed over, for -// composing into the caption. The window is time-based (recencyWindowMonths) but -// capped per partner (recentContentMaxPerPartner); when that cap bites, say so -// explicitly instead of implying full coverage. +// Noun phrase for the rank window (time-based but capped per partner); says so +// explicitly when the cap bites instead of implying full coverage. function formatRankWindowPhrase( summary: PartnerContentSearchPartner["matchSummary"] | undefined, ) { @@ -1083,58 +1034,6 @@ function formatMonthYear(iso: string | null) { }).format(date); } -function getContentTitle(chunk: PartnerContentSearchPartner["chunks"][number]) { - return ( - chunk.content.title?.trim() || - chunk.content.description?.trim().split(/\r?\n/)[0] || - "Untitled content" - ); -} - -function getContentHref(chunk: PartnerContentSearchPartner["chunks"][number]) { - if (chunk.platform.type === "instagram") { - return getInstagramContentHref(chunk); - } - - if (chunk.platform.type !== "youtube" || chunk.chunk.startMs === null) { - return chunk.content.url; - } - - try { - const url = new URL(chunk.content.url); - url.searchParams.set("t", `${Math.floor(chunk.chunk.startMs / 1000)}s`); - return url.toString(); - } catch { - return chunk.content.url; - } -} - -function getInstagramContentHref( - chunk: PartnerContentSearchPartner["chunks"][number], -) { - const shortcode = - extractInstagramShortcode(chunk.content.url) || - chunk.content.platformContentId; - - return `https://www.instagram.com/${chunk.content.type === "reel" ? "reel" : "p"}/${shortcode}/`; -} - -function extractInstagramShortcode(url: string) { - try { - return ( - new URL(url).pathname.match( - /^\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, - )?.[1] ?? null - ); - } catch { - return ( - url.match( - /instagram\.com\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, - )?.[1] ?? null - ); - } -} - function hasTimedTranscriptMatch({ source, startMs, @@ -1156,48 +1055,6 @@ function formatChunkTimeRange({ return formatTimestamp(startMs ?? endMs ?? 0); } -function formatTimestamp(ms: number) { - const totalSeconds = Math.max(0, Math.floor(ms / 1000)); - const minutes = Math.floor(totalSeconds / 60); - const seconds = totalSeconds % 60; - - return `${minutes}:${seconds.toString().padStart(2, "0")}`; -} - -function formatDuration(durationMs: number | null) { - if (!durationMs || durationMs <= 0) return null; - - const totalSeconds = Math.floor(durationMs / 1000); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor((totalSeconds % 3600) / 60); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - - return `${minutes}:${seconds.toString().padStart(2, "0")}`; -} - -function formatPublishedDate(publishedAt: string | null) { - if (!publishedAt) return null; - - const date = new Date(publishedAt); - if (Number.isNaN(date.getTime())) return null; - - return new Intl.DateTimeFormat(undefined, { - month: "short", - day: "numeric", - year: "numeric", - }).format(date); -} - -function formatMatchPercent(score: number) { - return `${Math.round(Math.min(1, Math.max(0, score)) * 100)}%`; -} - function getBackQueryString(searchParams: { toString(): string }) { const params = new URLSearchParams(searchParams.toString()); const queryString = params.toString(); diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts new file mode 100644 index 00000000000..579e2c7ecaf --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts @@ -0,0 +1,120 @@ +import type { PartnerContentTopicFitBand } from "@/lib/partner-content-search/constants"; +import { getPartnerContentThumbnailUrl } from "@/lib/partner-content-search/thumbnail-url"; +import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; + +// Shared display helpers for the network search surfaces (detail page + results +// list): formatters, link/thumbnail resolvers, and the band label map. + +export type ContentSearchChunk = PartnerContentSearchPartner["chunks"][number]; + +export const BAND_LABELS: Record = { + consistent: "Consistent", + frequent: "Frequent", + occasional: "Occasional", + "one-off": "One-off", + none: "No recent match", +}; + +export function formatTimestamp(ms: number) { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +export function formatDuration(durationMs: number | null) { + if (!durationMs || durationMs <= 0) return null; + + const totalSeconds = Math.floor(durationMs / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds + .toString() + .padStart(2, "0")}`; + } + + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + +export function formatPublishedDate(publishedAt: string | null) { + if (!publishedAt) return null; + + const date = new Date(publishedAt); + if (Number.isNaN(date.getTime())) return null; + + return new Intl.DateTimeFormat(undefined, { + month: "short", + day: "numeric", + year: "numeric", + }).format(date); +} + +export function formatMatchPercent(score: number) { + return `${Math.round(Math.min(1, Math.max(0, score)) * 100)}%`; +} + +export function getContentThumbnail(chunk: ContentSearchChunk) { + if (chunk.content.thumbnailUrl) { + return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); + } + + if (chunk.platform.type === "youtube") { + return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; + } + + return null; +} + +export function getContentTitle(chunk: ContentSearchChunk) { + return ( + chunk.content.title?.trim() || + chunk.content.description?.trim().split(/\r?\n/)[0] || + "Untitled content" + ); +} + +export function getContentHref(chunk: ContentSearchChunk) { + if (chunk.platform.type === "instagram") { + return getInstagramContentHref(chunk); + } + + if (chunk.platform.type !== "youtube" || chunk.chunk.startMs === null) { + return chunk.content.url; + } + + try { + const url = new URL(chunk.content.url); + url.searchParams.set("t", `${Math.floor(chunk.chunk.startMs / 1000)}s`); + return url.toString(); + } catch { + return chunk.content.url; + } +} + +export function getInstagramContentHref(chunk: ContentSearchChunk) { + const shortcode = + extractInstagramShortcode(chunk.content.url) || + chunk.content.platformContentId; + + return `https://www.instagram.com/${chunk.content.type === "reel" ? "reel" : "p"}/${shortcode}/`; +} + +export function extractInstagramShortcode(url: string) { + try { + return ( + new URL(url).pathname.match( + /^\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1] ?? null + ); + } catch { + return ( + url.match( + /instagram\.com\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, + )?.[1] ?? null + ); + } +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index f8e6cca4306..8faa26a6fb7 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -1,7 +1,6 @@ "use client"; import type { PartnerContentTopicFitBand } from "@/lib/partner-content-search/constants"; -import { getPartnerContentThumbnailUrl } from "@/lib/partner-content-search/thumbnail-url"; import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; import { Tooltip } from "@dub/ui"; import { @@ -16,6 +15,17 @@ import { import { cn, nFormatter } from "@dub/utils"; import type { PlatformType } from "@prisma/client"; import { useEffect, useState } from "react"; +import { + BAND_LABELS, + type ContentSearchChunk, + formatDuration, + formatMatchPercent, + formatPublishedDate, + formatTimestamp, + getContentHref, + getContentThumbnail, + getContentTitle, +} from "./content-display-utils"; import { NetworkPartnerCard } from "./network-partner-card"; const PLATFORM_LABELS: Partial> = { @@ -34,7 +44,6 @@ function contentLabel(platform?: PlatformType) { const TOP_CONTENT_PREVIEW_COUNT = 2; -type ContentSearchChunk = PartnerContentSearchPartner["chunks"][number]; type ContentSearchBar = NonNullable< PartnerContentSearchPartner["matchSummary"] >["contentBars"][number]; @@ -170,8 +179,7 @@ function NetworkPartnerContentMatch({ : summary?.platforms ?? [platform].filter(Boolean) ) as string[]; - // List mode (no query): there's no topic fit to score, so show the partner's - // platforms + how recently they've published. + // List mode (no query): no topic fit, so show platforms + last-published recency. if (!hasQuery) { return (
@@ -457,67 +465,6 @@ function PlatformIcon({ return ; } -function getContentThumbnail(chunk: ContentSearchChunk) { - if (chunk.content.thumbnailUrl) { - return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); - } - - if (chunk.platform.type === "youtube") { - return `https://i.ytimg.com/vi/${chunk.content.platformContentId}/hqdefault.jpg`; - } - - return null; -} - -function getContentTitle(chunk: ContentSearchChunk) { - return ( - chunk.content.title?.trim() || - chunk.content.description?.trim().split(/\r?\n/)[0] || - "Untitled content" - ); -} - -function getContentHref(chunk: ContentSearchChunk) { - if (chunk.platform.type === "instagram") { - return getInstagramContentHref(chunk); - } - - if (chunk.platform.type !== "youtube" || chunk.chunk.startMs === null) { - return chunk.content.url; - } - - try { - const url = new URL(chunk.content.url); - url.searchParams.set("t", `${Math.floor(chunk.chunk.startMs / 1000)}s`); - return url.toString(); - } catch { - return chunk.content.url; - } -} - -function getInstagramContentHref(chunk: ContentSearchChunk) { - const shortcode = - extractInstagramShortcode(chunk.content.url) || chunk.content.platformContentId; - - return `https://www.instagram.com/${chunk.content.type === "reel" ? "reel" : "p"}/${shortcode}/`; -} - -function extractInstagramShortcode(url: string) { - try { - return ( - new URL(url).pathname.match( - /^\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, - )?.[1] ?? null - ); - } catch { - return ( - url.match( - /instagram\.com\/(?:(?:[^/]+)\/)?(?:p|reel|tv)\/([^/?#]+)/, - )?.[1] ?? null - ); - } -} - function getContentEngagementMetrics( chunk: ContentSearchChunk, bar: ContentSearchBar | undefined, @@ -556,54 +503,7 @@ function formatChunkMatchLocation(chunk: ContentSearchChunk) { )}`; } -function formatTimestamp(ms: number) { - const totalSeconds = Math.max(0, Math.floor(ms / 1000)); - const minutes = Math.floor(totalSeconds / 60); - const seconds = totalSeconds % 60; - - return `${minutes}:${seconds.toString().padStart(2, "0")}`; -} - -function formatDuration(durationMs: number | null) { - if (!durationMs || durationMs <= 0) return null; - - const totalSeconds = Math.floor(durationMs / 1000); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor((totalSeconds % 3600) / 60); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `${hours}:${minutes.toString().padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - - return `${minutes}:${seconds.toString().padStart(2, "0")}`; -} - -function formatPublishedDate(publishedAt: string | null) { - if (!publishedAt) return null; - - const date = new Date(publishedAt); - if (Number.isNaN(date.getTime())) return null; - - return new Intl.DateTimeFormat(undefined, { - month: "short", - day: "numeric", - year: "numeric", - }).format(date); -} - -const BAND_LABELS: Record = { - consistent: "Consistent", - frequent: "Frequent", - occasional: "Occasional", - "one-off": "One-off", - none: "No recent match", -}; - -// Number + chip colors per band. Number color follows the band tier so the score -// reads at a glance (green = consistent, down to gray = one-off/none). +// Number + chip colors per band (number color follows the tier). const BAND_STYLES: Record< PartnerContentTopicFitBand, { number: string; chip: string } @@ -649,9 +549,7 @@ function PlatformIcons({ platforms }: { platforms: string[] }) { } return ( - // No text-color override here: the YouTube/TikTok/Instagram icons carry their - // own brand colors, so we let them render in color rather than desaturating - // them to the surrounding muted text color. + // No text-color override: the brand icons carry their own colors. {icons.map((Icon, index) => ( @@ -660,8 +558,7 @@ function PlatformIcons({ platforms }: { platforms: string[] }) { ); } -// Coarse "Nd / Nw / Nmo ago" label for the last publish date (timeAgo from -// @dub/utils switches to absolute dates past ~23h, which we don't want here). +// Coarse "Nd/Nw/Nmo ago" (timeAgo from @dub/utils goes absolute past ~23h). function lastPublishedLabel(iso: string | null | undefined) { if (!iso) return null; @@ -673,22 +570,13 @@ function lastPublishedLabel(iso: string | null | undefined) { return `${Math.floor(days / 365)}y ago`; } -function clampScore(score: number) { - return Math.min(1, Math.max(0, score)); -} - -function formatMatchPercent(score: number) { - return `${Math.round(clampScore(score) * 100)}%`; -} - function formatMatchEvidenceLabel( summary: PartnerContentSearchPartner["matchSummary"] | undefined, ) { const matchingPosts = summary?.matchedContentCount ?? 0; const recentPosts = summary?.recentContentCount ?? 0; - // Aggregate coverage rather than splitting evidence by source (transcript vs - // creator text), which read as noisy on the compact card. + // Aggregate coverage; splitting by source reads noisy on the compact card. if (recentPosts > 0) { return `${matchingPosts} of ${recentPosts} matching`; } diff --git a/apps/web/lib/partner-content-search/chunk-transcript.ts b/apps/web/lib/partner-content-search/chunk-transcript.ts index 156d488e3fb..0bfadaf81b2 100644 --- a/apps/web/lib/partner-content-search/chunk-transcript.ts +++ b/apps/web/lib/partner-content-search/chunk-transcript.ts @@ -46,8 +46,7 @@ export function estimateTokenCount(text: string) { const normalized = normalizeTranscriptText(text); if (!normalized) return 0; - // A lightweight approximation keeps this utility dependency-free. The - // ingestion job can swap in a provider tokenizer later if needed. + // Lightweight approximation keeps this dependency-free; swap in a tokenizer later if needed. const words = normalized.split(/\s+/).length; return Math.max(1, Math.ceil(words * 1.35)); } diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts index 012ef1e48b7..ddb0cda510b 100644 --- a/apps/web/lib/partner-content-search/constants.ts +++ b/apps/web/lib/partner-content-search/constants.ts @@ -1,17 +1,6 @@ -export const PARTNER_CONTENT_SEARCH_FEATURE_FLAG = - "PARTNER_CONTENT_SEARCH_ENABLED"; - -export const PARTNER_CONTENT_SEARCH_ENV_VARS = { - scrapeCreatorsApiKey: "SCRAPECREATORS_API_KEY", - voyageApiKey: "VOYAGE_API_KEY", -} as const; - -// Name of the manually-created PlanetScale VECTOR index on -// PartnerContentChunk.embedding, and the distance metric it's built with. These -// are the single source of truth: the search routes feed them into the FORCE -// INDEX hint and the DISTANCE() call, and they must match the @@index(map:) -// files that can't import this constant use a drift check test in -// tests/partner-content-search/vector-index-sync.test.ts. +// Manually-created PlanetScale VECTOR index on PartnerContentChunk.embedding and its +// distance metric. Keep in sync with the schema @@index(map:) and the FORCE INDEX / +// DISTANCE() calls — vector-index-sync.test.ts guards the drift. export const PARTNER_CONTENT_CHUNK_VECTOR_INDEX = "partner_content_chunk_embedding_cosine_idx"; @@ -40,19 +29,10 @@ export const PARTNER_CONTENT_SEARCH_LIMITS = { chunkMaxTokens: 800, chunkOverlapTokens: 80, chunkCandidateCount: 200, - // Safety cap on recent posts per partner feeding the topic-fit window. The - // window itself is time-based (recencyWindowMonths); this just bounds the - // per-partner item set so the exact per-item scoring query stays cheap for - // extremely prolific creators. recentContentMaxPerPartner: 200, - // Pre-dedup ANN over-fetch for retrieval. We pull limit * multiplier raw - // chunk candidates via the vector index, capped so the user-facing path does - // not request a large slice of the corpus as the chunk table grows. vectorSearchChunkPoolMultiplier: 3, vectorSearchChunkPoolMaxSize: 600, rerankerCandidateCount: 150, - // Cap each document sent to the reranker. Chunks are <=800 tokens; trimming the - // tail keeps the rerank payload (and thus latency) down with little quality loss. rerankerMaxDocChars: 2_000, partnerScorePoolSize: 3, } as const; @@ -62,35 +42,18 @@ export const PARTNER_CONTENT_SEARCH_FUSION_WEIGHTS = { existingPartnerRank: 0.4, } as const; -// "Topic fit" is the aggregate card score. Reranker relevance gates which recent -// posts count as on-topic, then the displayed score is based on coverage. -// `bandThresholds` are coverage cutoffs that drive the qualitative label + color. +// Tuning for the aggregate "Topic Fit" card score. Reranker relevance gates which +// recent posts count as on-topic; the score is then coverage-based. export const PARTNER_CONTENT_SEARCH_TOPIC_FIT = { - // A recent post counts as "on topic" when its reranker relevance clears this. - // Reranker scores for a broad keyword query top out well below 1 (~0.6-0.8 for - // a clearly on-topic post), so magnitude gates membership rather than scaling - // the score. Tune against a known off-topic creator to set the floor. rerankMatchThreshold: 0.4, - // Topic fit = 100 * adjustedCoverage^exponent, where adjusted coverage is - // strength-weighted evidence coverage after the small-sample confidence - // adjustment. The <1 exponent is a concave lift so mid-coverage creators don't - // read punishingly low. + // topicFit = 100 * coverage^exponent (concave lift, so mid-coverage isn't punished). coverageCurveExponent: 0.5, - // A matched source at or above this score gets full per-source topic credit. - // Scores below this but above the match threshold still count, but only - // partially. This prevents reranker magnitude from capping creators who are - // clearly and consistently on-topic. strongMatchScore: 0.7, - // A just-over-threshold source gets this much per-source topic credit. minimumMatchedEvidenceStrength: 0.35, - // Small recent-content samples should not rank as confidently as larger - // bodies of work with the same match ratio. Keep this light so a creator with - // 25-50 consistently on-topic posts can still reach the high 90s. - sampleConfidenceContentCount: 2, - // Creator-entered text on videos/reels (titles, descriptions, captions) is - // useful evidence, but weaker than transcript evidence because it can contain - // SEO copy, link blocks, or repeated channel boilerplate. Static posts get full - // credit because creator text is the main searchable content there. + // depthConfidence = 1 - exp(-weightedMatchedContentScore / this); lower saturates faster. + depthSaturationScore: 4, + // Credit kept when on-topic posts are diluted (0 = pure ratio, 1 = ignore dilution). + dilutionForgiveness: 0.25, creatorTextOnlyVideoWeight: 0.4, bandThresholds: { consistent: 0.5, @@ -98,22 +61,13 @@ export const PARTNER_CONTENT_SEARCH_TOPIC_FIT = { }, } as const; -// Ranking for the detail-pane "Top content" section: a relevance-led blend of -// how on-topic a matched post is with how well it performed (views), normalized -// per creator and robust to a single viral outlier. This only orders that list; -// it never feeds Topic Fit, the bars, or partner ordering. +// Tuning for the detail-pane "Top content" list only — never feeds Topic Fit, the +// bars, or partner ordering. export const PARTNER_CONTENT_SEARCH_TOP_CONTENT = { - // How many matched posts the "Top content" section highlights. topContentCount: 5, - // Blend weights. Relevance leads because this is a topic search; views break - // ties and surface the strongest proof points without overriding relevance. relevanceWeight: 0.7, engagementWeight: 0.3, - // Spread (in log10 view units) of the logistic that maps a post's views to a - // [0,1] engagement score relative to the creator's median. ~0.5 means roughly - // a 3x swing in views moves the score one logistic unit. Median post -> ~0.5, - // a viral outlier saturates toward 1 (never dominates), a below-median post - // lands near 0 but is never punished into negative territory. + // log10-view spread of the logistic mapping views to a [0,1] score vs. the median. engagementLogSpread: 0.5, } as const; @@ -124,23 +78,9 @@ export type PartnerContentTopicFitBand = | "one-off" | "none"; -// Max number of partners returned by the network content search (and the cap the -// route enforces). Shared so the client request and the route stay in sync. export const PARTNER_CONTENT_SEARCH_PARTNER_LIMIT = 50; - -// Default number of chunks surfaced per partner in the network content search. export const PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER = 2; - -// Max number of content-level matches surfaced for one partner. This lines up -// with the card's "last 28 posts" content match visualization. -export const PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER = 28; - -// Timeout for the query-embedding call on user-facing search routes. Fails fast -// through the normal error path instead of hanging until the function's -// maxDuration (30s) is hit. +export const PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER = 50; +export const PARTNER_CONTENT_SEARCH_DETAIL_CHUNKS_PER_PARTNER = 40; export const PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS = 10_000; - -// Timeout for the second-stage reranker call. Unlike the embedding timeout this -// fails *soft*: on timeout the search falls back to cosine ordering so a slow -// rerank never breaks results, it just doesn't improve them. export const PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS = 4_000; diff --git a/apps/web/lib/partner-content-search/ingestion/chunk-counts.ts b/apps/web/lib/partner-content-search/ingestion/chunk-counts.ts index 950eb140612..ea29d7fc9a1 100644 --- a/apps/web/lib/partner-content-search/ingestion/chunk-counts.ts +++ b/apps/web/lib/partner-content-search/ingestion/chunk-counts.ts @@ -40,10 +40,8 @@ export async function refreshPartnerContentItemChunkCounts( }; } -// Bulk variant of refreshPartnerContentItemChunkCounts: recomputes and writes -// the chunk counts for many items in a single round-trip, so callers processing -// a batch don't fan out to one query + one update per item. Covers items with -// zero remaining chunks (counts reset to 0) via the correlated subqueries. +// Bulk variant: recompute + write chunk counts for many items in one round-trip +// (covers items reset to 0 via the correlated subqueries). export async function refreshPartnerContentItemChunkCountsBulk( partnerContentItemIds: string[], ) { diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index fa80d76860e..0b682f8ebab 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -1,4 +1,5 @@ import { qstash } from "@/lib/cron"; +import { prisma } from "@/lib/prisma"; import type { Prisma } from "@prisma/client"; import { APP_DOMAIN_WITH_NGROK } from "@dub/utils"; import { logAndRespond } from "app/(ee)/api/cron/utils"; @@ -9,9 +10,8 @@ import { PartnerContentPlatform, } from "../types"; -// Partners enumerated per page / per page-worker job. Defaults to 500; override -// with the PARTNER_CONTENT_ENUMERATE_PAGE_SIZE env var (e.g. 3) to exercise the -// self-continuation chain locally without seeding 500+ eligible partners. +// Partners per enumerate page (default 500). Override via env to exercise the +// self-continuation chain locally without seeding 500+ partners. export const PARTNER_CONTENT_ENUMERATE_PAGE_SIZE = (() => { const raw = process.env.PARTNER_CONTENT_ENUMERATE_PAGE_SIZE; const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; @@ -66,11 +66,9 @@ export const partnerContentEnumeratePayloadSchema = z.object({ filter: partnerContentIngestionFilterSchema, runStamp: z.string().min(1), dryRun: z.boolean().default(false), - // Cursor handed back to this same route for self-continued enumeration; - // absent on the initial dispatch from the admin trigger. + // Cursor for self-continued enumeration; absent on the initial dispatch. startingAfter: z.string().optional(), - // Remaining partner budget carried across continuation hops. Seeded from - // filter.limitPartners on the first hop; omitted means unbounded. + // Partner budget carried across continuation hops; omitted = unbounded. remainingPartners: z.number().int().nonnegative().optional(), }); @@ -90,8 +88,7 @@ export const partnerContentFetchPayloadSchema = z.object({ runStamp: z.string().min(1), dryRun: z.boolean().default(false), forceTranscriptJobs: z.boolean().default(false), - // Manual/direct fetch escape hatch for linked-but-unverified platforms. - // Enumerated backfills still only enqueue verified platforms. + // Escape hatch for linked-but-unverified platforms; backfills stay verified-only. ignoreUnverified: z.boolean().default(false), partnerId: z.string().min(1), partnerPlatformId: z.string().min(1), @@ -102,9 +99,8 @@ export const partnerContentTranscriptPayloadSchema = z.object({ mode: partnerContentIngestionModeSchema, runStamp: z.string().min(1), dryRun: z.boolean().default(false), - // Re-fetch the transcript even if one was already fetched. Off by default so - // QStash redelivery doesn't re-burn ScrapeCreators credits; set by admin - // re-ingestion. + // Re-fetch even if already fetched. Off by default so QStash redelivery doesn't + // re-burn ScrapeCreators credits. forceRefetch: z.boolean().default(false), partnerId: z.string().min(1), partnerPlatformId: z.string().min(1), @@ -166,15 +162,136 @@ export async function enqueuePartnerContentEnumerate( }); } +// Enqueue one embed job for an already-chunked item. Throws on publish failure so +// the caller fails the job and QStash retries (else chunks stay unsearchable). +export async function enqueueEmbedJob({ + mode, + runStamp, + partnerId, + partnerContentItemId, +}: { + mode: PartnerContentIngestionMode; + runStamp: string; + partnerId: string; + partnerContentItemId: string; +}) { + try { + await qstash.publishJSON({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), + method: "POST", + body: { + mode, + runStamp, + partnerId, + partnerContentItemId, + }, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-embed", + mode, + runStamp, + partnerContentItemId, + ), + }); + } catch (error) { + // Don't swallow: chunks are already committed, so a dropped embed leaves them + // unsearchable. Rethrow → withCron 500 → QStash retries; forceRefetch=false skips + // the re-fetch and the embed dedup id keeps it idempotent. + console.error("[PartnerContentSearch] Failed to enqueue embed job", { + error, + mode, + runStamp, + partnerId, + partnerContentItemId, + }); + + throw error; + } +} + +// Fan out one embed job per pending content item on a partner platform. +export async function enqueueEmbedJobsForPartnerPlatform({ + mode, + runStamp, + partnerId, + partnerPlatformId, + limitContentItems, + maxChunks, +}: { + mode: PartnerContentIngestionMode; + runStamp: string; + partnerId: string; + partnerPlatformId?: string; + limitContentItems: number; + maxChunks: number; +}) { + if (!partnerPlatformId) { + return logAndRespond( + `[PartnerContentSearch] Embed enqueue requires partnerPlatformId for ${mode} run ${runStamp}.`, + { status: 400, logLevel: "warn" }, + ); + } + + const contentItems = await prisma.partnerContentItem.findMany({ + where: { + partnerId, + partnerPlatformId, + totalChunkCount: { + gt: 0, + }, + }, + select: { + id: true, + totalChunkCount: true, + embeddedChunkCount: true, + }, + orderBy: { + id: "asc", + }, + take: limitContentItems, + }); + + const pendingContentItems = contentItems.filter( + ({ totalChunkCount, embeddedChunkCount }) => + embeddedChunkCount < totalChunkCount, + ); + + const messages = pendingContentItems.map((contentItem) => ({ + url: getPartnerContentUrl(PARTNER_CONTENT_SEARCH_ROUTES.embed), + method: "POST" as const, + flowControl: PARTNER_CONTENT_EMBED_FLOW_CONTROL, + body: { + mode, + runStamp, + partnerId, + partnerContentItemId: contentItem.id, + maxChunks, + }, + deduplicationId: createPartnerContentDeduplicationId( + "partner-content-embed", + mode, + runStamp, + contentItem.id, + ), + })); + + if (messages.length > 0) { + await qstash.batchJSON(messages); + } + + return logAndRespond( + `[PartnerContentSearch] Enqueued ${messages.length} embed jobs (${pendingContentItems.length} pending of ${contentItems.length} inspected) for partner platform ${partnerPlatformId} on ${mode} run ${runStamp}.`, + ); +} + export function getIncrementalRefreshCutoff() { return new Date( Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, ); } -// Shared platform-eligibility predicate for the enumerate fan-out. The -// partner-level (enumerate) and platform-level (enumerate/page) routes both -// filter platforms on the same verified + incremental-recency rules. +// Shared platform-eligibility predicate (verified + incremental-recency rules), +// used by both the enumerate and enumerate/page routes. export function buildEligiblePartnerPlatformWhere({ mode, platforms, @@ -204,10 +321,42 @@ export function buildEligiblePartnerPlatformWhere({ }; } -// Parse a QStash payload for a cron route. A malformed payload is a permanent -// error QStash should not retry, so respond 400 rather than letting it bubble -// to withCron's 500 + error alert. Returns the parsed payload, or a Response to -// return directly from the handler. +// Partner-level eligibility for the enumerate route — composes +// buildEligiblePartnerPlatformWhere; gates on approved/trusted status. +export function buildEligiblePartnerWhere({ + mode, + filter, +}: { + mode: PartnerContentIngestionMode; + filter: { + partnerId?: string; + partnerIds?: string[]; + platforms: PartnerContentPlatform[]; + }; +}): Prisma.PartnerWhereInput { + return { + networkStatus: { + in: ["approved", "trusted"], + }, + ...(filter.partnerId && { + id: filter.partnerId, + }), + ...(filter.partnerIds?.length && { + id: { + in: filter.partnerIds, + }, + }), + platforms: { + some: buildEligiblePartnerPlatformWhere({ + mode, + platforms: filter.platforms, + }), + }, + }; +} + +// Parse a cron payload. A malformed body is permanent, so respond 400 (not a +// withCron 500 + alert). Returns the payload, or a Response to return directly. export function parsePartnerContentCronPayload( schema: T, rawBody: string, diff --git a/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts b/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts index cf06b963559..b9accc57fd3 100644 --- a/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts +++ b/apps/web/lib/partner-content-search/ingestion/write-fetched-content-items.ts @@ -209,10 +209,8 @@ export async function writeFetchedContentItems({ }; } -// Post-commit QStash dispatch. The content items are already written, so a -// dropped batch would silently strand transcript/embed jobs. Log structured -// context before rethrowing so the failure is observable + alertable (the -// rethrow surfaces it as withCron's 500); deduplicationIds keep the retry safe. +// Post-commit dispatch: items are already written, so a dropped batch strands +// transcript/embed jobs. Log + rethrow (withCron 500); dedup ids keep retry safe. async function dispatchContentJobs< T extends Parameters[0], >({ @@ -274,10 +272,8 @@ async function writeMetadataChunks({ const createdPlatformContentIds = new Set(); const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.id; - // Collect the items whose metadata actually changed, plus the new chunks to - // create, so we can write everything in one batched transaction instead of a - // per-item transaction + count refresh (which fanned out to ~100-250 serial - // DB round-trips on a full backfill). + // Batch changed items + new chunks into one transaction instead of per-item + // (which fanned out to ~100-250 serial round-trips on a full backfill). const changedItems: { id: string; sourceItem: NormalizedPartnerContentItem; diff --git a/apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts b/apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts new file mode 100644 index 00000000000..0907fc6091f --- /dev/null +++ b/apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts @@ -0,0 +1,118 @@ +import { createId } from "@/lib/api/create-id"; +import { + chunkTranscriptSegments, + hashTranscript, +} from "@/lib/partner-content-search/chunk-transcript"; +import { PARTNER_CONTENT_SEARCH_MODELS } from "@/lib/partner-content-search/constants"; +import { refreshPartnerContentItemChunkCounts } from "@/lib/partner-content-search/ingestion/chunk-counts"; +import { fetchPlatformTranscriptSegments } from "@/lib/partner-content-search/ingestion/platforms"; +import type { PartnerContentPlatform } from "@/lib/partner-content-search/types"; +import { prisma } from "@/lib/prisma"; + +// Fetch a content item's transcript, (re)chunk it, and replace its transcript chunks +// in one transaction. Returns a summary; marks notAvailable when there's no transcript. +export async function writeTranscriptChunks(contentItem: { + id: string; + partnerId: string; + url: string; + platform: PartnerContentPlatform; +}) { + const transcriptSegments = await fetchPlatformTranscriptSegments({ + platform: contentItem.platform, + url: contentItem.url, + }); + const normalizedTranscript = transcriptSegments + .map(({ text }) => text.trim()) + .filter(Boolean) + .join(" "); + const transcriptHash = + transcriptSegments.length > 0 ? hashTranscript(transcriptSegments) : null; + + if (!transcriptHash) { + await prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "notAvailable", + transcriptLastAttemptedAt: new Date(), + normalizedTranscript: null, + transcriptHash: null, + transcriptHasTimestamps: false, + }, + }); + + await prisma.partnerContentChunk.deleteMany({ + where: { + partnerContentItemId: contentItem.id, + source: "transcript", + }, + }); + + await refreshPartnerContentItemChunkCounts(contentItem.id); + + return { + transcriptAvailable: false, + segmentCount: 0, + normalizedTranscriptLength: 0, + transcriptHash: null, + chunkCount: 0, + chunksCreated: 0, + }; + } + + const chunks = chunkTranscriptSegments(transcriptSegments); + const transcriptHasTimestamps = transcriptSegments.some( + ({ startMs, endMs }) => startMs !== null || endMs !== null, + ); + const embeddingModel = PARTNER_CONTENT_SEARCH_MODELS.embedding.id; + + const [, deletedChunks, createChunksResult] = await prisma.$transaction([ + prisma.partnerContentItem.update({ + where: { + id: contentItem.id, + }, + data: { + transcriptFetchStatus: "fetched", + transcriptLastAttemptedAt: new Date(), + normalizedTranscript, + transcriptHash, + transcriptHasTimestamps, + embeddingModel, + lastFetchedAt: new Date(), + }, + }), + prisma.partnerContentChunk.deleteMany({ + where: { + partnerContentItemId: contentItem.id, + source: "transcript", + }, + }), + prisma.partnerContentChunk.createMany({ + data: chunks.map((chunk) => ({ + id: createId({ prefix: "pcc_" }), + partnerContentItemId: contentItem.id, + partnerId: contentItem.partnerId, + source: "transcript", + chunkIndex: chunk.chunkIndex, + chunkText: chunk.text, + startMs: chunk.startMs, + endMs: chunk.endMs, + textHash: transcriptHash, + embeddingModel, + })), + }), + ]); + + await refreshPartnerContentItemChunkCounts(contentItem.id); + + return { + transcriptAvailable: true, + segmentCount: transcriptSegments.length, + normalizedTranscriptLength: normalizedTranscript.length, + transcriptHash, + chunkCount: chunks.length, + chunksDeleted: deletedChunks.count, + chunksCreated: createChunksResult.count, + }; +} diff --git a/apps/web/lib/partner-content-search/listing.ts b/apps/web/lib/partner-content-search/listing.ts index 61a73e8989e..5f7a70a093b 100644 --- a/apps/web/lib/partner-content-search/listing.ts +++ b/apps/web/lib/partner-content-search/listing.ts @@ -3,9 +3,8 @@ import { PlatformType, Prisma } from "@prisma/client"; import { PARTNER_CONTENT_SEARCH_MODELS } from "./constants"; import type { PartnerContentSearchRow } from "./search-utils"; -// Empty-query path: the most recent embedded content across the eligible network, -// shaped into the same per-chunk row as the vector search so the downstream -// grouping/match-summary code is identical for both modes. +// Empty-query path: most recent embedded content across the eligible network, +// shaped into the same per-chunk row as the vector search so downstream is identical. export async function listPartnerNetworkContent({ programId, platforms, diff --git a/apps/web/lib/partner-content-search/match-summaries.ts b/apps/web/lib/partner-content-search/match-summaries.ts index 13abaa38ec9..2aa0f627ed6 100644 --- a/apps/web/lib/partner-content-search/match-summaries.ts +++ b/apps/web/lib/partner-content-search/match-summaries.ts @@ -7,23 +7,17 @@ import { import { createContentMatchEvidence, deriveTopicFit, - getEntityCreatorTextBoost, - getEntityTranscriptScore, getEvidenceMatchScore, getEvidenceSource, getEvidenceTopicScore, getMatchedSourceScore, - getPartnerContentSearchQuerySignals, getSourceScore, - hasExactQueryMention, - maxNullableScore, setSourceScore, type PartnerContentMatchEvidence, type PartnerContentMatchSource, - type PartnerContentSearchQuerySignals, type SourceScoreByContentItemId, } from "./ranking"; -import { toScore, type PartnerContentSearchRow } from "./search-utils"; +import { median, toScore, type PartnerContentSearchRow } from "./search-utils"; import type { PartnerContentSearchTimingLogger } from "./timing"; type PartnerRecentContentBarRow = { @@ -33,10 +27,8 @@ type PartnerRecentContentBarRow = { platformContentId: string; contentType: string; contentTitle: string | null; - contentDescription: string | null; contentUrl: string | null; contentDurationMs: number | null; - transcriptFetchStatus: string | null; publishedAt: Date | null; viewCount: bigint | number | null; likeCount: bigint | number | null; @@ -70,7 +62,6 @@ type PartnerContentMatchBar = { }; type QueryContentMatchContext = { - querySignals: PartnerContentSearchQuerySignals; cutoffDistance?: number | null; recentItemSourceBestDistance: SourceScoreByContentItemId; rerankByItemSource: SourceScoreByContentItemId; @@ -82,17 +73,6 @@ type ListContentMatchContext = { type ContentMatchContext = QueryContentMatchContext | ListContentMatchContext; -// Median of a numeric list (robust to viral outliers vs. a mean). Returns null -// for an empty list so callers can omit the signal when there's no data. -function median(values: number[]): number | null { - if (values.length === 0) return null; - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 - ? Math.round((sorted[mid - 1] + sorted[mid]) / 2) - : sorted[mid]; -} - function groupRowsBy(rows: T[], getKey: (row: T) => K) { const rowsByKey = new Map(); @@ -148,30 +128,23 @@ function getQueryContentMatch({ row: PartnerRecentContentBarRow; context: QueryContentMatchContext; }) { - // On-topic gate: prefer the reranker's calibrated relevance, which cleanly - // separates on- from off-topic. Fall back to the cosine cutoff only for items - // the reranker didn't score (rerank disabled/failed, or the item never reached - // the candidate pool). - const transcriptScore = getEntityTranscriptScore({ - currentScore: getMatchedSourceScore({ - rerankScore: getSourceScore( - context.rerankByItemSource, - row.partnerContentItemId, - "transcript", - ), - bestDistance: getSourceScore( - context.recentItemSourceBestDistance, - row.partnerContentItemId, - "transcript", - ), - cutoffDistance: context.cutoffDistance, - }), - // Transcript-level substring matching was removed from the search path; only - // title/description exact mentions (computed in memory below) feed scoring now. - hasExactQueryMention: false, - queryIntent: context.querySignals.intent, + // On-topic gate: prefer the reranker's calibrated relevance; fall back to the + // cosine cutoff only for items the reranker didn't score. Both sources rely + // solely on embedding retrieval + reranking — no lexical title/description boost. + const transcriptScore = getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "transcript", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "transcript", + ), + cutoffDistance: context.cutoffDistance, }); - const vectorCreatorTextScore = getMatchedSourceScore({ + const creatorTextScore = getMatchedSourceScore({ rerankScore: getSourceScore( context.rerankByItemSource, row.partnerContentItemId, @@ -184,35 +157,11 @@ function getQueryContentMatch({ ), cutoffDistance: context.cutoffDistance, }); - const titleHasExactQueryMention = hasExactQueryMention( - row.contentTitle, - context.querySignals, - ); - const descriptionHasExactQueryMention = hasExactQueryMention( - row.contentDescription, - context.querySignals, - ); - const creatorTextBoost = getEntityCreatorTextBoost({ - platformType: row.platformType, - contentType: row.contentType, - transcriptFetchStatus: row.transcriptFetchStatus, - titleHasExactQueryMention, - descriptionHasExactQueryMention, - // Chunk-level substring matching was removed from the search path (see above). - chunkHasExactQueryMention: false, - transcriptScore, - queryIntent: context.querySignals.intent, - }); - const creatorTextScore = maxNullableScore( - vectorCreatorTextScore, - creatorTextBoost?.score, - ); const matchEvidence = createContentMatchEvidence({ contentType: row.contentType, transcriptScore, creatorTextScore, - creatorTextWeightOverride: creatorTextBoost?.weight, }); return { @@ -317,7 +266,6 @@ export async function getPartnerMatchSummaries({ rows, partnerIds, platforms, - query, queryVector, cutoffDistance, itemSourceBestDistance, @@ -326,42 +274,30 @@ export async function getPartnerMatchSummaries({ rows: PartnerContentSearchRow[]; partnerIds: string[]; platforms?: PlatformType[]; - query?: string | null; - // Present only in query mode. When set, each shown partner's recent videos are - // counted as matched when at least as relevant as the weakest candidate - // (cutoffDistance). When null (list mode) we fall back to the retrieval rows for - // the match determination. + // Query mode only: gates a partner's recent posts as matched when at least as + // relevant as the weakest candidate (cutoffDistance). Null in list mode. queryVector?: string | null; cutoffDistance?: number | null; - // Best cosine distance per content-item + source across the candidate pool, - // computed once during retrieval. Reused here to gate matched recent content - // without a second per-item DISTANCE pass (any item that clears the cutoff is - // already in this map — see the producer in searchPartnerNetworkContent). + // Best distance per item+source from retrieval, reused to gate matched recent + // content without a second per-item DISTANCE pass. itemSourceBestDistance?: SourceScoreByContentItemId; logTiming?: PartnerContentSearchTimingLogger; }) { if (partnerIds.length === 0) return new Map(); - const querySignals = getPartnerContentSearchQuerySignals(query); const platformFilter = platforms?.length ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` : Prisma.empty; - // "Recent" is a time window (last recencyWindowMonths), not a fixed post count. - // A count window means wildly different time spans per creator (a daily poster's - // 28 posts span a month; a monthly poster's span 2+ years), so coverage over it - // conflates active and dormant creators. A time window normalizes that and keeps - // every platform with content in the period (incl. ones the creator has since - // moved on from; those still count toward topical authority). + // "Recent" is a time window (recencyWindowMonths), not a post count — a count + // window spans wildly different time per creator, conflating active and dormant. const recencyCutoff = new Date(); recencyCutoff.setMonth( recencyCutoff.getMonth() - PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, ); - // These three reads are independent, so issue them as one parallel wave rather - // than three serial round trips. (Prisma promises are lazy and don't execute - // until awaited, so they fan out together under the Promise.all below.) + // Independent reads — issued as one parallel wave (fan out under Promise.all). const contentCountsPromise = prisma.partnerContentItem .groupBy({ by: ["partnerId"], @@ -406,10 +342,8 @@ export async function getPartnerMatchSummaries({ platformContentId, contentType, contentTitle, - contentDescription, contentUrl, contentDurationMs, - transcriptFetchStatus, publishedAt, viewCount, likeCount, @@ -425,10 +359,8 @@ export async function getPartnerMatchSummaries({ pci.platformContentId, pci.contentType, pci.title AS contentTitle, - pci.description AS contentDescription, pci.url AS contentUrl, pci.durationMs AS contentDurationMs, - pci.transcriptFetchStatus, pci.publishedAt, pci.viewCount, pci.likeCount, @@ -461,8 +393,7 @@ export async function getPartnerMatchSummaries({ return recentContentRows; }); - // Reach: total followers across the partner's platforms (or just the filtered - // platform). The single signal brands most want and the card wasn't showing. + // Reach: total followers across the partner's platforms (or the filtered platform). const followerRowsPromise = prisma.partnerPlatform .groupBy({ by: ["partnerId"], @@ -514,20 +445,14 @@ export async function getPartnerMatchSummaries({ ({ partnerId }) => partnerId, ); - // Query mode only: gate which of each shown partner's recent videos count as - // "matched". A recent video is matched when its best chunk is at least as - // relevant as the weakest candidate (cutoffDistance). Because cutoffDistance is - // itself a candidate-pool item's distance, every recent video that could clear - // it is already in itemSourceBestDistance — so we reuse that map instead of - // re-running a per-item DISTANCE pass over the recent set. A recent video absent - // from the map is provably beyond the cutoff (not matched). + // Query mode: reuse itemSourceBestDistance to gate matched recent posts (a post + // absent from it is provably beyond the cutoff — see retrieval's producer). const recentItemSourceBestDistance = queryVector && itemSourceBestDistance ? itemSourceBestDistance : new Map>(); - // Calibrated reranker score per item + source (from the candidate pool), used - // to gate which recent posts count as on-topic. + // Per-item+source reranker scores from the candidate pool, to gate on-topic posts. const rerankByItemSource: SourceScoreByContentItemId = new Map(); for (const row of rows) { if (row.rerankScore != null) { @@ -542,7 +467,6 @@ export async function getPartnerMatchSummaries({ const queryMatchContext: QueryContentMatchContext | null = queryVector ? { - querySignals, cutoffDistance, recentItemSourceBestDistance, rerankByItemSource, @@ -585,12 +509,12 @@ export async function getPartnerMatchSummaries({ weightedMatchedContentScore, recentContentCount, }); - // Brand-facing signals over the on-topic posts: typical reach (median views, - // robust to a viral outlier) and how recently they last posted on topic. + // Brand-facing signals over on-topic posts: median views + last on-topic date. const medianViews = median( matchedBars .map((bar) => bar.viewCount) .filter((views): views is number => views != null && views > 0), + { round: true }, ); const matchedTimestamps = matchedBars .map((bar) => bar.publishedAt) @@ -600,8 +524,7 @@ export async function getPartnerMatchSummaries({ ? new Date(Math.max(...matchedTimestamps)).toISOString() : null; const followers = followersByPartner.get(partnerId) ?? null; - // Top platforms ranked by matched-post frequency, falling back to all recent - // posts when nothing matched (so the card can still show a platform). + // Top platforms by matched-post frequency (falls back to all recent when none matched). const platformFrequency = new Map(); for (const bar of matchedBars.length ? matchedBars : contentBars) { platformFrequency.set( diff --git a/apps/web/lib/partner-content-search/network-partners.ts b/apps/web/lib/partner-content-search/network-partners.ts index 4a16810621f..c84b8ddca90 100644 --- a/apps/web/lib/partner-content-search/network-partners.ts +++ b/apps/web/lib/partner-content-search/network-partners.ts @@ -3,9 +3,8 @@ import { parseRankedNetworkPartners } from "@/lib/api/network/normalize-ranked-n import type { ReachTier } from "@/lib/api/network/reach-tiers"; import type { PlatformType } from "@prisma/client"; -// Hydrate the shown partner candidates into full network-partner cards, applying -// the same ranking + post-retrieval user filters (reach/country/starred/platform) -// the listing endpoints use, keyed by id for the caller to merge back in. +// Hydrate partner candidates into full network-partner cards (same ranking + +// reach/country/starred/platform filters as the listing), keyed by id. export async function getNetworkPartnersById({ programId, partnerIds, diff --git a/apps/web/lib/partner-content-search/ranking.ts b/apps/web/lib/partner-content-search/ranking.ts index db88fcd7ff4..30245abb354 100644 --- a/apps/web/lib/partner-content-search/ranking.ts +++ b/apps/web/lib/partner-content-search/ranking.ts @@ -1,24 +1,19 @@ import type { PartnerContentTopicFitBand } from "./constants"; import { PARTNER_CONTENT_SEARCH_TOPIC_FIT } from "./constants"; -import { toScore, type PartnerContentSearchRow } from "./search-utils"; +import { + effectiveRowScore, + toScore, + type PartnerContentSearchRow, +} from "./search-utils"; export type PartnerContentMatchSource = "transcript" | "creatorText"; -// Best score (rerank or distance) per content item, split by evidence source. -// Produced during retrieval and reused by the match-summary pass; the source-map -// helpers below (get/set) all operate on exactly this shape. +// Best score per content item, split by evidence source (get/set helpers below). export type SourceScoreByContentItemId = Map< string, Map >; -export type PartnerContentSearchQueryIntent = "entity" | "semantic"; - -export type PartnerContentSearchQuerySignals = { - normalizedQuery: string; - intent: PartnerContentSearchQueryIntent; -}; - export type PartnerContentMatchEvidence = { primarySource: PartnerContentMatchSource | null; sources: PartnerContentMatchSource[]; @@ -28,51 +23,6 @@ export type PartnerContentMatchEvidence = { weight: number; }; -const ENTITY_EXACT_MATCH_SCORE = 0.95; - -const SEMANTIC_QUERY_TERMS = new Set([ - "athlete", - "athletes", - "beauty", - "blogger", - "bloggers", - "coach", - "coaches", - "cooking", - "creator", - "creators", - "dad", - "fashion", - "finance", - "fitness", - "food", - "gamer", - "gamers", - "gaming", - "health", - "influencer", - "influencers", - "lifestyle", - "mom", - "music", - "nutrition", - "outdoor", - "outdoors", - "parenting", - "personal", - "photography", - "podcast", - "podcaster", - "recipe", - "recipes", - "running", - "skincare", - "sports", - "travel", - "wellness", - "yoga", -]); - export function deriveTopicFit({ matchedContentCount, weightedMatchedContentCount, @@ -88,22 +38,28 @@ export function deriveTopicFit({ return { topicFit: 0, band: "none" }; } - // Topic fit is driven by strength-adjusted weighted coverage. Transcript - // evidence and creator text on static posts get full credit; creator text on - // videos is weighted by source context so repeated descriptions stay weaker - // than transcript evidence while exact captions can still count. Small samples - // are discounted so 3/3 weak matches do not read as equivalent to 30/30 strong - // matches. - const rawCoverage = weightedMatchedContentScore / recentContentCount; + // Coverage blends two signals: purity (strength-weighted share of recent posts + // on-topic) and depthConfidence (how much on-topic evidence there is, saturating + // so one post can't read as strong). depthConfidence gates the score; purity only + // modulates it between dilutionForgiveness and 1, so a deep on-topic body scores + // well even when mixed with off-topic posts. const { coverageCurveExponent, bandThresholds, - sampleConfidenceContentCount, + depthSaturationScore, + dilutionForgiveness, } = PARTNER_CONTENT_SEARCH_TOPIC_FIT; - const sampleConfidence = Math.sqrt( - recentContentCount / (recentContentCount + sampleConfidenceContentCount), + const purity = Math.min(1, weightedMatchedContentScore / recentContentCount); + const depthConfidence = + 1 - Math.exp(-weightedMatchedContentScore / depthSaturationScore); + const coverage = Math.min( + 1, + Math.max( + 0, + depthConfidence * + (dilutionForgiveness + (1 - dilutionForgiveness) * purity), + ), ); - const coverage = Math.min(1, Math.max(0, rawCoverage * sampleConfidence)); const topicFit = Math.round(100 * Math.pow(coverage, coverageCurveExponent)); const band: PartnerContentTopicFitBand = @@ -118,9 +74,8 @@ export function deriveTopicFit({ return { topicFit, band }; } -export function getRowRelevanceScore(row: PartnerContentSearchRow) { - return row.rerankScore ?? toScore(Number(row.distance)); -} +// Ranking-domain alias of effectiveRowScore (single implementation, no drift). +export const getRowRelevanceScore = effectiveRowScore; export function getSourceAwareRowScore(row: PartnerContentSearchRow) { const score = getRowRelevanceScore(row); @@ -142,142 +97,43 @@ export function sortRowsBySourceAwareScore(rows: PartnerContentSearchRow[]) { ); } -export function getEvidenceSource(source: string): PartnerContentMatchSource { - return source === "transcript" ? "transcript" : "creatorText"; -} - -export function getPartnerContentSearchQuerySignals( - query?: string | null, -): PartnerContentSearchQuerySignals { - const normalizedQuery = normalizeSearchText(query); - const terms = normalizedQuery.split(" ").filter(Boolean); - const intent: PartnerContentSearchQueryIntent = - normalizedQuery.length === 0 || - terms.some((term) => SEMANTIC_QUERY_TERMS.has(term)) || - terms.length > 3 - ? "semantic" - : "entity"; - - return { - normalizedQuery, - intent, - }; -} - -export function normalizeSearchText(value?: string | null) { - return (value ?? "") - .toLowerCase() - .normalize("NFKD") - .replace(/[\u0300-\u036f]/g, "") - .replace(/[^a-z0-9]+/g, " ") - .trim() - .replace(/\s+/g, " "); -} - -export function hasExactQueryMention( - value: string | null | undefined, - querySignals: PartnerContentSearchQuerySignals, -) { - return ( - querySignals.normalizedQuery.length > 0 && - normalizeSearchText(value).includes(querySignals.normalizedQuery) - ); -} - -export function getEntityTranscriptScore({ - currentScore, - hasExactQueryMention, - queryIntent, -}: { - currentScore: number | null; - hasExactQueryMention: boolean; - queryIntent: PartnerContentSearchQueryIntent; -}) { - if (queryIntent !== "entity" || !hasExactQueryMention) return currentScore; - return maxNullableScore(currentScore, ENTITY_EXACT_MATCH_SCORE); +// Final query-mode partner ordering: topic fit, then match-strength tiebreakers, +// then row score and reach. Generic so the route passes its candidate objects. +export function sortPartnersByTopicFit< + T extends { + score: number; + matchSummary: { + topicFit: number; + weightedMatchedContentScore: number; + weightedMatchedContentCount: number; + transcriptMatchedContentCount: number; + matchedContentCount: number; + followers: number | null; + } | null; + }, +>(partners: T[]) { + return [...partners].sort((a, b) => { + const aSummary = a.matchSummary; + const bSummary = b.matchSummary; + + return ( + (bSummary?.topicFit ?? 0) - (aSummary?.topicFit ?? 0) || + (bSummary?.weightedMatchedContentScore ?? 0) - + (aSummary?.weightedMatchedContentScore ?? 0) || + (bSummary?.weightedMatchedContentCount ?? 0) - + (aSummary?.weightedMatchedContentCount ?? 0) || + (bSummary?.transcriptMatchedContentCount ?? 0) - + (aSummary?.transcriptMatchedContentCount ?? 0) || + (bSummary?.matchedContentCount ?? 0) - + (aSummary?.matchedContentCount ?? 0) || + b.score - a.score || + (bSummary?.followers ?? 0) - (aSummary?.followers ?? 0) + ); + }); } -export function getEntityCreatorTextBoost({ - platformType, - contentType, - transcriptFetchStatus, - titleHasExactQueryMention, - descriptionHasExactQueryMention, - chunkHasExactQueryMention, - transcriptScore, - queryIntent, -}: { - platformType: string; - contentType: string; - transcriptFetchStatus?: string | null; - titleHasExactQueryMention: boolean; - descriptionHasExactQueryMention: boolean; - chunkHasExactQueryMention: boolean; - transcriptScore: number | null; - queryIntent: PartnerContentSearchQueryIntent; -}): { score: number; weight: number } | null { - if ( - queryIntent !== "entity" || - (!titleHasExactQueryMention && - !descriptionHasExactQueryMention && - !chunkHasExactQueryMention) - ) { - return null; - } - - if (isStaticContentType(contentType)) { - return { - score: ENTITY_EXACT_MATCH_SCORE, - weight: 1, - }; - } - - const platform = platformType.toLowerCase(); - const transcriptUnavailable = - transcriptScore == null || - isUnavailableTranscriptStatus(transcriptFetchStatus); - - if (platform === "tiktok" || platform === "instagram") { - return transcriptUnavailable - ? { - score: ENTITY_EXACT_MATCH_SCORE, - weight: 0.95, - } - : { - score: 0.9, - weight: 0.85, - }; - } - - if (platform === "youtube") { - return titleHasExactQueryMention - ? { - score: 0.9, - weight: 0.85, - } - : { - score: 0.65, - weight: 0.35, - }; - } - - return titleHasExactQueryMention - ? { - score: 0.9, - weight: 0.85, - } - : { - score: 0.75, - weight: 0.6, - }; -} - -export function maxNullableScore(...scores: Array) { - const numericScores = scores.filter( - (score): score is number => typeof score === "number", - ); - - return numericScores.length ? Math.max(...numericScores) : null; +export function getEvidenceSource(source: string): PartnerContentMatchSource { + return source === "transcript" ? "transcript" : "creatorText"; } function getCreatorTextWeight(contentType: string) { @@ -292,28 +148,14 @@ function isVideoLikeContentType(contentType: string) { ); } -function isStaticContentType(contentType: string) { - return ["carousel", "image", "photo", "post"].includes( - contentType.toLowerCase(), - ); -} - -function isUnavailableTranscriptStatus(status?: string | null) { - return ( - status === "pending" || status === "notAvailable" || status === "error" - ); -} - export function createContentMatchEvidence({ contentType, transcriptScore, creatorTextScore, - creatorTextWeightOverride, }: { contentType: string; transcriptScore: number | null; creatorTextScore: number | null; - creatorTextWeightOverride?: number | null; }): PartnerContentMatchEvidence { const sources: PartnerContentMatchSource[] = []; if (transcriptScore != null) sources.push("transcript"); @@ -325,8 +167,7 @@ export function createContentMatchEvidence({ : creatorTextScore != null ? "creatorText" : null; - const creatorTextWeight = - creatorTextWeightOverride ?? getCreatorTextWeight(contentType); + const creatorTextWeight = getCreatorTextWeight(contentType); const weight = primarySource === null ? 0 @@ -361,9 +202,8 @@ export function getEvidenceTopicScore(evidence: PartnerContentMatchEvidence) { return Number( Math.max( - // A transcript/source-content match has already passed the relevance gate, - // so it gets full topic credit. Creator text uses the context-aware weight - // chosen when the evidence was created. + // Transcript match already passed the relevance gate → full credit; creator + // text uses its context-aware weight. evidence.transcriptScore != null ? 1 : 0, getEvidenceStrength(evidence.creatorTextScore) * evidence.creatorTextWeight, diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts index b9f0d762dae..56e6c6046c1 100644 --- a/apps/web/lib/partner-content-search/retrieval.ts +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -52,11 +52,8 @@ function countDistinctContentItems(rows: { partnerContentItemId: string }[]) { .size; } -// Resolves a program's partner-eligibility sets for inline ANN pre-filtering: -// the deny-set (enrolled ∪ ignored partners) excluded from every query, plus the -// starred set the starred filter includes/excludes. All are per-program and bounded -// by the program's roster, so they ride into the flat ANN as id-set predicates -// without dragging the relational joins onto the vector-index traversal path. +// Partner-eligibility id-sets for inline ANN pre-filtering: the deny-set +// (enrolled ∪ ignored) excluded from every query, plus the starred set. async function resolveProgramPartnerEligibility({ programId, starred, @@ -125,10 +122,8 @@ export async function searchPartnerNetworkContent({ rerank: boolean; logTiming?: PartnerContentSearchTimingLogger; }) { - // Resolve the program's partner-eligibility sets in parallel with the query - // embedding — they need only programId, so they run under the Voyage round-trip - // and add little to no incremental time delay. (PrismaPromise.all kicks the queries off - // synchronously here, before we await the embedding below.) + // Runs under the Voyage embedding round-trip (needs only programId), so the + // eligibility queries add ~no latency. const eligibilityPromise = resolveProgramPartnerEligibility({ programId, starred, @@ -158,14 +153,10 @@ export async function searchPartnerNetworkContent({ const queryVector = serializeEmbeddingForVector(queryEmbedding); const eligibility = await eligibilityPromise; - // Pre-filter eligibility + user filters INLINE on the flat ANN (no joins): exclude - // the program's enrolled + ignored partners, honor explicit partnerIds, apply the - // starred filter, and apply the country/platform user filters via the denormalized - // c.country / c.platformType columns. Pre-filtering (vs. the old post-hydration - // filter) keeps these from biasing/starving the candidate pool — enrolled partners - // are exactly a program's strongest matches, so post-filtering them silently - // dropped the best results as a program matured. networkStatus stays a post-filter - // (non-selective: ingestion only embeds approved/trusted partners). + // Eligibility + user filters are applied INLINE on the flat ANN (country/platform + // via the denormalized c.country / c.platformType columns). Post-filtering would + // starve the candidate pool — enrolled partners are a program's strongest matches. + // networkStatus stays a post-filter (ingestion already gates approved/trusted). const annFilters: Prisma.Sql[] = []; if (eligibility.excludedPartnerIds.length > 0) { annFilters.push( @@ -198,12 +189,9 @@ export async function searchPartnerNetworkContent({ const annFilter = annFilters.length > 0 ? Prisma.join(annFilters, " ") : Prisma.empty; - // Retrieval is split into two bounded phases. First, keep the ANN query flat so - // PlanetScale can use the vector index for a small chunk-candidate pool. Then - // hydrate/filter only those chunk ids through the relational partner/platform - // joins below. This keeps the expensive joins off the vector traversal path. - // The cheap phase also returns the content-item id + source so we can dedup to - // the best chunk per item+source BEFORE the join (see retrievePool). + // Two-phase retrieval: keep the ANN flat so PlanetScale uses the vector index, + // then hydrate only the returned chunk ids through the relational joins — keeping + // the joins off the vector traversal path. const fetchCandidateChunks = (poolSize: number) => prisma.$queryRaw(Prisma.sql` SELECT @@ -219,12 +207,8 @@ export async function searchPartnerNetworkContent({ LIMIT ${poolSize} `); - // Dedup to the best chunk per content-item + source BEFORE the relational - // hydration join. The post-ANN filters are all per-item/per-partner, so the - // surviving items are identical whether we filter-then-dedup or dedup-then- - // filter; this just keeps chunk-heavy items from sending redundant rows through - // the 5-way join. itemRows (cutoff) and rows (rerank) are both derived from the - // per-item+source set below, so nothing downstream changes. + // Dedup to the best chunk per item+source before the hydration join (filters are + // per-item, so this is equivalent and keeps chunk-heavy items off the join). const retrievePool = async (poolSize: number) => { const candidateRows = await fetchCandidateChunks(poolSize); logTiming?.("vector-candidate-search-complete", { @@ -252,12 +236,9 @@ export async function searchPartnerNetworkContent({ let poolSize = initialPoolSize; let { candidateRowCount, poolRows } = await retrievePool(poolSize); - // Eligibility + user filters run INLINE in the ANN, so the pool comes back - // already eligible; networkStatus is the only remaining post-ANN filter and it's - // non-selective (ingestion gates on approved/trusted), so the hydrated pool rarely - // shrinks. Keep a one-shot expansion as a safety net for the rare case it does - // (e.g. a cluster of demoted partners): if we under-filled and the ANN hadn't - // exhausted the index (it returned a full pool), widen to the cap once and retry. + // The pool comes back already eligible (filters run inline), so it rarely shrinks. + // Safety net for when it does (e.g. a cluster of demoted partners): if we + // under-filled but the ANN returned a full pool, widen to the cap once and retry. let distinctItemCount = countDistinctContentItems(poolRows); if ( poolSize < maxPoolSize && @@ -284,11 +265,9 @@ export async function searchPartnerNetworkContent({ retrievalShape: "two-phase", }); - // Best cosine distance per content-item + evidence source across the candidate - // pool. getPartnerMatchSummaries reuses this to gate which recent posts count as - // "matched" instead of recomputing DISTANCE per item in SQL: cutoffDistance is - // itself a pool item's distance, so any item that could clear it is already in - // this map (an item missing here is provably beyond the cutoff — not matched). + // Best distance per item+source. getPartnerMatchSummaries reuses this to gate + // matched recent posts instead of a second per-item DISTANCE pass — any item that + // could clear the cutoff is already here (one missing is provably beyond it). const itemSourceBestDistance: SourceScoreByContentItemId = new Map(); for (const row of poolRows) { setSourceDistance( @@ -299,20 +278,15 @@ export async function searchPartnerNetworkContent({ ); } - // Collapse to one best chunk per content item for the item-level cutoff, and - // one best chunk per content item + source for reranking/source-aware evidence. - // This keeps metadata and transcript evidence separable without letting one - // chunk-heavy video flood the candidate pool. + // One best chunk per item for the cutoff; one per item+source for rerank/evidence + // (keeps metadata and transcript evidence separable). const itemRows = dedupeBestChunkPerContentItem(poolRows).slice(0, limit); const rows = dedupeBestChunkPerContentItemSource(poolRows).slice( 0, Math.min(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, limit * 2), ); - // Cosine-distance cutoff = the least-relevant candidate kept. getPartnerMatchSummaries - // uses this to decide which of a partner's recent videos count as "matched" (i.e. - // at least as relevant as the weakest video in the candidate pool), computed - // exactly per item, independent of which raw chunk survived the global cap. + // Cutoff = the least-relevant candidate kept; the match gate for recent posts. const cutoffDistance = itemRows.length ? Number(itemRows[itemRows.length - 1].distance) : null; @@ -377,18 +351,9 @@ async function hydratePartnerContentSearchRows({ if (candidateRows.length === 0) return []; const chunkIds = candidateRows.map(({ chunkId }) => chunkId); - // Eligibility + user filters (enrolled/ignored/starred/country/platform) are all - // pre-filtered in the ANN now, so hydration is a plain metadata fetch over - // already-eligible chunk ids. networkStatus is the one remaining post-filter - // (cheap and non-selective — see the search query note); the partner/platform - // joins stay only to supply display columns and that networkStatus check. - - // Plain relational hydration over an already-eligible chunk-id set: the inner - // joins in the prior raw form were pure filters (all relations are required - // FKs), and the lone post-ANN predicate, networkStatus, maps to the partner - // relation filter below. `chunkText` is deliberately not selected here (it's - // hydrated separately in hydratePartnerContentChunkText); we backfill the "" - // placeholder when flattening to the row shape. + // Plain metadata fetch over already-eligible chunk ids (user/eligibility filters + // ran inline in the ANN). networkStatus is the one remaining post-filter, via the + // partner relation. chunkText is hydrated separately below, so we backfill "". const hydratedRows = await prisma.partnerContentChunk.findMany({ where: { id: { in: chunkIds }, @@ -538,3 +503,60 @@ async function hydratePartnerContentChunkText({ : row, ); } + +// Single-phase admin diagnostics query: one raw ANN with inline hydration, no +// eligibility/network pre-filtering (intentionally searches the full corpus). +export async function searchAdminPartnerContentChunks({ + queryVector, + limit, + partnerIds, + platform, +}: { + queryVector: string; + limit: number; + partnerIds?: string[]; + platform?: PlatformType; +}) { + const partnerFilter = partnerIds?.length + ? Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})` + : Prisma.empty; + const platformFilter = platform + ? Prisma.sql`AND pp.type = ${platform}` + : Prisma.empty; + + return await prisma.$queryRaw(Prisma.sql` + SELECT + c.id AS chunkId, + c.partnerContentItemId, + c.partnerId, + p.name AS partnerName, + p.username AS partnerUsername, + p.image AS partnerImage, + p.description AS partnerDescription, + pp.type AS platformType, + pp.identifier AS platformIdentifier, + pci.platformContentId, + pci.url AS contentUrl, + pci.contentType, + pci.title AS contentTitle, + pci.thumbnailUrl AS contentThumbnailUrl, + pci.publishedAt AS contentPublishedAt, + pci.durationMs AS contentDurationMs, + c.source AS chunkSource, + c.chunkIndex, + c.chunkText, + c.startMs, + c.endMs, + DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance + FROM PartnerContentChunk c + INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId + INNER JOIN Partner p ON p.id = c.partnerId + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + WHERE c.embedding IS NOT NULL + AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + ${partnerFilter} + ${platformFilter} + ORDER BY distance ASC + LIMIT ${limit} + `); +} diff --git a/apps/web/lib/partner-content-search/search-utils.ts b/apps/web/lib/partner-content-search/search-utils.ts index 3d0865e71e2..18b9c42c587 100644 --- a/apps/web/lib/partner-content-search/search-utils.ts +++ b/apps/web/lib/partner-content-search/search-utils.ts @@ -1,6 +1,5 @@ -// Shared helpers for the partner content search routes (admin + network). -// Both routes run the same raw vector query and group rows by partner; only the -// per-chunk shape differs, so each route passes its own `toChunkResult` mapper. +// Shared helpers for the admin + network content search routes: per-partner +// grouping (each route passes its own toChunkResult) and reranking. import { PARTNER_CONTENT_SEARCH_LIMITS, @@ -42,8 +41,7 @@ export type PartnerContentSearchRow = { startMs: number | null; endMs: number | null; distance: number | string; - // Set by the reranker (second stage) when it runs; absent for cosine-only - // results. `rerankPartnerSearchRows` mutates rows in place to attach it. + // Attached by the reranker (second stage); absent for cosine-only results. rerankScore?: number | null; }; @@ -51,21 +49,28 @@ export function toScore(distance: number) { return Number((1 - distance).toFixed(6)); } -// The score used for ranking + display: the reranker's relevance score when -// present, otherwise cosine similarity (1 - cosine distance). +// Median of a numeric list (robust center; null when empty). `round` rounds the +// even-length average for display; leave it off when the value feeds further math. +export function median( + values: number[], + { round = false }: { round?: boolean } = {}, +): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 !== 0) return sorted[mid]; + const average = (sorted[mid - 1] + sorted[mid]) / 2; + return round ? Math.round(average) : average; +} + +// Effective score: reranker relevance when present, else cosine similarity. export function effectiveRowScore(row: PartnerContentSearchRow) { return row.rerankScore ?? toScore(Number(row.distance)); } -// Collapse a flat, distance-ascending chunk pool to the single best chunk per -// content item. Because rows are sorted best-first, the first occurrence of each -// content item is its best chunk. This turns a top-k *chunk* pool into a per-video -// candidate set in app code, avoiding a SQL window-function dedup that would force -// a full distance scan and defeat the ANN vector index. -// -// Generic over the row shape so callers can dedup lightweight candidate rows -// (chunkId + ids + distance) before the hydration join as well as fully-hydrated -// rows. +// Collapse a distance-ascending chunk pool to the best chunk per content item (first +// occurrence wins). Done in app code, not SQL, so a window-function dedup doesn't +// defeat the ANN index. Generic so it works pre- or post-hydration. export function dedupeBestChunkPerContentItem< T extends { partnerContentItemId: string }, >(rows: T[]) { @@ -173,11 +178,8 @@ export function groupPartnerSearchResults({ .map(({ chunkKeys, ...partner }) => partner); } -// Re-score the top vector-search candidates with the Voyage reranker. Mutates -// rows in place to attach `rerankScore`, then returns them sorted by effective -// score (rerank when present) descending. Fails *soft*: on a reranker timeout or -// API error it logs and returns the original cosine ordering, so the second-stage -// call can never break search; it can only improve it. +// Re-score the top candidates with the Voyage reranker, sorted by effective score. +// Fails soft: on timeout/API error, returns the original cosine ordering. export async function rerankPartnerSearchRows({ query, rows, @@ -187,8 +189,7 @@ export async function rerankPartnerSearchRows({ }): Promise<{ rows: PartnerContentSearchRow[]; reranked: boolean }> { if (rows.length === 0) return { rows, reranked: false }; - // Only the top-N cosine candidates are reranked, to bound the rerank payload - // (and latency). Rows beyond the cap keep their cosine score. + // Only the top-N candidates are reranked, to bound payload/latency. const candidates = rows.slice( 0, PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, @@ -226,9 +227,8 @@ export async function rerankPartnerSearchRows({ throw error; } - // Reranked candidates always sort ahead of the cosine-only tail (rows beyond - // the rerank cap): their cosine score was already higher, and we don't want a - // weak un-reranked row leapfrogging a reranked one on a different score scale. + // Reranked candidates sort ahead of the cosine-only tail (don't let an + // un-reranked row leapfrog a reranked one on a different score scale). const reranked = [...rows].sort((a, b) => { const aReranked = a.rerankScore != null; const bReranked = b.rerankScore != null; diff --git a/apps/web/lib/partner-content-search/timing.ts b/apps/web/lib/partner-content-search/timing.ts index 14afc8a0538..3e08d3e219c 100644 --- a/apps/web/lib/partner-content-search/timing.ts +++ b/apps/web/lib/partner-content-search/timing.ts @@ -1,7 +1,5 @@ -// Stage timing logger for the partner content search route. Emits one structured -// console line per pipeline stage with the elapsed/delta timing plus a shared -// request context, throttling noisy sub-5ms stages unless they're on the -// always-log allowlist below. +// Stage timing logger for the content search route: one structured console line +// per pipeline stage (elapsed/delta), throttling sub-5ms stages off the allowlist. const MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS = 5; const PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES = new Set([ diff --git a/apps/web/lib/partner-content-search/top-content-ranking.ts b/apps/web/lib/partner-content-search/top-content-ranking.ts index 311f40d96f0..3237503b80d 100644 --- a/apps/web/lib/partner-content-search/top-content-ranking.ts +++ b/apps/web/lib/partner-content-search/top-content-ranking.ts @@ -1,29 +1,12 @@ -// Pure ranking helpers for the partner detail-pane "Top content" section. -// -// This module is intentionally dependency-free (only constants) so it can run on -// both the server and the client without dragging server-only code (voyage, db) -// into the client bundle. It blends a matched post's relevance with how well it -// performed (views), normalized per creator and robust to a single viral post. -// -// Scope note: this orders the "Top content" list only. It does not feed Topic -// Fit, the content-match bars, the X-of-Y counts, or partner ordering. +// Pure ranking helpers for the partner detail-pane "Top content" list. Dependency- +// free (constants only) so it stays client-safe — no server-only code (voyage, db) +// in the bundle. Orders that list only; never feeds Topic Fit, the bars, or counts. import { PARTNER_CONTENT_SEARCH_TOP_CONTENT } from "./constants"; +import { median } from "./search-utils"; -// Median of a numeric list (robust center, unlike a mean which a viral outlier -// would drag upward). Returns null for an empty list. -function median(values: number[]): number | null { - if (values.length === 0) return null; - const sorted = [...values].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 - ? (sorted[mid - 1] + sorted[mid]) / 2 - : sorted[mid]; -} - -// The creator's "typical" view count, used as the engagement baseline. Computed -// over all of their recent posts (matched + unmatched) with real view data, so a -// single viral hit can't move it and posts aren't normalized against each other. +// Creator's "typical" view count (median over recent posts with view data), used as +// the engagement baseline — robust to a single viral hit. export function getViewBaseline( views: Array, ): number | null { @@ -32,13 +15,9 @@ export function getViewBaseline( ); } -// Maps a post's views to a [0,1] engagement score relative to the creator's -// median, on a log scale through a logistic squash: -// - a median post scores ~0.5, -// - a viral outlier saturates toward 1 (bounded, never dominates), -// - a below-median post lands near 0 but is never negative (not punished). -// Returns null when there's no usable signal (no baseline, or no views on the -// post) so callers can fall back to pure relevance instead of guessing. +// Maps a post's views to a [0,1] engagement score vs. the creator's median, via a +// log-scale logistic: median ~0.5, viral saturates toward 1, below-median near 0 +// (never negative). Null when there's no usable signal (no baseline or no views). export function getEngagementScore({ views, baselineViews, @@ -57,10 +36,8 @@ export function getEngagementScore({ return 1 / (1 + Math.exp(-z)); } -// Relevance-led blend used to order the "Top content" list. When there's no -// usable engagement signal (no creator baseline or no views on the post), this -// degrades gracefully to pure relevance so ranking never depends on view data -// being present. +// Relevance-led blend ordering the "Top content" list; degrades to pure relevance +// when there's no engagement signal. export function getBlendedTopContentScore({ relevance, views, diff --git a/apps/web/lib/partner-content-search/voyage.ts b/apps/web/lib/partner-content-search/voyage.ts index f3a884f4099..413192a7ede 100644 --- a/apps/web/lib/partner-content-search/voyage.ts +++ b/apps/web/lib/partner-content-search/voyage.ts @@ -28,8 +28,7 @@ type VoyageRerankResponse = { type VoyageFetch = typeof fetch; -// Typed error so callers can branch on the HTTP status (e.g. 429 rate limits) -// instead of string-matching the message. +// Typed error so callers can branch on HTTP status (e.g. 429), not the message. export class VoyageApiError extends Error { constructor( readonly status: number, @@ -40,10 +39,8 @@ export class VoyageApiError extends Error { } } -// Thrown when a Voyage request exceeds the caller-supplied timeout. User-facing -// search routes catch this to fail fast (through the normal error path, with -// rate-limit headers + logging) instead of hanging until Vercel kills the -// function at maxDuration. +// Thrown when a Voyage request exceeds the caller's timeout. Search routes catch +// it to fail fast instead of hanging until Vercel kills the function at maxDuration. export class VoyageTimeoutError extends Error { constructor( readonly endpoint: string, @@ -61,8 +58,7 @@ function isAbortOrTimeoutError(error: unknown): boolean { ); } -// Serialize an embedding for use in raw SQL TO_VECTOR(...) statements. Shared by -// the embed cron route (writes) and the admin search route (query vector). +// Serialize an embedding for raw SQL TO_VECTOR(...). Used by the embed + search routes. export function serializeEmbeddingForVector(embedding: number[]) { const expectedDimensions = PARTNER_CONTENT_SEARCH_MODELS.embedding.dimensions; @@ -143,8 +139,7 @@ export async function embedPartnerContentTexts({ inputType: VoyageInputType; apiKey?: string; fetchImpl?: VoyageFetch; - // When set, aborts the request after this many ms (opt-in: the bulk embed cron - // omits it so large batches aren't cut off; user-facing search routes pass it). + // Opt-in abort timeout; the bulk embed cron omits it so large batches aren't cut off. timeoutMs?: number; }) { if (!apiKey) throw new Error("VOYAGE_API_KEY is not set."); @@ -192,12 +187,10 @@ export async function rerankPartnerContent({ topK?: number; apiKey?: string; fetchImpl?: VoyageFetch; - // When set, aborts the request after this many ms. User-facing search routes - // pass it so a slow rerank fails fast and falls back to cosine ordering. + // Opt-in abort timeout; search routes pass it so a slow rerank falls back to cosine. timeoutMs?: number; }): Promise { - // AI SDK Core supports embeddings, but not Voyage reranking. Keep this direct - // wrapper until Dub has a shared reranker abstraction. + // AI SDK Core doesn't support Voyage reranking; direct wrapper until we share one. if (!apiKey) throw new Error("VOYAGE_API_KEY is not set."); if (documents.length === 0) return []; diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index 0a1e294eac1..623aa80817d 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -162,3 +162,36 @@ export const partnerNetworkContentSearchSchema = z.object({ // Second-stage reranking is on by default; pass `false` for diagnostics. rerank: z.boolean().default(true), }); + +const PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT = 10; +const PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER = 3; + +// Request body for the admin diagnostics route POST /api/admin/partner-content/search. +// Simpler than the network schema: a required query, a single platform filter, and +// it surfaces more chunks per partner for inspection. +export const partnerAdminContentSearchSchema = z.object({ + query: z.string().trim().min(1).max(500), + limit: z + .number() + .int() + .positive() + .max(50) + .default(PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT), + chunksPerPartner: z + .number() + .int() + .positive() + .max(10) + .default(PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER), + candidateChunkCount: z + .number() + .int() + .positive() + .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) + .optional(), + partnerIds: z.array(z.string()).min(1).max(100).optional(), + platform: z.enum(PlatformType).optional(), + // Second-stage reranking is on by default; pass `false` to inspect cosine-only + // ranking for comparison. + rerank: z.boolean().default(true), +}); diff --git a/apps/web/tests/partner-content-search/ranking.test.ts b/apps/web/tests/partner-content-search/ranking.test.ts index 18e26de8109..26ca5b569a5 100644 --- a/apps/web/tests/partner-content-search/ranking.test.ts +++ b/apps/web/tests/partner-content-search/ranking.test.ts @@ -1,11 +1,11 @@ import { createContentMatchEvidence, deriveTopicFit, - getEntityCreatorTextBoost, getEvidenceMatchScore, getEvidenceTopicScore, - getPartnerContentSearchQuerySignals, + getMatchedSourceScore, } from "@/lib/partner-content-search/ranking"; +import { PARTNER_CONTENT_SEARCH_TOPIC_FIT } from "@/lib/partner-content-search/constants"; import { describe, expect, test, vi } from "vitest"; // Mock server-only module (ranking.ts imports search-utils.ts -> voyage.ts). @@ -37,55 +37,31 @@ describe("partner content ranking", () => { expect(getEvidenceTopicScore(transcriptVideo)).toBe(1); }); - test("lets exact entity captions override the default creator text video discount", () => { - const evidence = createContentMatchEvidence({ - contentType: "video", - transcriptScore: null, - creatorTextScore: 0.95, - creatorTextWeightOverride: 0.95, - }); - - expect(evidence.creatorTextWeight).toBe(0.95); - expect(evidence.weight).toBe(0.95); - expect(getEvidenceMatchScore(evidence)).toBe(0.95); - expect(getEvidenceTopicScore(evidence)).toBe(0.95); - }); - - test("only boosts exact creator text for entity-style queries", () => { - const entityQuery = getPartnerContentSearchQuerySignals("Framer"); - const semanticQuery = - getPartnerContentSearchQuerySignals("fitness influencer"); + test("gates a recent post as matched only via reranker threshold or cutoff distance", () => { + const { rerankMatchThreshold } = PARTNER_CONTENT_SEARCH_TOPIC_FIT; - expect(entityQuery.intent).toBe("entity"); - expect(semanticQuery.intent).toBe("semantic"); + // Reranker score is authoritative when present: at/above the threshold matches, + // below it does not — regardless of distance. + expect(getMatchedSourceScore({ rerankScore: rerankMatchThreshold })).toBe( + rerankMatchThreshold, + ); + expect( + getMatchedSourceScore({ rerankScore: rerankMatchThreshold - 0.01 }), + ).toBeNull(); + // Without a reranker score, fall back to the cosine cutoff: within cutoff + // matches (scored as 1 - distance), beyond it does not. expect( - getEntityCreatorTextBoost({ - platformType: "tiktok", - contentType: "video", - transcriptFetchStatus: "pending", - titleHasExactQueryMention: false, - descriptionHasExactQueryMention: true, - chunkHasExactQueryMention: true, - transcriptScore: null, - queryIntent: entityQuery.intent, - }), - ).toEqual({ - score: 0.95, - weight: 0.95, - }); + getMatchedSourceScore({ bestDistance: 0.3, cutoffDistance: 0.4 }), + ).toBe(0.7); expect( - getEntityCreatorTextBoost({ - platformType: "tiktok", - contentType: "video", - transcriptFetchStatus: "pending", - titleHasExactQueryMention: false, - descriptionHasExactQueryMention: true, - chunkHasExactQueryMention: true, - transcriptScore: null, - queryIntent: semanticQuery.intent, - }), + getMatchedSourceScore({ bestDistance: 0.5, cutoffDistance: 0.4 }), ).toBeNull(); + + // No retrieval evidence at all → no match (lexical title/description text + // alone can no longer create or boost a match). + expect(getMatchedSourceScore({})).toBeNull(); + expect(getMatchedSourceScore({ bestDistance: 0.3 })).toBeNull(); }); test("does not let high creator text override weaker transcript evidence at full weight", () => { @@ -144,4 +120,44 @@ describe("partner content ranking", () => { expect(topicFit.topicFit).toBeGreaterThanOrEqual(98); }); + + test("topic fit demotes a thin one-off below a deep but diluted creator", () => { + // One on-topic post, nothing else: topically "pure" but unproven. + const oneOff = deriveTopicFit({ + matchedContentCount: 1, + weightedMatchedContentCount: 1, + weightedMatchedContentScore: 0.9, + recentContentCount: 1, + }); + // Many on-topic posts diluted by off-topic ones (the creator also posts + // about other things). Lower purity, but a real body of on-topic work. + const deepButDiluted = deriveTopicFit({ + matchedContentCount: 30, + weightedMatchedContentCount: 30, + weightedMatchedContentScore: 30, + recentContentCount: 100, + }); + + expect(oneOff.band).toBe("one-off"); + expect(oneOff.topicFit).toBeLessThan(50); + expect(deepButDiluted.topicFit).toBeGreaterThan(oneOff.topicFit); + }); + + test("topic fit rewards on-topic depth over a shallow diluted creator", () => { + const deepButDiluted = deriveTopicFit({ + matchedContentCount: 30, + weightedMatchedContentCount: 30, + weightedMatchedContentScore: 30, + recentContentCount: 100, + }); + // Only a couple of on-topic posts amid many off-topic ones: little evidence. + const shallowDiluted = deriveTopicFit({ + matchedContentCount: 2, + weightedMatchedContentCount: 2, + weightedMatchedContentScore: 2, + recentContentCount: 50, + }); + + expect(deepButDiluted.topicFit).toBeGreaterThan(shallowDiluted.topicFit); + }); }); From 45bb8156970e1d126d0aa3f5fa8c8b66dedc8ce5 Mon Sep 17 00:00:00 2001 From: David Clark Date: Tue, 23 Jun 2026 08:31:07 -0400 Subject: [PATCH 16/25] refactor partner network UI and content-search into smaller modules --- .../admin/partner-content/backfill/route.ts | 4 - .../api/admin/partner-content/search/route.ts | 126 --- .../api/cron/partner-content/embed/route.ts | 6 +- .../cron/partner-content/enumerate/route.ts | 2 +- .../cron/partner-content/transcript/route.ts | 12 +- .../network/partners/content-search/route.ts | 16 +- .../(ee)/api/network/partners/count/route.ts | 29 +- .../api/partner-content/thumbnail/route.ts | 49 +- .../network/[partnerId]/content-match-row.tsx | 257 ++++++ .../[partnerId]/network-invite-control.tsx | 46 + .../network/[partnerId]/page-client.tsx | 853 +----------------- .../network/[partnerId]/search-fit-bars.tsx | 150 +++ .../network/[partnerId]/search-fit-panel.tsx | 296 ++++++ .../network/[partnerId]/search-fit-utils.ts | 113 +++ .../program/network/content-display-utils.ts | 13 + .../network-content-search-results.tsx | 110 +-- .../network-partner-audience-section.tsx | 70 ++ .../network/network-partner-card-actions.tsx | 84 ++ .../network/network-partner-card-types.ts | 9 + .../program/network/network-partner-card.tsx | 404 +-------- .../network/network-platform-filter.tsx | 36 +- .../program/network/overflow-pill-list.tsx | 116 +++ .../(ee)/program/network/page-client.tsx | 234 +---- .../(ee)/program/network/platform-icon.tsx | 26 + .../program/network/platform-stat-card.tsx | 129 +++ .../network/use-network-detail-sheet.ts | 67 ++ .../use-network-partner-filters-state.ts | 71 ++ .../network/use-toggle-partner-starred.ts | 140 +++ .../network/partner-network-listing-where.ts | 49 + .../content-match-bars.ts | 225 +++++ .../ingestion/eligibility-where.ts | 75 ++ .../ingestion/enqueue.ts | 246 +---- ...s => fetch-and-write-transcript-chunks.ts} | 2 +- .../ingestion/payload-schemas.ts | 114 +++ .../ingestion/routes.ts | 45 + .../partner-content-search/match-summaries.ts | 398 +------- .../match-summary-queries.ts | 182 ++++ apps/web/lib/partner-content-search/rerank.ts | 76 ++ .../lib/partner-content-search/retrieval.ts | 59 +- .../partner-content-search/search-utils.ts | 99 +- .../partner-content-search/thumbnail-url.ts | 3 +- .../web/lib/swr/use-partner-content-search.ts | 4 - apps/web/lib/zod/schemas/partner-network.ts | 40 - apps/web/ui/partners/partner-basic-fields.tsx | 165 ++++ apps/web/ui/partners/partner-info-cards.tsx | 407 +-------- .../ui/partners/partner-info-rewards-card.tsx | 170 ++++ .../ui/partners/partner-info-tags-card.tsx | 46 + apps/web/ui/partners/referred-by-partner.tsx | 72 ++ 48 files changed, 3042 insertions(+), 2903 deletions(-) delete mode 100644 apps/web/app/(ee)/api/admin/partner-content/search/route.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/network-invite-control.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-audience-section.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-actions.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-types.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/overflow-pill-list.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-icon.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-stat-card.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-detail-sheet.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts create mode 100644 apps/web/lib/partner-content-search/content-match-bars.ts create mode 100644 apps/web/lib/partner-content-search/ingestion/eligibility-where.ts rename apps/web/lib/partner-content-search/ingestion/{write-transcript-chunks.ts => fetch-and-write-transcript-chunks.ts} (98%) create mode 100644 apps/web/lib/partner-content-search/ingestion/payload-schemas.ts create mode 100644 apps/web/lib/partner-content-search/ingestion/routes.ts create mode 100644 apps/web/lib/partner-content-search/match-summary-queries.ts create mode 100644 apps/web/lib/partner-content-search/rerank.ts create mode 100644 apps/web/ui/partners/partner-basic-fields.tsx create mode 100644 apps/web/ui/partners/partner-info-rewards-card.tsx create mode 100644 apps/web/ui/partners/partner-info-tags-card.tsx create mode 100644 apps/web/ui/partners/referred-by-partner.tsx diff --git a/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts b/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts index 8bc2c3c46b5..509db840a90 100644 --- a/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts +++ b/apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts @@ -23,8 +23,6 @@ const backfillTriggerSchema = z.object({ export const POST = withAdmin( async ({ req }) => { try { - // Fail loud on malformed JSON instead of silently widening to a - // full backfill; parseRequestBody throws a 400 DubApiError. const body = backfillTriggerSchema.parse(await parseRequestBody(req)); const runStamp = body.runStamp ?? createPartnerContentRunStamp(); @@ -57,8 +55,6 @@ export const POST = withAdmin( qstashResponse, }); } catch (error) { - // Normalize ZodError -> 422, DubApiError -> 4xx, QStash failure -> 500 - // (the withAdmin wrapper does not do this for us). return handleAndReturnErrorResponse(error); } }, diff --git a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts b/apps/web/app/(ee)/api/admin/partner-content/search/route.ts deleted file mode 100644 index 3734b90dccc..00000000000 --- a/apps/web/app/(ee)/api/admin/partner-content/search/route.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { DubApiError, handleAndReturnErrorResponse } from "@/lib/api/errors"; -import { parseRequestBody } from "@/lib/api/utils"; -import { withAdmin } from "@/lib/auth"; -import { - PARTNER_CONTENT_SEARCH_LIMITS, - PARTNER_CONTENT_SEARCH_MODELS, - PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, -} from "@/lib/partner-content-search/constants"; -import { searchAdminPartnerContentChunks } from "@/lib/partner-content-search/retrieval"; -import { - groupPartnerSearchResults, - rerankPartnerSearchRows, - toScore, - type PartnerContentSearchRow, -} from "@/lib/partner-content-search/search-utils"; -import { - embedPartnerContentTexts, - serializeEmbeddingForVector, - VoyageTimeoutError, -} from "@/lib/partner-content-search/voyage"; -import { partnerAdminContentSearchSchema } from "@/lib/zod/schemas/partner-network"; -import { NextResponse } from "next/server"; - -export const dynamic = "force-dynamic"; -export const maxDuration = 30; - -// POST /api/admin/partner-content/search -export const POST = withAdmin( - async ({ req }) => { - try { - const body = partnerAdminContentSearchSchema.parse( - await parseRequestBody(req), - ); - const candidateChunkCount = - body.candidateChunkCount ?? - Math.min( - PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, - Math.max(25, body.limit * body.chunksPerPartner * 5), - ); - - let queryEmbedding: number[]; - try { - [queryEmbedding] = await embedPartnerContentTexts({ - input: [body.query], - inputType: "query", - timeoutMs: PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, - }); - } catch (error) { - if (error instanceof VoyageTimeoutError) { - throw new DubApiError({ - code: "internal_server_error", - message: "Partner content search timed out. Please try again.", - }); - } - throw error; - } - - const queryVector = serializeEmbeddingForVector(queryEmbedding); - const candidateRows = await searchAdminPartnerContentChunks({ - queryVector, - limit: candidateChunkCount, - partnerIds: body.partnerIds, - platform: body.platform, - }); - const { rows, reranked } = body.rerank - ? await rerankPartnerSearchRows({ - query: body.query, - rows: candidateRows, - }) - : { rows: candidateRows, reranked: false }; - - return NextResponse.json({ - success: true, - query: body.query, - candidateChunkCount, - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - reranked, - rerankModel: reranked - ? PARTNER_CONTENT_SEARCH_MODELS.reranker.model - : null, - resultCount: rows.length, - partners: groupPartnerSearchResults({ - rows, - limit: body.limit, - chunksPerPartner: body.chunksPerPartner, - toChunkResult, - }), - }); - } catch (error) { - return handleAndReturnErrorResponse(error); - } - }, - { - requiredRoles: ["owner"], - }, -); - -function toChunkResult(row: PartnerContentSearchRow, distance: number) { - return { - chunkId: row.chunkId, - partnerContentItemId: row.partnerContentItemId, - platform: { - type: row.platformType, - identifier: row.platformIdentifier, - }, - content: { - platformContentId: row.platformContentId, - url: row.contentUrl, - title: row.contentTitle, - thumbnailUrl: row.contentThumbnailUrl, - publishedAt: row.contentPublishedAt?.toISOString() ?? null, - durationMs: row.contentDurationMs, - }, - chunk: { - source: row.chunkSource, - index: row.chunkIndex, - chunkText: row.chunkText, - startMs: row.startMs, - endMs: row.endMs, - }, - distance, - score: row.rerankScore ?? toScore(distance), - cosineScore: toScore(distance), - rerankScore: row.rerankScore ?? null, - }; -} diff --git a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts index 33c079fc3ba..aa57cfd262f 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/embed/route.ts @@ -52,8 +52,7 @@ export const POST = withCron(async ({ rawBody }) => { partnerId: true, totalChunkCount: true, embeddedChunkCount: true, - // Source values for the denormalized pre-filter columns (country, platformType), - // stamped on the chunk at embed time — the point it becomes searchable. + // need country and platform type for denormalized pre-filter columns partner: { select: { country: true, @@ -146,8 +145,7 @@ export const POST = withCron(async ({ rawBody }) => { ); } - // Denormalized pre-filter values stamped with the embedding (constant across the - // batch — one content item → one partner/platform). Both are ~static + // Denormalized pre-filter values saved with embeddings. Both are ~static const country = contentItem.partner.country; const platformType = contentItem.partnerPlatform.type; diff --git a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts index adefdb8b0f8..6b7778f8833 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/enumerate/route.ts @@ -88,7 +88,7 @@ export const POST = withCron(async ({ rawBody }) => { partnerIds, }, }, - // Self-continuation hop: re-enter from the last id instead of draining in one go. + // Pick up the next batch on a separate run when there are more partners. ...(hasMore ? [ { diff --git a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts index da60ac65c42..86959d76d14 100644 --- a/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts +++ b/apps/web/app/(ee)/api/cron/partner-content/transcript/route.ts @@ -4,7 +4,7 @@ import { parsePartnerContentCronPayload, partnerContentTranscriptPayloadSchema, } from "@/lib/partner-content-search/ingestion/enqueue"; -import { writeTranscriptChunks } from "@/lib/partner-content-search/ingestion/write-transcript-chunks"; +import { fetchAndWriteTranscriptChunks } from "@/lib/partner-content-search/ingestion/fetch-and-write-transcript-chunks"; import { prisma } from "@/lib/prisma"; import { logAndRespond } from "../../utils"; @@ -70,10 +70,12 @@ export const POST = withCron(async ({ rawBody }) => { ); } - let transcriptWriteResult: Awaited>; + let transcriptResult: Awaited< + ReturnType + >; try { - transcriptWriteResult = await writeTranscriptChunks({ + transcriptResult = await fetchAndWriteTranscriptChunks({ ...contentItem, platform: payload.platform, }); @@ -91,7 +93,7 @@ export const POST = withCron(async ({ rawBody }) => { throw error; } - if (!transcriptWriteResult.transcriptAvailable) { + if (!transcriptResult.transcriptAvailable) { await enqueueEmbedJob({ mode: payload.mode, runStamp: payload.runStamp, @@ -112,6 +114,6 @@ export const POST = withCron(async ({ rawBody }) => { }); return logAndRespond( - `[PartnerContentSearch] Transcribed content item ${contentItem.id}: ${transcriptWriteResult.chunkCount} chunks (${transcriptWriteResult.chunksCreated} created), embed enqueued for ${payload.mode} run ${payload.runStamp}.`, + `[PartnerContentSearch] Transcribed content item ${contentItem.id}: ${transcriptResult.chunkCount} chunks (${transcriptResult.chunksCreated} created), embed enqueued for ${payload.mode} run ${payload.runStamp}.`, ); }); diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index 5face1e6740..ac8466f5224 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -3,7 +3,6 @@ import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-progr import { parseRequestBody } from "@/lib/api/utils"; import { withWorkspace } from "@/lib/auth"; import { - PARTNER_CONTENT_SEARCH_LIMITS, PARTNER_CONTENT_SEARCH_MODELS, PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, } from "@/lib/partner-content-search/constants"; @@ -18,6 +17,7 @@ import { } from "@/lib/partner-content-search/ranking"; import { searchPartnerNetworkContent } from "@/lib/partner-content-search/retrieval"; import { + getCandidateChunkCount, groupPartnerSearchResults, toScore, type PartnerContentSearchRow, @@ -30,7 +30,7 @@ import { NextResponse } from "next/server"; export const dynamic = "force-dynamic"; export const maxDuration = 30; -// POST /api/network/partners/content-search - semantic search over indexed partner content +// POST /api/network/partners/content-search - network discover content search; returns ranked partners with matching chunks export const POST = withWorkspace( async ({ workspace, req }) => { const programId = getDefaultProgramIdOrThrow(workspace); @@ -54,13 +54,11 @@ export const POST = withWorkspace( const body = partnerNetworkContentSearchSchema.parse( await parseRequestBody(req), ); - const candidateChunkCount = body.query - ? body.candidateChunkCount ?? - Math.min( - PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, - Math.max(25, body.limit * body.chunksPerPartner * 6), - ) - : body.limit * body.chunksPerPartner * 2; + const candidateChunkCount = getCandidateChunkCount({ + hasQuery: Boolean(body.query), + limit: body.limit, + chunksPerPartner: body.chunksPerPartner, + }); const logTiming = createPartnerContentSearchTimingLogger({ workspaceId: workspace.id, programId, diff --git a/apps/web/app/(ee)/api/network/partners/count/route.ts b/apps/web/app/(ee)/api/network/partners/count/route.ts index 508a813e476..f7ee03dd2f2 100644 --- a/apps/web/app/(ee)/api/network/partners/count/route.ts +++ b/apps/web/app/(ee)/api/network/partners/count/route.ts @@ -1,14 +1,13 @@ import { DubApiError } from "@/lib/api/errors"; import { partnerNetworkListingParts, + partnerReachWhere, partnerWhereFromListingParts, } from "@/lib/api/network/partner-network-listing-where"; -import { reachTiersToRanges } from "@/lib/api/network/reach-tiers"; import { getDefaultProgramIdOrThrow } from "@/lib/api/programs/get-default-program-id-or-throw"; import { withWorkspace } from "@/lib/auth"; import { prisma } from "@/lib/prisma"; import { getNetworkPartnersCountQuerySchema } from "@/lib/zod/schemas/partner-network"; -import { Prisma } from "@prisma/client"; import { NextResponse } from "next/server"; // GET /api/network/partners/count - get the number of available partners in the network @@ -43,27 +42,8 @@ export const GET = withWorkspace( const commonWhere = partnerWhereFromListingParts(listingParts); - // Reach is a discover-only filter. Approximate the ranking's "max subscribers - // across selected platforms in tier" with a `some` test (a selected platform - // whose subscribers fall in a chosen tier) so pagination totals track the - // filtered discover results. Applied only to discover-scoped counts below. - const reachRanges = reach?.length ? reachTiersToRanges(reach) : []; - const reachWhere: Prisma.PartnerWhereInput = reachRanges.length - ? { - platforms: { - some: { - verifiedAt: { not: null }, - ...(platform?.length && { type: { in: platform } }), - OR: reachRanges.map(({ min, max }) => ({ - subscribers: { - gte: BigInt(min), - ...(max != null && { lt: BigInt(max) }), - }, - })), - }, - }, - } - : {}; + // Same max-subscribers-in-tier rule as the listing. Applied to discover only. + const reachWhere = partnerReachWhere({ reach, platform }); const statusWheres = { discover: { @@ -128,7 +108,7 @@ export const GET = withWorkspace( where: { ...commonWhere, ...statusWheres.discover, - ...reachWhere, + ...reachWhere, // reach filter only applies to discover }, }) : undefined, @@ -171,6 +151,7 @@ export const GET = withWorkspace( where: { ...commonWhere, ...statusWhereForFacet, + // reach filter only applies to discover ...(!status || status === "discover" ? reachWhere : {}), }, orderBy: { diff --git a/apps/web/app/api/partner-content/thumbnail/route.ts b/apps/web/app/api/partner-content/thumbnail/route.ts index 4d88f241cc0..a5d1f7b235e 100644 --- a/apps/web/app/api/partner-content/thumbnail/route.ts +++ b/apps/web/app/api/partner-content/thumbnail/route.ts @@ -1,6 +1,11 @@ import { isInstagramCdnUrl } from "@/lib/partner-content-search/thumbnail-url"; +import { fetchWithTimeout } from "@dub/utils"; import { NextRequest, NextResponse } from "next/server"; +export const maxDuration = 15; + +const THUMBNAIL_FETCH_TIMEOUT_MS = 5000; + export async function GET(req: NextRequest) { const url = req.nextUrl.searchParams.get("url"); @@ -8,30 +13,38 @@ export async function GET(req: NextRequest) { return new NextResponse("Invalid thumbnail URL", { status: 400 }); } - const response = await fetch(url, { - headers: { - Accept: - "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", - "User-Agent": - "Mozilla/5.0 (compatible; DubPartnerContentPreview/1.0; +https://dub.co)", - }, - redirect: "error", - }); - - if (!response.ok || !response.body) { + let response: Response; + try { + response = await fetchWithTimeout( + url, + { + headers: { + Accept: + "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "User-Agent": + "Mozilla/5.0 (compatible; DubPartnerContentPreview/1.0; +https://dub.co)", + }, + // Block redirect-based SSRF off the allowlisted host. + redirect: "error", + }, + THUMBNAIL_FETCH_TIMEOUT_MS, + ); + } catch { return new NextResponse("Failed to fetch thumbnail", { status: 502 }); } const contentType = response.headers.get("content-type") ?? ""; - if (!contentType.startsWith("image/")) { - return new NextResponse("Invalid thumbnail content type", { status: 502 }); + if (!response.ok || !response.body || !contentType.startsWith("image/")) { + return new NextResponse("Failed to fetch thumbnail", { status: 502 }); } - return new NextResponse(response.body, { - headers: { - "Cache-Control": "public, max-age=3600, s-maxage=86400", - "Content-Type": contentType, - }, + const headers = new Headers({ + "Cache-Control": "public, max-age=3600, s-maxage=86400", + "Content-Type": contentType, }); + const contentLength = response.headers.get("content-length"); + if (contentLength) headers.set("Content-Length", contentLength); + + return new NextResponse(response.body, { headers }); } diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx new file mode 100644 index 00000000000..fa743d182e6 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx @@ -0,0 +1,257 @@ +"use client"; + +import { PARTNER_CONTENT_SEARCH_LIMITS } from "@/lib/partner-content-search/constants"; +import { + type PartnerContentMatchEvidence, + type PartnerContentSearchPartner, +} from "@/lib/swr/use-partner-content-search"; +import { cn, nFormatter } from "@dub/utils"; +import { + formatDuration, + formatMatchPercent, + formatPublishedDate, + formatTimestamp, + getContentHref, + getContentThumbnail, + getContentTitle, +} from "../content-display-utils"; +import { PlatformIcon } from "../platform-icon"; +import { type MatchedContentItem } from "./search-fit-utils"; + +export function ContentMatchSkeletons({ count }: { count: number }) { + return ( + <> + {[...Array(count)].map((_, index) => ( +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ))} + + ); +} + +export function ContentMatchRow({ item }: { item: MatchedContentItem }) { + const { chunk } = item; + const evidence = item.matchEvidence; + const isTimedTranscriptMatch = chunk + ? hasTimedTranscriptMatch(chunk.chunk) + : false; + const timeLabel = + chunk && isTimedTranscriptMatch + ? formatChunkTimeRange(chunk.chunk) + : formatDuration(item.durationMs); + const dateLabel = formatPublishedDate(item.publishedAt); + const matchTags = getMatchTags( + evidence, + chunk?.chunk.source ?? + (evidence.primarySource === "creatorText" ? "metadata" : "transcript"), + ); + const snippet = chunk ? getMatchSnippet(chunk) : null; + const excerpt = snippet ? `"…${snippet.slice(0, 130).trimEnd()}…"` : null; + const thumbnail = getItemThumbnail(item); + const meta = [timeLabel, dateLabel].filter(Boolean).join(" · "); + const score = item.relevance; + const title = getItemTitle(item); + const viewCount = item.viewCount; + + return ( + + {/* Preview image */} +
+ {thumbnail ? ( + + ) : ( +
+ +
+ )} +
+
+
+ + + {title} + +
+
+ {meta && {meta}} + {matchTags.length > 0 && ( + <> + {meta && ·} + + {matchTags.map((tag) => ( + + {tag.label} + + ))} + + + )} + {viewCount != null && viewCount > 0 && ( + <> + · + {nFormatter(viewCount)} views + + )} +
+ {excerpt && ( +

+ {excerpt} +

+ )} +
+
+ + {formatMatchPercent(score)} + + + match + +
+ + ); +} + +function getMatchTags( + evidence: PartnerContentMatchEvidence | undefined, + fallbackSource: string, +) { + const sources = evidence?.sources.length + ? evidence.sources + : fallbackSource === "metadata" + ? ["creatorText" as const] + : ["transcript" as const]; + + return sources.map((source) => ({ + source, + label: source === "transcript" ? "Transcript" : "Creator text", + })); +} + +// Displayed snippet. Transcript chunks are prose; creator-text chunks store the raw +// embedding input, so we surface just the creator-entered text. +function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { + const text = (chunk.chunk.text ?? "").trim(); + if (!text) return null; + if (chunk.chunk.source !== "metadata") return text; + + const description = text.match(/Description:\s*([\s\S]+)$/i)?.[1]; + return ( + ( + description ?? text.replace(/^Content type:[\s\S]*?Title:\s*/i, "") + ).trim() || null + ); +} + +// Item thumbnail: the loaded chunk's preview, else a YouTube thumbnail from the id. +function getItemThumbnail(item: MatchedContentItem) { + if (item.chunk) return getContentThumbnail(item.chunk); + if (item.platform === "youtube" && item.platformContentId) { + return `https://i.ytimg.com/vi/${item.platformContentId}/hqdefault.jpg`; + } + return null; +} + +function getItemTitle(item: MatchedContentItem) { + if (item.chunk) return getContentTitle(item.chunk); + return item.title?.trim() || "Untitled content"; +} + +// Link for a matched item: the chunk-aware href (YouTube deep-link timestamp, +// Instagram normalization) when enriched, otherwise the bar's canonical URL. +function getItemHref(item: MatchedContentItem) { + if (item.chunk) return getContentHref(item.chunk); + return item.url ?? "#"; +} + +// Noun phrase for the rank window (time-based but capped per partner); says so +// explicitly when the cap bites instead of implying full coverage. +export function formatRankWindowPhrase( + summary: PartnerContentSearchPartner["matchSummary"] | undefined, +) { + if (!summary || !summary.recentContentCount) return null; + + const { recentContentCount, oldestPublishedAt, newestPublishedAt } = summary; + const oldest = formatMonthYear(oldestPublishedAt); + const newest = formatMonthYear(newestPublishedAt); + const countCapped = + recentContentCount >= PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner; + + if (countCapped) { + return oldest + ? `the ${recentContentCount} most recent posts, back to ${oldest}` + : `the ${recentContentCount} most recent posts`; + } + + if (oldest && newest) { + return oldest === newest + ? `${recentContentCount} ${ + recentContentCount === 1 ? "post" : "posts" + } from ${oldest}` + : `${recentContentCount} posts, ${oldest} – ${newest}`; + } + + return `${recentContentCount} recent ${ + recentContentCount === 1 ? "post" : "posts" + }`; +} + +function formatMonthYear(iso: string | null) { + if (!iso) return null; + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return null; + + return new Intl.DateTimeFormat(undefined, { + month: "short", + year: "numeric", + }).format(date); +} + +function hasTimedTranscriptMatch({ + source, + startMs, + endMs, +}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { + return source !== "metadata" && (startMs !== null || endMs !== null); +} + +function formatChunkTimeRange({ + source, + startMs, + endMs, +}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { + if (source === "metadata") return "Creator text match"; + if (startMs === null && endMs === null) return "Transcript match"; + if (startMs !== null && endMs !== null) { + return `${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; + } + return formatTimestamp(startMs ?? endMs ?? 0); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/network-invite-control.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/network-invite-control.tsx new file mode 100644 index 00000000000..3fb4d84c8a2 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/network-invite-control.tsx @@ -0,0 +1,46 @@ +"use client"; + +import { NetworkPartnerProps } from "@/lib/types"; +import { Button } from "@dub/ui"; +import { EnvelopeArrowRight } from "@dub/ui/icons"; +import { EmailContent } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-email-preview"; +import { InviteNetworkPartnerSheet } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-network-partner-sheet"; +import { useState } from "react"; + +export function NetworkInviteControl({ + partner, + nested, + onSuccess, +}: { + partner: NetworkPartnerProps; + nested?: boolean; + onSuccess: () => void; +}) { + const [showInviteSheet, setShowInviteSheet] = useState(false); + const [inviteEmailContent, setInviteEmailContent] = + useState(null); + + if (partner.invitedAt || partner.recruitedAt) return null; + + return ( + <> + +
- ) : null} -
- )} -
-
- ); -} - -function SearchFitPanelSkeleton() { - return ( -
-
-
-
-
-
-
-
-
- -
- -
-
-
- {[...Array(18)].map((_, index) => ( -
- ))} -
-
-
-
- -
-
-
-
-
-
- -
-
-
- ); -} - -function ContentMatchSkeletons({ count }: { count: number }) { - return ( - <> - {[...Array(count)].map((_, index) => ( -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ))} - - ); -} - -// A matched post for the detail lists: from the cached summary's bars, enriched -// with a loaded chunk (snippet, timed transcript, thumbnail) when available. -type MatchedContentItem = { - contentItemId: string; - platform: string; - platformContentId: string; - title: string | null; - url: string | null; - durationMs: number | null; - publishedAt: string | null; - viewCount: number | null; - // The displayed relevance rating (0-1); also feeds the blend. - relevance: number; - // Relevance + reach blend, used only to order the Top content list. - blendedScore: number; - matchEvidence: PartnerContentMatchEvidence; - chunk?: PartnerContentSearchPartner["chunks"][number]; -}; - -// Per-item relevance from the scoped reranked summary (one scale; the cached global -// summary mixes rerank + cosine). Includes any item with evidence, not just matched. -function buildUnifiedRelevanceMap( - relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, -) { - const map = new Map(); - for (const bar of relevanceSummary?.contentBars ?? []) { - const score = getEvidenceDisplayScore(bar.matchEvidence); - if (score != null) map.set(bar.partnerContentItemId, score); - } - return map; -} - -function buildMatchedContentItems( - summary: PartnerContentSearchPartner["matchSummary"] | undefined, - chunks: PartnerContentSearchPartner["chunks"], - unifiedRelevanceByItemId?: Map, -): MatchedContentItem[] { - const bars = summary?.contentBars ?? []; - - // Best loaded chunk per content item, for snippet/thumbnail enrichment. - const chunkByContentItemId = new Map< - string, - PartnerContentSearchPartner["chunks"][number] - >(); - for (const chunk of chunks) { - const current = chunkByContentItemId.get(chunk.partnerContentItemId); - if (!current || chunk.score > current.score) { - chunkByContentItemId.set(chunk.partnerContentItemId, chunk); - } - } - - // Per-creator engagement baseline: median recent views (matched + unmatched). - const baselineViews = getViewBaseline(bars.map((bar) => bar.viewCount)); - - return bars - .filter((bar) => bar.matched) - .map((bar) => { - // Prefer the unified single-scale relevance once the rerank lands; else cached. - const relevance = - unifiedRelevanceByItemId?.get(bar.partnerContentItemId) ?? - getEvidenceDisplayScore(bar.matchEvidence) ?? - bar.matchScore ?? - 0; - - return { - contentItemId: bar.partnerContentItemId, - platform: bar.platform, - platformContentId: bar.platformContentId, - title: bar.title, - url: bar.url, - durationMs: bar.durationMs, - publishedAt: bar.publishedAt, - viewCount: bar.viewCount, - relevance, - blendedScore: getBlendedTopContentScore({ - relevance, - views: bar.viewCount, - baselineViews, - }), - matchEvidence: bar.matchEvidence, - chunk: chunkByContentItemId.get(bar.partnerContentItemId), - }; - }); -} - -function publishedAtMs(iso: string | null) { - if (!iso) return -Infinity; - const ms = new Date(iso).getTime(); - return Number.isNaN(ms) ? -Infinity : ms; -} - -function ContentMatchRow({ item }: { item: MatchedContentItem }) { - const { chunk } = item; - const evidence = item.matchEvidence; - const isTimedTranscriptMatch = chunk - ? hasTimedTranscriptMatch(chunk.chunk) - : false; - const timeLabel = - chunk && isTimedTranscriptMatch - ? formatChunkTimeRange(chunk.chunk) - : formatDuration(item.durationMs); - const dateLabel = formatPublishedDate(item.publishedAt); - const matchTags = getMatchTags( - evidence, - chunk?.chunk.source ?? - (evidence.primarySource === "creatorText" ? "metadata" : "transcript"), - ); - const snippet = chunk ? getMatchSnippet(chunk) : null; - const excerpt = snippet ? `"…${snippet.slice(0, 130).trimEnd()}…"` : null; - const thumbnail = getItemThumbnail(item); - const meta = [timeLabel, dateLabel].filter(Boolean).join(" · "); - const score = item.relevance; - const title = getItemTitle(item); - const viewCount = item.viewCount; - - return ( - - {/* Preview image */} -
- {thumbnail ? ( - - ) : ( -
- -
- )} -
-
-
- - - {title} - -
-
- {meta && {meta}} - {matchTags.length > 0 && ( - <> - {meta && ·} - - {matchTags.map((tag) => ( - - {tag.label} - - ))} - - - )} - {viewCount != null && viewCount > 0 && ( - <> - · - {nFormatter(viewCount)} views - - )} -
- {excerpt && ( -

- {excerpt} -

- )} -
-
- - {formatMatchPercent(score)} - - - match - -
- - ); -} - -function getEvidenceDisplayScore( - evidence: PartnerContentMatchEvidence | undefined, -) { - if (!evidence || evidence.sources.length === 0) return null; - - return Math.max( - evidence.transcriptScore ?? 0, - evidence.creatorTextScore ?? 0, - ); -} - -function getMatchTags( - evidence: PartnerContentMatchEvidence | undefined, - fallbackSource: string, -) { - const sources = evidence?.sources.length - ? evidence.sources - : fallbackSource === "metadata" - ? ["creatorText" as const] - : ["transcript" as const]; - - return sources.map((source) => ({ - source, - label: source === "transcript" ? "Transcript" : "Creator text", - })); -} - -// Displayed snippet. Transcript chunks are prose; creator-text chunks store the raw -// embedding input, so we surface just the creator-entered text. -function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { - const text = (chunk.chunk.text ?? "").trim(); - if (!text) return null; - if (chunk.chunk.source !== "metadata") return text; - - const description = text.match(/Description:\s*([\s\S]+)$/i)?.[1]; - return ( - ( - description ?? text.replace(/^Content type:[\s\S]*?Title:\s*/i, "") - ).trim() || null - ); -} - -function PlatformIcon({ - platform, - className, -}: { - platform: string; - className?: string; -}) { - const Icon = - platform === "youtube" - ? YouTube - : platform === "instagram" - ? Instagram - : platform === "tiktok" - ? TikTok - : null; - - return Icon ? : null; -} - -// Item thumbnail: the loaded chunk's preview, else a YouTube thumbnail from the id. -function getItemThumbnail(item: MatchedContentItem) { - if (item.chunk) return getContentThumbnail(item.chunk); - if (item.platform === "youtube" && item.platformContentId) { - return `https://i.ytimg.com/vi/${item.platformContentId}/hqdefault.jpg`; - } - return null; -} - -function getItemTitle(item: MatchedContentItem) { - if (item.chunk) return getContentTitle(item.chunk); - return item.title?.trim() || "Untitled content"; -} - -// Link for a matched item: the chunk-aware href (YouTube deep-link timestamp, -// Instagram normalization) when enriched, otherwise the bar's canonical URL. -function getItemHref(item: MatchedContentItem) { - if (item.chunk) return getContentHref(item.chunk); - return item.url ?? "#"; -} - -// Noun phrase for the rank window (time-based but capped per partner); says so -// explicitly when the cap bites instead of implying full coverage. -function formatRankWindowPhrase( - summary: PartnerContentSearchPartner["matchSummary"] | undefined, -) { - if (!summary || !summary.recentContentCount) return null; - - const { recentContentCount, oldestPublishedAt, newestPublishedAt } = summary; - const oldest = formatMonthYear(oldestPublishedAt); - const newest = formatMonthYear(newestPublishedAt); - const countCapped = - recentContentCount >= PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner; - - if (countCapped) { - return oldest - ? `the ${recentContentCount} most recent posts, back to ${oldest}` - : `the ${recentContentCount} most recent posts`; - } - - if (oldest && newest) { - return oldest === newest - ? `${recentContentCount} ${ - recentContentCount === 1 ? "post" : "posts" - } from ${oldest}` - : `${recentContentCount} posts, ${oldest} – ${newest}`; - } - - return `${recentContentCount} recent ${ - recentContentCount === 1 ? "post" : "posts" - }`; -} - -function formatMonthYear(iso: string | null) { - if (!iso) return null; - const date = new Date(iso); - if (Number.isNaN(date.getTime())) return null; - - return new Intl.DateTimeFormat(undefined, { - month: "short", - year: "numeric", - }).format(date); -} - -function hasTimedTranscriptMatch({ - source, - startMs, - endMs, -}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { - return source !== "metadata" && (startMs !== null || endMs !== null); -} - -function formatChunkTimeRange({ - source, - startMs, - endMs, -}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { - if (source === "metadata") return "Creator text match"; - if (startMs === null && endMs === null) return "Transcript match"; - if (startMs !== null && endMs !== null) { - return `${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; - } - return formatTimestamp(startMs ?? endMs ?? 0); -} - function getBackQueryString(searchParams: { toString(): string }) { const params = new URLSearchParams(searchParams.toString()); const queryString = params.toString(); diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx new file mode 100644 index 00000000000..efca77e38a6 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx @@ -0,0 +1,150 @@ +"use client"; + +import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { Tooltip } from "@dub/ui"; +import { cn, nFormatter } from "@dub/utils"; +import { useState } from "react"; +import { + formatDuration, + formatPublishedDate, +} from "../content-display-utils"; +import { PlatformIcon } from "../platform-icon"; + +// Per-post match bars: height = match magnitude, color = platform. Muted, evenly- +// weighted palette so platforms read as a calm spectrum (none looks like a non-match). +const PLATFORM_BAR_COLORS: Record = { + youtube: "bg-[#bd8488]", + instagram: "bg-[#b083a2]", + tiktok: "bg-[#74a09a]", + twitter: "bg-[#8589b3]", + x: "bg-[#8589b3]", + linkedin: "bg-[#7c9cbd]", + website: "bg-[#bda77c]", +}; + +// Glanceable recent-activity strip, capped to the most-recent N so columns stay +// hoverable. Topic Fit + the "X of Y" counts still use the full server-side set. +const MAX_VISIBLE_CONTENT_BARS = 40; + +export function ContentMatchBars({ + summary, +}: { + summary: PartnerContentSearchPartner["matchSummary"] | undefined; +}) { + // One open tooltip at a time — driving every bar's open state from one value + // prevents a fast cursor flick leaving several open. + const [openBarId, setOpenBarId] = useState(null); + + const allBars = summary?.contentBars ?? []; + if (!allBars.length) return null; + const bars = allBars.slice(0, MAX_VISIBLE_CONTENT_BARS); + + return ( +
setOpenBarId(null)} + > + {bars.map((bar) => { + const score = + bar.matched && bar.matchScore != null + ? Math.min(1, Math.max(0, bar.matchScore)) + : 0; + // Magnitude → height (matched posts get a floor so they stay legible). + const height = bar.matched ? Math.round(10 + score * 26) : 5; + const isCreatorTextOnlyVideoMatch = + bar.matchEvidence.primarySource === "creatorText" && + bar.matchEvidence.weight < 1; + // The whole column is the hover/click target (easier to land on); on hover + // a gold wash fills it and the bar turns gold. + const columnClassName = cn( + "group flex h-full min-w-[5px] flex-1 items-end rounded-[3px] transition-colors duration-75 hover:bg-amber-100/70", + bar.url && "cursor-pointer", + ); + const fill = ( + + ); + + return ( + } + // Snappy bar-to-bar hover: open instantly, no grace area, no close animation. + delayDuration={0} + disableHoverableContent + disableAnimation + open={openBarId === bar.partnerContentItemId} + onOpenChange={(nextOpen) => + setOpenBarId((current) => + nextOpen + ? bar.partnerContentItemId + : current === bar.partnerContentItemId + ? null + : current, + ) + } + > + {bar.url ? ( + + {fill} + + ) : ( +
{fill}
+ )} + + ); + })} +
+ ); +} + +// Hover card for a content bar: platform, title, date · length · views (all cached). +function BarTooltip({ + bar, +}: { + bar: NonNullable< + PartnerContentSearchPartner["matchSummary"] + >["contentBars"][number]; +}) { + const title = bar.title?.trim() || "Untitled content"; + const meta = [ + formatPublishedDate(bar.publishedAt), + formatDuration(bar.durationMs), + bar.viewCount && bar.viewCount > 0 + ? `${nFormatter(bar.viewCount)} views` + : null, + ] + .filter(Boolean) + .join(" · "); + + return ( +
+
+ + + {title} + +
+ {meta && ( +
{meta}
+ )} +
+ ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx new file mode 100644 index 00000000000..81c8a66447c --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx @@ -0,0 +1,296 @@ +"use client"; + +import { + PARTNER_CONTENT_SEARCH_TOP_CONTENT, + type PartnerContentTopicFitBand, +} from "@/lib/partner-content-search/constants"; +import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { Button } from "@dub/ui"; +import { cn, nFormatter } from "@dub/utils"; +import { useState } from "react"; +import { BAND_LABELS, lastPostedLabel } from "../content-display-utils"; +import { PlatformIcon } from "../platform-icon"; +import { + ContentMatchRow, + ContentMatchSkeletons, + formatRankWindowPhrase, +} from "./content-match-row"; +import { ContentMatchBars } from "./search-fit-bars"; +import { + buildMatchedContentItems, + buildUnifiedRelevanceMap, + DETAIL_CONTENT_INITIAL_MATCH_COUNT, + DETAIL_CONTENT_MATCH_INCREMENT, + publishedAtMs, +} from "./search-fit-utils"; + +const TOPIC_FIT_BAND_STYLES: Record< + PartnerContentTopicFitBand, + { number: string; chip: string } +> = { + consistent: { number: "text-green-600", chip: "bg-green-50 text-green-700" }, + frequent: { number: "text-blue-600", chip: "bg-blue-50 text-blue-700" }, + occasional: { number: "text-amber-600", chip: "bg-amber-50 text-amber-700" }, + "one-off": { + number: "text-neutral-500", + chip: "bg-neutral-100 text-neutral-600", + }, + none: { number: "text-neutral-400", chip: "bg-neutral-100 text-neutral-500" }, +}; + +export function SearchFitPanel({ + error, + isLoading, + isRefining = false, + summary: initialSummary, + relevanceSummary, + searchPartner, +}: { + error: unknown; + isLoading: boolean; + // Scoped rerank in flight; when it lands, every row's relevance moves to one scale. + isRefining?: boolean; + summary?: PartnerContentSearchPartner["matchSummary"]; + // Scoped reranked summary, only to put per-row relevance on one scale. Null until it resolves. + relevanceSummary?: PartnerContentSearchPartner["matchSummary"] | null; + searchPartner?: PartnerContentSearchPartner; +}) { + const [visibleAllCount, setVisibleAllCount] = useState( + DETAIL_CONTENT_INITIAL_MATCH_COUNT, + ); + const summary = initialSummary ?? searchPartner?.matchSummary; + + if (isLoading && !summary) { + return ; + } + + const isLoadingRows = isLoading || isRefining; + // Per-item relevance on a single scale, from the scoped reranked summary. + const unifiedRelevanceByItemId = buildUnifiedRelevanceMap(relevanceSummary); + // Hold row rendering until the scoped run finishes so snippets/order don't swap in. + const items = buildMatchedContentItems( + summary, + isLoadingRows ? [] : (searchPartner?.chunks ?? []), + unifiedRelevanceByItemId, + ); + + // Top content: relevance + reach blend. All content: same set, newest-first. + const topContent = [...items] + .sort((a, b) => b.blendedScore - a.blendedScore) + .slice(0, PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount); + const allContent = [...items].sort( + (a, b) => publishedAtMs(b.publishedAt) - publishedAtMs(a.publishedAt), + ); + const visibleAll = allContent.slice(0, visibleAllCount); + const hiddenAllCount = Math.max(0, allContent.length - visibleAll.length); + // "All content" only earns its place when it adds rows beyond the top set. + const showAllSection = + !isLoadingRows && + allContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; + + const band = summary?.band ?? "none"; + const bandStyles = TOPIC_FIT_BAND_STYLES[band]; + const topPlatforms = summary?.topPlatforms?.length + ? summary.topPlatforms + : summary?.platforms ?? []; + const lastOnTopic = lastPostedLabel(summary?.lastOnTopicAt ?? null); + const rankWindowPhrase = formatRankWindowPhrase(summary); + const topContentCaption = rankWindowPhrase + ? `Ranked by relevance + reach across ${rankWindowPhrase}.` + : "Ranked by relevance + reach."; + + return ( +
+ {/* Topic fit summary row — layout adapted from the Partner Search hi-fi ref */} +
+
+
+ Topic fit +
+
+ + {summary?.topicFit ?? "—"} + + + {BAND_LABELS[band]} + +
+
+ +
+ +
+
+ {summary?.followers ? ( + {nFormatter(summary.followers)} followers + ) : null} + {summary?.followers && summary?.medianViews ? ( + · + ) : null} + {summary?.medianViews ? ( + {nFormatter(summary.medianViews)} median views + ) : null} +
+ + {summary && ( +
+ + {summary.matchedContentCount} of {summary.recentContentCount}{" "} + matching + + {topPlatforms.length > 0 && ( + <> + · + matched on + + {topPlatforms.map((platform) => ( + + ))} + + + )} + {lastOnTopic && ( + <> + · + last post {lastOnTopic} + + )} +
+ )} +
+
+ +
+ {/* Top content — ranked by the relevance + reach blend */} +
+

+ Top content +

+

+ {topContentCaption} + {isRefining && ( + · refining match… + )} +

+
+ +
+ {error ? ( +
+ Failed to load search matches +
+ ) : isLoadingRows ? ( + + ) : topContent.length ? ( + topContent.map((item) => ( + + )) + ) : ( +
+ No matching content found for this partner. +
+ )} +
+ + {/* All content — same matched set, simply most-recent-first */} + {showAllSection && ( +
+
+

+ All content +

+

+ Most recent first +

+
+ +
+ {visibleAll.map((item) => ( + + ))} +
+ + {hiddenAllCount > 0 ? ( +
+
+ ) : null} +
+ )} +
+
+ ); +} + +export function SearchFitPanelSkeleton() { + return ( +
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+ {[...Array(18)].map((_, index) => ( +
+ ))} +
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+ ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts new file mode 100644 index 00000000000..06bcebb48a9 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts @@ -0,0 +1,113 @@ +import { + getBlendedTopContentScore, + getViewBaseline, +} from "@/lib/partner-content-search/top-content-ranking"; +import { + type PartnerContentMatchEvidence, + type PartnerContentSearchPartner, +} from "@/lib/swr/use-partner-content-search"; + +export const DETAIL_CONTENT_INITIAL_MATCH_COUNT = 8; +export const DETAIL_CONTENT_MATCH_INCREMENT = 8; + +// A matched post for the detail lists: from the cached summary's bars, enriched +// with a loaded chunk (snippet, timed transcript, thumbnail) when available. +export type MatchedContentItem = { + contentItemId: string; + platform: string; + platformContentId: string; + title: string | null; + url: string | null; + durationMs: number | null; + publishedAt: string | null; + viewCount: number | null; + // The displayed relevance rating (0-1); also feeds the blend. + relevance: number; + // Relevance + reach blend, used only to order the Top content list. + blendedScore: number; + matchEvidence: PartnerContentMatchEvidence; + chunk?: PartnerContentSearchPartner["chunks"][number]; +}; + +export function getEvidenceDisplayScore( + evidence: PartnerContentMatchEvidence | undefined, +) { + if (!evidence || evidence.sources.length === 0) return null; + + return Math.max( + evidence.transcriptScore ?? 0, + evidence.creatorTextScore ?? 0, + ); +} + +// Per-item relevance from the scoped reranked summary (one scale; the cached global +// summary mixes rerank + cosine). Includes any item with evidence, not just matched. +export function buildUnifiedRelevanceMap( + relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, +) { + const map = new Map(); + for (const bar of relevanceSummary?.contentBars ?? []) { + const score = getEvidenceDisplayScore(bar.matchEvidence); + if (score != null) map.set(bar.partnerContentItemId, score); + } + return map; +} + +export function buildMatchedContentItems( + summary: PartnerContentSearchPartner["matchSummary"] | undefined, + chunks: PartnerContentSearchPartner["chunks"], + unifiedRelevanceByItemId?: Map, +): MatchedContentItem[] { + const bars = summary?.contentBars ?? []; + + // Best loaded chunk per content item, for snippet/thumbnail enrichment. + const chunkByContentItemId = new Map< + string, + PartnerContentSearchPartner["chunks"][number] + >(); + for (const chunk of chunks) { + const current = chunkByContentItemId.get(chunk.partnerContentItemId); + if (!current || chunk.score > current.score) { + chunkByContentItemId.set(chunk.partnerContentItemId, chunk); + } + } + + // Per-creator engagement baseline: median recent views (matched + unmatched). + const baselineViews = getViewBaseline(bars.map((bar) => bar.viewCount)); + + return bars + .filter((bar) => bar.matched) + .map((bar) => { + // Prefer the unified single-scale relevance once the rerank lands; else cached. + const relevance = + unifiedRelevanceByItemId?.get(bar.partnerContentItemId) ?? + getEvidenceDisplayScore(bar.matchEvidence) ?? + bar.matchScore ?? + 0; + + return { + contentItemId: bar.partnerContentItemId, + platform: bar.platform, + platformContentId: bar.platformContentId, + title: bar.title, + url: bar.url, + durationMs: bar.durationMs, + publishedAt: bar.publishedAt, + viewCount: bar.viewCount, + relevance, + blendedScore: getBlendedTopContentScore({ + relevance, + views: bar.viewCount, + baselineViews, + }), + matchEvidence: bar.matchEvidence, + chunk: chunkByContentItemId.get(bar.partnerContentItemId), + }; + }); +} + +export function publishedAtMs(iso: string | null) { + if (!iso) return -Infinity; + const ms = new Date(iso).getTime(); + return Number.isNaN(ms) ? -Infinity : ms; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts index 579e2c7ecaf..7451d1bf888 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts @@ -57,6 +57,19 @@ export function formatMatchPercent(score: number) { return `${Math.round(Math.min(1, Math.max(0, score)) * 100)}%`; } +// Coarse "Nd / Nw / Nmo ago" relative label for the last on-topic/published post. +// timeAgo from @dub/utils goes absolute past ~23h, so we roll our own. +export function lastPostedLabel(iso: string | null | undefined) { + if (!iso) return null; + + const days = Math.floor((Date.now() - new Date(iso).getTime()) / 86_400_000); + if (days <= 0) return "today"; + if (days < 7) return `${days}d ago`; + if (days < 8 * 7) return `${Math.floor(days / 7)}w ago`; + if (days < 365) return `${Math.floor(days / 30)}mo ago`; + return `${Math.floor(days / 365)}y ago`; +} + export function getContentThumbnail(chunk: ContentSearchChunk) { if (chunk.content.thumbnailUrl) { return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index 8faa26a6fb7..dcb1e06d25d 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -3,15 +3,7 @@ import type { PartnerContentTopicFitBand } from "@/lib/partner-content-search/constants"; import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; import { Tooltip } from "@dub/ui"; -import { - Globe, - Instagram, - LinkedIn, - TikTok, - Twitter, - User, - YouTube, -} from "@dub/ui/icons"; +import { User } from "@dub/ui/icons"; import { cn, nFormatter } from "@dub/utils"; import type { PlatformType } from "@prisma/client"; import { useEffect, useState } from "react"; @@ -25,8 +17,10 @@ import { getContentHref, getContentThumbnail, getContentTitle, + lastPostedLabel, } from "./content-display-utils"; import { NetworkPartnerCard } from "./network-partner-card"; +import { PLATFORM_ICONS, PlatformIcon } from "./platform-icon"; const PLATFORM_LABELS: Partial> = { youtube: "YouTube", @@ -81,42 +75,10 @@ export function NetworkContentSearchResults({ return (
{[...Array(8)].map((_, idx) => ( -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- {[...Array(TOP_CONTENT_PREVIEW_COUNT)].map((_, index) => ( -
- ))} -
-
-
-
-
-
+ bottomContent={} + /> ))}
); @@ -204,7 +166,7 @@ function NetworkPartnerContentMatch({ const followers = summary?.followers ?? null; const medianViews = summary?.medianViews ?? null; const matchLabel = formatMatchEvidenceLabel(summary); - const lastOnTopic = lastPublishedLabel(summary?.lastOnTopicAt); + const lastOnTopic = lastPostedLabel(summary?.lastOnTopicAt); const previewItems = getContentPreviewItems(partner); return ( @@ -270,6 +232,31 @@ function NetworkPartnerContentMatch({ ); } +function NetworkPartnerContentMatchSkeleton() { + return ( +
+
+
+
+
+
+
+
+
+ {[...Array(TOP_CONTENT_PREVIEW_COUNT)].map((_, index) => ( +
+ ))} +
+
+
+
+
+ ); +} + function getContentPreviewItems( partner: PartnerContentSearchPartner, ): ContentPreviewItem[] { @@ -453,18 +440,6 @@ function ContentPreviewTooltip({ item }: { item: ContentPreviewItem }) { ); } -function PlatformIcon({ - platform, - className, -}: { - platform: string; - className?: string; -}) { - const Icon = PLATFORM_ICONS[platform as PlatformType] ?? User; - - return ; -} - function getContentEngagementMetrics( chunk: ContentSearchChunk, bar: ContentSearchBar | undefined, @@ -530,15 +505,6 @@ const BAND_STYLES: Record< }, }; -const PLATFORM_ICONS: Partial> = { - youtube: YouTube, - instagram: Instagram, - tiktok: TikTok, - twitter: Twitter, - linkedin: LinkedIn, - website: Globe, -}; - function PlatformIcons({ platforms }: { platforms: string[] }) { const icons = platforms .map((platform) => PLATFORM_ICONS[platform as PlatformType]) @@ -558,18 +524,6 @@ function PlatformIcons({ platforms }: { platforms: string[] }) { ); } -// Coarse "Nd/Nw/Nmo ago" (timeAgo from @dub/utils goes absolute past ~23h). -function lastPublishedLabel(iso: string | null | undefined) { - if (!iso) return null; - - const days = Math.floor((Date.now() - new Date(iso).getTime()) / 86_400_000); - if (days <= 0) return "today"; - if (days < 7) return `${days}d ago`; - if (days < 8 * 7) return `${Math.floor(days / 7)}w ago`; - if (days < 365) return `${Math.floor(days / 30)}mo ago`; - return `${Math.floor(days / 365)}y ago`; -} - function formatMatchEvidenceLabel( summary: PartnerContentSearchPartner["matchSummary"] | undefined, ) { diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-audience-section.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-audience-section.tsx new file mode 100644 index 00000000000..82abba0244f --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-audience-section.tsx @@ -0,0 +1,70 @@ +"use client"; + +import { NetworkPartnerProps } from "@/lib/types"; +import { cn } from "@dub/utils"; +import { PartnerPlatformDisplayData } from "./network-partner-card-types"; +import { PlatformStatCard } from "./platform-stat-card"; + +export function NetworkPartnerAudienceSection({ + partner, + partnerPlatformsData, +}: { + partner?: NetworkPartnerProps; + partnerPlatformsData: PartnerPlatformDisplayData[] | null; +}) { + return ( +
+ + Audience + + +
+ {partnerPlatformsData + ? partnerPlatformsData.map( + ({ + label, + icon: PlatformIcon, + verified, + stat, + value, + href, + info, + verifiedAt, + }) => ( + + ), + ) + : [...Array(6)].map((_, idx) => ( +
+ ))} +
+
+ ); +} + +export function getPlatformSortOrder({ + verified, + value, +}: { + verified: boolean; + value?: string | null; +}) { + if (verified) return 0; + if (value) return 1; + return 2; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-actions.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-actions.tsx new file mode 100644 index 00000000000..4db3d52a582 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-actions.tsx @@ -0,0 +1,84 @@ +"use client"; + +import { mutatePrefix } from "@/lib/swr/mutate"; +import usePartnerNetworkInvitesUsage from "@/lib/swr/use-partner-network-invites-usage"; +import useWorkspace from "@/lib/swr/use-workspace"; +import { NetworkPartnerProps } from "@/lib/types"; +import { useTrialLimitActivateModal } from "@/ui/modals/trial-limit-activate-modal"; +import { PartnerStarButton } from "@/ui/partners/partner-star-button"; +import { Button } from "@dub/ui"; +import { isWorkspaceBillingTrialActive } from "@dub/utils"; +import { EmailContent } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-email-preview"; +import { InviteNetworkPartnerSheet } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-network-partner-sheet"; +import { useState } from "react"; + +export function NetworkPartnerCardActions({ + partner, + onToggleStarred, + showInvite, +}: { + partner: NetworkPartnerProps; + onToggleStarred?: (starred: boolean) => void; + showInvite: boolean; +}) { + const { trialEndsAt } = useWorkspace(); + const { openTrialLimitModal, TrialLimitActivateModal } = + useTrialLimitActivateModal(); + const trialActive = isWorkspaceBillingTrialActive(trialEndsAt); + + const [showInviteSheet, setShowInviteSheet] = useState(false); + const [inviteEmailContent, setInviteEmailContent] = + useState(null); + + const { remaining: remainingInvites } = usePartnerNetworkInvitesUsage(); + const atNetworkInviteLimit = remainingInvites === 0; + const disabled = atNetworkInviteLimit && !trialActive; + + const handleInvitePress = () => { + if (trialActive && atNetworkInviteLimit) { + openTrialLimitModal("networkInvites"); + return; + } + setShowInviteSheet(true); + }; + + return ( + <> + + { + mutatePrefix("/api/network/partners"); + }} + {...(trialActive && { + onInviteLimitError: () => openTrialLimitModal("networkInvites"), + })} + /> +
+ {showInvite && ( +
+ + ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-types.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-types.ts new file mode 100644 index 00000000000..9e99dac5141 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card-types.ts @@ -0,0 +1,9 @@ +import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms"; +import type { Icon } from "@dub/ui/icons"; + +export type PartnerPlatformDisplayData = ReturnType< + (typeof PARTNER_PLATFORM_FIELDS)[number]["data"] +> & { + label: string; + icon: Icon; +}; diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx index ea62d127161..d5a03eba925 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-partner-card.tsx @@ -1,43 +1,26 @@ "use client"; import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms"; -import { mutatePrefix } from "@/lib/swr/mutate"; -import usePartnerNetworkInvitesUsage from "@/lib/swr/use-partner-network-invites-usage"; import useWorkspace from "@/lib/swr/use-workspace"; import { NetworkPartnerProps } from "@/lib/types"; -import { useTrialLimitActivateModal } from "@/ui/modals/trial-limit-activate-modal"; import { PartnerAvatar } from "@/ui/partners/partner-avatar"; -import { PartnerStarButton } from "@/ui/partners/partner-star-button"; import { TrustedPartnerBadge } from "@/ui/partners/trusted-partner-badge"; -import { - BadgeCheck2Fill, - Button, - DynamicTooltipWrapper, - Tooltip, - UserPlus, - useResizeObserver, - useRouterStuff, -} from "@dub/ui"; -import type { Icon } from "@dub/ui/icons"; +import { UserPlus, useRouterStuff } from "@dub/ui"; import { ChartActivity2, EnvelopeArrowRight, Globe } from "@dub/ui/icons"; import { COUNTRIES, cn, isClickOnInteractiveChild, - isWorkspaceBillingTrialActive, timeAgo, } from "@dub/utils"; -import { EmailContent } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-email-preview"; -import { InviteNetworkPartnerSheet } from "app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/invite-network-partner-sheet"; import { usePathname } from "next/navigation"; -import { type ReactNode, useEffect, useMemo, useRef, useState } from "react"; - -type PartnerPlatformDisplayData = ReturnType< - (typeof PARTNER_PLATFORM_FIELDS)[number]["data"] -> & { - label: string; - icon: Icon; -}; +import { type ReactNode, useMemo } from "react"; +import { + NetworkPartnerAudienceSection, + getPlatformSortOrder, +} from "./network-partner-audience-section"; +import { NetworkPartnerCardActions } from "./network-partner-card-actions"; +import { ListRow } from "./overflow-pill-list"; export function NetworkPartnerCard({ bottomContent, @@ -212,374 +195,3 @@ export function NetworkPartnerCard({
); } - -function NetworkPartnerAudienceSection({ - partner, - partnerPlatformsData, -}: { - partner?: NetworkPartnerProps; - partnerPlatformsData: PartnerPlatformDisplayData[] | null; -}) { - return ( -
- - Audience - - -
- {partnerPlatformsData - ? partnerPlatformsData.map( - ({ - label, - icon: PlatformIcon, - verified, - stat, - value, - href, - info, - verifiedAt, - }) => ( - - ), - ) - : [...Array(6)].map((_, idx) => ( -
- ))} -
-
- ); -} - -function getPlatformSortOrder({ - verified, - value, -}: { - verified: boolean; - value?: string | null; -}) { - if (verified) return 0; - if (value) return 1; - return 2; -} - -function NetworkPartnerCardActions({ - partner, - onToggleStarred, - showInvite, -}: { - partner: NetworkPartnerProps; - onToggleStarred?: (starred: boolean) => void; - showInvite: boolean; -}) { - const { trialEndsAt } = useWorkspace(); - const { openTrialLimitModal, TrialLimitActivateModal } = - useTrialLimitActivateModal(); - const trialActive = isWorkspaceBillingTrialActive(trialEndsAt); - - const [showInviteSheet, setShowInviteSheet] = useState(false); - const [inviteEmailContent, setInviteEmailContent] = - useState(null); - - const { remaining: remainingInvites } = usePartnerNetworkInvitesUsage(); - const atNetworkInviteLimit = remainingInvites === 0; - const disabled = atNetworkInviteLimit && !trialActive; - - const handleInvitePress = () => { - if (trialActive && atNetworkInviteLimit) { - openTrialLimitModal("networkInvites"); - return; - } - setShowInviteSheet(true); - }; - - return ( - <> - - { - mutatePrefix("/api/network/partners"); - }} - {...(trialActive && { - onInviteLimitError: () => openTrialLimitModal("networkInvites"), - })} - /> -
- {showInvite && ( -
- - ); -} - -function PlatformStatCard({ - label, - icon: PlatformIcon, - verified, - stat, - value, - info, - verifiedAt, - href, -}: { - label: string; - icon: Icon; - verified: boolean; - stat?: string | null; - value?: string | null; - info?: string[]; - verifiedAt?: Date | null; - href?: string | null; -}) { - const content = ( -
-
- - {verified && ( - - )} -
- - {verified && stat ? stat : "—"} - - {label} -
- ); - - const As = href ? "a" : "div"; - - return ( - - ), - } - : undefined - } - > - e.stopPropagation()} - > - {content} - - - ); -} - -function PlatformStatTooltipContent({ - icon: PlatformIcon, - value, - stat, - info, - verifiedAt, -}: { - icon: Icon; - value?: string | null; - stat?: string | null; - info?: string[]; - verifiedAt?: Date | null; -}) { - return ( -
-
-
- -
-
-
- {value} -
- {(info?.[0] ?? stat) && ( -
- {info?.[0] ?? stat} -
- )} -
-
-
- {verifiedAt ? ( - <> - - Verified {timeAgo(verifiedAt, { withAgo: true })} - - ) : ( - "Not verified" - )} -
-
- ); -} - -function ListRow({ - items, - className, -}: { - items?: { icon?: Icon; label: string }[]; - className?: string; -}) { - const containerRef = useRef(null); - - const [isReady, setIsReady] = useState(false); - - const [shownItems, setShownItems] = useState< - { icon?: Icon; label: string }[] | undefined - >(items); - - useEffect(() => { - if (isReady) return; - - setIsReady(false); - setShownItems(items); - }, [items, isReady]); - - useEffect(() => { - if (!containerRef.current) return; - - if ( - shownItems?.length && - containerRef.current.scrollWidth > containerRef.current.clientWidth - ) { - setIsReady(false); - setShownItems(shownItems?.slice(0, -1)); - } else { - setIsReady(true); - } - }, [shownItems]); - - const entry = useResizeObserver(containerRef); - useEffect(() => { - if (!containerRef.current) return; - - if (containerRef.current.scrollWidth > containerRef.current.clientWidth) - setIsReady(false); - }, [entry]); - - return ( -
-
- {items ? ( - items.length ? ( - <> - {shownItems?.map(({ icon, label }) => ( - - ))} - {(shownItems?.length ?? 0) < items.length && ( - - {items - .filter( - ({ label }) => - !shownItems?.some( - ({ label: shownLabel }) => shownLabel === label, - ), - ) - .map(({ icon, label }) => ( - - ))} -
- } - > -
- +{items.length - (shownItems?.length ?? 0)} -
- - )} - - ) : ( -
- - No categories - -
- ) - ) : ( - [...Array(2)].map((_, idx) => ( -
- )) - )} -
-
- ); -} - -function ListPill({ icon: Icon, label }: { icon?: Icon; label: string }) { - return ( -
- {Icon && } - - {label} - -
- ); -} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx index f215288b4fe..7a6b75e03e1 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-platform-filter.tsx @@ -1,6 +1,7 @@ import { Popover } from "@dub/ui"; import { Globe, + GlobePointer, Instagram, LinkedIn, TikTok, @@ -144,32 +145,27 @@ export function NetworkPlatformFilter({ ); diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/overflow-pill-list.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/overflow-pill-list.tsx new file mode 100644 index 00000000000..400d13e0e4b --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/overflow-pill-list.tsx @@ -0,0 +1,116 @@ +"use client"; + +import { Tooltip, useResizeObserver } from "@dub/ui"; +import type { Icon } from "@dub/ui/icons"; +import { cn } from "@dub/utils"; +import { useEffect, useRef, useState } from "react"; + +export function ListRow({ + items, + className, +}: { + items?: { icon?: Icon; label: string }[]; + className?: string; +}) { + const containerRef = useRef(null); + + const [isReady, setIsReady] = useState(false); + + const [shownItems, setShownItems] = useState< + { icon?: Icon; label: string }[] | undefined + >(items); + + useEffect(() => { + if (isReady) return; + + setIsReady(false); + setShownItems(items); + }, [items, isReady]); + + useEffect(() => { + if (!containerRef.current) return; + + if ( + shownItems?.length && + containerRef.current.scrollWidth > containerRef.current.clientWidth + ) { + setIsReady(false); + setShownItems(shownItems?.slice(0, -1)); + } else { + setIsReady(true); + } + }, [shownItems]); + + const entry = useResizeObserver(containerRef); + useEffect(() => { + if (!containerRef.current) return; + + if (containerRef.current.scrollWidth > containerRef.current.clientWidth) + setIsReady(false); + }, [entry]); + + return ( +
+
+ {items ? ( + items.length ? ( + <> + {shownItems?.map(({ icon, label }) => ( + + ))} + {(shownItems?.length ?? 0) < items.length && ( + + {items + .filter( + ({ label }) => + !shownItems?.some( + ({ label: shownLabel }) => shownLabel === label, + ), + ) + .map(({ icon, label }) => ( + + ))} +
+ } + > +
+ +{items.length - (shownItems?.length ?? 0)} +
+ + )} + + ) : ( +
+ + No categories + +
+ ) + ) : ( + [...Array(2)].map((_, idx) => ( +
+ )) + )} +
+
+ ); +} + +function ListPill({ icon: Icon, label }: { icon?: Icon; label: string }) { + return ( +
+ {Icon && } + + {label} + +
+ ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index eb7027bf3ea..5bec67b5c6b 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -1,7 +1,5 @@ "use client"; -import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner"; -import { parseReachTiers } from "@/lib/api/network/reach-tiers"; import useNetworkPartnersCount from "@/lib/swr/use-network-partners-count"; import usePartnerContentSearch from "@/lib/swr/use-partner-content-search"; import useWorkspace from "@/lib/swr/use-workspace"; @@ -18,10 +16,6 @@ import { } from "@dub/ui"; import { Star, StarFill } from "@dub/ui/icons"; import { cn, fetcher } from "@dub/utils"; -import { PlatformType } from "@prisma/client"; -import { useAction } from "next-safe-action/hooks"; -import { useEffect, useMemo, useState } from "react"; -import { toast } from "sonner"; import { useDebounce } from "use-debounce"; import useSWR from "swr"; import { NetworkContentSearchResults } from "./network-content-search-results"; @@ -29,19 +23,16 @@ import { NetworkEmptyState } from "./network-empty-state"; import { NetworkPartnerCard } from "./network-partner-card"; import { NetworkPartnerDetailSheet } from "./network-partner-detail-sheet"; import { NetworkPlatformFilter } from "./network-platform-filter"; +import { useNetworkDetailSheet } from "./use-network-detail-sheet"; import { - getContentSearchPlatforms, - isAllPlatformsSelected, - parseSelectedPlatforms, - platformFilterParam, -} from "./platform-filter-utils"; + FILTER_FETCH_DEBOUNCE_MS, + useNetworkPartnerFiltersState, +} from "./use-network-partner-filters-state"; import { usePartnerNetworkFilters } from "./use-partner-network-filters"; - -// Filter changes update the URL instantly (snappy controls) but the data fetch is -// debounced so a rapid burst of toggles collapses into a single request — sparing -// the heavy ranking SQL and (in search mode) the billed Voyage embed+rerank call. -// keepPreviousData holds the prior results on screen during the brief wait. -const FILTER_FETCH_DEBOUNCE_MS = 250; +import { + useToggleStarredContentSearch, + useToggleStarredRankedList, +} from "./use-toggle-partner-starred"; const tabs = [ { @@ -67,23 +58,18 @@ export function ProgramPartnerNetworkPageClient({ }: ProgramPartnerNetworkPageClientProps = {}) { const { id: workspaceId } = useWorkspace(); const { searchParams, getQueryString, queryParams } = useRouterStuff(); - const selectedPlatforms = useMemo( - () => parseSelectedPlatforms(searchParams.get("platform")), - [searchParams], - ); - const platformFilter = platformFilterParam(selectedPlatforms); - // The content-searchable subset of the selection. Semantic search runs only - // when at least one searchable platform is selected; otherwise we fall back to - // the ranked partner list (filtered to the chosen platforms). - const contentSearchPlatforms = getContentSearchPlatforms(selectedPlatforms); - const selectedReach = useMemo( - () => parseReachTiers(searchParams.get("reach")), - [searchParams], - ); - const reachFilter = selectedReach.length ? selectedReach : undefined; - const search = searchParams.get("search")?.trim() ?? ""; - const country = searchParams.get("country") ?? undefined; - const starred = searchParams.get("starred") === "true"; + + const { + selectedPlatforms, + platformFilter, + contentSearchPlatforms, + reachFilter, + search, + country, + starred, + updateSearchParams, + onPlatformsChange, + } = useNetworkPartnerFiltersState(); const status = variant === "ignored" @@ -94,26 +80,6 @@ export function ProgramPartnerNetworkPageClient({ search.length > 0 && contentSearchPlatforms.length > 0; - // Update filter params via the History API instead of router.push. These are - // query-only changes that drive client-side SWR; page.tsx reads no searchParams, - // so a full RSC navigation per click only adds latency before useSearchParams() - // (and thus the control's checked state) updates. pushState updates it - // synchronously — instant feedback — while SWR still refetches on key change. - const updateSearchParams = (opts: { - set?: Record; - del?: string | string[]; - }) => { - const newPath = queryParams({ ...opts, getNewPath: true }) as string; - window.history.pushState(null, "", newPath); - }; - - const onPlatformsChange = (platforms: PlatformType[]) => - updateSearchParams( - isAllPlatformsSelected(platforms) - ? { del: ["platform", "page"] } - : { set: { platform: platforms.join(",") }, del: "page" }, - ); - const { data: partnerCounts, error: countError } = useNetworkPartnersCount(); const { @@ -162,10 +128,6 @@ export function ProgramPartnerNetworkPageClient({ keepPreviousData: true, }); - const { executeAsync: updateDiscoveredPartner } = useAction( - updateDiscoveredPartnerAction, - ); - const { pagination, setPagination } = usePagination( PARTNER_NETWORK_MAX_PAGE_SIZE, ); @@ -182,49 +144,25 @@ export function ProgramPartnerNetworkPageClient({ const isStarred = searchParams.get("starred") === "true"; const selectedPartnerId = searchParams.get("partnerId"); - const [detailsSheetState, setDetailsSheetState] = useState< - | { open: false; partnerId: string | null } - | { open: true; partnerId: string } - >( - selectedPartnerId - ? { open: true, partnerId: selectedPartnerId } - : { open: false, partnerId: null }, - ); - - useEffect(() => { - if (selectedPartnerId) { - setDetailsSheetState({ open: true, partnerId: selectedPartnerId }); - } else { - setDetailsSheetState({ open: false, partnerId: null }); - } - }, [selectedPartnerId]); + const { + detailsSheetState, + setDetailsSheetState, + previousPartnerId, + nextPartnerId, + } = useNetworkDetailSheet({ + selectedPartnerId, + isContentSearchMode, + contentSearchPartners: contentSearchResults?.partners, + partners, + }); - const sheetPartnerIds = useMemo( - () => - isContentSearchMode - ? contentSearchResults?.partners.map(({ partnerId }) => partnerId) - : partners?.map(({ id }) => id), - [contentSearchResults?.partners, isContentSearchMode, partners], + const toggleStarredContentSearch = + useToggleStarredContentSearch(mutateContentSearch); + const toggleStarredRankedList = useToggleStarredRankedList( + mutatePartners, + partners, ); - const [previousPartnerId, nextPartnerId] = useMemo(() => { - if (!sheetPartnerIds?.length || !detailsSheetState.partnerId) { - return [null, null]; - } - - const currentIndex = sheetPartnerIds.findIndex( - (id) => id === detailsSheetState.partnerId, - ); - if (currentIndex === -1) return [null, null]; - - return [ - currentIndex > 0 ? sheetPartnerIds[currentIndex - 1] : null, - currentIndex < sheetPartnerIds.length - 1 - ? sheetPartnerIds[currentIndex + 1] - : null, - ]; - }, [detailsSheetState.partnerId, sheetPartnerIds]); - return (
{detailsSheetState.partnerId && ( @@ -368,62 +306,7 @@ export function ProgramPartnerNetworkPageClient({ : undefined } onToggleStarred={ - variant === "ignored" - ? undefined - : (partnerId, starred) => { - mutateContentSearch( - // @ts-ignore SWR doesn't seem to have proper typing for partial data results w/ `populateCache` - async () => { - const result = await updateDiscoveredPartner({ - workspaceId: workspaceId!, - partnerId, - starred, - }); - if (!result?.data) { - toast.error("Failed to star partner"); - throw new Error("Failed to star partner"); - } - - return result.data; - }, - { - optimisticData: (data) => - data && { - ...data, - partners: data.partners.map((p) => - p.partnerId === partnerId - ? { - ...p, - partner: { - ...p.partner, - starredAt: starred ? new Date() : null, - }, - } - : p, - ), - }, - populateCache: ( - result: { starredAt: Date | null }, - data, - ) => - data && { - ...data, - partners: data.partners.map((p) => - p.partnerId === partnerId - ? { - ...p, - partner: { - ...p.partner, - starredAt: result.starredAt, - }, - } - : p, - ), - }, - revalidate: false, - }, - ); - } + variant === "ignored" ? undefined : toggleStarredContentSearch } /> ) : error || countError ? ( @@ -446,47 +329,8 @@ export function ProgramPartnerNetworkPageClient({ onToggleStarred={ variant === "ignored" ? undefined - : (starred) => { - mutatePartners( - // @ts-ignore SWR doesn't seem to have proper typing for partial data results w/ `populateCache` - async () => { - const result = await updateDiscoveredPartner({ - workspaceId: workspaceId!, - partnerId: partner.id, - starred, - }); - if (!result?.data) { - toast.error("Failed to star partner"); - throw new Error("Failed to star partner"); - } - - return result.data; - }, - { - optimisticData: (data) => - (data || partners).map((p) => - p.id === partner.id - ? { - ...p, - starredAt: starred - ? new Date() - : null, - } - : p, - ), - populateCache: ( - result: { starredAt: Date | null }, - data, - ) => - (data || partners).map((p) => - p.id === partner.id - ? { ...p, starredAt: result.starredAt } - : p, - ), - revalidate: false, - }, - ); - } + : (starred) => + toggleStarredRankedList(partner.id, starred) } /> )) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-icon.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-icon.tsx new file mode 100644 index 00000000000..781bfb25480 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-icon.tsx @@ -0,0 +1,26 @@ +import { Globe, Instagram, LinkedIn, TikTok, Twitter, User, YouTube } from "@dub/ui/icons"; +import { cn } from "@dub/utils"; +import type { PlatformType } from "@prisma/client"; + +// Shared platform → brand icon map for the network search surfaces (detail page + +// results list). Unknown platforms fall back to a generic person icon. +export const PLATFORM_ICONS: Partial> = { + youtube: YouTube, + instagram: Instagram, + tiktok: TikTok, + twitter: Twitter, + linkedin: LinkedIn, + website: Globe, +}; + +export function PlatformIcon({ + platform, + className, +}: { + platform: string; + className?: string; +}) { + const Icon = PLATFORM_ICONS[platform as PlatformType] ?? User; + + return ; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-stat-card.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-stat-card.tsx new file mode 100644 index 00000000000..71d1ac50484 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-stat-card.tsx @@ -0,0 +1,129 @@ +import { BadgeCheck2Fill, DynamicTooltipWrapper } from "@dub/ui"; +import type { Icon } from "@dub/ui/icons"; +import { cn, timeAgo } from "@dub/utils"; + +export function PlatformStatCard({ + label, + icon: PlatformIcon, + verified, + stat, + value, + info, + verifiedAt, + href, +}: { + label: string; + icon: Icon; + verified: boolean; + stat?: string | null; + value?: string | null; + info?: string[]; + verifiedAt?: Date | null; + href?: string | null; +}) { + const content = ( +
+
+ + {verified && ( + + )} +
+ + {verified && stat ? stat : "—"} + + {label} +
+ ); + + const As = href ? "a" : "div"; + + return ( + + ), + } + : undefined + } + > + e.stopPropagation()} + > + {content} + + + ); +} + +function PlatformStatTooltipContent({ + icon: PlatformIcon, + value, + stat, + info, + verifiedAt, +}: { + icon: Icon; + value?: string | null; + stat?: string | null; + info?: string[]; + verifiedAt?: Date | null; +}) { + return ( +
+
+
+ +
+
+
+ {value} +
+ {(info?.[0] ?? stat) && ( +
+ {info?.[0] ?? stat} +
+ )} +
+
+
+ {verifiedAt ? ( + <> + + Verified {timeAgo(verifiedAt, { withAgo: true })} + + ) : ( + "Not verified" + )} +
+
+ ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-detail-sheet.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-detail-sheet.ts new file mode 100644 index 00000000000..bb729d7204a --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-detail-sheet.ts @@ -0,0 +1,67 @@ +"use client"; + +import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { NetworkPartnerProps } from "@/lib/types"; +import { useEffect, useMemo, useState } from "react"; + +export function useNetworkDetailSheet({ + selectedPartnerId, + isContentSearchMode, + contentSearchPartners, + partners, +}: { + selectedPartnerId: string | null; + isContentSearchMode: boolean; + contentSearchPartners: PartnerContentSearchPartner[] | undefined; + partners: NetworkPartnerProps[] | undefined; +}) { + const [detailsSheetState, setDetailsSheetState] = useState< + | { open: false; partnerId: string | null } + | { open: true; partnerId: string } + >( + selectedPartnerId + ? { open: true, partnerId: selectedPartnerId } + : { open: false, partnerId: null }, + ); + + useEffect(() => { + if (selectedPartnerId) { + setDetailsSheetState({ open: true, partnerId: selectedPartnerId }); + } else { + setDetailsSheetState({ open: false, partnerId: null }); + } + }, [selectedPartnerId]); + + const sheetPartnerIds = useMemo( + () => + isContentSearchMode + ? contentSearchPartners?.map(({ partnerId }) => partnerId) + : partners?.map(({ id }) => id), + [contentSearchPartners, isContentSearchMode, partners], + ); + + const [previousPartnerId, nextPartnerId] = useMemo(() => { + if (!sheetPartnerIds?.length || !detailsSheetState.partnerId) { + return [null, null]; + } + + const currentIndex = sheetPartnerIds.findIndex( + (id) => id === detailsSheetState.partnerId, + ); + if (currentIndex === -1) return [null, null]; + + return [ + currentIndex > 0 ? sheetPartnerIds[currentIndex - 1] : null, + currentIndex < sheetPartnerIds.length - 1 + ? sheetPartnerIds[currentIndex + 1] + : null, + ]; + }, [detailsSheetState.partnerId, sheetPartnerIds]); + + return { + detailsSheetState, + setDetailsSheetState, + previousPartnerId, + nextPartnerId, + }; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts new file mode 100644 index 00000000000..7a3931082bc --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts @@ -0,0 +1,71 @@ +"use client"; + +import { parseReachTiers } from "@/lib/api/network/reach-tiers"; +import { useRouterStuff } from "@dub/ui"; +import { PlatformType } from "@prisma/client"; +import { useMemo } from "react"; +import { + getContentSearchPlatforms, + isAllPlatformsSelected, + parseSelectedPlatforms, + platformFilterParam, +} from "./platform-filter-utils"; + +// Filter changes update the URL instantly (snappy controls) but the data fetch is +// debounced avoiding ranking SQL + Voyage embed+rerank call. +export const FILTER_FETCH_DEBOUNCE_MS = 250; + +// keeps the prior results on screen during the brief wait. +export function useNetworkPartnerFiltersState() { + const { searchParams, queryParams } = useRouterStuff(); + + const selectedPlatforms = useMemo( + () => parseSelectedPlatforms(searchParams.get("platform")), + [searchParams], + ); + const platformFilter = platformFilterParam(selectedPlatforms); + // The content-searchable subset of the selection. Semantic search runs only + // when at least one searchable platform is selected; otherwise we fall back to + // the ranked partner list (filtered to the chosen platforms). + const contentSearchPlatforms = getContentSearchPlatforms(selectedPlatforms); + const selectedReach = useMemo( + () => parseReachTiers(searchParams.get("reach")), + [searchParams], + ); + const reachFilter = selectedReach.length ? selectedReach : undefined; + const search = searchParams.get("search")?.trim() ?? ""; + const country = searchParams.get("country") ?? undefined; + const starred = searchParams.get("starred") === "true"; + + // Update filter params via the History API instead of router.push. These are + // query-only changes that drive client-side SWR; page.tsx reads no searchParams, + // so a full RSC navigation per click only adds latency before useSearchParams() + // (and thus the control's checked state) updates. pushState updates it + // synchronously — instant feedback — while SWR still refetches on key change. + const updateSearchParams = (opts: { + set?: Record; + del?: string | string[]; + }) => { + const newPath = queryParams({ ...opts, getNewPath: true }) as string; + window.history.pushState(null, "", newPath); + }; + + const onPlatformsChange = (platforms: PlatformType[]) => + updateSearchParams( + isAllPlatformsSelected(platforms) + ? { del: ["platform", "page"] } + : { set: { platform: platforms.join(",") }, del: "page" }, + ); + + return { + selectedPlatforms, + platformFilter, + contentSearchPlatforms, + reachFilter, + search, + country, + starred, + updateSearchParams, + onPlatformsChange, + }; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts new file mode 100644 index 00000000000..3bc933ea843 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts @@ -0,0 +1,140 @@ +"use client"; + +import { updateDiscoveredPartnerAction } from "@/lib/actions/partners/update-discovered-partner"; +import type { PartnerContentSearchResponse } from "@/lib/swr/use-partner-content-search"; +import useWorkspace from "@/lib/swr/use-workspace"; +import { NetworkPartnerProps } from "@/lib/types"; +import { useAction } from "next-safe-action/hooks"; +import { useCallback } from "react"; +import { toast } from "sonner"; +import type { KeyedMutator } from "swr"; + +// Two near-identical optimistic star-toggle mutations live here. They share the +// same star-update action, toast-on-failure, optimisticData/populateCache/revalidate +// semantics, and rollback-on-error — they differ only in the SWR cache shape they +// mutate (the content-search response's nested `partner.starredAt` vs the ranked +// list's flat `starredAt`). Kept as two hooks rather than one parameterized hook so +// each cache transform stays verbatim and behavior is provably unchanged. + +/** + * Optimistic star toggle for content-search mode. Mutates the + * `PartnerContentSearchResponse` cache, locating the partner by `partnerId` and + * updating its nested `partner.starredAt`. + */ +export function useToggleStarredContentSearch( + mutateContentSearch: KeyedMutator, +) { + const { id: workspaceId } = useWorkspace(); + const { executeAsync: updateDiscoveredPartner } = useAction( + updateDiscoveredPartnerAction, + ); + + return useCallback( + (partnerId: string, starred: boolean) => { + mutateContentSearch( + // @ts-ignore SWR doesn't seem to have proper typing for partial data results w/ `populateCache` + async () => { + const result = await updateDiscoveredPartner({ + workspaceId: workspaceId!, + partnerId, + starred, + }); + if (!result?.data) { + toast.error("Failed to star partner"); + throw new Error("Failed to star partner"); + } + + return result.data; + }, + { + optimisticData: (data) => + data && { + ...data, + partners: data.partners.map((p) => + p.partnerId === partnerId + ? { + ...p, + partner: { + ...p.partner, + starredAt: starred ? new Date() : null, + }, + } + : p, + ), + }, + populateCache: (result: { starredAt: Date | null }, data) => + data && { + ...data, + partners: data.partners.map((p) => + p.partnerId === partnerId + ? { + ...p, + partner: { + ...p.partner, + starredAt: result.starredAt, + }, + } + : p, + ), + }, + revalidate: false, + }, + ); + }, + [mutateContentSearch, updateDiscoveredPartner, workspaceId], + ); +} + +/** + * Optimistic star toggle for ranked-list mode. Mutates the + * `NetworkPartnerProps[]` cache, locating the partner by `id` and updating its + * flat `starredAt`. `partners` is used as the fallback when the cache is empty, + * preserving the original closure's `data || partners` behavior. + */ +export function useToggleStarredRankedList( + mutatePartners: KeyedMutator, + partners: NetworkPartnerProps[] | undefined, +) { + const { id: workspaceId } = useWorkspace(); + const { executeAsync: updateDiscoveredPartner } = useAction( + updateDiscoveredPartnerAction, + ); + + return useCallback( + (partnerId: string, starred: boolean) => { + mutatePartners( + // @ts-ignore SWR doesn't seem to have proper typing for partial data results w/ `populateCache` + async () => { + const result = await updateDiscoveredPartner({ + workspaceId: workspaceId!, + partnerId, + starred, + }); + if (!result?.data) { + toast.error("Failed to star partner"); + throw new Error("Failed to star partner"); + } + + return result.data; + }, + { + optimisticData: (data) => + (data || partners)!.map((p) => + p.id === partnerId + ? { + ...p, + starredAt: starred ? new Date() : null, + } + : p, + ), + populateCache: (result: { starredAt: Date | null }, data) => + (data || partners)!.map((p) => + p.id === partnerId ? { ...p, starredAt: result.starredAt } : p, + ), + revalidate: false, + }, + ); + }, + [mutatePartners, partners, updateDiscoveredPartner, workspaceId], + ); +} diff --git a/apps/web/lib/api/network/partner-network-listing-where.ts b/apps/web/lib/api/network/partner-network-listing-where.ts index 8ad90a1f341..e533383685a 100644 --- a/apps/web/lib/api/network/partner-network-listing-where.ts +++ b/apps/web/lib/api/network/partner-network-listing-where.ts @@ -1,4 +1,6 @@ import { PlatformType, Prisma } from "@prisma/client"; +import type { ReachTier } from "./reach-tiers"; +import { reachTiersToRanges } from "./reach-tiers"; /** Query params shared by `/api/network/partners` and `/count` for listing. */ export type PartnerNetworkListingParams = { @@ -62,3 +64,50 @@ export function partnerNetworkListingWhere( ): Prisma.PartnerWhereInput { return partnerWhereFromListingParts(partnerNetworkListingParts(params)); } + +// Reach tier filter: the partner's MAX subscriber count across verified platforms +// (optionally scoped to selected platform types) must fall in a chosen tier. +// "some platform >= min" AND "no scoped platform >= max" matches +// calculatePartnerRanking's MAX-subscribers semantics. +export function partnerReachWhere({ + reach, + platform, +}: { + reach?: ReachTier[]; + platform?: PlatformType[]; +}): Prisma.PartnerWhereInput { + if (!reach?.length) return {}; + + const ranges = reachTiersToRanges(reach); + const platformScope: Prisma.PartnerPlatformWhereInput = { + verifiedAt: { not: null }, + ...(platform?.length && { type: { in: platform } }), + }; + + return { + OR: ranges.map(({ min, max }) => ({ + AND: [ + { + platforms: { + some: { + ...platformScope, + subscribers: { gte: BigInt(min) }, + }, + }, + }, + ...(max != null + ? [ + { + platforms: { + none: { + ...platformScope, + subscribers: { gte: BigInt(max) }, + }, + }, + }, + ] + : []), + ], + })), + }; +} diff --git a/apps/web/lib/partner-content-search/content-match-bars.ts b/apps/web/lib/partner-content-search/content-match-bars.ts new file mode 100644 index 00000000000..d48b324f7b6 --- /dev/null +++ b/apps/web/lib/partner-content-search/content-match-bars.ts @@ -0,0 +1,225 @@ +import { + createContentMatchEvidence, + getEvidenceMatchScore, + getEvidenceSource, + getEvidenceTopicScore, + getMatchedSourceScore, + getSourceScore, + type PartnerContentMatchEvidence, + type SourceScoreByContentItemId, +} from "./ranking"; +import { toScore, type PartnerContentSearchRow } from "./search-utils"; +import type { PartnerRecentContentBarRow } from "./match-summary-queries"; + +export type PartnerContentItemBestMatch = { + transcriptScore: number | null; + creatorTextScore: number | null; +}; + +export type PartnerContentMatchBar = { + partnerContentItemId: string; + platform: string; + platformContentId: string; + title: string | null; + url: string | null; + durationMs: number | null; + publishedAt: string | null; + viewCount: number | null; + likeCount: number | null; + commentCount: number | null; + shareCount: number | null; + saveCount: number | null; + matched: boolean; + matchScore: number | null; + matchEvidence: PartnerContentMatchEvidence; +}; + +export type QueryContentMatchContext = { + cutoffDistance?: number | null; + recentItemSourceBestDistance: SourceScoreByContentItemId; + rerankByItemSource: SourceScoreByContentItemId; +}; + +export type ListContentMatchContext = { + bestMatchByContentItemId: Map; +}; + +export type ContentMatchContext = + | QueryContentMatchContext + | ListContentMatchContext; + +export function getBestMatchByContentItemId(rows: PartnerContentSearchRow[]) { + const bestMatchByContentItemId = new Map< + string, + PartnerContentItemBestMatch + >(); + + for (const row of rows) { + // Use the effective score (rerank when present) so the content-match bars + // stay consistent with the reranked partner ordering. + const score = row.rerankScore ?? toScore(Number(row.distance)); + const evidenceSource = getEvidenceSource(row.chunkSource); + const existing = bestMatchByContentItemId.get(row.partnerContentItemId); + const next = existing ?? { + transcriptScore: null, + creatorTextScore: null, + }; + + if (evidenceSource === "transcript") { + next.transcriptScore = + next.transcriptScore == null + ? score + : Math.max(next.transcriptScore, score); + } else { + next.creatorTextScore = + next.creatorTextScore == null + ? score + : Math.max(next.creatorTextScore, score); + } + + bestMatchByContentItemId.set(row.partnerContentItemId, next); + } + + return bestMatchByContentItemId; +} + +export function getQueryContentMatch({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: QueryContentMatchContext; +}) { + // On-topic gate: prefer the reranker's calibrated relevance; fall back to the + // cosine cutoff only for items the reranker didn't score. Both sources rely + // solely on embedding retrieval + reranking — no lexical title/description boost. + const transcriptScore = getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "transcript", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "transcript", + ), + cutoffDistance: context.cutoffDistance, + }); + const creatorTextScore = getMatchedSourceScore({ + rerankScore: getSourceScore( + context.rerankByItemSource, + row.partnerContentItemId, + "creatorText", + ), + bestDistance: getSourceScore( + context.recentItemSourceBestDistance, + row.partnerContentItemId, + "creatorText", + ), + cutoffDistance: context.cutoffDistance, + }); + + const matchEvidence = createContentMatchEvidence({ + contentType: row.contentType, + transcriptScore, + creatorTextScore, + }); + + return { + matchEvidence, + matchScore: getEvidenceMatchScore(matchEvidence), + }; +} + +export function getListContentMatch({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: ListContentMatchContext; +}) { + const match = context.bestMatchByContentItemId.get(row.partnerContentItemId); + const matchEvidence = createContentMatchEvidence({ + contentType: row.contentType, + transcriptScore: match?.transcriptScore ?? null, + creatorTextScore: match?.creatorTextScore ?? null, + }); + + return { + matchEvidence, + matchScore: getEvidenceMatchScore(matchEvidence), + }; +} + +export function toContentMatchBar({ + row, + context, +}: { + row: PartnerRecentContentBarRow; + context: ContentMatchContext; +}): PartnerContentMatchBar { + const { matchEvidence, matchScore } = + "bestMatchByContentItemId" in context + ? getListContentMatch({ row, context }) + : getQueryContentMatch({ row, context }); + const matched = matchEvidence.sources.length > 0; + + return { + partnerContentItemId: row.partnerContentItemId, + platform: row.platformType, + platformContentId: row.platformContentId, + title: row.contentTitle, + url: row.contentUrl, + durationMs: + row.contentDurationMs != null ? Number(row.contentDurationMs) : null, + publishedAt: row.publishedAt?.toISOString() ?? null, + viewCount: row.viewCount != null ? Number(row.viewCount) : null, + likeCount: row.likeCount != null ? Number(row.likeCount) : null, + commentCount: row.commentCount != null ? Number(row.commentCount) : null, + shareCount: row.shareCount != null ? Number(row.shareCount) : null, + saveCount: row.saveCount != null ? Number(row.saveCount) : null, + matched, + matchScore, + matchEvidence, + }; +} + +export function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { + const matchedBars = contentBars.filter((bar) => bar.matched); + const matchedContentCount = matchedBars.length; + const transcriptMatchedContentCount = contentBars.filter( + ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), + ).length; + const creatorTextMatchedContentCount = contentBars.filter( + ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), + ).length; + const creatorTextOnlyContentCount = contentBars.filter( + ({ matchEvidence }) => + matchEvidence.primarySource === "creatorText" && + matchEvidence.sources.length === 1, + ).length; + const weightedMatchedContentCount = Number( + contentBars + .reduce((total, bar) => total + bar.matchEvidence.weight, 0) + .toFixed(3), + ); + const weightedMatchedContentScore = Number( + contentBars + .reduce( + (total, bar) => total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), + 0, + ) + .toFixed(3), + ); + + return { + matchedBars, + matchedContentCount, + transcriptMatchedContentCount, + creatorTextMatchedContentCount, + creatorTextOnlyContentCount, + weightedMatchedContentCount, + weightedMatchedContentScore, + }; +} diff --git a/apps/web/lib/partner-content-search/ingestion/eligibility-where.ts b/apps/web/lib/partner-content-search/ingestion/eligibility-where.ts new file mode 100644 index 00000000000..e2e9fa2db4e --- /dev/null +++ b/apps/web/lib/partner-content-search/ingestion/eligibility-where.ts @@ -0,0 +1,75 @@ +import type { Prisma } from "@prisma/client"; +import { PartnerContentPlatform } from "../types"; +import type { PartnerContentIngestionMode } from "./payload-schemas"; +import { PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS } from "./routes"; + +export function getIncrementalRefreshCutoff() { + return new Date( + Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, + ); +} + +// Shared platform-eligibility predicate (verified + incremental-recency rules), +// used by both the enumerate and enumerate/page routes. +export function buildEligiblePartnerPlatformWhere({ + mode, + platforms, +}: { + mode: PartnerContentIngestionMode; + platforms: PartnerContentPlatform[]; +}): Prisma.PartnerPlatformWhereInput { + return { + type: { + in: platforms, + }, + verifiedAt: { + not: null, + }, + ...(mode === "incremental" && { + OR: [ + { + contentLastFetchedAt: null, + }, + { + contentLastFetchedAt: { + lt: getIncrementalRefreshCutoff(), + }, + }, + ], + }), + }; +} + +// Partner-level eligibility for the enumerate route — composes +// buildEligiblePartnerPlatformWhere; gates on approved/trusted status. +export function buildEligiblePartnerWhere({ + mode, + filter, +}: { + mode: PartnerContentIngestionMode; + filter: { + partnerId?: string; + partnerIds?: string[]; + platforms: PartnerContentPlatform[]; + }; +}): Prisma.PartnerWhereInput { + return { + networkStatus: { + in: ["approved", "trusted"], + }, + ...(filter.partnerId && { + id: filter.partnerId, + }), + ...(filter.partnerIds?.length && { + id: { + in: filter.partnerIds, + }, + }), + platforms: { + some: buildEligiblePartnerPlatformWhere({ + mode, + platforms: filter.platforms, + }), + }, + }; +} diff --git a/apps/web/lib/partner-content-search/ingestion/enqueue.ts b/apps/web/lib/partner-content-search/ingestion/enqueue.ts index 0b682f8ebab..ac674a44ed3 100644 --- a/apps/web/lib/partner-content-search/ingestion/enqueue.ts +++ b/apps/web/lib/partner-content-search/ingestion/enqueue.ts @@ -1,151 +1,23 @@ import { qstash } from "@/lib/cron"; import { prisma } from "@/lib/prisma"; -import type { Prisma } from "@prisma/client"; -import { APP_DOMAIN_WITH_NGROK } from "@dub/utils"; import { logAndRespond } from "app/(ee)/api/cron/utils"; import "server-only"; -import * as z from "zod/v4"; +import type { + PartnerContentEnumeratePayload, + PartnerContentIngestionMode, +} from "./payload-schemas"; import { - PARTNER_CONTENT_SEARCH_PLATFORMS, - PartnerContentPlatform, -} from "../types"; - -// Partners per enumerate page (default 500). Override via env to exercise the -// self-continuation chain locally without seeding 500+ partners. -export const PARTNER_CONTENT_ENUMERATE_PAGE_SIZE = (() => { - const raw = process.env.PARTNER_CONTENT_ENUMERATE_PAGE_SIZE; - const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; - return Number.isInteger(parsed) && parsed > 0 ? parsed : 500; -})(); -export const PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS = 7; - -export const PARTNER_CONTENT_EMBED_FLOW_CONTROL = { - key: "partner-content-embed-voyage", - parallelism: 20, - rate: 600, - period: "1m", -} as const; - -export const PARTNER_CONTENT_SEARCH_ROUTES = { - enumerate: "/api/cron/partner-content/enumerate", - enumeratePage: "/api/cron/partner-content/enumerate/page", - fetch: "/api/cron/partner-content/fetch", - transcript: "/api/cron/partner-content/transcript", - embed: "/api/cron/partner-content/embed", -} as const; - -type PartnerContentSearchRoute = - (typeof PARTNER_CONTENT_SEARCH_ROUTES)[keyof typeof PARTNER_CONTENT_SEARCH_ROUTES]; - -export const partnerContentIngestionModeSchema = z.enum([ - "incremental", - "backfill", -]); - -export type PartnerContentIngestionMode = z.infer< - typeof partnerContentIngestionModeSchema ->; - -export const partnerContentIngestionFilterSchema = z - .object({ - partnerId: z.string().optional(), - partnerIds: z.array(z.string()).max(500).optional(), - platforms: z - .array(z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS)) - .min(1) - .default([...PARTNER_CONTENT_SEARCH_PLATFORMS]), - limitPartners: z.number().int().positive().max(100_000).optional(), - }) - .prefault({}) - .refine((filter) => !(filter.partnerId && filter.partnerIds?.length), { - message: "Use either partnerId or partnerIds, not both.", - }); - -export const partnerContentEnumeratePayloadSchema = z.object({ - mode: partnerContentIngestionModeSchema, - filter: partnerContentIngestionFilterSchema, - runStamp: z.string().min(1), - dryRun: z.boolean().default(false), - // Cursor for self-continued enumeration; absent on the initial dispatch. - startingAfter: z.string().optional(), - // Partner budget carried across continuation hops; omitted = unbounded. - remainingPartners: z.number().int().nonnegative().optional(), -}); - -export const partnerContentEnumeratePagePayloadSchema = z.object({ - mode: partnerContentIngestionModeSchema, - filter: partnerContentIngestionFilterSchema, - runStamp: z.string().min(1), - dryRun: z.boolean().default(false), - partnerIds: z - .array(z.string()) - .min(1) - .max(PARTNER_CONTENT_ENUMERATE_PAGE_SIZE), -}); - -export const partnerContentFetchPayloadSchema = z.object({ - mode: partnerContentIngestionModeSchema, - runStamp: z.string().min(1), - dryRun: z.boolean().default(false), - forceTranscriptJobs: z.boolean().default(false), - // Escape hatch for linked-but-unverified platforms; backfills stay verified-only. - ignoreUnverified: z.boolean().default(false), - partnerId: z.string().min(1), - partnerPlatformId: z.string().min(1), - platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), -}); - -export const partnerContentTranscriptPayloadSchema = z.object({ - mode: partnerContentIngestionModeSchema, - runStamp: z.string().min(1), - dryRun: z.boolean().default(false), - // Re-fetch even if already fetched. Off by default so QStash redelivery doesn't - // re-burn ScrapeCreators credits. - forceRefetch: z.boolean().default(false), - partnerId: z.string().min(1), - partnerPlatformId: z.string().min(1), - partnerContentItemId: z.string().min(1), - platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), -}); - -export const partnerContentEmbedPayloadSchema = z - .object({ - mode: partnerContentIngestionModeSchema, - runStamp: z.string().min(1), - partnerId: z.string().min(1), - partnerPlatformId: z.string().min(1).optional(), - partnerContentItemId: z.string().min(1).optional(), - limitContentItems: z.number().int().positive().max(500).default(100), - maxChunks: z.number().int().positive().max(128).default(96), - }) - .refine( - ({ partnerPlatformId, partnerContentItemId }) => - Boolean(partnerPlatformId) !== Boolean(partnerContentItemId), - { - message: "Use exactly one of partnerPlatformId or partnerContentItemId.", - }, - ); - -export type PartnerContentEnumeratePayload = z.infer< - typeof partnerContentEnumeratePayloadSchema ->; - -export function createPartnerContentRunStamp() { - return new Date().toISOString().replace(/[:.]/g, "-"); -} - -export function getPartnerContentUrl(route: PartnerContentSearchRoute) { - return `${APP_DOMAIN_WITH_NGROK}${route}`; -} - -export function createPartnerContentDeduplicationId( - ...parts: Array -) { - return parts - .filter((part): part is string | number => part !== undefined) - .map((part) => String(part).replace(/[^a-zA-Z0-9_-]/g, "-")) - .join("-"); -} + PARTNER_CONTENT_EMBED_FLOW_CONTROL, + PARTNER_CONTENT_SEARCH_ROUTES, + createPartnerContentDeduplicationId, + getPartnerContentUrl, +} from "./routes"; + +// Re-export every symbol previously defined here so existing import paths stay +// valid with zero call-site changes. +export * from "./payload-schemas"; +export * from "./routes"; +export * from "./eligibility-where"; export async function enqueuePartnerContentEnumerate( payload: PartnerContentEnumeratePayload, @@ -283,91 +155,3 @@ export async function enqueueEmbedJobsForPartnerPlatform({ `[PartnerContentSearch] Enqueued ${messages.length} embed jobs (${pendingContentItems.length} pending of ${contentItems.length} inspected) for partner platform ${partnerPlatformId} on ${mode} run ${runStamp}.`, ); } - -export function getIncrementalRefreshCutoff() { - return new Date( - Date.now() - PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS * 24 * 60 * 60 * 1000, - ); -} - -// Shared platform-eligibility predicate (verified + incremental-recency rules), -// used by both the enumerate and enumerate/page routes. -export function buildEligiblePartnerPlatformWhere({ - mode, - platforms, -}: { - mode: PartnerContentIngestionMode; - platforms: PartnerContentPlatform[]; -}): Prisma.PartnerPlatformWhereInput { - return { - type: { - in: platforms, - }, - verifiedAt: { - not: null, - }, - ...(mode === "incremental" && { - OR: [ - { - contentLastFetchedAt: null, - }, - { - contentLastFetchedAt: { - lt: getIncrementalRefreshCutoff(), - }, - }, - ], - }), - }; -} - -// Partner-level eligibility for the enumerate route — composes -// buildEligiblePartnerPlatformWhere; gates on approved/trusted status. -export function buildEligiblePartnerWhere({ - mode, - filter, -}: { - mode: PartnerContentIngestionMode; - filter: { - partnerId?: string; - partnerIds?: string[]; - platforms: PartnerContentPlatform[]; - }; -}): Prisma.PartnerWhereInput { - return { - networkStatus: { - in: ["approved", "trusted"], - }, - ...(filter.partnerId && { - id: filter.partnerId, - }), - ...(filter.partnerIds?.length && { - id: { - in: filter.partnerIds, - }, - }), - platforms: { - some: buildEligiblePartnerPlatformWhere({ - mode, - platforms: filter.platforms, - }), - }, - }; -} - -// Parse a cron payload. A malformed body is permanent, so respond 400 (not a -// withCron 500 + alert). Returns the payload, or a Response to return directly. -export function parsePartnerContentCronPayload( - schema: T, - rawBody: string, -): z.infer | Response { - try { - return schema.parse(JSON.parse(rawBody)) as z.infer; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return logAndRespond( - `[PartnerContentSearch] Ignoring invalid cron payload: ${message}`, - { status: 400, logLevel: "warn" }, - ); - } -} diff --git a/apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts b/apps/web/lib/partner-content-search/ingestion/fetch-and-write-transcript-chunks.ts similarity index 98% rename from apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts rename to apps/web/lib/partner-content-search/ingestion/fetch-and-write-transcript-chunks.ts index 0907fc6091f..abc56a0bafc 100644 --- a/apps/web/lib/partner-content-search/ingestion/write-transcript-chunks.ts +++ b/apps/web/lib/partner-content-search/ingestion/fetch-and-write-transcript-chunks.ts @@ -11,7 +11,7 @@ import { prisma } from "@/lib/prisma"; // Fetch a content item's transcript, (re)chunk it, and replace its transcript chunks // in one transaction. Returns a summary; marks notAvailable when there's no transcript. -export async function writeTranscriptChunks(contentItem: { +export async function fetchAndWriteTranscriptChunks(contentItem: { id: string; partnerId: string; url: string; diff --git a/apps/web/lib/partner-content-search/ingestion/payload-schemas.ts b/apps/web/lib/partner-content-search/ingestion/payload-schemas.ts new file mode 100644 index 00000000000..ec3b0e35751 --- /dev/null +++ b/apps/web/lib/partner-content-search/ingestion/payload-schemas.ts @@ -0,0 +1,114 @@ +import * as z from "zod/v4"; +import { logAndRespond } from "app/(ee)/api/cron/utils"; +import { PARTNER_CONTENT_SEARCH_PLATFORMS } from "../types"; +import { PARTNER_CONTENT_ENUMERATE_PAGE_SIZE } from "./routes"; + +export const partnerContentIngestionModeSchema = z.enum([ + "incremental", + "backfill", +]); + +export type PartnerContentIngestionMode = z.infer< + typeof partnerContentIngestionModeSchema +>; + +export const partnerContentIngestionFilterSchema = z + .object({ + partnerId: z.string().optional(), + partnerIds: z.array(z.string()).max(500).optional(), + platforms: z + .array(z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS)) + .min(1) + .default([...PARTNER_CONTENT_SEARCH_PLATFORMS]), + limitPartners: z.number().int().positive().max(100_000).optional(), + }) + .prefault({}) + .refine((filter) => !(filter.partnerId && filter.partnerIds?.length), { + message: "Use either partnerId or partnerIds, not both.", + }); + +export const partnerContentEnumeratePayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + filter: partnerContentIngestionFilterSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + // Cursor for self-continued enumeration; absent on the initial dispatch. + startingAfter: z.string().optional(), + // Partner budget carried across continuation hops; omitted = unbounded. + remainingPartners: z.number().int().nonnegative().optional(), +}); + +export const partnerContentEnumeratePagePayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + filter: partnerContentIngestionFilterSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + partnerIds: z + .array(z.string()) + .min(1) + .max(PARTNER_CONTENT_ENUMERATE_PAGE_SIZE), +}); + +export const partnerContentFetchPayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + forceTranscriptJobs: z.boolean().default(false), + // Escape hatch for linked-but-unverified platforms; backfills stay verified-only. + ignoreUnverified: z.boolean().default(false), + partnerId: z.string().min(1), + partnerPlatformId: z.string().min(1), + platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), +}); + +export const partnerContentTranscriptPayloadSchema = z.object({ + mode: partnerContentIngestionModeSchema, + runStamp: z.string().min(1), + dryRun: z.boolean().default(false), + // Re-fetch even if already fetched. Off by default so QStash redelivery doesn't + // re-burn ScrapeCreators credits. + forceRefetch: z.boolean().default(false), + partnerId: z.string().min(1), + partnerPlatformId: z.string().min(1), + partnerContentItemId: z.string().min(1), + platform: z.enum(PARTNER_CONTENT_SEARCH_PLATFORMS), +}); + +export const partnerContentEmbedPayloadSchema = z + .object({ + mode: partnerContentIngestionModeSchema, + runStamp: z.string().min(1), + partnerId: z.string().min(1), + partnerPlatformId: z.string().min(1).optional(), + partnerContentItemId: z.string().min(1).optional(), + limitContentItems: z.number().int().positive().max(500).default(100), + maxChunks: z.number().int().positive().max(128).default(96), + }) + .refine( + ({ partnerPlatformId, partnerContentItemId }) => + Boolean(partnerPlatformId) !== Boolean(partnerContentItemId), + { + message: "Use exactly one of partnerPlatformId or partnerContentItemId.", + }, + ); + +export type PartnerContentEnumeratePayload = z.infer< + typeof partnerContentEnumeratePayloadSchema +>; + +// Parse a cron payload. A malformed body is permanent, so respond 400 (not a +// withCron 500 + alert). Returns the payload, or a Response to return directly. +export function parsePartnerContentCronPayload( + schema: T, + rawBody: string, +): z.infer | Response { + try { + return schema.parse(JSON.parse(rawBody)) as z.infer; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return logAndRespond( + `[PartnerContentSearch] Ignoring invalid cron payload: ${message}`, + { status: 400, logLevel: "warn" }, + ); + } +} diff --git a/apps/web/lib/partner-content-search/ingestion/routes.ts b/apps/web/lib/partner-content-search/ingestion/routes.ts new file mode 100644 index 00000000000..7e0db5f2854 --- /dev/null +++ b/apps/web/lib/partner-content-search/ingestion/routes.ts @@ -0,0 +1,45 @@ +import { APP_DOMAIN_WITH_NGROK } from "@dub/utils"; + +// Partners per enumerate page (default 500). Override via env to exercise the +// self-continuation chain locally without seeding 500+ partners. +export const PARTNER_CONTENT_ENUMERATE_PAGE_SIZE = (() => { + const raw = process.env.PARTNER_CONTENT_ENUMERATE_PAGE_SIZE; + const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; + return Number.isInteger(parsed) && parsed > 0 ? parsed : 500; +})(); +export const PARTNER_CONTENT_INCREMENTAL_REFRESH_DAYS = 7; + +export const PARTNER_CONTENT_EMBED_FLOW_CONTROL = { + key: "partner-content-embed-voyage", + parallelism: 20, + rate: 600, + period: "1m", +} as const; + +export const PARTNER_CONTENT_SEARCH_ROUTES = { + enumerate: "/api/cron/partner-content/enumerate", + enumeratePage: "/api/cron/partner-content/enumerate/page", + fetch: "/api/cron/partner-content/fetch", + transcript: "/api/cron/partner-content/transcript", + embed: "/api/cron/partner-content/embed", +} as const; + +export type PartnerContentSearchRoute = + (typeof PARTNER_CONTENT_SEARCH_ROUTES)[keyof typeof PARTNER_CONTENT_SEARCH_ROUTES]; + +export function createPartnerContentRunStamp() { + return new Date().toISOString().replace(/[:.]/g, "-"); +} + +export function getPartnerContentUrl(route: PartnerContentSearchRoute) { + return `${APP_DOMAIN_WITH_NGROK}${route}`; +} + +export function createPartnerContentDeduplicationId( + ...parts: Array +) { + return parts + .filter((part): part is string | number => part !== undefined) + .map((part) => String(part).replace(/[^a-zA-Z0-9_-]/g, "-")) + .join("-"); +} diff --git a/apps/web/lib/partner-content-search/match-summaries.ts b/apps/web/lib/partner-content-search/match-summaries.ts index 2aa0f627ed6..43f03661700 100644 --- a/apps/web/lib/partner-content-search/match-summaries.ts +++ b/apps/web/lib/partner-content-search/match-summaries.ts @@ -1,78 +1,22 @@ -import { prisma } from "@/lib/prisma"; -import { PlatformType, Prisma } from "@prisma/client"; +import { PlatformType } from "@prisma/client"; import { - PARTNER_CONTENT_SEARCH_LIMITS, - PARTNER_CONTENT_SEARCH_MODELS, -} from "./constants"; + ContentMatchContext, + getBestMatchByContentItemId, + getContentBarMatchStats, + QueryContentMatchContext, + toContentMatchBar, +} from "./content-match-bars"; +import { fetchMatchSummaryBaseData } from "./match-summary-queries"; import { - createContentMatchEvidence, deriveTopicFit, - getEvidenceMatchScore, getEvidenceSource, - getEvidenceTopicScore, - getMatchedSourceScore, - getSourceScore, setSourceScore, - type PartnerContentMatchEvidence, type PartnerContentMatchSource, type SourceScoreByContentItemId, } from "./ranking"; -import { median, toScore, type PartnerContentSearchRow } from "./search-utils"; +import { median, type PartnerContentSearchRow } from "./search-utils"; import type { PartnerContentSearchTimingLogger } from "./timing"; -type PartnerRecentContentBarRow = { - partnerId: string; - partnerContentItemId: string; - platformType: string; - platformContentId: string; - contentType: string; - contentTitle: string | null; - contentUrl: string | null; - contentDurationMs: number | null; - publishedAt: Date | null; - viewCount: bigint | number | null; - likeCount: bigint | number | null; - commentCount: bigint | number | null; - shareCount: bigint | number | null; - saveCount: bigint | number | null; - rowNumber: bigint | number; -}; - -type PartnerContentItemBestMatch = { - transcriptScore: number | null; - creatorTextScore: number | null; -}; - -type PartnerContentMatchBar = { - partnerContentItemId: string; - platform: string; - platformContentId: string; - title: string | null; - url: string | null; - durationMs: number | null; - publishedAt: string | null; - viewCount: number | null; - likeCount: number | null; - commentCount: number | null; - shareCount: number | null; - saveCount: number | null; - matched: boolean; - matchScore: number | null; - matchEvidence: PartnerContentMatchEvidence; -}; - -type QueryContentMatchContext = { - cutoffDistance?: number | null; - recentItemSourceBestDistance: SourceScoreByContentItemId; - rerankByItemSource: SourceScoreByContentItemId; -}; - -type ListContentMatchContext = { - bestMatchByContentItemId: Map; -}; - -type ContentMatchContext = QueryContentMatchContext | ListContentMatchContext; - function groupRowsBy(rows: T[], getKey: (row: T) => K) { const rowsByKey = new Map(); @@ -86,182 +30,6 @@ function groupRowsBy(rows: T[], getKey: (row: T) => K) { return rowsByKey; } -function getBestMatchByContentItemId(rows: PartnerContentSearchRow[]) { - const bestMatchByContentItemId = new Map< - string, - PartnerContentItemBestMatch - >(); - - for (const row of rows) { - // Use the effective score (rerank when present) so the content-match bars - // stay consistent with the reranked partner ordering. - const score = row.rerankScore ?? toScore(Number(row.distance)); - const evidenceSource = getEvidenceSource(row.chunkSource); - const existing = bestMatchByContentItemId.get(row.partnerContentItemId); - const next = existing ?? { - transcriptScore: null, - creatorTextScore: null, - }; - - if (evidenceSource === "transcript") { - next.transcriptScore = - next.transcriptScore == null - ? score - : Math.max(next.transcriptScore, score); - } else { - next.creatorTextScore = - next.creatorTextScore == null - ? score - : Math.max(next.creatorTextScore, score); - } - - bestMatchByContentItemId.set(row.partnerContentItemId, next); - } - - return bestMatchByContentItemId; -} - -function getQueryContentMatch({ - row, - context, -}: { - row: PartnerRecentContentBarRow; - context: QueryContentMatchContext; -}) { - // On-topic gate: prefer the reranker's calibrated relevance; fall back to the - // cosine cutoff only for items the reranker didn't score. Both sources rely - // solely on embedding retrieval + reranking — no lexical title/description boost. - const transcriptScore = getMatchedSourceScore({ - rerankScore: getSourceScore( - context.rerankByItemSource, - row.partnerContentItemId, - "transcript", - ), - bestDistance: getSourceScore( - context.recentItemSourceBestDistance, - row.partnerContentItemId, - "transcript", - ), - cutoffDistance: context.cutoffDistance, - }); - const creatorTextScore = getMatchedSourceScore({ - rerankScore: getSourceScore( - context.rerankByItemSource, - row.partnerContentItemId, - "creatorText", - ), - bestDistance: getSourceScore( - context.recentItemSourceBestDistance, - row.partnerContentItemId, - "creatorText", - ), - cutoffDistance: context.cutoffDistance, - }); - - const matchEvidence = createContentMatchEvidence({ - contentType: row.contentType, - transcriptScore, - creatorTextScore, - }); - - return { - matchEvidence, - matchScore: getEvidenceMatchScore(matchEvidence), - }; -} - -function getListContentMatch({ - row, - context, -}: { - row: PartnerRecentContentBarRow; - context: ListContentMatchContext; -}) { - const match = context.bestMatchByContentItemId.get(row.partnerContentItemId); - const matchEvidence = createContentMatchEvidence({ - contentType: row.contentType, - transcriptScore: match?.transcriptScore ?? null, - creatorTextScore: match?.creatorTextScore ?? null, - }); - - return { - matchEvidence, - matchScore: getEvidenceMatchScore(matchEvidence), - }; -} - -function toContentMatchBar({ - row, - context, -}: { - row: PartnerRecentContentBarRow; - context: ContentMatchContext; -}): PartnerContentMatchBar { - const { matchEvidence, matchScore } = - "bestMatchByContentItemId" in context - ? getListContentMatch({ row, context }) - : getQueryContentMatch({ row, context }); - const matched = matchEvidence.sources.length > 0; - - return { - partnerContentItemId: row.partnerContentItemId, - platform: row.platformType, - platformContentId: row.platformContentId, - title: row.contentTitle, - url: row.contentUrl, - durationMs: - row.contentDurationMs != null ? Number(row.contentDurationMs) : null, - publishedAt: row.publishedAt?.toISOString() ?? null, - viewCount: row.viewCount != null ? Number(row.viewCount) : null, - likeCount: row.likeCount != null ? Number(row.likeCount) : null, - commentCount: row.commentCount != null ? Number(row.commentCount) : null, - shareCount: row.shareCount != null ? Number(row.shareCount) : null, - saveCount: row.saveCount != null ? Number(row.saveCount) : null, - matched, - matchScore, - matchEvidence, - }; -} - -function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { - const matchedBars = contentBars.filter((bar) => bar.matched); - const matchedContentCount = matchedBars.length; - const transcriptMatchedContentCount = contentBars.filter( - ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), - ).length; - const creatorTextMatchedContentCount = contentBars.filter( - ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), - ).length; - const creatorTextOnlyContentCount = contentBars.filter( - ({ matchEvidence }) => - matchEvidence.primarySource === "creatorText" && - matchEvidence.sources.length === 1, - ).length; - const weightedMatchedContentCount = Number( - contentBars - .reduce((total, bar) => total + bar.matchEvidence.weight, 0) - .toFixed(3), - ); - const weightedMatchedContentScore = Number( - contentBars - .reduce( - (total, bar) => total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), - 0, - ) - .toFixed(3), - ); - - return { - matchedBars, - matchedContentCount, - transcriptMatchedContentCount, - creatorTextMatchedContentCount, - creatorTextOnlyContentCount, - weightedMatchedContentCount, - weightedMatchedContentScore, - }; -} - export async function getPartnerMatchSummaries({ rows, partnerIds, @@ -285,150 +53,12 @@ export async function getPartnerMatchSummaries({ }) { if (partnerIds.length === 0) return new Map(); - const platformFilter = platforms?.length - ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` - : Prisma.empty; - - // "Recent" is a time window (recencyWindowMonths), not a post count — a count - // window spans wildly different time per creator, conflating active and dormant. - const recencyCutoff = new Date(); - recencyCutoff.setMonth( - recencyCutoff.getMonth() - - PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, - ); - - // Independent reads — issued as one parallel wave (fan out under Promise.all). - const contentCountsPromise = prisma.partnerContentItem - .groupBy({ - by: ["partnerId"], - where: { - partnerId: { - in: partnerIds, - }, - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - embeddedChunkCount: { - gt: 0, - }, - ...(platforms?.length && { - partnerPlatform: { - type: { in: platforms }, - }, - }), - }, - _count: { - _all: true, - }, - _min: { - publishedAt: true, - }, - _max: { - publishedAt: true, - }, - }) - .then((contentCounts) => { - logTiming?.("match-summary-content-counts-complete", { - contentCountRows: contentCounts.length, - }); - return contentCounts; - }); - - const recentContentRowsPromise = prisma - .$queryRaw( - Prisma.sql` - SELECT - partnerId, - partnerContentItemId, - platformType, - platformContentId, - contentType, - contentTitle, - contentUrl, - contentDurationMs, - publishedAt, - viewCount, - likeCount, - commentCount, - shareCount, - saveCount, - rowNumber - FROM ( - SELECT - pci.partnerId, - pci.id AS partnerContentItemId, - pp.type AS platformType, - pci.platformContentId, - pci.contentType, - pci.title AS contentTitle, - pci.url AS contentUrl, - pci.durationMs AS contentDurationMs, - pci.publishedAt, - pci.viewCount, - pci.likeCount, - pci.commentCount, - pci.shareCount, - pci.saveCount, - ROW_NUMBER() OVER ( - PARTITION BY pci.partnerId - ORDER BY pci.publishedAt DESC, pci.createdAt DESC, pci.id ASC - ) AS rowNumber - FROM PartnerContentItem pci - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - WHERE pci.partnerId IN (${Prisma.join(partnerIds)}) - ${platformFilter} - AND ( - pci.publishedAt >= ${recencyCutoff} - OR (pci.publishedAt IS NULL AND pci.createdAt >= ${recencyCutoff}) - ) - AND pci.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - AND pci.embeddedChunkCount > 0 - ) recentContent - WHERE rowNumber <= ${PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner} - ORDER BY partnerId ASC, rowNumber ASC - `, - ) - .then((recentContentRows) => { - logTiming?.("match-summary-recent-content-complete", { - recentContentRows: recentContentRows.length, - }); - return recentContentRows; + const { contentCounts, recentContentRows, followerRows } = + await fetchMatchSummaryBaseData({ + partnerIds, + platforms, + logTiming, }); - - // Reach: total followers across the partner's platforms (or the filtered platform). - const followerRowsPromise = prisma.partnerPlatform - .groupBy({ - by: ["partnerId"], - where: { - partnerId: { - in: partnerIds, - }, - ...(platforms?.length && { - type: { in: platforms }, - }), - }, - _sum: { - subscribers: true, - }, - }) - .then((followerRows) => { - logTiming?.("match-summary-followers-complete", { - followerRows: followerRows.length, - }); - return followerRows; - }); - - logTiming?.("match-summary-base-queries-start", { - partnerCount: partnerIds.length, - }); - const [contentCounts, recentContentRows, followerRows] = await Promise.all([ - contentCountsPromise, - recentContentRowsPromise, - followerRowsPromise, - ]); - logTiming?.("match-summary-base-queries-complete", { - contentCountRows: contentCounts.length, - recentContentRows: recentContentRows.length, - followerRows: followerRows.length, - }); const followersByPartner = new Map( followerRows.map((row) => [ row.partnerId, diff --git a/apps/web/lib/partner-content-search/match-summary-queries.ts b/apps/web/lib/partner-content-search/match-summary-queries.ts new file mode 100644 index 00000000000..ee015beaccc --- /dev/null +++ b/apps/web/lib/partner-content-search/match-summary-queries.ts @@ -0,0 +1,182 @@ +import { prisma } from "@/lib/prisma"; +import { PlatformType, Prisma } from "@prisma/client"; +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_MODELS, +} from "./constants"; +import type { PartnerContentSearchTimingLogger } from "./timing"; + +export type PartnerRecentContentBarRow = { + partnerId: string; + partnerContentItemId: string; + platformType: string; + platformContentId: string; + contentType: string; + contentTitle: string | null; + contentUrl: string | null; + contentDurationMs: number | null; + publishedAt: Date | null; + viewCount: bigint | number | null; + likeCount: bigint | number | null; + commentCount: bigint | number | null; + shareCount: bigint | number | null; + saveCount: bigint | number | null; + rowNumber: bigint | number; +}; + +export async function fetchMatchSummaryBaseData({ + partnerIds, + platforms, + logTiming, +}: { + partnerIds: string[]; + platforms?: PlatformType[]; + logTiming?: PartnerContentSearchTimingLogger; +}) { + const platformFilter = platforms?.length + ? Prisma.sql`AND pp.type IN (${Prisma.join(platforms)})` + : Prisma.empty; + + // "Recent" is a time window (recencyWindowMonths), not a post count — a count + // window spans wildly different time per creator, conflating active and dormant. + const recencyCutoff = new Date(); + recencyCutoff.setMonth( + recencyCutoff.getMonth() - + PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, + ); + + // Independent reads — issued as one parallel wave (fan out under Promise.all). + const contentCountsPromise = prisma.partnerContentItem + .groupBy({ + by: ["partnerId"], + where: { + partnerId: { + in: partnerIds, + }, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + embeddedChunkCount: { + gt: 0, + }, + ...(platforms?.length && { + partnerPlatform: { + type: { in: platforms }, + }, + }), + }, + _count: { + _all: true, + }, + _min: { + publishedAt: true, + }, + _max: { + publishedAt: true, + }, + }) + .then((contentCounts) => { + logTiming?.("match-summary-content-counts-complete", { + contentCountRows: contentCounts.length, + }); + return contentCounts; + }); + + const recentContentRowsPromise = prisma + .$queryRaw( + Prisma.sql` + SELECT + partnerId, + partnerContentItemId, + platformType, + platformContentId, + contentType, + contentTitle, + contentUrl, + contentDurationMs, + publishedAt, + viewCount, + likeCount, + commentCount, + shareCount, + saveCount, + rowNumber + FROM ( + SELECT + pci.partnerId, + pci.id AS partnerContentItemId, + pp.type AS platformType, + pci.platformContentId, + pci.contentType, + pci.title AS contentTitle, + pci.url AS contentUrl, + pci.durationMs AS contentDurationMs, + pci.publishedAt, + pci.viewCount, + pci.likeCount, + pci.commentCount, + pci.shareCount, + pci.saveCount, + ROW_NUMBER() OVER ( + PARTITION BY pci.partnerId + ORDER BY pci.publishedAt DESC, pci.createdAt DESC, pci.id ASC + ) AS rowNumber + FROM PartnerContentItem pci + INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId + WHERE pci.partnerId IN (${Prisma.join(partnerIds)}) + ${platformFilter} + AND ( + pci.publishedAt >= ${recencyCutoff} + OR (pci.publishedAt IS NULL AND pci.createdAt >= ${recencyCutoff}) + ) + AND pci.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} + AND pci.embeddedChunkCount > 0 + ) recentContent + WHERE rowNumber <= ${PARTNER_CONTENT_SEARCH_LIMITS.recentContentMaxPerPartner} + ORDER BY partnerId ASC, rowNumber ASC + `, + ) + .then((recentContentRows) => { + logTiming?.("match-summary-recent-content-complete", { + recentContentRows: recentContentRows.length, + }); + return recentContentRows; + }); + + // Reach: total followers across the partner's platforms (or the filtered platform). + const followerRowsPromise = prisma.partnerPlatform + .groupBy({ + by: ["partnerId"], + where: { + partnerId: { + in: partnerIds, + }, + ...(platforms?.length && { + type: { in: platforms }, + }), + }, + _sum: { + subscribers: true, + }, + }) + .then((followerRows) => { + logTiming?.("match-summary-followers-complete", { + followerRows: followerRows.length, + }); + return followerRows; + }); + + logTiming?.("match-summary-base-queries-start", { + partnerCount: partnerIds.length, + }); + const [contentCounts, recentContentRows, followerRows] = await Promise.all([ + contentCountsPromise, + recentContentRowsPromise, + followerRowsPromise, + ]); + logTiming?.("match-summary-base-queries-complete", { + contentCountRows: contentCounts.length, + recentContentRows: recentContentRows.length, + followerRows: followerRows.length, + }); + + return { contentCounts, recentContentRows, followerRows }; +} diff --git a/apps/web/lib/partner-content-search/rerank.ts b/apps/web/lib/partner-content-search/rerank.ts new file mode 100644 index 00000000000..9fe3b99b38b --- /dev/null +++ b/apps/web/lib/partner-content-search/rerank.ts @@ -0,0 +1,76 @@ +// Server-only second-stage reranking via the Voyage reranker. Kept separate from +// search-utils so the pure ranking/grouping helpers there stay client-safe (the +// content-search UI imports them transitively). +import "server-only"; + +import { + PARTNER_CONTENT_SEARCH_LIMITS, + PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS, +} from "./constants"; +import { effectiveRowScore, type PartnerContentSearchRow } from "./search-utils"; +import { + rerankPartnerContent, + VoyageApiError, + VoyageTimeoutError, +} from "./voyage"; + +// Re-score the top candidates with the Voyage reranker, sorted by effective score. +// Fails soft: on timeout/API error, returns the original cosine ordering. +export async function rerankPartnerSearchRows({ + query, + rows, +}: { + query: string; + rows: PartnerContentSearchRow[]; +}): Promise<{ rows: PartnerContentSearchRow[]; reranked: boolean }> { + if (rows.length === 0) return { rows, reranked: false }; + + // Only the top-N candidates are reranked, to bound payload/latency. + const candidates = rows.slice( + 0, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, + ); + const documents = candidates.map((row) => + (row.chunkText ?? "").slice( + 0, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerMaxDocChars, + ), + ); + + try { + const results = await rerankPartnerContent({ + query, + documents, + topK: candidates.length, + timeoutMs: PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS, + }); + + for (const { index, relevanceScore } of results) { + const row = candidates[index]; + if (row) row.rerankScore = relevanceScore; + } + } catch (error) { + if ( + error instanceof VoyageTimeoutError || + error instanceof VoyageApiError + ) { + console.error( + "[partner-content-search] reranker failed, falling back to cosine:", + error.message, + ); + return { rows, reranked: false }; + } + throw error; + } + + // Reranked candidates sort ahead of the cosine-only tail (don't let an + // un-reranked row leapfrog a reranked one on a different score scale). + const reranked = [...rows].sort((a, b) => { + const aReranked = a.rerankScore != null; + const bReranked = b.rerankScore != null; + if (aReranked !== bReranked) return aReranked ? -1 : 1; + return effectiveRowScore(b) - effectiveRowScore(a); + }); + + return { rows: reranked, reranked: true }; +} diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts index 56e6c6046c1..ba9faffd997 100644 --- a/apps/web/lib/partner-content-search/retrieval.ts +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -14,10 +14,10 @@ import { sortRowsByRelevanceScore, type SourceScoreByContentItemId, } from "./ranking"; +import { rerankPartnerSearchRows } from "./rerank"; import { dedupeBestChunkPerContentItem, dedupeBestChunkPerContentItemSource, - rerankPartnerSearchRows, type PartnerContentSearchRow, } from "./search-utils"; import type { PartnerContentSearchTimingLogger } from "./timing"; @@ -503,60 +503,3 @@ async function hydratePartnerContentChunkText({ : row, ); } - -// Single-phase admin diagnostics query: one raw ANN with inline hydration, no -// eligibility/network pre-filtering (intentionally searches the full corpus). -export async function searchAdminPartnerContentChunks({ - queryVector, - limit, - partnerIds, - platform, -}: { - queryVector: string; - limit: number; - partnerIds?: string[]; - platform?: PlatformType; -}) { - const partnerFilter = partnerIds?.length - ? Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})` - : Prisma.empty; - const platformFilter = platform - ? Prisma.sql`AND pp.type = ${platform}` - : Prisma.empty; - - return await prisma.$queryRaw(Prisma.sql` - SELECT - c.id AS chunkId, - c.partnerContentItemId, - c.partnerId, - p.name AS partnerName, - p.username AS partnerUsername, - p.image AS partnerImage, - p.description AS partnerDescription, - pp.type AS platformType, - pp.identifier AS platformIdentifier, - pci.platformContentId, - pci.url AS contentUrl, - pci.contentType, - pci.title AS contentTitle, - pci.thumbnailUrl AS contentThumbnailUrl, - pci.publishedAt AS contentPublishedAt, - pci.durationMs AS contentDurationMs, - c.source AS chunkSource, - c.chunkIndex, - c.chunkText, - c.startMs, - c.endMs, - DISTANCE(TO_VECTOR(${queryVector}), c.embedding, ${Prisma.raw(`'${PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE}'`)}) AS distance - FROM PartnerContentChunk c - INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId - INNER JOIN Partner p ON p.id = c.partnerId - INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId - WHERE c.embedding IS NOT NULL - AND c.embeddingModel = ${PARTNER_CONTENT_SEARCH_MODELS.embedding.id} - ${partnerFilter} - ${platformFilter} - ORDER BY distance ASC - LIMIT ${limit} - `); -} diff --git a/apps/web/lib/partner-content-search/search-utils.ts b/apps/web/lib/partner-content-search/search-utils.ts index 18b9c42c587..b0cd9af2d1f 100644 --- a/apps/web/lib/partner-content-search/search-utils.ts +++ b/apps/web/lib/partner-content-search/search-utils.ts @@ -1,15 +1,8 @@ -// Shared helpers for the admin + network content search routes: per-partner -// grouping (each route passes its own toChunkResult) and reranking. - -import { - PARTNER_CONTENT_SEARCH_LIMITS, - PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS, -} from "./constants"; -import { - rerankPartnerContent, - VoyageApiError, - VoyageTimeoutError, -} from "./voyage"; +// Pure (client-safe) helpers for the admin + network content search routes: +// per-partner grouping (each route passes its own toChunkResult), candidate +// sizing, and score utilities. Server-only reranking lives in ./rerank. + +import { PARTNER_CONTENT_SEARCH_LIMITS } from "./constants"; export type PartnerContentSearchRow = { chunkId: string; @@ -49,6 +42,27 @@ export function toScore(distance: number) { return Number((1 - distance).toFixed(6)); } +// How many candidate chunks to retrieve to satisfy `limit` partners at +// `chunksPerPartner` each. Query mode over-fetches (×6, floored at 25, capped at +// the candidate ceiling) so reranking + topic-fit have a deep pool to work from; +// list mode has no relevance gate, so a smaller pool (×2) suffices. +export function getCandidateChunkCount({ + hasQuery, + limit, + chunksPerPartner, +}: { + hasQuery: boolean; + limit: number; + chunksPerPartner: number; +}) { + if (!hasQuery) return limit * chunksPerPartner * 2; + + return Math.min( + PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, + Math.max(25, limit * chunksPerPartner * 6), + ); +} + // Median of a numeric list (robust center; null when empty). `round` rounds the // even-length average for display; leave it off when the value feeds further math. export function median( @@ -177,64 +191,3 @@ export function groupPartnerSearchResults({ .slice(0, limit) .map(({ chunkKeys, ...partner }) => partner); } - -// Re-score the top candidates with the Voyage reranker, sorted by effective score. -// Fails soft: on timeout/API error, returns the original cosine ordering. -export async function rerankPartnerSearchRows({ - query, - rows, -}: { - query: string; - rows: PartnerContentSearchRow[]; -}): Promise<{ rows: PartnerContentSearchRow[]; reranked: boolean }> { - if (rows.length === 0) return { rows, reranked: false }; - - // Only the top-N candidates are reranked, to bound payload/latency. - const candidates = rows.slice( - 0, - PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, - ); - const documents = candidates.map((row) => - (row.chunkText ?? "").slice( - 0, - PARTNER_CONTENT_SEARCH_LIMITS.rerankerMaxDocChars, - ), - ); - - try { - const results = await rerankPartnerContent({ - query, - documents, - topK: candidates.length, - timeoutMs: PARTNER_CONTENT_SEARCH_RERANK_TIMEOUT_MS, - }); - - for (const { index, relevanceScore } of results) { - const row = candidates[index]; - if (row) row.rerankScore = relevanceScore; - } - } catch (error) { - if ( - error instanceof VoyageTimeoutError || - error instanceof VoyageApiError - ) { - console.error( - "[partner-content-search] reranker failed, falling back to cosine:", - error.message, - ); - return { rows, reranked: false }; - } - throw error; - } - - // Reranked candidates sort ahead of the cosine-only tail (don't let an - // un-reranked row leapfrog a reranked one on a different score scale). - const reranked = [...rows].sort((a, b) => { - const aReranked = a.rerankScore != null; - const bReranked = b.rerankScore != null; - if (aReranked !== bReranked) return aReranked ? -1 : 1; - return effectiveRowScore(b) - effectiveRowScore(a); - }); - - return { rows: reranked, reranked: true }; -} diff --git a/apps/web/lib/partner-content-search/thumbnail-url.ts b/apps/web/lib/partner-content-search/thumbnail-url.ts index 620d55a36a8..111f6e2171d 100644 --- a/apps/web/lib/partner-content-search/thumbnail-url.ts +++ b/apps/web/lib/partner-content-search/thumbnail-url.ts @@ -10,7 +10,8 @@ export function getPartnerContentThumbnailUrl(thumbnailUrl: string | null) { export function isInstagramCdnUrl(url: string) { try { - const { hostname } = new URL(url); + const { hostname, protocol } = new URL(url); + if (protocol !== "https:") return false; return ( hostname === "cdninstagram.com" || hostname.endsWith(".cdninstagram.com") || diff --git a/apps/web/lib/swr/use-partner-content-search.ts b/apps/web/lib/swr/use-partner-content-search.ts index 638fad9ac32..b880067f3d9 100644 --- a/apps/web/lib/swr/use-partner-content-search.ts +++ b/apps/web/lib/swr/use-partner-content-search.ts @@ -121,7 +121,6 @@ export default function usePartnerContentSearch({ country, starred, partnerIds, - candidateChunkCount, limit = PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, chunksPerPartner = PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, debounceMs = 0, @@ -133,7 +132,6 @@ export default function usePartnerContentSearch({ country?: string; starred: boolean; partnerIds?: string[]; - candidateChunkCount?: number; limit?: number; chunksPerPartner?: number; // Coalesce rapid filter changes into one request. Voyage search is billed and @@ -155,7 +153,6 @@ export default function usePartnerContentSearch({ country, starred, partnerIds?.join(",") ?? "", - candidateChunkCount, limit, chunksPerPartner, ]) @@ -180,7 +177,6 @@ export default function usePartnerContentSearch({ query, limit, chunksPerPartner, - ...(candidateChunkCount && { candidateChunkCount }), ...(platforms?.length && { platforms }), ...(reach?.length && { reach }), ...(country && { country }), diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index 623aa80817d..c532501db00 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -2,7 +2,6 @@ import { REACH_TIER_KEYS } from "@/lib/api/network/reach-tiers"; import { processKey } from "@/lib/api/links/utils"; import { PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, - PARTNER_CONTENT_SEARCH_LIMITS, PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, } from "@/lib/partner-content-search/constants"; @@ -153,45 +152,6 @@ export const partnerNetworkContentSearchSchema = z.object({ .positive() .max(PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER) .default(PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER), - candidateChunkCount: z - .number() - .int() - .positive() - .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) - .optional(), // Second-stage reranking is on by default; pass `false` for diagnostics. rerank: z.boolean().default(true), }); - -const PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT = 10; -const PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER = 3; - -// Request body for the admin diagnostics route POST /api/admin/partner-content/search. -// Simpler than the network schema: a required query, a single platform filter, and -// it surfaces more chunks per partner for inspection. -export const partnerAdminContentSearchSchema = z.object({ - query: z.string().trim().min(1).max(500), - limit: z - .number() - .int() - .positive() - .max(50) - .default(PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT), - chunksPerPartner: z - .number() - .int() - .positive() - .max(10) - .default(PARTNER_ADMIN_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER), - candidateChunkCount: z - .number() - .int() - .positive() - .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount) - .optional(), - partnerIds: z.array(z.string()).min(1).max(100).optional(), - platform: z.enum(PlatformType).optional(), - // Second-stage reranking is on by default; pass `false` to inspect cosine-only - // ranking for comparison. - rerank: z.boolean().default(true), -}); diff --git a/apps/web/ui/partners/partner-basic-fields.tsx b/apps/web/ui/partners/partner-basic-fields.tsx new file mode 100644 index 00000000000..aef4987ea00 --- /dev/null +++ b/apps/web/ui/partners/partner-basic-fields.tsx @@ -0,0 +1,165 @@ +import { + AdminNetworkPartner, + EnrolledPartnerExtendedProps, + NetworkPartnerProps, +} from "@/lib/types"; +import { CalendarIcon, Globe, OfficeBuilding, TimestampTooltip } from "@dub/ui"; +import { + TriangleWarning, + UserArrowRight, + Users, + VerifiedBadge, +} from "@dub/ui/icons"; +import { COUNTRIES, formatDate, formatDateTimeSmart } from "@dub/utils"; +import { CircleMinus } from "lucide-react"; +import { ReactNode, createElement } from "react"; +import { + getPayoutMethodIconConfig, + getPayoutMethodLabel, +} from "./payouts/payout-method-config"; +import { ReferredByPartner } from "./referred-by-partner"; + +export type BasicField = { + id: string; + icon: React.ReactElement; + text: ReactNode | null | undefined; + /** When set, the row is wrapped in TimestampTooltip (local / UTC / unix). */ + timestamp?: Date | string | number; + /** Optional outer wrapper (e.g. ConversionScoreTooltip) around the row content. */ + wrapper?: React.ComponentType<{ children: React.ReactNode }>; +}; + +export function getBasicFields({ + partner, + isEnrolled, + isAdmin, + isNetwork, + canCreateReferralReward, +}: { + partner?: + | EnrolledPartnerExtendedProps + | NetworkPartnerProps + | AdminNetworkPartner; + isEnrolled: boolean; + isAdmin: boolean; + isNetwork: boolean; + canCreateReferralReward: boolean; +}): BasicField[] { + let basicFields: BasicField[] = [ + { + id: "country", + icon: partner?.country ? ( + {`Flag + ) : ( + + ), + text: partner?.country ? COUNTRIES[partner.country] : "Planet Earth", + }, + { + id: "companyName", + icon: , + text: partner ? partner.companyName || null : undefined, + }, + ]; + + if ((isEnrolled || isAdmin) && partner) { + const isPendingApplication = + "status" in partner && partner.status === "pending"; + + // This branch runs only for enrolled/admin partners. Cast to surface the + // payout/referral fields that NetworkPartnerProps lacks — compile-time only, + // with no runtime effect (absent fields already read as undefined). + const detailPartner = partner as EnrolledPartnerExtendedProps; + + basicFields = basicFields.concat([ + { + id: "createdAt", + icon: isPendingApplication ? ( + + ) : ( + + ), + text: isPendingApplication ? ( + + + Applied {formatDate(partner.createdAt)} + + + ) : ( + `${isPendingApplication ? "Applied" : "Partner since"} ${formatDate(partner.createdAt)}` + ), + timestamp: isPendingApplication ? undefined : partner.createdAt, + }, + { + id: "payoutMethod" as const, + icon: detailPartner.defaultPayoutMethod ? ( + createElement( + getPayoutMethodIconConfig(detailPartner.defaultPayoutMethod).Icon, + { className: "size-3.5 shrink-0" }, + ) + ) : ( + + ), + text: + detailPartner.defaultPayoutMethod && detailPartner.payoutsEnabledAt + ? `${getPayoutMethodLabel(detailPartner.defaultPayoutMethod)} connected ${formatDateTimeSmart(detailPartner.payoutsEnabledAt)}` + : "No payout method connected", + ...(detailPartner.payoutsEnabledAt + ? { timestamp: detailPartner.payoutsEnabledAt } + : {}), + }, + // TODO: once more partners verify their identity, we can show this by default + ...(partner.identityVerifiedAt + ? [ + { + id: "identityVerifiedAt", + icon: partner.identityVerifiedAt ? ( + + ) : ( + + ), + text: partner.identityVerifiedAt + ? `Identity verified ${formatDate(partner.identityVerifiedAt, { month: "short" })}` + : "Identity not verified", + ...(partner.identityVerifiedAt + ? { timestamp: partner.identityVerifiedAt } + : {}), + }, + ] + : []), + + // Referred by + ...(isEnrolled && canCreateReferralReward + ? [ + { + id: "referredBy", + icon: , + text: , + }, + ] + : []), + ]); + } + + if (isNetwork) { + basicFields = basicFields.concat([ + { + id: "joinedAt", + icon: , + text: partner ? `Joined ${formatDate(partner.createdAt!)}` : undefined, + timestamp: partner?.createdAt, + }, + ]); + } + + return basicFields; +} diff --git a/apps/web/ui/partners/partner-info-cards.tsx b/apps/web/ui/partners/partner-info-cards.tsx index 31afe13ccf9..c01a5ec6d8e 100644 --- a/apps/web/ui/partners/partner-info-cards.tsx +++ b/apps/web/ui/partners/partner-info-cards.tsx @@ -1,45 +1,15 @@ -import { useAttributeReferringPartnerModal } from "@/lib/partner-referrals/components/attribute-referring-partner-modal"; -import { usePartnerReferral } from "@/lib/partner-referrals/hooks/use-partner-referral"; import { getPlanCapabilities } from "@/lib/plan-capabilities"; import useGroup from "@/lib/swr/use-group"; import useWorkspace from "@/lib/swr/use-workspace"; import { AdminNetworkPartner, - BountyListProps, EnrolledPartnerExtendedProps, NetworkPartnerProps, - RewardProps, } from "@/lib/types"; import { DEFAULT_PARTNER_GROUP } from "@/lib/zod/schemas/groups"; -import { INACTIVE_ENROLLMENT_STATUSES } from "@/lib/zod/schemas/partners"; import { usePartnerGroupHistorySheet } from "@/ui/activity-logs/partner-group-history-sheet"; -import { - Button, - CalendarIcon, - CopyButton, - Globe, - Heart, - InfoTooltip, - OfficeBuilding, - TimestampTooltip, - Trophy, -} from "@dub/ui"; -import { - TriangleWarning, - UserArrowRight, - Users, - VerifiedBadge, -} from "@dub/ui/icons"; -import { - COUNTRIES, - fetcher, - formatDate, - formatDateTimeSmart, -} from "@dub/utils"; -import { CircleMinus } from "lucide-react"; -import Link from "next/link"; -import { Fragment, ReactNode, createElement } from "react"; -import useSWR from "swr"; +import { CopyButton, InfoTooltip, TimestampTooltip } from "@dub/ui"; +import { Fragment, ReactNode } from "react"; import { PartnerApplicationRiskSummary } from "./fraud-risks/partner-application-risk-summary"; import { PartnerApplicationRiskBanner, @@ -49,20 +19,12 @@ import { PartnerRiskIndicator } from "./fraud-risks/partner-risk-indicator"; import { PartnerAvatar } from "./partner-avatar"; import { PARTNER_PLATFORM_FIELDS } from "@/lib/partners/partner-platforms"; import { monthlyTrafficAmountsMap } from "@/lib/partners/partner-profile"; -import { PartnerInfoGroup } from "./partner-info-group"; +import { getBasicFields } from "./partner-basic-fields"; +import { PartnerInfoRewardsCard } from "./partner-info-rewards-card"; +import { PartnerInfoTagsCard } from "./partner-info-tags-card"; import { PartnerNetworkStatusBadge } from "./partner-network/partner-network-status-badge"; import { PartnerStarButton } from "./partner-star-button"; import { PartnerStatusBadgeWithTooltip } from "./partner-status-badge-with-tooltip"; -import { PartnerTagsList } from "./partner-tags-list"; -import { - getPayoutMethodIconConfig, - getPayoutMethodLabel, -} from "./payouts/payout-method-config"; -import { ProgramRewardList } from "./program-reward-list"; -import { - UpdatePartnerTagsModal, - useUpdatePartnerTagsModal, -} from "./update-partner-tags-modal"; type PartnerInfoCardsProps = { showFraudIndicator?: boolean; @@ -88,16 +50,6 @@ type PartnerInfoCardsProps = { | { type: "admin"; partner?: AdminNetworkPartner } ); -type BasicField = { - id: string; - icon: React.ReactElement; - text: ReactNode | null | undefined; - /** When set, the row is wrapped in TimestampTooltip (local / UTC / unix). */ - timestamp?: Date | string | number; - /** Optional outer wrapper (e.g. ConversionScoreTooltip) around the row content. */ - wrapper?: React.ComponentType<{ children: React.ReactNode }>; -}; - export function PartnerInfoCards({ type, partner, @@ -134,123 +86,13 @@ export function PartnerInfoCards({ { keepPreviousData: false }, ); - const { data: bounties, error: errorBounties } = useSWR( - workspaceId && partner && isEnrolled - ? `/api/bounties?workspaceId=${workspaceId}&partnerId=${partner.id}` - : null, - fetcher, - ); - - let basicFields: BasicField[] = [ - { - id: "country", - icon: partner?.country ? ( - {`Flag - ) : ( - - ), - text: partner?.country ? COUNTRIES[partner.country] : "Planet Earth", - }, - { - id: "companyName", - icon: , - text: partner ? partner.companyName || null : undefined, - }, - ]; - - if ((isEnrolled || isAdmin) && partner) { - const isPendingApplication = - "status" in partner && partner.status === "pending"; - - basicFields = basicFields.concat([ - { - id: "createdAt", - icon: isPendingApplication ? ( - - ) : ( - - ), - text: isPendingApplication ? ( - - - Applied {formatDate(partner.createdAt)} - - - ) : ( - `${isPendingApplication ? "Applied" : "Partner since"} ${formatDate(partner.createdAt)}` - ), - timestamp: isPendingApplication ? undefined : partner.createdAt, - }, - { - id: "payoutMethod" as const, - icon: partner.defaultPayoutMethod ? ( - createElement( - getPayoutMethodIconConfig(partner.defaultPayoutMethod).Icon, - { className: "size-3.5 shrink-0" }, - ) - ) : ( - - ), - text: - partner.defaultPayoutMethod && partner.payoutsEnabledAt - ? `${getPayoutMethodLabel(partner.defaultPayoutMethod)} connected ${formatDateTimeSmart(partner.payoutsEnabledAt)}` - : "No payout method connected", - ...(partner.payoutsEnabledAt - ? { timestamp: partner.payoutsEnabledAt } - : {}), - }, - // TODO: once more partners verify their identity, we can show this by default - ...(partner.identityVerifiedAt - ? [ - { - id: "identityVerifiedAt", - icon: partner.identityVerifiedAt ? ( - - ) : ( - - ), - text: partner.identityVerifiedAt - ? `Identity verified ${formatDate(partner.identityVerifiedAt, { month: "short" })}` - : "Identity not verified", - ...(partner.identityVerifiedAt - ? { timestamp: partner.identityVerifiedAt } - : {}), - }, - ] - : []), - - // Referred by - ...(isEnrolled && canCreateReferralReward - ? [ - { - id: "referredBy", - icon: , - text: , - }, - ] - : []), - ]); - } - - if (isNetwork) { - basicFields = basicFields.concat([ - { - id: "joinedAt", - icon: , - text: partner ? `Joined ${formatDate(partner.createdAt!)}` : undefined, - timestamp: partner?.createdAt, - }, - ]); - } + const basicFields = getBasicFields({ + partner, + isEnrolled, + isAdmin, + isNetwork, + canCreateReferralReward, + }); // Browse mode: website/socials rendered as compact rows (matching the detail // rows above) instead of the bordered platform cards. @@ -457,7 +299,7 @@ export function PartnerInfoCards({
)} - {isEnrolled && partner && } + {isEnrolled && partner && } {partner && isEnrolled && showApplicationRiskAnalysis && ( @@ -466,218 +308,19 @@ export function PartnerInfoCards({
{!isAdmin && !browseMode && ( -
- {/* Group */} -
- {isEnrolled && ( -
-

- Group -

- - {partner && partner.status !== "pending" && hasActivityLogs && ( -
- )} - - {partnerGroupHistorySheet} - {partner ? ( - - ) : ( -
- )} -
- - {isEnrolled && partner?.status === "approved" && ( - <> - {/* Rewards */} -
-

- Rewards -

- {group ? ( - group.clickReward || - group.leadReward || - group.saleReward || - group.discount ? ( - r !== null)} - discount={group.discount} - variant="plain" - className="text-content-subtle gap-2 text-xs leading-4" - iconClassName="size-3.5" - /> - ) : ( - - No rewards - - ) - ) : ( -
- )} -
- {/* Eligible bounties */} -
-

- Eligible Bounties -

- {bounties ? ( - bounties.length ? ( -
- {bounties.map((bounty) => { - const Icon = - bounty.type === "performance" ? Trophy : Heart; - return ( - - - - {bounty.name} - - - ); - })} -
- ) : ( -

- No eligible bounties -

- ) - ) : errorBounties ? ( -

- Failed to load bounties -

- ) : ( -
- )} -
- - )} -
+ )}
); } - -function TagsList({ partner }: { partner: EnrolledPartnerExtendedProps }) { - const { showUpdatePartnerTagsModal, setShowUpdatePartnerTagsModal } = - useUpdatePartnerTagsModal(); - - return ( -
- -
- - Tags - - - -
- setShowUpdatePartnerTagsModal(true)} - mode="link" - /> -
- ); -} - -function ReferredByPartner({ - partner, -}: { - partner: Pick< - EnrolledPartnerExtendedProps, - "id" | "name" | "image" | "email" | "groupId" | "totalCommissions" - >; -}) { - const { slug } = useWorkspace(); - - const { referral, loading, error } = usePartnerReferral({ - partnerId: partner?.id, - }); - - const { - AttributeReferringPartnerModal, - setShowAttributeReferringPartnerModal, - } = useAttributeReferringPartnerModal({ partner }); - - if (error) { - return null; - } - - if (loading) { - return ( - -
-
-
- - ); - } - - // Has a referring partner - if (referral && referral.referredBy) { - return ( - - Referred by - - - {referral.referredBy.name} - - - ); - } - - return ( - <> - - - - ); -} diff --git a/apps/web/ui/partners/partner-info-rewards-card.tsx b/apps/web/ui/partners/partner-info-rewards-card.tsx new file mode 100644 index 00000000000..98a28b015dc --- /dev/null +++ b/apps/web/ui/partners/partner-info-rewards-card.tsx @@ -0,0 +1,170 @@ +"use client"; + +import { + AdminNetworkPartner, + BountyListProps, + EnrolledPartnerExtendedProps, + GroupProps, + NetworkPartnerProps, + RewardProps, +} from "@/lib/types"; +import { INACTIVE_ENROLLMENT_STATUSES } from "@/lib/zod/schemas/partners"; +import { Button, Heart, Trophy } from "@dub/ui"; +import { fetcher } from "@dub/utils"; +import Link from "next/link"; +import { ReactNode } from "react"; +import useSWR from "swr"; +import { PartnerInfoGroup } from "./partner-info-group"; +import { ProgramRewardList } from "./program-reward-list"; + +export function PartnerInfoRewardsCard({ + partner, + workspaceId, + workspaceSlug, + isEnrolled, + group, + partnerGroupHistorySheet, + setGroupHistoryOpen, + hasActivityLogs, + selectedGroupId, + setSelectedGroupId, +}: { + partner?: + | EnrolledPartnerExtendedProps + | NetworkPartnerProps + | AdminNetworkPartner; + workspaceId?: string; + workspaceSlug?: string; + isEnrolled: boolean; + group?: GroupProps; + partnerGroupHistorySheet: ReactNode; + setGroupHistoryOpen: (open: boolean) => void; + hasActivityLogs: boolean; + selectedGroupId?: string | null; + setSelectedGroupId?: (groupId: string) => void; +}) { + const { data: bounties, error: errorBounties } = useSWR( + workspaceId && partner && isEnrolled + ? `/api/bounties?workspaceId=${workspaceId}&partnerId=${partner.id}` + : null, + fetcher, + ); + + return ( +
+ {/* Group */} +
+ {isEnrolled && ( +
+

+ Group +

+ + {partner && + (partner as EnrolledPartnerExtendedProps).status !== "pending" && + hasActivityLogs && ( +
+ )} + + {partnerGroupHistorySheet} + {partner ? ( + + ) : ( +
+ )} +
+ + {isEnrolled && + (partner as EnrolledPartnerExtendedProps | undefined)?.status === + "approved" && ( + <> + {/* Rewards */} +
+

+ Rewards +

+ {group ? ( + group.clickReward || + group.leadReward || + group.saleReward || + group.discount ? ( + r !== null)} + discount={group.discount} + variant="plain" + className="text-content-subtle gap-2 text-xs leading-4" + iconClassName="size-3.5" + /> + ) : ( + No rewards + ) + ) : ( +
+ )} +
+ {/* Eligible bounties */} +
+

+ Eligible Bounties +

+ {bounties ? ( + bounties.length ? ( +
+ {bounties.map((bounty) => { + const Icon = + bounty.type === "performance" ? Trophy : Heart; + return ( + + + + {bounty.name} + + + ); + })} +
+ ) : ( +

+ No eligible bounties +

+ ) + ) : errorBounties ? ( +

+ Failed to load bounties +

+ ) : ( +
+ )} +
+ + )} +
+ ); +} diff --git a/apps/web/ui/partners/partner-info-tags-card.tsx b/apps/web/ui/partners/partner-info-tags-card.tsx new file mode 100644 index 00000000000..5743c8bede7 --- /dev/null +++ b/apps/web/ui/partners/partner-info-tags-card.tsx @@ -0,0 +1,46 @@ +"use client"; + +import { EnrolledPartnerExtendedProps } from "@/lib/types"; +import { PartnerTagsList } from "./partner-tags-list"; +import { + UpdatePartnerTagsModal, + useUpdatePartnerTagsModal, +} from "./update-partner-tags-modal"; + +export function PartnerInfoTagsCard({ + partner, +}: { + partner: EnrolledPartnerExtendedProps; +}) { + const { showUpdatePartnerTagsModal, setShowUpdatePartnerTagsModal } = + useUpdatePartnerTagsModal(); + + return ( +
+ +
+ + Tags + + + +
+ setShowUpdatePartnerTagsModal(true)} + mode="link" + /> +
+ ); +} diff --git a/apps/web/ui/partners/referred-by-partner.tsx b/apps/web/ui/partners/referred-by-partner.tsx new file mode 100644 index 00000000000..dae25d9d2c2 --- /dev/null +++ b/apps/web/ui/partners/referred-by-partner.tsx @@ -0,0 +1,72 @@ +"use client"; + +import { useAttributeReferringPartnerModal } from "@/lib/partner-referrals/components/attribute-referring-partner-modal"; +import { usePartnerReferral } from "@/lib/partner-referrals/hooks/use-partner-referral"; +import useWorkspace from "@/lib/swr/use-workspace"; +import { EnrolledPartnerExtendedProps } from "@/lib/types"; +import Link from "next/link"; +import { PartnerAvatar } from "./partner-avatar"; + +export function ReferredByPartner({ + partner, +}: { + partner: Pick< + EnrolledPartnerExtendedProps, + "id" | "name" | "image" | "email" | "groupId" | "totalCommissions" + >; +}) { + const { slug } = useWorkspace(); + + const { referral, loading, error } = usePartnerReferral({ + partnerId: partner?.id, + }); + + const { + AttributeReferringPartnerModal, + setShowAttributeReferringPartnerModal, + } = useAttributeReferringPartnerModal({ partner }); + + if (error) { + return null; + } + + if (loading) { + return ( + +
+
+
+ + ); + } + + // Has a referring partner + if (referral && referral.referredBy) { + return ( + + Referred by + + + {referral.referredBy.name} + + + ); + } + + return ( + <> + + + + ); +} From 83337abbdd03e6c689258211c216852df6f70b08 Mon Sep 17 00:00:00 2001 From: David Clark Date: Tue, 23 Jun 2026 16:52:31 -0400 Subject: [PATCH 17/25] further refactor partner content search; centralize schemas and refactor loading UX --- .../network/partners/content-search/route.ts | 35 +-- .../app/(ee)/api/network/partners/route.ts | 2 - .../network/[partnerId]/page-client.tsx | 8 +- .../network-content-search-results.tsx | 94 ++++---- .../(ee)/program/network/page-client.tsx | 21 +- .../program/network/platform-filter-utils.ts | 5 +- .../use-network-partner-filters-state.ts | 8 +- .../network/use-toggle-partner-starred.ts | 23 +- .../api/network/calculate-partner-ranking.ts | 104 ++++++--- .../network/partner-network-listing-where.ts | 19 +- apps/web/lib/api/network/reach-tiers.ts | 7 +- apps/web/lib/api/scrape-creators/client.ts | 15 +- .../lib/partner-content-search/constants.ts | 15 +- .../web/lib/partner-content-search/listing.ts | 3 +- .../network-partners.ts | 5 +- .../web/lib/partner-content-search/ranking.ts | 9 +- apps/web/lib/partner-content-search/rerank.ts | 30 +-- .../lib/partner-content-search/retrieval.ts | 210 ++++++------------ .../partner-content-search/search-utils.ts | 9 +- apps/web/lib/partner-content-search/timing.ts | 2 - .../web/lib/swr/use-partner-content-search.ts | 195 +++++----------- apps/web/lib/zod/schemas/partner-network.ts | 141 ++++++++++++ .../partner-content-search-vector-index.sql | 31 +-- .../schema/partner-content-search.prisma | 19 +- .../vector-index-sync.test.ts | 7 +- apps/web/ui/partners/partner-info-cards.tsx | 5 +- packages/ui/src/tooltip.tsx | 5 +- 27 files changed, 503 insertions(+), 524 deletions(-) diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index ac8466f5224..6f660fb306a 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -24,7 +24,10 @@ import { } from "@/lib/partner-content-search/search-utils"; import { createPartnerContentSearchTimingLogger } from "@/lib/partner-content-search/timing"; import { prisma } from "@/lib/prisma"; -import { partnerNetworkContentSearchSchema } from "@/lib/zod/schemas/partner-network"; +import { + partnerNetworkContentSearchResponseSchema, + partnerNetworkContentSearchSchema, +} from "@/lib/zod/schemas/partner-network"; import { NextResponse } from "next/server"; export const dynamic = "force-dynamic"; @@ -174,20 +177,22 @@ export const POST = withWorkspace( returnedPartnerCount: Math.min(sortedPartners.length, body.limit), }); - return NextResponse.json({ - success: true, - query: body.query ?? null, - platforms: body.platforms ?? null, - country: body.country ?? null, - candidateChunkCount, - embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, - reranked, - rerankModel: reranked - ? PARTNER_CONTENT_SEARCH_MODELS.reranker.model - : null, - resultCount: rows.length, - partners: sortedPartners.slice(0, body.limit), - }); + return NextResponse.json( + partnerNetworkContentSearchResponseSchema.parse({ + success: true, + query: body.query ?? null, + platforms: body.platforms ?? null, + country: body.country ?? null, + candidateChunkCount, + embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, + reranked, + rerankModel: reranked + ? PARTNER_CONTENT_SEARCH_MODELS.reranker.model + : null, + resultCount: rows.length, + partners: sortedPartners.slice(0, body.limit), + }), + ); }, { requiredPlan: ["enterprise", "advanced"], diff --git a/apps/web/app/(ee)/api/network/partners/route.ts b/apps/web/app/(ee)/api/network/partners/route.ts index 0cc30353a8b..2a723ccbe29 100644 --- a/apps/web/app/(ee)/api/network/partners/route.ts +++ b/apps/web/app/(ee)/api/network/partners/route.ts @@ -115,7 +115,6 @@ export const GET = withWorkspace( similarityScore: sp.similarityScore, })); - console.time("calculatePartnerRanking"); const partners = await calculatePartnerRanking({ programId, partnerIds, @@ -128,7 +127,6 @@ export const GET = withWorkspace( reach: reach ?? undefined, similarPrograms, }); - console.timeEnd("calculatePartnerRanking"); return NextResponse.json(parseRankedNetworkPartners(partners)); }, diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 5a673436eee..8fdee3c7cd0 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -78,11 +78,9 @@ export function NetworkPartnerDetailContent({ ); const loadedPartner = partners?.[0]; - // The results page already ran the global search and handed us this partner's - // match data, so the detail opens instantly. In the background we run a scoped - // single-partner rerank so the matched list shares ONE relevance scale — the - // global search only reranks its top-N pool, mixing rerank + cosine otherwise. - // Non-blocking: the cached Topic Fit headline paints now; rows skeleton until it lands. + // Open instantly from the cached match data; the background refetch reranks just + // this partner so every row lands on one relevance scale (global search only + // reranks its top-N pool). const shouldFetchSearch = hasContentSearch; const { data: searchResults, diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index dcb1e06d25d..541b00be4c0 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -49,14 +49,16 @@ type ContentPreviewItem = { export function NetworkContentSearchResults({ error, hasQuery, - isLoading, + isFetching, partners, platform, onToggleStarred, }: { error: unknown; hasQuery: boolean; - isLoading: boolean; + // A search is pending or in flight. Drives the first-search skeletons (no + // results yet) and dims the prior results during a re-search. + isFetching: boolean; partners?: PartnerContentSearchPartner[]; platform?: PlatformType; onToggleStarred?: (partnerId: string, starred: boolean) => void; @@ -71,7 +73,9 @@ export function NetworkContentSearchResults({ ); } - if (!partners && isLoading) { + // First search (no results yet): skeletons. On re-search we keep the previous + // results on screen, dimmed, instead — see the grid below. + if (isFetching && !partners) { return (
{[...Array(8)].map((_, idx) => ( @@ -102,7 +106,13 @@ export function NetworkContentSearchResults({ } return ( -
+
{partners.map((partner) => ( +
+
+
+
+
+
+
+
+ {[...Array(TOP_CONTENT_PREVIEW_COUNT)].map((_, index) => ( +
+ ))} +
+
+
+
+
+ ); +} + function NetworkPartnerContentMatch({ hasQuery, partner, @@ -163,8 +198,6 @@ function NetworkPartnerContentMatch({ const band = summary?.band ?? "none"; const styles = BAND_STYLES[band]; - const followers = summary?.followers ?? null; - const medianViews = summary?.medianViews ?? null; const matchLabel = formatMatchEvidenceLabel(summary); const lastOnTopic = lastPostedLabel(summary?.lastOnTopicAt); const previewItems = getContentPreviewItems(partner); @@ -183,7 +216,7 @@ function NetworkPartnerContentMatch({
@@ -201,21 +234,6 @@ function NetworkPartnerContentMatch({
- {(followers || medianViews) && ( -
- {followers ? ( - {nFormatter(followers)} followers - ) : null} - {followers && medianViews ? ( - · - ) : null} - {medianViews ? ( - - {nFormatter(medianViews)} median views - - ) : null} -
- )}
{matchLabel} @@ -232,31 +250,6 @@ function NetworkPartnerContentMatch({ ); } -function NetworkPartnerContentMatchSkeleton() { - return ( -
-
-
-
-
-
-
-
-
- {[...Array(TOP_CONTENT_PREVIEW_COUNT)].map((_, index) => ( -
- ))} -
-
-
-
-
- ); -} - function getContentPreviewItems( partner: PartnerContentSearchPartner, ): ContentPreviewItem[] { @@ -306,6 +299,11 @@ function ContentPreviewTile({ item }: { item: ContentPreviewItem }) { } delayDuration={100} + // Content is informational (no links/buttons to hover onto), so skip + // Radix's hoverable "grace area". Otherwise the open tile's grace polygon + // — which spans up to its large content panel — covers the adjacent tile + // and suppresses its tooltip when you move between them. + disableHoverableContent > - + {metric.value} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index 5bec67b5c6b..9a94973880c 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -10,14 +10,15 @@ import { AnimatedSizeContainer, Button, Filter, + InfoTooltip, PaginationControls, usePagination, useRouterStuff, } from "@dub/ui"; import { Star, StarFill } from "@dub/ui/icons"; import { cn, fetcher } from "@dub/utils"; -import { useDebounce } from "use-debounce"; import useSWR from "swr"; +import { useDebounce } from "use-debounce"; import { NetworkContentSearchResults } from "./network-content-search-results"; import { NetworkEmptyState } from "./network-empty-state"; import { NetworkPartnerCard } from "./network-partner-card"; @@ -85,7 +86,7 @@ export function ProgramPartnerNetworkPageClient({ const { data: contentSearchResults, error: contentSearchError, - isLoading: isSearchingContent, + isPending: isContentSearchPending, mutate: mutateContentSearch, } = usePartnerContentSearch({ enabled: isContentSearchMode, @@ -271,11 +272,15 @@ export function ProgramPartnerNetworkPageClient({ onChange={onPlatformsChange} />
-
- +
+ +
+ +
@@ -298,7 +303,7 @@ export function ProgramPartnerNetworkPageClient({ 0} - isLoading={isSearchingContent} + isFetching={isContentSearchPending} partners={contentSearchResults?.partners} platform={ contentSearchPlatforms.length === 1 diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts index e0af82a1936..51907394cd9 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/platform-filter-utils.ts @@ -1,10 +1,7 @@ import { isPartnerContentSearchPlatform } from "@/lib/partner-content-search/types"; import type { PlatformType } from "@prisma/client"; -// The platforms a partner can be filtered by, in display order. This is the -// inclusion set: with all of them selected the filter is a no-op (we omit the -// `platform` param entirely), and deselecting narrows results to partners -// present on any of the still-selected platforms. +// Filterable platforms, in display order. All selected = no filter export const NETWORK_FILTER_PLATFORMS = [ "youtube", "instagram", diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts index 7a3931082bc..ce1a679c01f 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-network-partner-filters-state.ts @@ -37,11 +37,9 @@ export function useNetworkPartnerFiltersState() { const country = searchParams.get("country") ?? undefined; const starred = searchParams.get("starred") === "true"; - // Update filter params via the History API instead of router.push. These are - // query-only changes that drive client-side SWR; page.tsx reads no searchParams, - // so a full RSC navigation per click only adds latency before useSearchParams() - // (and thus the control's checked state) updates. pushState updates it - // synchronously — instant feedback — while SWR still refetches on key change. + // Filter updates use history.pushState instead of router.push. Params only + // drive client-side SWR — page.tsx doesn't read them — so router.push would + // add an RSC round-trip for no gain. SWR refetches when the URL key changes. const updateSearchParams = (opts: { set?: Record; del?: string | string[]; diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts index 3bc933ea843..a7cc20f9c69 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/use-toggle-partner-starred.ts @@ -9,18 +9,11 @@ import { useCallback } from "react"; import { toast } from "sonner"; import type { KeyedMutator } from "swr"; -// Two near-identical optimistic star-toggle mutations live here. They share the -// same star-update action, toast-on-failure, optimisticData/populateCache/revalidate -// semantics, and rollback-on-error — they differ only in the SWR cache shape they -// mutate (the content-search response's nested `partner.starredAt` vs the ranked -// list's flat `starredAt`). Kept as two hooks rather than one parameterized hook so -// each cache transform stays verbatim and behavior is provably unchanged. +// Two optimistic star-toggle hooks for the two cache shapes (content-search's nested +// `partner.starredAt` vs the ranked list's flat `starredAt`). Left unparameterized so +// each cache transform stays verbatim. -/** - * Optimistic star toggle for content-search mode. Mutates the - * `PartnerContentSearchResponse` cache, locating the partner by `partnerId` and - * updating its nested `partner.starredAt`. - */ +// Content-search cache: find the partner by `partnerId`, update nested `partner.starredAt`. export function useToggleStarredContentSearch( mutateContentSearch: KeyedMutator, ) { @@ -85,12 +78,8 @@ export function useToggleStarredContentSearch( ); } -/** - * Optimistic star toggle for ranked-list mode. Mutates the - * `NetworkPartnerProps[]` cache, locating the partner by `id` and updating its - * flat `starredAt`. `partners` is used as the fallback when the cache is empty, - * preserving the original closure's `data || partners` behavior. - */ +// Ranked-list cache: find the partner by `id`, update flat `starredAt`. `partners` +// is the fallback when the cache is empty (`data || partners`). export function useToggleStarredRankedList( mutatePartners: KeyedMutator, partners: NetworkPartnerProps[] | undefined, diff --git a/apps/web/lib/api/network/calculate-partner-ranking.ts b/apps/web/lib/api/network/calculate-partner-ranking.ts index 99cf2283917..e55e32168cc 100644 --- a/apps/web/lib/api/network/calculate-partner-ranking.ts +++ b/apps/web/lib/api/network/calculate-partner-ranking.ts @@ -12,6 +12,36 @@ export interface PartnerRankingParams extends PartnerRankingFilters { similarPrograms?: Array<{ programId: string; similarityScore: number }>; } +// Raw row from the ranking query: Partner columns (p.*) vary, so the base is an +// index signature; `platforms` is the JSON_ARRAYAGG aggregate parsed below. +type RankedPartnerRawRow = Record & { + id: string; + platforms?: string | unknown[] | null; +}; + +// One platform inside the JSON aggregate. MySQL JSON returns BigInt as numbers and +// DateTime as ISO strings. +type RawPlatformAggregate = { + partnerId: string; + type: string; + identifier: string | null; + verifiedAt: string | null; + platformId: string | null; + subscribers: number | string | null; + posts: number | string | null; + views: number | string | null; +}; + +type SerializedPartnerPlatform = Omit< + RawPlatformAggregate, + "subscribers" | "posts" | "views" | "verifiedAt" +> & { + subscribers: bigint; + posts: bigint; + views: bigint; + verifiedAt: Date | null; +}; + /** * Partner Ranking Algorithm (only used for the "discover" tab) * Ranks partners based on performance in similar programs only. @@ -92,10 +122,8 @@ export async function calculatePartnerRanking({ conditions.push(Prisma.sql`(dp.starredAt IS NULL OR dp.id IS NULL)`); } - // Filter by audience-size tier(s): the partner's MAX subscriber count across - // the selected platforms (their headline reach) must fall in one of the chosen - // tiers. Scoped to the same platform selection so reach reflects the audience - // the brand is actually filtering for. + // Reach-tier filter: the partner's max subscriber count across the selected + // platforms must fall in a chosen tier (see reach-tiers.ts). if (reach && reach.length > 0) { const ranges = reachTiersToRanges(reach); const reachPlatformScope = @@ -174,9 +202,10 @@ export async function calculatePartnerRanking({ GROUP BY pe_all.partnerId ) allProgramMetrics ON allProgramMetrics.partnerId = p.id`; - const similarProgramMetricsJoin = - similarPrograms.length > 0 - ? Prisma.sql`LEFT JOIN ( + const hasSimilarPrograms = similarPrograms.length > 0; + + const similarProgramMetricsJoin = hasSimilarPrograms + ? Prisma.sql`LEFT JOIN ( SELECT pe2.partnerId, -- Similarity score: Sum weighted performance (0-50 points, no averaging) @@ -222,15 +251,18 @@ export async function calculatePartnerRanking({ AND pe2.status = 'approved' GROUP BY pe2.partnerId ) similarProgramMetrics ON similarProgramMetrics.partnerId = p.id` - : Prisma.sql`LEFT JOIN ( - SELECT - NULL as partnerId, - NULL as similarityScore, - NULL as programMatchScore - WHERE FALSE - ) similarProgramMetrics ON similarProgramMetrics.partnerId = p.id`; - - const partners = await prisma.$queryRaw>` + : Prisma.empty; + + // No similar programs → omit the join entirely and score these contributions as 0. + // A `FROM`-less `SELECT NULL … WHERE FALSE` dual-derived table joined to another is + // a Vitess planner hazard (vtgate can emit "ERROR 1054 Unknown column"), so we drop + // the join rather than emit a degenerate one. See references/planetscale-mysql.md. + const similarProgramScore = hasSimilarPrograms + ? Prisma.sql`COALESCE(similarProgramMetrics.similarityScore, 0) + + COALESCE(similarProgramMetrics.programMatchScore, 0)` + : Prisma.sql`0`; + + const partners = await prisma.$queryRaw` SELECT p.*, COALESCE(pe.lastConversionAt, allProgramMetrics.lastConversionAt) as lastConversionAt, @@ -255,8 +287,7 @@ export async function calculatePartnerRanking({ CASE WHEN ${hasProfileCheck} THEN 500 ELSE 0 END + -- Trusted partner bonus: 200 points for partners with networkStatus = "trusted" CASE WHEN p.networkStatus = "trusted" THEN 200 ELSE 0 END + - COALESCE(similarProgramMetrics.similarityScore, 0) + - COALESCE(similarProgramMetrics.programMatchScore, 0) + ${similarProgramScore} ) as finalScore FROM ( -- OPTIMIZATION: Filter to discoverable partners FIRST using subquery @@ -353,8 +384,8 @@ export async function calculatePartnerRanking({ LIMIT ${pageSize} OFFSET ${offset} `; - return partners.map((partner: any) => { - let platforms: any[] = []; + return partners.map((partner) => { + let platforms: SerializedPartnerPlatform[] = []; if (partner.platforms) { try { // Handle both string and already-parsed JSON @@ -365,20 +396,27 @@ export async function calculatePartnerRanking({ // Transform platforms to match Prisma types // MySQL JSON returns BigInt as numbers and DateTime as strings - platforms = (Array.isArray(parsedPlatforms) ? parsedPlatforms : []).map( - (platform: any) => ({ - ...platform, - subscribers: platform.subscribers - ? BigInt(platform.subscribers) - : BigInt(0), - posts: platform.posts ? BigInt(platform.posts) : BigInt(0), - views: platform.views ? BigInt(platform.views) : BigInt(0), - verifiedAt: platform.verifiedAt - ? new Date(platform.verifiedAt) - : null, - }), + platforms = ( + Array.isArray(parsedPlatforms) + ? (parsedPlatforms as RawPlatformAggregate[]) + : [] + ).map((platform) => ({ + ...platform, + subscribers: platform.subscribers + ? BigInt(platform.subscribers) + : BigInt(0), + posts: platform.posts ? BigInt(platform.posts) : BigInt(0), + views: platform.views ? BigInt(platform.views) : BigInt(0), + verifiedAt: platform.verifiedAt + ? new Date(platform.verifiedAt) + : null, + })); + } catch (error) { + // A malformed platform aggregate shouldn't silently drop the partner. + console.error( + `[calculatePartnerRanking] Failed to parse platforms JSON for partner ${partner.id}`, + error, ); - } catch { platforms = []; } } diff --git a/apps/web/lib/api/network/partner-network-listing-where.ts b/apps/web/lib/api/network/partner-network-listing-where.ts index e533383685a..6390f7ed48a 100644 --- a/apps/web/lib/api/network/partner-network-listing-where.ts +++ b/apps/web/lib/api/network/partner-network-listing-where.ts @@ -1,7 +1,16 @@ -import { PlatformType, Prisma } from "@prisma/client"; +import { PartnerNetworkStatus, PlatformType, Prisma } from "@prisma/client"; import type { ReachTier } from "./reach-tiers"; import { reachTiersToRanges } from "./reach-tiers"; +// Partner network statuses that count as "discoverable" in the network. Ingestion +// only embeds partners in these statuses, and the discover listing + content-search +// hydrate filter on them. Single source of truth for the Prisma call sites; the raw +// $queryRaw paths (e.g. calculatePartnerRanking) keep their own inline literals. +export const DISCOVERABLE_NETWORK_STATUSES: PartnerNetworkStatus[] = [ + "approved", + "trusted", +]; + /** Query params shared by `/api/network/partners` and `/count` for listing. */ export type PartnerNetworkListingParams = { partnerIds?: string[]; @@ -34,7 +43,7 @@ export function partnerNetworkListingParts( return { listingPartnerBase: { - networkStatus: { in: ["approved", "trusted"] }, + networkStatus: { in: DISCOVERABLE_NETWORK_STATUSES }, ...(params.partnerIds && { id: { in: params.partnerIds }, }), @@ -65,10 +74,8 @@ export function partnerNetworkListingWhere( return partnerWhereFromListingParts(partnerNetworkListingParts(params)); } -// Reach tier filter: the partner's MAX subscriber count across verified platforms -// (optionally scoped to selected platform types) must fall in a chosen tier. -// "some platform >= min" AND "no scoped platform >= max" matches -// calculatePartnerRanking's MAX-subscribers semantics. +// Reach-tier filter (see reach-tiers.ts). "some platform >= min AND no scoped +// platform >= max" is the WHERE-clause equivalent of calculatePartnerRanking's max. export function partnerReachWhere({ reach, platform, diff --git a/apps/web/lib/api/network/reach-tiers.ts b/apps/web/lib/api/network/reach-tiers.ts index ebde6f66830..3784ba6bce3 100644 --- a/apps/web/lib/api/network/reach-tiers.ts +++ b/apps/web/lib/api/network/reach-tiers.ts @@ -1,7 +1,6 @@ -// Audience-size tiers for partner discovery. Reach is gauged as the partner's -// MAX subscriber count across the *selected* platforms (their headline audience), -// so the tier filter composes with topic relevance instead of overriding it and -// stays well-defined for any platform selection (unlike a per-platform sort). +// Audience-size tiers for partner discovery. Reach is the partner's max subscriber +// count across the selected platforms, so the filter composes with topic relevance +// and stays well-defined for any platform selection. export const REACH_TIER_KEYS = [ "nano", diff --git a/apps/web/lib/api/scrape-creators/client.ts b/apps/web/lib/api/scrape-creators/client.ts index de1a2fccc15..6fad792bb02 100644 --- a/apps/web/lib/api/scrape-creators/client.ts +++ b/apps/web/lib/api/scrape-creators/client.ts @@ -1,4 +1,8 @@ +import "server-only"; + +import { logger } from "@/lib/axiom/server"; import { createFetch, createSchema } from "@better-fetch/fetch"; +import { prettyPrint } from "@dub/utils"; import { PlatformType } from "@prisma/client"; import * as z from "zod/v4"; import { @@ -12,15 +16,19 @@ import { youtubeTranscriptSchema, } from "./schema"; +// Bound each request so a hung vendor call can't run past the caller's +// (cron/route) maxDuration. Aborts the fetch; the single linear retry still applies. +const SCRAPE_CREATORS_REQUEST_TIMEOUT_MS = 30_000; export const scrapeCreatorsFetch = createFetch({ baseURL: "https://api.scrapecreators.com", + timeout: SCRAPE_CREATORS_REQUEST_TIMEOUT_MS, retry: { type: "linear", attempts: 1, delay: 3000, }, headers: { - "x-api-key": process.env.SCRAPECREATORS_API_KEY!, + "x-api-key": process.env.SCRAPECREATORS_API_KEY ?? "", }, schema: createSchema( { @@ -130,6 +138,9 @@ export const scrapeCreatorsFetch = createFetch({ }, ), onError: ({ error }) => { - console.error("[ScrapeCreators] Error", error); + logger.error("[ScrapeCreators] Error", { + error: prettyPrint(error), + }); + void logger.flush(); }, }); diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts index ddb0cda510b..ace2ac10422 100644 --- a/apps/web/lib/partner-content-search/constants.ts +++ b/apps/web/lib/partner-content-search/constants.ts @@ -23,18 +23,13 @@ export const PARTNER_CONTENT_SEARCH_MODELS = { export const PARTNER_CONTENT_SEARCH_LIMITS = { recencyWindowMonths: 12, contentItemsPerPartnerPlatform: 50, - lowTranscriptCoverageThreshold: 0.3, - maxTranscriptBytesInPlanetscale: 500_000, chunkMinTokens: 400, chunkMaxTokens: 800, - chunkOverlapTokens: 80, - chunkCandidateCount: 200, - recentContentMaxPerPartner: 200, - vectorSearchChunkPoolMultiplier: 3, - vectorSearchChunkPoolMaxSize: 600, - rerankerCandidateCount: 150, - rerankerMaxDocChars: 2_000, - partnerScorePoolSize: 3, + chunkOverlapTokens: 80, // Cross-chunk context. + recentContentMaxPerPartner: 200, // Detail topic-fit eval cap. + vectorSearchChunkPoolSize: 600, // Raw ANN pool. + rerankerCandidateCount: 150, // Query/rerank cap. + rerankerDocumentCharSafetyLimit: 20_000, // Bug guard. } as const; export const PARTNER_CONTENT_SEARCH_FUSION_WEIGHTS = { diff --git a/apps/web/lib/partner-content-search/listing.ts b/apps/web/lib/partner-content-search/listing.ts index 5f7a70a093b..d4bc77e6df0 100644 --- a/apps/web/lib/partner-content-search/listing.ts +++ b/apps/web/lib/partner-content-search/listing.ts @@ -1,3 +1,4 @@ +import { DISCOVERABLE_NETWORK_STATUSES } from "@/lib/api/network/partner-network-listing-where"; import { prisma } from "@/lib/prisma"; import { PlatformType, Prisma } from "@prisma/client"; import { PARTNER_CONTENT_SEARCH_MODELS } from "./constants"; @@ -75,7 +76,7 @@ export async function listPartnerNetworkContent({ }), partner: { networkStatus: { - in: ["approved", "trusted"], + in: DISCOVERABLE_NETWORK_STATUSES, }, ...(country && { country }), programs: { diff --git a/apps/web/lib/partner-content-search/network-partners.ts b/apps/web/lib/partner-content-search/network-partners.ts index c84b8ddca90..dd4f9abdb83 100644 --- a/apps/web/lib/partner-content-search/network-partners.ts +++ b/apps/web/lib/partner-content-search/network-partners.ts @@ -3,8 +3,9 @@ import { parseRankedNetworkPartners } from "@/lib/api/network/normalize-ranked-n import type { ReachTier } from "@/lib/api/network/reach-tiers"; import type { PlatformType } from "@prisma/client"; -// Hydrate partner candidates into full network-partner cards (same ranking + -// reach/country/starred/platform filters as the listing), keyed by id. +// Hydrates ranked partner candidates into network-partner cards (keyed by id) with the +// listing's reach/country/starred/platform filters. Only place search touches +// calculatePartnerRanking; the ranking it returns is discarded (topicFit reorders). export async function getNetworkPartnersById({ programId, partnerIds, diff --git a/apps/web/lib/partner-content-search/ranking.ts b/apps/web/lib/partner-content-search/ranking.ts index 30245abb354..2fe92b3fda5 100644 --- a/apps/web/lib/partner-content-search/ranking.ts +++ b/apps/web/lib/partner-content-search/ranking.ts @@ -38,11 +38,10 @@ export function deriveTopicFit({ return { topicFit: 0, band: "none" }; } - // Coverage blends two signals: purity (strength-weighted share of recent posts - // on-topic) and depthConfidence (how much on-topic evidence there is, saturating - // so one post can't read as strong). depthConfidence gates the score; purity only - // modulates it between dilutionForgiveness and 1, so a deep on-topic body scores - // well even when mixed with off-topic posts. + // Coverage = depthConfidence (saturating on-topic evidence, so one post can't read + // as strong) modulated by purity (strength-weighted on-topic share). depthConfidence + // gates the score; purity only moves it between dilutionForgiveness and 1, so a deep + // on-topic body still scores well when mixed with off-topic posts. const { coverageCurveExponent, bandThresholds, diff --git a/apps/web/lib/partner-content-search/rerank.ts b/apps/web/lib/partner-content-search/rerank.ts index 9fe3b99b38b..8e8158f366b 100644 --- a/apps/web/lib/partner-content-search/rerank.ts +++ b/apps/web/lib/partner-content-search/rerank.ts @@ -30,12 +30,14 @@ export async function rerankPartnerSearchRows({ 0, PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, ); - const documents = candidates.map((row) => - (row.chunkText ?? "").slice( - 0, - PARTNER_CONTENT_SEARCH_LIMITS.rerankerMaxDocChars, - ), - ); + const documentCharSafetyLimit = + PARTNER_CONTENT_SEARCH_LIMITS.rerankerDocumentCharSafetyLimit; + const documents = candidates.map(({ chunkText }) => { + const document = chunkText ?? ""; + return document.length > documentCharSafetyLimit + ? document.slice(0, documentCharSafetyLimit) + : document; + }); try { const results = await rerankPartnerContent({ @@ -63,14 +65,14 @@ export async function rerankPartnerSearchRows({ throw error; } - // Reranked candidates sort ahead of the cosine-only tail (don't let an - // un-reranked row leapfrog a reranked one on a different score scale). - const reranked = [...rows].sort((a, b) => { - const aReranked = a.rerankScore != null; - const bReranked = b.rerankScore != null; - if (aReranked !== bReranked) return aReranked ? -1 : 1; - return effectiveRowScore(b) - effectiveRowScore(a); - }); + // Reranker-only ordering: return just the reranked candidates, sorted by their + // calibrated relevance. We drop the un-reranked cosine tail (rows beyond + // rerankerCandidateCount) rather than appending it — the cosine and rerank score + // scales must never be blended in one ordering. The fail-soft path above returns + // the full cosine ordering instead. + const reranked = candidates + .filter((row) => row.rerankScore != null) + .sort((a, b) => effectiveRowScore(b) - effectiveRowScore(a)); return { rows: reranked, reranked: true }; } diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts index ba9faffd997..de4130115f4 100644 --- a/apps/web/lib/partner-content-search/retrieval.ts +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -1,4 +1,5 @@ import { DubApiError } from "@/lib/api/errors"; +import { DISCOVERABLE_NETWORK_STATUSES } from "@/lib/api/network/partner-network-listing-where"; import { prisma } from "@/lib/prisma"; import { PlatformType, Prisma } from "@prisma/client"; import { @@ -37,70 +38,6 @@ type PartnerContentSearchHydrationRow = Omit< "distance" >; -function getVectorSearchChunkPoolSize(limit: number) { - return Math.max( - limit, - Math.min( - PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize, - limit * PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMultiplier, - ), - ); -} - -function countDistinctContentItems(rows: { partnerContentItemId: string }[]) { - return new Set(rows.map(({ partnerContentItemId }) => partnerContentItemId)) - .size; -} - -// Partner-eligibility id-sets for inline ANN pre-filtering: the deny-set -// (enrolled ∪ ignored) excluded from every query, plus the starred set. -async function resolveProgramPartnerEligibility({ - programId, - starred, - logTiming, -}: { - programId: string; - starred?: boolean; - logTiming?: PartnerContentSearchTimingLogger; -}) { - const [enrolledRows, ignoredRows, starredRows] = await Promise.all([ - prisma.programEnrollment.findMany({ - where: { programId }, - select: { partnerId: true }, - }), - prisma.discoveredPartner.findMany({ - where: { programId, ignoredAt: { not: null } }, - select: { partnerId: true }, - }), - // Only needed when the starred filter is active. - starred !== undefined - ? prisma.discoveredPartner.findMany({ - where: { programId, starredAt: { not: null } }, - select: { partnerId: true }, - }) - : Promise.resolve<{ partnerId: string }[]>([]), - ]); - - const excludedPartnerIds = [ - ...new Set([ - ...enrolledRows.map(({ partnerId }) => partnerId), - ...ignoredRows.map(({ partnerId }) => partnerId), - ]), - ]; - const starredPartnerIds = [ - ...new Set(starredRows.map(({ partnerId }) => partnerId)), - ]; - - logTiming?.("partner-eligibility-resolved", { - enrolledCount: enrolledRows.length, - ignoredCount: ignoredRows.length, - excludedPartnerCount: excludedPartnerIds.length, - starredPartnerCount: starredPartnerIds.length, - }); - - return { excludedPartnerIds, starredPartnerIds }; -} - export async function searchPartnerNetworkContent({ programId, query, @@ -122,14 +59,6 @@ export async function searchPartnerNetworkContent({ rerank: boolean; logTiming?: PartnerContentSearchTimingLogger; }) { - // Runs under the Voyage embedding round-trip (needs only programId), so the - // eligibility queries add ~no latency. - const eligibilityPromise = resolveProgramPartnerEligibility({ - programId, - starred, - logTiming, - }); - let queryEmbedding: number[]; logTiming?.("query-embedding-start"); try { @@ -151,32 +80,43 @@ export async function searchPartnerNetworkContent({ embeddingDimensions: queryEmbedding.length, }); const queryVector = serializeEmbeddingForVector(queryEmbedding); - const eligibility = await eligibilityPromise; - // Eligibility + user filters are applied INLINE on the flat ANN (country/platform - // via the denormalized c.country / c.platformType columns). Post-filtering would - // starve the candidate pool — enrolled partners are a program's strongest matches. - // networkStatus stays a post-filter (ingestion already gates approved/trusted). - const annFilters: Prisma.Sql[] = []; - if (eligibility.excludedPartnerIds.length > 0) { - annFilters.push( - Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.excludedPartnerIds)})`, - ); - } + // Push eligibility + user filters into the ANN (not after — post-filtering would + // starve the candidate pool). Eligibility rides as fixed-shape semi/anti-joins + // rather than materialized id-sets: the per-program enrolled/ignored/starred sets + // grow unbounded (some programs enroll 1k+ partners), so a NOT IN literal would + // balloon the SQL and can't reuse the cached plan. EXPLAIN confirms PlanetScale + // materializes each subquery once and probes it by hash while keeping the vector + // index (type=vector) under FORCE INDEX. networkStatus stays a post-filter since + // ingestion already gates approved/trusted. country/platformType are denormalized + // columns and partnerIds is a bounded UI selection, so those stay inline. + const annFilters: Prisma.Sql[] = [ + Prisma.sql`AND NOT EXISTS ( + SELECT 1 FROM ProgramEnrollment pe + WHERE pe.partnerId = c.partnerId AND pe.programId = ${programId} + )`, + Prisma.sql`AND NOT EXISTS ( + SELECT 1 FROM DiscoveredPartner dpi + WHERE dpi.partnerId = c.partnerId AND dpi.programId = ${programId} + AND dpi.ignoredAt IS NOT NULL + )`, + ]; if (partnerIds?.length) { annFilters.push(Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})`); } if (starred === true) { - // starred filter on with no starred partners → no eligible candidates. - annFilters.push( - eligibility.starredPartnerIds.length > 0 - ? Prisma.sql`AND c.partnerId IN (${Prisma.join(eligibility.starredPartnerIds)})` - : Prisma.sql`AND 1 = 0`, - ); - } else if (starred === false && eligibility.starredPartnerIds.length > 0) { - annFilters.push( - Prisma.sql`AND c.partnerId NOT IN (${Prisma.join(eligibility.starredPartnerIds)})`, - ); + // No starred partners → the EXISTS matches nothing → no candidates (intended). + annFilters.push(Prisma.sql`AND EXISTS ( + SELECT 1 FROM DiscoveredPartner dps + WHERE dps.partnerId = c.partnerId AND dps.programId = ${programId} + AND dps.starredAt IS NOT NULL + )`); + } else if (starred === false) { + annFilters.push(Prisma.sql`AND NOT EXISTS ( + SELECT 1 FROM DiscoveredPartner dps + WHERE dps.partnerId = c.partnerId AND dps.programId = ${programId} + AND dps.starredAt IS NOT NULL + )`); } if (country) { annFilters.push(Prisma.sql`AND c.country = ${country}`); @@ -186,8 +126,7 @@ export async function searchPartnerNetworkContent({ Prisma.sql`AND c.platformType IN (${Prisma.join(platforms)})`, ); } - const annFilter = - annFilters.length > 0 ? Prisma.join(annFilters, " ") : Prisma.empty; + const annFilter = Prisma.join(annFilters, " "); // Two-phase retrieval: keep the ANN flat so PlanetScale uses the vector index, // then hydrate only the returned chunk ids through the relational joins — keeping @@ -207,60 +146,33 @@ export async function searchPartnerNetworkContent({ LIMIT ${poolSize} `); - // Dedup to the best chunk per item+source before the hydration join (filters are - // per-item, so this is equivalent and keeps chunk-heavy items off the join). - const retrievePool = async (poolSize: number) => { - const candidateRows = await fetchCandidateChunks(poolSize); - logTiming?.("vector-candidate-search-complete", { - candidateRowCount: candidateRows.length, - poolSize, - vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, - }); - const dedupedCandidates = - dedupeBestChunkPerContentItemSource(candidateRows); - const poolRows = await hydratePartnerContentSearchRows({ - candidateRows: dedupedCandidates, - logTiming, - }); - return { candidateRowCount: candidateRows.length, poolRows }; - }; - - const initialPoolSize = getVectorSearchChunkPoolSize(limit); - const maxPoolSize = PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolMaxSize; + // Fixed pool: the request-derived candidate count always resolves to the cap in + // practice, and the ANN refills the pool from deeper in the ranking when eligibility + // filters exclude near-neighbors (verified against PlanetScale SPANN), so there's no + // need to size it per-request or widen it adaptively. + const poolSize = PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolSize; logTiming?.("vector-search-start", { - poolSize: initialPoolSize, - maxPoolSize, + poolSize, retrievalShape: "two-phase", }); - let poolSize = initialPoolSize; - let { candidateRowCount, poolRows } = await retrievePool(poolSize); - - // The pool comes back already eligible (filters run inline), so it rarely shrinks. - // Safety net for when it does (e.g. a cluster of demoted partners): if we - // under-filled but the ANN returned a full pool, widen to the cap once and retry. - let distinctItemCount = countDistinctContentItems(poolRows); - if ( - poolSize < maxPoolSize && - candidateRowCount === poolSize && - distinctItemCount < limit - ) { - poolSize = maxPoolSize; - logTiming?.("vector-search-pool-expanded", { - previousPoolSize: initialPoolSize, - poolSize, - distinctItemCount, - limit, - }); - ({ candidateRowCount, poolRows } = await retrievePool(poolSize)); - distinctItemCount = countDistinctContentItems(poolRows); - } + const candidateRows = await fetchCandidateChunks(poolSize); + logTiming?.("vector-candidate-search-complete", { + candidateRowCount: candidateRows.length, + poolSize, + vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, + }); + // Dedup to the best chunk per item+source before the hydration join (filters are + // per-item, so this is equivalent and keeps chunk-heavy items off the join). + const dedupedCandidates = dedupeBestChunkPerContentItemSource(candidateRows); + const poolRows = await hydratePartnerContentSearchRows({ + candidateRows: dedupedCandidates, + logTiming, + }); logTiming?.("vector-search-complete", { - candidateRowCount, + candidateRowCount: candidateRows.length, poolRowCount: poolRows.length, - distinctItemCount, poolSize, - expanded: poolSize !== initialPoolSize, vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, retrievalShape: "two-phase", }); @@ -283,7 +195,7 @@ export async function searchPartnerNetworkContent({ const itemRows = dedupeBestChunkPerContentItem(poolRows).slice(0, limit); const rows = dedupeBestChunkPerContentItemSource(poolRows).slice( 0, - Math.min(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, limit * 2), + Math.min(PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, limit * 2), ); // Cutoff = the least-relevant candidate kept; the match gate for recent posts. @@ -314,6 +226,12 @@ export async function searchPartnerNetworkContent({ }; } + // NOTE: reach / verified-platform / conversion filters run downstream in + // calculatePartnerRanking (via getNetworkPartnersById), after this rerank — so when + // they're active we pay to rerank chunks from partners that get dropped later. Left + // as-is on purpose: pre-filtering them here needs the canonical discover-eligibility + // predicate that only exists once calculatePartnerRanking is split into hydrate-card + // vs rank. Revisit at that consolidation. logTiming?.("rerank-start", { rowCount: rowsWithChunkText.length, rerankerCandidateCount: Math.min( @@ -333,8 +251,10 @@ export async function searchPartnerNetworkContent({ ).length, }); return { + // rerankResult.rows is already reranker-ordered on success, or the full cosine + // ordering on fail-soft. Never re-sort here — sorting by effective score would + // blend the rerank and cosine scales we just took care to keep separate. ...rerankResult, - rows: sortRowsByRelevanceScore(rerankResult.rows), queryVector, cutoffDistance, itemSourceBestDistance, @@ -359,7 +279,7 @@ async function hydratePartnerContentSearchRows({ id: { in: chunkIds }, embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.id, partner: { - networkStatus: { in: ["approved", "trusted"] }, + networkStatus: { in: DISCOVERABLE_NETWORK_STATUSES }, }, }, select: { diff --git a/apps/web/lib/partner-content-search/search-utils.ts b/apps/web/lib/partner-content-search/search-utils.ts index b0cd9af2d1f..da64e1915a0 100644 --- a/apps/web/lib/partner-content-search/search-utils.ts +++ b/apps/web/lib/partner-content-search/search-utils.ts @@ -42,10 +42,9 @@ export function toScore(distance: number) { return Number((1 - distance).toFixed(6)); } -// How many candidate chunks to retrieve to satisfy `limit` partners at -// `chunksPerPartner` each. Query mode over-fetches (×6, floored at 25, capped at -// the candidate ceiling) so reranking + topic-fit have a deep pool to work from; -// list mode has no relevance gate, so a smaller pool (×2) suffices. +// Candidate chunks to retrieve for `limit` partners at `chunksPerPartner` each. +// Query mode over-fetches (×6, min 25, capped at rerank input size); +// list mode has no relevance gate so ×2 is enough. export function getCandidateChunkCount({ hasQuery, limit, @@ -58,7 +57,7 @@ export function getCandidateChunkCount({ if (!hasQuery) return limit * chunksPerPartner * 2; return Math.min( - PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount, + PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, Math.max(25, limit * chunksPerPartner * 6), ); } diff --git a/apps/web/lib/partner-content-search/timing.ts b/apps/web/lib/partner-content-search/timing.ts index 3e08d3e219c..dac47788239 100644 --- a/apps/web/lib/partner-content-search/timing.ts +++ b/apps/web/lib/partner-content-search/timing.ts @@ -4,10 +4,8 @@ const MIN_PARTNER_CONTENT_SEARCH_TIMING_DELTA_MS = 5; const PARTNER_CONTENT_SEARCH_ALWAYS_LOG_TIMING_STAGES = new Set([ "query-embedding-complete", - "partner-eligibility-resolved", "vector-candidate-search-complete", "vector-candidate-hydration-complete", - "vector-search-pool-expanded", "vector-search-complete", "candidate-dedupe-complete", "chunk-text-hydration-complete", diff --git a/apps/web/lib/swr/use-partner-content-search.ts b/apps/web/lib/swr/use-partner-content-search.ts index b880067f3d9..0cedbd1dfb7 100644 --- a/apps/web/lib/swr/use-partner-content-search.ts +++ b/apps/web/lib/swr/use-partner-content-search.ts @@ -2,116 +2,23 @@ import type { ReachTier } from "@/lib/api/network/reach-tiers"; import { PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER, PARTNER_CONTENT_SEARCH_PARTNER_LIMIT, - type PartnerContentTopicFitBand, } from "@/lib/partner-content-search/constants"; -import type { NetworkPartnerProps } from "@/lib/types"; +import type { + PartnerContentMatchEvidence, + PartnerContentMatchSource, + PartnerNetworkContentSearchPartner, + PartnerNetworkContentSearchResponse, +} from "@/lib/zod/schemas/partner-network"; import type { PlatformType } from "@prisma/client"; import useSWR from "swr"; import { useDebounce } from "use-debounce"; import useWorkspace from "./use-workspace"; -export type PartnerContentMatchSource = "transcript" | "creatorText"; - -export type PartnerContentMatchEvidence = { - primarySource: PartnerContentMatchSource | null; - sources: PartnerContentMatchSource[]; - transcriptScore: number | null; - creatorTextScore: number | null; - creatorTextWeight: number; - weight: number; -}; - -export type PartnerContentSearchPartner = { - partnerId: string; - name: string; - username: string | null; - image: string | null; - description: string | null; - score: number; - // Debug comparison for validating reranker impact against raw vector search. - cosineScore?: number | null; - rerankScore?: number | null; - partner: NetworkPartnerProps; - chunks: { - chunkId: string; - partnerContentItemId: string; - platform: { - type: string; - identifier: string; - }; - content: { - platformContentId: string; - url: string; - type: string; - title: string | null; - description: string | null; - thumbnailUrl: string | null; - publishedAt: string | null; - durationMs: number | null; - viewCount: number | null; - likeCount: number | null; - commentCount: number | null; - shareCount: number | null; - saveCount: number | null; - }; - chunk: { - source: "metadata" | "transcript" | string; - text: string; - startMs: number | null; - endMs: number | null; - }; - score: number; - cosineScore?: number; - rerankScore?: number | null; - matchEvidence?: PartnerContentMatchEvidence; - }[]; - matchSummary: { - matchedContentCount: number; - transcriptMatchedContentCount: number; - creatorTextMatchedContentCount: number; - creatorTextOnlyContentCount: number; - weightedMatchedContentCount: number; - weightedMatchedContentScore: number; - recentContentCount: number; - totalContentCount: number; - // Aggregate card score (0-100) + its coverage-based band, and the partner's - // platforms ranked by matched-post frequency. - topicFit: number; - band: PartnerContentTopicFitBand; - topPlatforms: string[]; - // Brand-facing reach/activity signals over the on-topic posts. - followers: number | null; - medianViews: number | null; - lastOnTopicAt: string | null; - platforms: string[]; - sources: string[]; - oldestPublishedAt: string | null; - newestPublishedAt: string | null; - contentBars: { - partnerContentItemId: string; - platform: string; - platformContentId: string; - title: string | null; - url: string | null; - durationMs: number | null; - publishedAt: string | null; - viewCount: number | null; - likeCount: number | null; - commentCount: number | null; - shareCount: number | null; - saveCount: number | null; - matched: boolean; - matchScore: number | null; - matchEvidence: PartnerContentMatchEvidence; - }[]; - } | null; -}; - -export type PartnerContentSearchResponse = { - success: boolean; - reranked?: boolean; - partners: PartnerContentSearchPartner[]; -}; +// Derived from the response schema (single source of truth in partner-network.ts) +// and re-exported under these names so existing consumers keep importing them here. +export type { PartnerContentMatchEvidence, PartnerContentMatchSource }; +export type PartnerContentSearchPartner = PartnerNetworkContentSearchPartner; +export type PartnerContentSearchResponse = PartnerNetworkContentSearchResponse; export default function usePartnerContentSearch({ enabled, @@ -159,45 +66,51 @@ export default function usePartnerContentSearch({ : null; const [debouncedKeySignature] = useDebounce(keySignature, debounceMs); - const { data, error, isLoading, mutate } = useSWR( - debouncedKeySignature - ? ["partner-content-search", debouncedKeySignature] - : null, - // The shared `fetcher` from @dub/utils is GET-only; this endpoint takes a - // POST body, so it needs its own fetcher. - async () => { - const response = await fetch( - `/api/network/partners/content-search?workspaceId=${workspaceId}`, - { - method: "POST", - headers: { - "content-type": "application/json", + const { data, error, isLoading, isValidating, mutate } = + useSWR( + debouncedKeySignature + ? ["partner-content-search", debouncedKeySignature] + : null, + // The shared `fetcher` from @dub/utils is GET-only; this endpoint takes a + // POST body, so it needs its own fetcher. + async () => { + const response = await fetch( + `/api/network/partners/content-search?workspaceId=${workspaceId}`, + { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify({ + query, + limit, + chunksPerPartner, + ...(platforms?.length && { platforms }), + ...(reach?.length && { reach }), + ...(country && { country }), + ...(partnerIds?.length && { partnerIds }), + starred: starred || undefined, + }), }, - body: JSON.stringify({ - query, - limit, - chunksPerPartner, - ...(platforms?.length && { platforms }), - ...(reach?.length && { reach }), - ...(country && { country }), - ...(partnerIds?.length && { partnerIds }), - starred: starred || undefined, - }), - }, - ); + ); + + if (!response.ok) { + throw new Error("Failed to search partner content"); + } - if (!response.ok) { - throw new Error("Failed to search partner content"); - } + return response.json(); + }, + { + keepPreviousData: true, + revalidateOnFocus: false, + dedupingInterval: 20000, + }, + ); - return response.json(); - }, - { - keepPreviousData: true, - revalidateOnFocus: false, - dedupingInterval: 20000, - }, - ); + // True from the moment the inputs change — while the fetch key is still + // debouncing AND during the request itself — so the UI can show a loading + // state immediately instead of after the debounce settles. + const isPending = keySignature !== debouncedKeySignature || isValidating; - return { data, error, isLoading, mutate }; + return { data, error, isLoading, isPending, mutate }; } diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index c532501db00..0a3e32f55dd 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -155,3 +155,144 @@ export const partnerNetworkContentSearchSchema = z.object({ // Second-stage reranking is on by default; pass `false` for diagnostics. rerank: z.boolean().default(true), }); + +// ---- Response for POST /api/network/partners/content-search ---- +// Single source of truth: the route validates its response against this schema +// and the SWR hook derives its types from it (no hand-maintained duplicate). + +export const partnerContentMatchSourceSchema = z.enum([ + "transcript", + "creatorText", +]); + +export const partnerContentMatchEvidenceSchema = z.object({ + primarySource: partnerContentMatchSourceSchema.nullable(), + sources: z.array(partnerContentMatchSourceSchema), + transcriptScore: z.number().nullable(), + creatorTextScore: z.number().nullable(), + creatorTextWeight: z.number(), + weight: z.number(), +}); + +const partnerContentSearchChunkSchema = z.object({ + chunkId: z.string(), + partnerContentItemId: z.string(), + platform: z.object({ + type: z.string(), + identifier: z.string(), + }), + content: z.object({ + platformContentId: z.string(), + url: z.string(), + type: z.string(), + title: z.string().nullable(), + description: z.string().nullable(), + thumbnailUrl: z.string().nullable(), + publishedAt: z.string().nullable(), + durationMs: z.number().nullable(), + viewCount: z.number().nullable(), + likeCount: z.number().nullable(), + commentCount: z.number().nullable(), + shareCount: z.number().nullable(), + saveCount: z.number().nullable(), + }), + chunk: z.object({ + source: z.string(), + text: z.string(), + startMs: z.number().nullable(), + endMs: z.number().nullable(), + }), + score: z.number(), + cosineScore: z.number().nullish(), + rerankScore: z.number().nullish(), + matchEvidence: partnerContentMatchEvidenceSchema.optional(), +}); + +const partnerContentSearchContentBarSchema = z.object({ + partnerContentItemId: z.string(), + platform: z.string(), + platformContentId: z.string(), + title: z.string().nullable(), + url: z.string().nullable(), + durationMs: z.number().nullable(), + publishedAt: z.string().nullable(), + viewCount: z.number().nullable(), + likeCount: z.number().nullable(), + commentCount: z.number().nullable(), + shareCount: z.number().nullable(), + saveCount: z.number().nullable(), + matched: z.boolean(), + matchScore: z.number().nullable(), + matchEvidence: partnerContentMatchEvidenceSchema, +}); + +const partnerContentTopicFitBandSchema = z.enum([ + "consistent", + "frequent", + "occasional", + "one-off", + "none", +]); + +const partnerContentSearchMatchSummarySchema = z.object({ + matchedContentCount: z.number(), + transcriptMatchedContentCount: z.number(), + creatorTextMatchedContentCount: z.number(), + creatorTextOnlyContentCount: z.number(), + weightedMatchedContentCount: z.number(), + weightedMatchedContentScore: z.number(), + recentContentCount: z.number(), + totalContentCount: z.number(), + topicFit: z.number(), + band: partnerContentTopicFitBandSchema, + followers: z.number().nullable(), + medianViews: z.number().nullable(), + lastOnTopicAt: z.string().nullable(), + topPlatforms: z.array(z.string()), + platforms: z.array(z.string()), + sources: z.array(z.string()), + oldestPublishedAt: z.string().nullable(), + newestPublishedAt: z.string().nullable(), + contentBars: z.array(partnerContentSearchContentBarSchema), +}); + +const partnerContentSearchResponsePartnerSchema = z.object({ + partnerId: z.string(), + name: z.string(), + username: z.string().nullable(), + image: z.string().nullable(), + description: z.string().nullable(), + score: z.number(), + cosineScore: z.number().nullish(), + rerankScore: z.number().nullish(), + // Already validated upstream by parseRankedNetworkPartners; pass through as-is + // so the partner subtree's schema transforms aren't re-applied here. + partner: z.custom>(), + chunks: z.array(partnerContentSearchChunkSchema), + matchSummary: partnerContentSearchMatchSummarySchema.nullable(), +}); + +export const partnerNetworkContentSearchResponseSchema = z.object({ + success: z.boolean(), + query: z.string().nullable(), + platforms: z.array(z.enum(PlatformType)).nullable(), + country: z.string().nullable(), + candidateChunkCount: z.number(), + embeddingModel: z.string(), + reranked: z.boolean(), + rerankModel: z.string().nullable(), + resultCount: z.number(), + partners: z.array(partnerContentSearchResponsePartnerSchema), +}); + +export type PartnerNetworkContentSearchResponse = z.infer< + typeof partnerNetworkContentSearchResponseSchema +>; +export type PartnerNetworkContentSearchPartner = + PartnerNetworkContentSearchResponse["partners"][number]; +export type PartnerContentMatchEvidence = z.infer< + typeof partnerContentMatchEvidenceSchema +>; +export type PartnerContentMatchSource = z.infer< + typeof partnerContentMatchSourceSchema +>; diff --git a/apps/web/prisma/manual/partner-content-search-vector-index.sql b/apps/web/prisma/manual/partner-content-search-vector-index.sql index 00fc127e704..52ef6bbc9ab 100644 --- a/apps/web/prisma/manual/partner-content-search-vector-index.sql +++ b/apps/web/prisma/manual/partner-content-search-vector-index.sql @@ -1,29 +1,10 @@ --- Manual PlanetScale index DDL for partner natural-language search. --- --- Prisma manages the PartnerContentChunk.embedding VECTOR(1024) column through --- Unsupported("vector(1024)") in apps/web/prisma/schema/partner-content-search.prisma. --- Run this after Prisma has created the column on the target PlanetScale branch. --- The schema includes a same-named @@index only so Prisma db push preserves this --- manual index; Prisma cannot define the VECTOR INDEX distance/options below. --- --- The index name and distance metric below are the same values the search routes --- feed into their FORCE INDEX hint and DISTANCE() call (if they diverge, the --- planner falls back to a full scan). They mirror PARTNER_CONTENT_CHUNK_VECTOR_INDEX --- and PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE in --- apps/web/lib/partner-content-search/constants.ts; the match is enforced by --- apps/web/tests/partner-content-search/vector-index-sync.test.ts. +-- Manual PlanetScale VECTOR index for partner natural-language search. Prisma can't +-- express the VECTOR INDEX distance/options, so create it here after Prisma has pushed +-- the embedding column (the same-named @@index in the schema just stops db push from +-- dropping it). The name + distance must match PARTNER_CONTENT_CHUNK_VECTOR_INDEX / +-- _DISTANCE in lib/partner-content-search/constants.ts; vector-index-sync.test.ts +-- enforces it, and the search routes' FORCE INDEX hint depends on it. CREATE /*vt+ QUERY_TIMEOUT_MS=0 */ VECTOR INDEX partner_content_chunk_embedding_cosine_idx ON PartnerContentChunk(embedding) SECONDARY_ENGINE_ATTRIBUTE='{"type":"spann", "distance":"cosine"}'; - --- Optional verification once the table has enough rows for the planner to use --- the vector index: --- --- EXPLAIN --- SELECT c.id, --- DISTANCE(TO_VECTOR('[0, ...]'), c.embedding, 'cosine') AS distance --- FROM PartnerContentChunk c --- WHERE c.embedding IS NOT NULL --- ORDER BY distance ASC --- LIMIT 10; diff --git a/apps/web/prisma/schema/partner-content-search.prisma b/apps/web/prisma/schema/partner-content-search.prisma index 291864cfd1b..781f5a49536 100644 --- a/apps/web/prisma/schema/partner-content-search.prisma +++ b/apps/web/prisma/schema/partner-content-search.prisma @@ -1,12 +1,8 @@ // Partner natural-language search schema. -// -// Prisma does not expose PartnerContentChunk.embedding in Prisma Client because -// VECTOR is modeled as an unsupported database type. Prisma still manages the -// column for schema drift, while all vector reads/writes stay in raw SQL. -// Keep the matching @@index below so Prisma db push preserves the manual index, -// but create/recreate the real cosine vector index with -// apps/web/prisma/manual/partner-content-search-vector-index.sql because Prisma -// cannot express PlanetScale VECTOR INDEX options. +// PartnerContentChunk.embedding is an Unsupported VECTOR column, so it's absent from +// Prisma Client and all vector reads/writes are raw SQL. The real cosine index is +// created via prisma/manual/partner-content-search-vector-index.sql (Prisma can't +// express VECTOR INDEX options); the @@index below just preserves it across db push. enum PartnerContentTranscriptFetchStatus { pending @@ -89,10 +85,7 @@ model PartnerContentChunk { // Covers the empty-query "recent content" list path, which filters on // embeddingModel while joining chunks by partnerContentItemId. @@index([partnerContentItemId, embeddingModel]) - // Drift guard only: Prisma cannot express PlanetScale VECTOR INDEX options. - // Create/recreate the real cosine ANN index with prisma/manual/partner-content-search-vector-index.sql. - // The map name mirrors PARTNER_CONTENT_CHUNK_VECTOR_INDEX in - // apps/web/lib/partner-content-search/constants.ts (the FORCE INDEX hint); the - // match is enforced by tests/partner-content-search/vector-index-sync.test.ts. + // Drift guard so db push keeps the manual cosine index (see file header and + // partner-content-search-vector-index.sql). Name verified by vector-index-sync.test.ts. @@index([embedding], map: "partner_content_chunk_embedding_cosine_idx") } diff --git a/apps/web/tests/partner-content-search/vector-index-sync.test.ts b/apps/web/tests/partner-content-search/vector-index-sync.test.ts index c3e15efae70..8bcbcbf805d 100644 --- a/apps/web/tests/partner-content-search/vector-index-sync.test.ts +++ b/apps/web/tests/partner-content-search/vector-index-sync.test.ts @@ -6,11 +6,8 @@ import { PARTNER_CONTENT_CHUNK_VECTOR_INDEX, } from "@/lib/partner-content-search/constants"; -// The manual PlanetScale VECTOR index DDL and the Prisma @@index drift guard both -// hardcode the index name + distance metric because neither a .sql file nor the -// Prisma schema can import a TS constant. These assertions are the enforcement -// that replaces the old "keep in sync" comments: if the constant and either file -// disagree, this fails instead of silently shipping an unusable index. +// The index name + distance metric are hardcoded in both the .sql DDL and the Prisma +// @@index (neither can import the TS constant), so assert they match the constants. describe("partner content search vector index", () => { const read = (relativePath: string) => readFileSync(join(process.cwd(), relativePath), "utf8"); diff --git a/apps/web/ui/partners/partner-info-cards.tsx b/apps/web/ui/partners/partner-info-cards.tsx index c01a5ec6d8e..8664070b43e 100644 --- a/apps/web/ui/partners/partner-info-cards.tsx +++ b/apps/web/ui/partners/partner-info-cards.tsx @@ -39,9 +39,8 @@ type PartnerInfoCardsProps = { setSelectedGroupId?: (groupId: string) => void; /** - * Network-browse mode (marketplace discovery): the partner isn't enrolled in a - * group, so hide the group/rewards/bounties block and surface website & socials - * in the sidebar instead. + * Network-browse (marketplace) mode: partner isn't enrolled, so hide the + * group/rewards/bounties block and show website & socials in the sidebar instead. */ browseMode?: boolean; } & ( diff --git a/packages/ui/src/tooltip.tsx b/packages/ui/src/tooltip.tsx index cbc578af191..bcccae00b95 100644 --- a/packages/ui/src/tooltip.tsx +++ b/packages/ui/src/tooltip.tsx @@ -72,10 +72,7 @@ export interface TooltipProps disabled?: boolean; disableHoverableContent?: TooltipPrimitive.TooltipProps["disableHoverableContent"]; delayDuration?: TooltipPrimitive.TooltipProps["delayDuration"]; - /** - * Controlled open state. Pass `open` and `onOpenChange` together to control - * it from the parent; omit both to let the tooltip manage its own. - */ + /** Controlled open state — pass with `onOpenChange`, or omit both to self-manage. */ open?: boolean; onOpenChange?: (open: boolean) => void; /** Drop the entrance animation so the tooltip closes instantly without lingering. */ From 2b9096cd79842430148a4c614539a31857546212 Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 12:14:51 -0400 Subject: [PATCH 18/25] replace ranking-based hydration in content search; parallelize enrichment --- .../network/partners/content-search/route.ts | 48 +++--- .../web/lib/partner-content-search/listing.ts | 12 -- .../network-partners.ts | 109 ++++++++++--- .../lib/partner-content-search/retrieval.ts | 149 +++--------------- .../partner-content-search/search-utils.ts | 32 +--- apps/web/lib/zod/schemas/partner-network.ts | 15 +- 6 files changed, 145 insertions(+), 220 deletions(-) diff --git a/apps/web/app/(ee)/api/network/partners/content-search/route.ts b/apps/web/app/(ee)/api/network/partners/content-search/route.ts index 6f660fb306a..265f39232b4 100644 --- a/apps/web/app/(ee)/api/network/partners/content-search/route.ts +++ b/apps/web/app/(ee)/api/network/partners/content-search/route.ts @@ -129,33 +129,33 @@ export const POST = withWorkspace( partnerCandidateCount: partnerCandidates.length, partnerCandidateLimit, }); - logTiming("match-summaries-start", { + const partnerIdsToHydrate = partnerCandidates.map( + ({ partnerId }) => partnerId, + ); + logTiming("partner-enrichment-start", { partnerCandidateCount: partnerCandidates.length, }); - const matchSummaries = await getPartnerMatchSummaries({ - rows, - partnerIds: partnerCandidates.map(({ partnerId }) => partnerId), - platforms: body.platforms, - queryVector, - cutoffDistance, - itemSourceBestDistance, - logTiming, - }); - logTiming("match-summaries-complete", { + // Match summaries and card hydration share no inputs; run them in parallel + const [matchSummaries, networkPartners] = await Promise.all([ + getPartnerMatchSummaries({ + rows, + partnerIds: partnerIdsToHydrate, + platforms: body.platforms, + queryVector, + cutoffDistance, + itemSourceBestDistance, + logTiming, + }), + getNetworkPartnersById({ + programId, + partnerIds: partnerIdsToHydrate, + platforms: body.platforms, + reach: body.reach, + country: body.country, + }), + ]); + logTiming("partner-enrichment-complete", { summaryCount: matchSummaries.size, - }); - logTiming("partner-hydration-start", { - partnerCandidateCount: partnerCandidates.length, - }); - const networkPartners = await getNetworkPartnersById({ - programId, - partnerIds: partnerCandidates.map(({ partnerId }) => partnerId), - platforms: body.platforms, - reach: body.reach, - country: body.country, - starred: body.starred, - }); - logTiming("partner-hydration-complete", { hydratedPartnerCount: networkPartners.size, }); const partners = partnerCandidates diff --git a/apps/web/lib/partner-content-search/listing.ts b/apps/web/lib/partner-content-search/listing.ts index d4bc77e6df0..675e3fe26fe 100644 --- a/apps/web/lib/partner-content-search/listing.ts +++ b/apps/web/lib/partner-content-search/listing.ts @@ -103,14 +103,6 @@ export async function listPartnerNetworkContent({ commentCount: true, shareCount: true, saveCount: true, - partner: { - select: { - name: true, - username: true, - image: true, - description: true, - }, - }, partnerPlatform: { select: { type: true, @@ -150,10 +142,6 @@ export async function listPartnerNetworkContent({ chunkId: chunk.id, partnerContentItemId: contentItem.id, partnerId: contentItem.partnerId, - partnerName: contentItem.partner.name, - partnerUsername: contentItem.partner.username, - partnerImage: contentItem.partner.image, - partnerDescription: contentItem.partner.description, platformType: contentItem.partnerPlatform.type, platformIdentifier: contentItem.partnerPlatform.identifier, platformContentId: contentItem.platformContentId, diff --git a/apps/web/lib/partner-content-search/network-partners.ts b/apps/web/lib/partner-content-search/network-partners.ts index dd4f9abdb83..6c123e1efe8 100644 --- a/apps/web/lib/partner-content-search/network-partners.ts +++ b/apps/web/lib/partner-content-search/network-partners.ts @@ -1,40 +1,109 @@ -import { calculatePartnerRanking } from "@/lib/api/network/calculate-partner-ranking"; -import { parseRankedNetworkPartners } from "@/lib/api/network/normalize-ranked-network-partner"; +import { + partnerNetworkListingWhere, + partnerReachWhere, +} from "@/lib/api/network/partner-network-listing-where"; import type { ReachTier } from "@/lib/api/network/reach-tiers"; +import { prisma } from "@/lib/prisma"; +import { NetworkPartnerSchema } from "@/lib/zod/schemas/partner-network"; import type { PlatformType } from "@prisma/client"; +import type * as z from "zod/v4"; -// Hydrates ranked partner candidates into network-partner cards (keyed by id) with the -// listing's reach/country/starred/platform filters. Only place search touches -// calculatePartnerRanking; the ranking it returns is discarded (topicFit reorders). +type NetworkPartnerCard = z.infer; + +// Hydrate partner cards for content search export async function getNetworkPartnersById({ programId, partnerIds, platforms, reach, country, - starred, }: { programId: string; partnerIds: string[]; platforms?: PlatformType[]; reach?: ReachTier[]; country?: string; - starred?: boolean; -}) { +}): Promise> { if (partnerIds.length === 0) return new Map(); - const rankedPartners = await calculatePartnerRanking({ - programId, - partnerIds, - status: "discover", - page: 1, - pageSize: partnerIds.length, - country, - starred: starred ?? undefined, - platform: platforms, - reach, + const partners = await prisma.partner.findMany({ + where: { + AND: [ + partnerNetworkListingWhere({ partnerIds, country, platform: platforms }), + partnerReachWhere({ reach, platform: platforms }), + ], + }, + select: { + id: true, + name: true, + companyName: true, + country: true, + profileType: true, + image: true, + description: true, + createdAt: true, + networkStatus: true, + monthlyTraffic: true, + identityVerificationStatus: true, + identityVerifiedAt: true, + platforms: { + select: { + type: true, + identifier: true, + verifiedAt: true, + platformId: true, + subscribers: true, + posts: true, + views: true, + }, + }, + programs: { + where: { status: "approved" }, + select: { + program: { select: { categories: { select: { category: true } } } }, + }, + }, + discoveredByPrograms: { + where: { programId }, + select: { starredAt: true, invitedAt: true, ignoredAt: true }, + take: 1, + }, + }, + }); + + const cards = partners.map((partner): NetworkPartnerCard => { + const discovered = partner.discoveredByPrograms[0]; + const categories = Array.from( + new Set( + partner.programs.flatMap(({ program }) => + program.categories.map(({ category }) => category), + ), + ), + ).sort(); + + return { + id: partner.id, + name: partner.name, + companyName: partner.companyName, + country: partner.country, + profileType: partner.profileType, + image: partner.image, + description: partner.description, + createdAt: partner.createdAt, + networkStatus: partner.networkStatus, + monthlyTraffic: partner.monthlyTraffic, + identityVerificationStatus: partner.identityVerificationStatus, + identityVerifiedAt: partner.identityVerifiedAt, + preferredEarningStructures: [], // schema-required; UI doesn't read them + salesChannels: [], + starredAt: discovered?.starredAt ?? null, + invitedAt: discovered?.invitedAt ?? null, + ignoredAt: discovered?.ignoredAt ?? null, + recruitedAt: null, // discover candidates aren't enrolled in this program + categories, + platforms: partner.platforms, + }; }); - const partners = parseRankedNetworkPartners(rankedPartners); - return new Map(partners.map((partner) => [partner.id, partner])); + return new Map(cards.map((card) => [card.id, card])); } diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts index de4130115f4..f6c1caa6609 100644 --- a/apps/web/lib/partner-content-search/retrieval.ts +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -81,15 +81,7 @@ export async function searchPartnerNetworkContent({ }); const queryVector = serializeEmbeddingForVector(queryEmbedding); - // Push eligibility + user filters into the ANN (not after — post-filtering would - // starve the candidate pool). Eligibility rides as fixed-shape semi/anti-joins - // rather than materialized id-sets: the per-program enrolled/ignored/starred sets - // grow unbounded (some programs enroll 1k+ partners), so a NOT IN literal would - // balloon the SQL and can't reuse the cached plan. EXPLAIN confirms PlanetScale - // materializes each subquery once and probes it by hash while keeping the vector - // index (type=vector) under FORCE INDEX. networkStatus stays a post-filter since - // ingestion already gates approved/trusted. country/platformType are denormalized - // columns and partnerIds is a bounded UI selection, so those stay inline. + // Eligibility filters inline in the ANN query; post-filtering would starve the candidate pool. const annFilters: Prisma.Sql[] = [ Prisma.sql`AND NOT EXISTS ( SELECT 1 FROM ProgramEnrollment pe @@ -105,7 +97,6 @@ export async function searchPartnerNetworkContent({ annFilters.push(Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})`); } if (starred === true) { - // No starred partners → the EXISTS matches nothing → no candidates (intended). annFilters.push(Prisma.sql`AND EXISTS ( SELECT 1 FROM DiscoveredPartner dps WHERE dps.partnerId = c.partnerId AND dps.programId = ${programId} @@ -128,9 +119,7 @@ export async function searchPartnerNetworkContent({ } const annFilter = Prisma.join(annFilters, " "); - // Two-phase retrieval: keep the ANN flat so PlanetScale uses the vector index, - // then hydrate only the returned chunk ids through the relational joins — keeping - // the joins off the vector traversal path. + // Lean ANN query first (vector index), then hydrate chunk ids — joins stay off the vector path. const fetchCandidateChunks = (poolSize: number) => prisma.$queryRaw(Prisma.sql` SELECT @@ -146,14 +135,11 @@ export async function searchPartnerNetworkContent({ LIMIT ${poolSize} `); - // Fixed pool: the request-derived candidate count always resolves to the cap in - // practice, and the ANN refills the pool from deeper in the ranking when eligibility - // filters exclude near-neighbors (verified against PlanetScale SPANN), so there's no - // need to size it per-request or widen it adaptively. + // Fixed pool size — ANN refills when eligibility filters skip near-neighbors. const poolSize = PARTNER_CONTENT_SEARCH_LIMITS.vectorSearchChunkPoolSize; logTiming?.("vector-search-start", { poolSize, - retrievalShape: "two-phase", + retrievalShape: "ann-then-topk-hydrate", }); const candidateRows = await fetchCandidateChunks(poolSize); @@ -162,26 +148,11 @@ export async function searchPartnerNetworkContent({ poolSize, vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, }); - // Dedup to the best chunk per item+source before the hydration join (filters are - // per-item, so this is equivalent and keeps chunk-heavy items off the join). - const dedupedCandidates = dedupeBestChunkPerContentItemSource(candidateRows); - const poolRows = await hydratePartnerContentSearchRows({ - candidateRows: dedupedCandidates, - logTiming, - }); - logTiming?.("vector-search-complete", { - candidateRowCount: candidateRows.length, - poolRowCount: poolRows.length, - poolSize, - vectorIndex: PARTNER_CONTENT_CHUNK_VECTOR_INDEX, - retrievalShape: "two-phase", - }); + const sourceDeduped = dedupeBestChunkPerContentItemSource(candidateRows); - // Best distance per item+source. getPartnerMatchSummaries reuses this to gate - // matched recent posts instead of a second per-item DISTANCE pass — any item that - // could clear the cutoff is already here (one missing is provably beyond it). + // Reused by getPartnerMatchSummaries to gate matched posts without a second DISTANCE pass. const itemSourceBestDistance: SourceScoreByContentItemId = new Map(); - for (const row of poolRows) { + for (const row of sourceDeduped) { setSourceDistance( itemSourceBestDistance, row.partnerContentItemId, @@ -190,35 +161,31 @@ export async function searchPartnerNetworkContent({ ); } - // One best chunk per item for the cutoff; one per item+source for rerank/evidence - // (keeps metadata and transcript evidence separable). - const itemRows = dedupeBestChunkPerContentItem(poolRows).slice(0, limit); - const rows = dedupeBestChunkPerContentItemSource(poolRows).slice( + // One best chunk per item for the cutoff; one per item+source for rerank/evidence. + const itemRows = dedupeBestChunkPerContentItem(candidateRows).slice(0, limit); + const candidateSourceRows = sourceDeduped.slice( 0, Math.min(PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, limit * 2), ); - // Cutoff = the least-relevant candidate kept; the match gate for recent posts. + // Match gate: worst distance among kept candidates (networkStatus drift dropped at hydrate). const cutoffDistance = itemRows.length ? Number(itemRows[itemRows.length - 1].distance) : null; logTiming?.("candidate-dedupe-complete", { - itemRowCount: itemRows.length, - sourceRowCount: rows.length, + candidateRowCount: candidateRows.length, + sourceRowCount: candidateSourceRows.length, cutoffDistance, }); - const rowsWithChunkText = await hydratePartnerContentChunkText({ - rows, - maxRows: rerank - ? PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount - : rows.length, + const rows = await hydratePartnerContentSearchRows({ + candidateRows: candidateSourceRows, logTiming, }); if (!rerank) { return { - rows: sortRowsByRelevanceScore(rowsWithChunkText), + rows: sortRowsByRelevanceScore(rows), reranked: false, queryVector, cutoffDistance, @@ -226,22 +193,17 @@ export async function searchPartnerNetworkContent({ }; } - // NOTE: reach / verified-platform / conversion filters run downstream in - // calculatePartnerRanking (via getNetworkPartnersById), after this rerank — so when - // they're active we pay to rerank chunks from partners that get dropped later. Left - // as-is on purpose: pre-filtering them here needs the canonical discover-eligibility - // predicate that only exists once calculatePartnerRanking is split into hydrate-card - // vs rank. Revisit at that consolidation. + // reach/verified-platform filter at card hydration (post-rerank); pre-filtering risks ANN starvation. logTiming?.("rerank-start", { - rowCount: rowsWithChunkText.length, + rowCount: rows.length, rerankerCandidateCount: Math.min( - rowsWithChunkText.length, + rows.length, PARTNER_CONTENT_SEARCH_LIMITS.rerankerCandidateCount, ), }); const rerankResult = await rerankPartnerSearchRows({ query, - rows: rowsWithChunkText, + rows, }); logTiming?.("rerank-complete", { reranked: rerankResult.reranked, @@ -251,9 +213,7 @@ export async function searchPartnerNetworkContent({ ).length, }); return { - // rerankResult.rows is already reranker-ordered on success, or the full cosine - // ordering on fail-soft. Never re-sort here — sorting by effective score would - // blend the rerank and cosine scales we just took care to keep separate. + // Don't re-sort — reranker order is intentional; blending scales would be wrong. ...rerankResult, queryVector, cutoffDistance, @@ -271,9 +231,7 @@ async function hydratePartnerContentSearchRows({ if (candidateRows.length === 0) return []; const chunkIds = candidateRows.map(({ chunkId }) => chunkId); - // Plain metadata fetch over already-eligible chunk ids (user/eligibility filters - // ran inline in the ANN). networkStatus is the one remaining post-filter, via the - // partner relation. chunkText is hydrated separately below, so we backfill "". + // networkStatus filtered post-ANN; catches status drift since embedding. const hydratedRows = await prisma.partnerContentChunk.findMany({ where: { id: { in: chunkIds }, @@ -289,14 +247,7 @@ async function hydratePartnerContentSearchRows({ source: true, startMs: true, endMs: true, - partner: { - select: { - name: true, - username: true, - image: true, - description: true, - }, - }, + chunkText: true, partnerContentItem: { select: { platformContentId: true, @@ -323,7 +274,7 @@ async function hydratePartnerContentSearchRows({ }, }); const hydratedByChunkId = new Map( - hydratedRows.map(({ partner, partnerContentItem: item, ...chunk }): [ + hydratedRows.map(({ partnerContentItem: item, ...chunk }): [ string, PartnerContentSearchHydrationRow, ] => [ @@ -332,10 +283,6 @@ async function hydratePartnerContentSearchRows({ chunkId: chunk.id, partnerContentItemId: chunk.partnerContentItemId, partnerId: chunk.partnerId, - partnerName: partner.name, - partnerUsername: partner.username, - partnerImage: partner.image, - partnerDescription: partner.description, platformType: item.partnerPlatform.type, platformIdentifier: item.partnerPlatform.identifier, platformContentId: item.platformContentId, @@ -352,7 +299,7 @@ async function hydratePartnerContentSearchRows({ contentShareCount: item.shareCount, contentSaveCount: item.saveCount, chunkSource: chunk.source, - chunkText: "", + chunkText: chunk.chunkText, startMs: chunk.startMs, endMs: chunk.endMs, }, @@ -377,49 +324,3 @@ async function hydratePartnerContentSearchRows({ return rows; } - -async function hydratePartnerContentChunkText({ - rows, - maxRows, - logTiming, -}: { - rows: PartnerContentSearchRow[]; - maxRows: number; - logTiming?: PartnerContentSearchTimingLogger; -}) { - const rowsToHydrate = rows.slice(0, maxRows); - const chunkIds = rowsToHydrate.map(({ chunkId }) => chunkId); - - if (chunkIds.length === 0) return rows; - - logTiming?.("chunk-text-hydration-start", { - requestedRowCount: rowsToHydrate.length, - totalRowCount: rows.length, - }); - const chunkTextRows = await prisma.partnerContentChunk.findMany({ - where: { - id: { - in: chunkIds, - }, - }, - select: { - id: true, - chunkText: true, - }, - }); - const chunkTextById = new Map( - chunkTextRows.map((row) => [row.id, row.chunkText]), - ); - logTiming?.("chunk-text-hydration-complete", { - hydratedRowCount: chunkTextById.size, - }); - - return rows.map((row, index) => - index < maxRows - ? { - ...row, - chunkText: chunkTextById.get(row.chunkId) ?? row.chunkText, - } - : row, - ); -} diff --git a/apps/web/lib/partner-content-search/search-utils.ts b/apps/web/lib/partner-content-search/search-utils.ts index da64e1915a0..0e99208606a 100644 --- a/apps/web/lib/partner-content-search/search-utils.ts +++ b/apps/web/lib/partner-content-search/search-utils.ts @@ -1,6 +1,4 @@ -// Pure (client-safe) helpers for the admin + network content search routes: -// per-partner grouping (each route passes its own toChunkResult), candidate -// sizing, and score utilities. Server-only reranking lives in ./rerank. +// Client-safe helpers for content search; server-only reranking lives in ./rerank. import { PARTNER_CONTENT_SEARCH_LIMITS } from "./constants"; @@ -8,10 +6,6 @@ export type PartnerContentSearchRow = { chunkId: string; partnerContentItemId: string; partnerId: string; - partnerName: string; - partnerUsername: string | null; - partnerImage: string | null; - partnerDescription: string | null; platformType: string; platformIdentifier: string; platformContentId: string; @@ -28,23 +22,18 @@ export type PartnerContentSearchRow = { contentShareCount?: bigint | number | null; contentSaveCount?: bigint | number | null; chunkSource: string; - // Only the admin query selects chunkIndex; the network query omits it. - chunkIndex?: number; chunkText: string; startMs: number | null; endMs: number | null; distance: number | string; - // Attached by the reranker (second stage); absent for cosine-only results. - rerankScore?: number | null; + rerankScore?: number | null; // rerank stage only; absent for cosine-only }; export function toScore(distance: number) { return Number((1 - distance).toFixed(6)); } -// Candidate chunks to retrieve for `limit` partners at `chunksPerPartner` each. -// Query mode over-fetches (×6, min 25, capped at rerank input size); -// list mode has no relevance gate so ×2 is enough. +// Query mode over-fetches (×6, min 25, rerank cap); list mode uses ×2. export function getCandidateChunkCount({ hasQuery, limit, @@ -62,8 +51,6 @@ export function getCandidateChunkCount({ ); } -// Median of a numeric list (robust center; null when empty). `round` rounds the -// even-length average for display; leave it off when the value feeds further math. export function median( values: number[], { round = false }: { round?: boolean } = {}, @@ -76,14 +63,11 @@ export function median( return round ? Math.round(average) : average; } -// Effective score: reranker relevance when present, else cosine similarity. export function effectiveRowScore(row: PartnerContentSearchRow) { return row.rerankScore ?? toScore(Number(row.distance)); } -// Collapse a distance-ascending chunk pool to the best chunk per content item (first -// occurrence wins). Done in app code, not SQL, so a window-function dedup doesn't -// defeat the ANN index. Generic so it works pre- or post-hydration. +// dedupe in app to preserve ANN distance ordering without circumventing index export function dedupeBestChunkPerContentItem< T extends { partnerContentItemId: string }, >(rows: T[]) { @@ -134,10 +118,6 @@ export function groupPartnerSearchResults({ string, { partnerId: string; - name: string; - username: string | null; - image: string | null; - description: string | null; score: number; cosineScore: number; rerankScore: number | null; @@ -155,10 +135,6 @@ export function groupPartnerSearchResults({ const existing = partners.get(row.partnerId); const partner = existing ?? { partnerId: row.partnerId, - name: row.partnerName, - username: row.partnerUsername, - image: row.partnerImage, - description: row.partnerDescription, score: effectiveScore, cosineScore, rerankScore, diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index 0a3e32f55dd..68a9506d200 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -131,8 +131,7 @@ export const invitePartnerFromNetworkSchema = z.object({ const PARTNER_NETWORK_CONTENT_SEARCH_DEFAULT_PARTNER_LIMIT = 20; -// Request body for POST /api/network/partners/content-search (semantic search -// over indexed partner content). +// POST /api/network/partners/content-search - semantic search over indexed partner content export const partnerNetworkContentSearchSchema = z.object({ query: z.string().trim().max(500).optional(), platforms: z.array(z.enum(PlatformType)).min(1).optional(), @@ -152,13 +151,10 @@ export const partnerNetworkContentSearchSchema = z.object({ .positive() .max(PARTNER_CONTENT_SEARCH_MAX_CHUNKS_PER_PARTNER) .default(PARTNER_CONTENT_SEARCH_DEFAULT_CHUNKS_PER_PARTNER), - // Second-stage reranking is on by default; pass `false` for diagnostics. rerank: z.boolean().default(true), }); -// ---- Response for POST /api/network/partners/content-search ---- -// Single source of truth: the route validates its response against this schema -// and the SWR hook derives its types from it (no hand-maintained duplicate). +// Content-search response schema — shared by the route validator and SWR types. export const partnerContentMatchSourceSchema = z.enum([ "transcript", @@ -258,15 +254,10 @@ const partnerContentSearchMatchSummarySchema = z.object({ const partnerContentSearchResponsePartnerSchema = z.object({ partnerId: z.string(), - name: z.string(), - username: z.string().nullable(), - image: z.string().nullable(), - description: z.string().nullable(), score: z.number(), cosineScore: z.number().nullish(), rerankScore: z.number().nullish(), - // Already validated upstream by parseRankedNetworkPartners; pass through as-is - // so the partner subtree's schema transforms aren't re-applied here. + // Profile fields on `partner` only; z.custom skips re-parsing nested transforms. partner: z.custom>(), chunks: z.array(partnerContentSearchChunkSchema), matchSummary: partnerContentSearchMatchSummarySchema.nullable(), From 4c91addb4020940e7192e0c2792e01e44e8cd271 Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 13:54:29 -0400 Subject: [PATCH 19/25] switch partner content detail to summary bar (remove vertical content bars); revert tooltip --- .../network/[partnerId]/content-match-row.tsx | 180 +++++-------- .../network/[partnerId]/content-thumbnail.tsx | 39 +++ .../network/[partnerId]/page-client.tsx | 32 ++- .../[partnerId]/recent-content-panel.tsx | 244 ++++++++++++++++++ .../network/[partnerId]/search-fit-bars.tsx | 171 +++--------- .../network/[partnerId]/search-fit-panel.tsx | 186 ++++++------- .../network/[partnerId]/search-fit-utils.ts | 54 ++-- .../network-content-search-results.tsx | 13 +- .../content-match-bars.ts | 33 +-- .../partner-content-search/match-summaries.ts | 13 +- .../match-summary-queries.ts | 27 +- apps/web/lib/zod/schemas/partner-network.ts | 10 +- packages/ui/src/tooltip.tsx | 19 +- 13 files changed, 533 insertions(+), 488 deletions(-) create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-thumbnail.tsx create mode 100644 apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx index fa743d182e6..e593ecab215 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx @@ -1,10 +1,7 @@ "use client"; import { PARTNER_CONTENT_SEARCH_LIMITS } from "@/lib/partner-content-search/constants"; -import { - type PartnerContentMatchEvidence, - type PartnerContentSearchPartner, -} from "@/lib/swr/use-partner-content-search"; +import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; import { cn, nFormatter } from "@dub/utils"; import { formatDuration, @@ -16,6 +13,7 @@ import { getContentTitle, } from "../content-display-utils"; import { PlatformIcon } from "../platform-icon"; +import { ContentThumbnail } from "./content-thumbnail"; import { type MatchedContentItem } from "./search-fit-utils"; export function ContentMatchSkeletons({ count }: { count: number }) { @@ -44,49 +42,31 @@ export function ContentMatchSkeletons({ count }: { count: number }) { export function ContentMatchRow({ item }: { item: MatchedContentItem }) { const { chunk } = item; - const evidence = item.matchEvidence; - const isTimedTranscriptMatch = chunk - ? hasTimedTranscriptMatch(chunk.chunk) - : false; - const timeLabel = - chunk && isTimedTranscriptMatch - ? formatChunkTimeRange(chunk.chunk) - : formatDuration(item.durationMs); - const dateLabel = formatPublishedDate(item.publishedAt); - const matchTags = getMatchTags( - evidence, - chunk?.chunk.source ?? - (evidence.primarySource === "creatorText" ? "metadata" : "transcript"), - ); - const snippet = chunk ? getMatchSnippet(chunk) : null; + // Only transcript chunks display in UI. + const isTranscriptChunk = chunk.chunk.source !== "metadata"; + const snippet = isTranscriptChunk ? getMatchSnippet(chunk) : null; const excerpt = snippet ? `"…${snippet.slice(0, 130).trimEnd()}…"` : null; - const thumbnail = getItemThumbnail(item); - const meta = [timeLabel, dateLabel].filter(Boolean).join(" · "); + const thumbnail = getContentThumbnail(chunk); + const baseMeta = [ + formatDuration(chunk.content.durationMs), + formatPublishedDate(item.publishedAt), + ] + .filter(Boolean) + .join(" · "); + const matchLocation = formatMatchLocation(chunk.chunk); + const engagement = formatEngagement(chunk.content); const score = item.relevance; - const title = getItemTitle(item); - const viewCount = item.viewCount; + const title = getContentTitle(chunk); return ( {/* Preview image */} -
- {thumbnail ? ( - - ) : ( -
- -
- )} -
+
@@ -95,30 +75,25 @@ export function ContentMatchRow({ item }: { item: MatchedContentItem }) {
- {meta && {meta}} - {matchTags.length > 0 && ( + {baseMeta && {baseMeta}} + {matchLocation && ( <> - {meta && ·} - - {matchTags.map((tag) => ( - - {tag.label} - - ))} + {baseMeta && ·} + + {matchLocation} )} - {viewCount != null && viewCount > 0 && ( + {engagement && ( <> - · - {nFormatter(viewCount)} views + {(baseMeta || matchLocation) && ( + · + )} + {engagement} )}
@@ -140,56 +115,38 @@ export function ContentMatchRow({ item }: { item: MatchedContentItem }) { ); } -function getMatchTags( - evidence: PartnerContentMatchEvidence | undefined, - fallbackSource: string, -) { - const sources = evidence?.sources.length - ? evidence.sources - : fallbackSource === "metadata" - ? ["creatorText" as const] - : ["transcript" as const]; - - return sources.map((source) => ({ - source, - label: source === "transcript" ? "Transcript" : "Creator text", - })); -} - -// Displayed snippet. Transcript chunks are prose; creator-text chunks store the raw -// embedding input, so we surface just the creator-entered text. -function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { - const text = (chunk.chunk.text ?? "").trim(); - if (!text) return null; - if (chunk.chunk.source !== "metadata") return text; - - const description = text.match(/Description:\s*([\s\S]+)$/i)?.[1]; - return ( - ( - description ?? text.replace(/^Content type:[\s\S]*?Title:\s*/i, "") - ).trim() || null - ); -} - -// Item thumbnail: the loaded chunk's preview, else a YouTube thumbnail from the id. -function getItemThumbnail(item: MatchedContentItem) { - if (item.chunk) return getContentThumbnail(item.chunk); - if (item.platform === "youtube" && item.platformContentId) { - return `https://i.ytimg.com/vi/${item.platformContentId}/hqdefault.jpg`; +// Where the match was found, for the row meta: "Matched transcript 1:23 - 1:45", +// "Matched transcript" (untimed), or "Matched creator text". +function formatMatchLocation({ + source, + startMs, + endMs, +}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { + if (source === "metadata") return "Matched creator text"; + if (startMs === null && endMs === null) return "Matched transcript"; + if (startMs !== null && endMs !== null) { + return `Matched transcript ${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; } - return null; + return `Matched transcript ${formatTimestamp(startMs ?? endMs ?? 0)}`; } -function getItemTitle(item: MatchedContentItem) { - if (item.chunk) return getContentTitle(item.chunk); - return item.title?.trim() || "Untitled content"; +// Reach + engagement we have for the post, in descending prominence. +function formatEngagement( + content: PartnerContentSearchPartner["chunks"][number]["content"], +) { + return [ + content.viewCount ? `${nFormatter(content.viewCount)} views` : null, + content.likeCount ? `${nFormatter(content.likeCount)} likes` : null, + content.commentCount ? `${nFormatter(content.commentCount)} comments` : null, + content.shareCount ? `${nFormatter(content.shareCount)} shares` : null, + content.saveCount ? `${nFormatter(content.saveCount)} saves` : null, + ] + .filter(Boolean) + .join(" · "); } -// Link for a matched item: the chunk-aware href (YouTube deep-link timestamp, -// Instagram normalization) when enriched, otherwise the bar's canonical URL. -function getItemHref(item: MatchedContentItem) { - if (item.chunk) return getContentHref(item.chunk); - return item.url ?? "#"; +function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { + return (chunk.chunk.text ?? "").trim() || null; } // Noun phrase for the rank window (time-based but capped per partner); says so @@ -234,24 +191,3 @@ function formatMonthYear(iso: string | null) { year: "numeric", }).format(date); } - -function hasTimedTranscriptMatch({ - source, - startMs, - endMs, -}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { - return source !== "metadata" && (startMs !== null || endMs !== null); -} - -function formatChunkTimeRange({ - source, - startMs, - endMs, -}: PartnerContentSearchPartner["chunks"][number]["chunk"]) { - if (source === "metadata") return "Creator text match"; - if (startMs === null && endMs === null) return "Transcript match"; - if (startMs !== null && endMs !== null) { - return `${formatTimestamp(startMs)} - ${formatTimestamp(endMs)}`; - } - return formatTimestamp(startMs ?? endMs ?? 0); -} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-thumbnail.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-thumbnail.tsx new file mode 100644 index 00000000000..719b88553b1 --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-thumbnail.tsx @@ -0,0 +1,39 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { PlatformIcon } from "../platform-icon"; + +// In case platform CDN thumbnails expire; fall back to the platform icon on load failure. +export function ContentThumbnail({ + thumbnail, + platform, +}: { + thumbnail: string | null; + platform: string; +}) { + const [hasImageError, setHasImageError] = useState(false); + const thumbnailSrc = hasImageError ? null : thumbnail; + + // Rows recycle as the list re-renders, so clear a prior failure when the source + // changes — otherwise a fresh/valid URL stays masked by an earlier load error. + useEffect(() => { + setHasImageError(false); + }, [thumbnail]); + + return ( +
+ {thumbnailSrc ? ( + setHasImageError(true)} + /> + ) : ( +
+ +
+ )} +
+ ); +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 8fdee3c7cd0..b7344f70e77 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -19,6 +19,7 @@ import { platformFilterParam, } from "../platform-filter-utils"; import { NetworkInviteControl } from "./network-invite-control"; +import { RecentContentPanel } from "./recent-content-panel"; import { SearchFitPanel } from "./search-fit-panel"; import { useSearchParams } from "next/navigation"; import { ArrowLeft } from "lucide-react"; @@ -78,9 +79,7 @@ export function NetworkPartnerDetailContent({ ); const loadedPartner = partners?.[0]; - // Open instantly from the cached match data; the background refetch reranks just - // this partner so every row lands on one relevance scale (global search only - // reranks its top-N pool). + // Background scoped rerank puts every row on one relevance scale. const shouldFetchSearch = hasContentSearch; const { data: searchResults, @@ -97,6 +96,21 @@ export function NetworkPartnerDetailContent({ chunksPerPartner: PARTNER_CONTENT_SEARCH_DETAIL_CHUNKS_PER_PARTNER, }); + const { + data: recentResults, + error: recentError, + isLoading: isLoadingRecent, + } = usePartnerContentSearch({ + enabled: Boolean(workspaceId && !hasContentSearch), + query: "", + country, + starred: false, + partnerIds: [partnerId], + limit: 1, + chunksPerPartner: PARTNER_CONTENT_SEARCH_DETAIL_CHUNKS_PER_PARTNER, + }); + const recentPartner = recentResults?.partners?.[0]; + const fetchedPartner = searchResults?.partners?.[0]; const partner = loadedPartner ?? initialSearchPartner?.partner ?? fetchedPartner?.partner; @@ -104,8 +118,6 @@ export function NetworkPartnerDetailContent({ const searchPartner = fetchedPartner ?? initialSearchPartner; const searchSummary = initialSearchPartner?.matchSummary ?? fetchedPartner?.matchSummary; - // Per-row relevance comes from the scoped reranked summary once it lands (one - // scale); null until then, so rows show cached scores and quietly upgrade. const relevanceSummary = fetchedPartner?.matchSummary ?? null; // Deep-link with nothing cached → full skeleton; otherwise a silent refinement. const isFallbackLoading = isLoadingSearch && !initialSearchPartner; @@ -180,15 +192,23 @@ export function NetworkPartnerDetailContent({ hideDescription hideMonthlyTraffic /> - {hasContentSearch && ( + {hasContentSearch ? ( + ) : ( + )}
)} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx new file mode 100644 index 00000000000..e96d67fbd4a --- /dev/null +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx @@ -0,0 +1,244 @@ +"use client"; + +import { PARTNER_CONTENT_SEARCH_TOP_CONTENT } from "@/lib/partner-content-search/constants"; +import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { type NetworkPartnerProps } from "@/lib/types"; +import { Button } from "@dub/ui"; +import { nFormatter } from "@dub/utils"; +import { useState } from "react"; +import { + formatDuration, + formatPublishedDate, + getContentHref, + getContentThumbnail, + getContentTitle, + lastPostedLabel, +} from "../content-display-utils"; +import { PlatformIcon } from "../platform-icon"; +import { ContentMatchSkeletons } from "./content-match-row"; +import { ContentThumbnail } from "./content-thumbnail"; +import { publishedAtMs } from "./search-fit-utils"; + +const RECENT_INITIAL_COUNT = 8; +const RECENT_INCREMENT = 8; +const MAX_NICHE_CHIPS = 3; + +type RecentContentItem = { + contentItemId: string; + platform: string; + title: string; + href: string; + thumbnail: string | null; + publishedAt: string | null; + durationMs: number | null; + viewCount: number | null; +}; + +export function RecentContentPanel({ + partner, + searchPartner, + isLoading, + error, +}: { + partner?: NetworkPartnerProps; + searchPartner?: PartnerContentSearchPartner; + isLoading: boolean; + error: unknown; +}) { + const [visibleRecentCount, setVisibleRecentCount] = + useState(RECENT_INITIAL_COUNT); + + const items = buildRecentItems(searchPartner?.chunks ?? []); + + const topContent = [...items] + .sort((a, b) => (b.viewCount ?? 0) - (a.viewCount ?? 0)) + .slice(0, PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount); + const recentContent = [...items].sort( + (a, b) => publishedAtMs(b.publishedAt) - publishedAtMs(a.publishedAt), + ); + const visibleRecent = recentContent.slice(0, visibleRecentCount); + const hiddenRecentCount = Math.max(0, recentContent.length - visibleRecent.length); + // "Recent content" only earns its own section when it adds rows beyond the top set. + const showRecentSection = + !isLoading && + recentContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; + + const categories = (partner?.categories ?? []).slice(0, MAX_NICHE_CHIPS); + const followers = sumFollowers(partner); + const medianViews = medianViewCount(items); + const lastPublished = lastPostedLabel( + recentContent.find((item) => item.publishedAt)?.publishedAt, + ); + const metaParts = [ + followers ? `${nFormatter(followers)} followers` : null, + medianViews ? `${nFormatter(medianViews)} median views` : null, + lastPublished ? `last post ${lastPublished}` : null, + ].filter((part): part is string => Boolean(part)); + + return ( +
+
+ {categories.length > 0 && ( +
+ {categories.map((category) => ( + + {category} + + ))} +
+ )} + {metaParts.length > 0 && ( +
+ {metaParts.join(" · ")} +
+ )} +
+ +
+
+

+ Top content +

+

+ Ranked by reach +

+
+ +
+ {error ? ( +
+ Failed to load recent content +
+ ) : isLoading ? ( + + ) : topContent.length ? ( + topContent.map((item) => ( + + )) + ) : ( +
+ No indexed content found for this partner. +
+ )} +
+ + {showRecentSection && ( +
+
+

+ Recent content +

+

+ Most recent first +

+
+ +
+ {visibleRecent.map((item) => ( + + ))} +
+ + {hiddenRecentCount > 0 ? ( +
+
+ ) : null} +
+ )} +
+
+ ); +} + +function RecentContentRow({ item }: { item: RecentContentItem }) { + const meta = [formatDuration(item.durationMs), formatPublishedDate(item.publishedAt)] + .filter(Boolean) + .join(" · "); + + return ( + + +
+
+ + + {item.title} + +
+ {meta && ( +
{meta}
+ )} +
+ {item.viewCount != null && item.viewCount > 0 && ( + + {nFormatter(item.viewCount)} views + + )} + + ); +} + +// One row per content item (chunks repeat per item — keep the first, its content +// fields are identical across its chunks). +function buildRecentItems( + chunks: PartnerContentSearchPartner["chunks"], +): RecentContentItem[] { + const byContentItemId = new Map(); + + for (const chunk of chunks) { + if (byContentItemId.has(chunk.partnerContentItemId)) continue; + byContentItemId.set(chunk.partnerContentItemId, { + contentItemId: chunk.partnerContentItemId, + platform: chunk.platform.type, + title: getContentTitle(chunk), + href: getContentHref(chunk), + thumbnail: getContentThumbnail(chunk), + publishedAt: chunk.content.publishedAt, + durationMs: chunk.content.durationMs, + viewCount: chunk.content.viewCount, + }); + } + + return [...byContentItemId.values()]; +} + +function sumFollowers(partner?: NetworkPartnerProps) { + if (!partner?.platforms?.length) return null; + const total = partner.platforms.reduce( + (sum, platform) => sum + Number(platform.subscribers ?? 0), + 0, + ); + return total > 0 ? total : null; +} + +function medianViewCount(items: RecentContentItem[]) { + const views = items + .map((item) => item.viewCount) + .filter((value): value is number => value != null && value > 0) + .sort((a, b) => a - b); + if (!views.length) return null; + + const mid = Math.floor(views.length / 2); + return views.length % 2 === 0 + ? Math.round((views[mid - 1] + views[mid]) / 2) + : views[mid]; +} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx index efca77e38a6..2e676b85a90 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx @@ -1,150 +1,55 @@ "use client"; import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; -import { Tooltip } from "@dub/ui"; -import { cn, nFormatter } from "@dub/utils"; -import { useState } from "react"; -import { - formatDuration, - formatPublishedDate, -} from "../content-display-utils"; -import { PlatformIcon } from "../platform-icon"; +import { cn } from "@dub/utils"; -// Per-post match bars: height = match magnitude, color = platform. Muted, evenly- -// weighted palette so platforms read as a calm spectrum (none looks like a non-match). -const PLATFORM_BAR_COLORS: Record = { - youtube: "bg-[#bd8488]", - instagram: "bg-[#b083a2]", - tiktok: "bg-[#74a09a]", - twitter: "bg-[#8589b3]", - x: "bg-[#8589b3]", - linkedin: "bg-[#7c9cbd]", - website: "bg-[#bda77c]", -}; - -// Glanceable recent-activity strip, capped to the most-recent N so columns stay -// hoverable. Topic Fit + the "X of Y" counts still use the full server-side set. -const MAX_VISIBLE_CONTENT_BARS = 40; +// Coverage bar from summary counts (strong / partial / no match). +const SEGMENTS = [ + { key: "strong", label: "Strong", color: "bg-[#1D9E75]" }, + { key: "partial", label: "Partial", color: "bg-[#EF9F27]" }, + { key: "none", label: "No match", color: "bg-neutral-300" }, +] as const; export function ContentMatchBars({ summary, }: { summary: PartnerContentSearchPartner["matchSummary"] | undefined; }) { - // One open tooltip at a time — driving every bar's open state from one value - // prevents a fast cursor flick leaving several open. - const [openBarId, setOpenBarId] = useState(null); - - const allBars = summary?.contentBars ?? []; - if (!allBars.length) return null; - const bars = allBars.slice(0, MAX_VISIBLE_CONTENT_BARS); + if (!summary || !summary.recentContentCount) return null; + + const { + recentContentCount, + matchedContentCount, + strongMatchedContentCount, + partialMatchedContentCount, + } = summary; + const noMatchCount = Math.max(0, recentContentCount - matchedContentCount); + + const counts: Record<(typeof SEGMENTS)[number]["key"], number> = { + strong: strongMatchedContentCount, + partial: partialMatchedContentCount, + none: noMatchCount, + }; return ( -
setOpenBarId(null)} - > - {bars.map((bar) => { - const score = - bar.matched && bar.matchScore != null - ? Math.min(1, Math.max(0, bar.matchScore)) - : 0; - // Magnitude → height (matched posts get a floor so they stay legible). - const height = bar.matched ? Math.round(10 + score * 26) : 5; - const isCreatorTextOnlyVideoMatch = - bar.matchEvidence.primarySource === "creatorText" && - bar.matchEvidence.weight < 1; - // The whole column is the hover/click target (easier to land on); on hover - // a gold wash fills it and the bar turns gold. - const columnClassName = cn( - "group flex h-full min-w-[5px] flex-1 items-end rounded-[3px] transition-colors duration-75 hover:bg-amber-100/70", - bar.url && "cursor-pointer", - ); - const fill = ( - +
+ {SEGMENTS.filter(({ key }) => counts[key] > 0).map(({ key, color }) => ( +
- ); - - return ( - } - // Snappy bar-to-bar hover: open instantly, no grace area, no close animation. - delayDuration={0} - disableHoverableContent - disableAnimation - open={openBarId === bar.partnerContentItemId} - onOpenChange={(nextOpen) => - setOpenBarId((current) => - nextOpen - ? bar.partnerContentItemId - : current === bar.partnerContentItemId - ? null - : current, - ) - } - > - {bar.url ? ( - - {fill} - - ) : ( -
{fill}
- )} - - ); - })} -
- ); -} - -// Hover card for a content bar: platform, title, date · length · views (all cached). -function BarTooltip({ - bar, -}: { - bar: NonNullable< - PartnerContentSearchPartner["matchSummary"] - >["contentBars"][number]; -}) { - const title = bar.title?.trim() || "Untitled content"; - const meta = [ - formatPublishedDate(bar.publishedAt), - formatDuration(bar.durationMs), - bar.viewCount && bar.viewCount > 0 - ? `${nFormatter(bar.viewCount)} views` - : null, - ] - .filter(Boolean) - .join(" · "); - - return ( -
-
- - - {title} - + ))} +
+
+ {SEGMENTS.map(({ key, label, color }) => ( + + + {label} {counts[key]} + + ))}
- {meta && ( -
{meta}
- )}
); } diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx index 81c8a66447c..4772f0d7bf1 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx @@ -8,8 +8,7 @@ import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content- import { Button } from "@dub/ui"; import { cn, nFormatter } from "@dub/utils"; import { useState } from "react"; -import { BAND_LABELS, lastPostedLabel } from "../content-display-utils"; -import { PlatformIcon } from "../platform-icon"; +import { lastPostedLabel } from "../content-display-utils"; import { ContentMatchRow, ContentMatchSkeletons, @@ -19,44 +18,57 @@ import { ContentMatchBars } from "./search-fit-bars"; import { buildMatchedContentItems, buildUnifiedRelevanceMap, - DETAIL_CONTENT_INITIAL_MATCH_COUNT, - DETAIL_CONTENT_MATCH_INCREMENT, + DETAIL_CONTENT_INITIAL_DISPLAY_COUNT, + DETAIL_CONTENT_PAGE_COUNT, publishedAtMs, } from "./search-fit-utils"; -const TOPIC_FIT_BAND_STYLES: Record< - PartnerContentTopicFitBand, - { number: string; chip: string } -> = { - consistent: { number: "text-green-600", chip: "bg-green-50 text-green-700" }, - frequent: { number: "text-blue-600", chip: "bg-blue-50 text-blue-700" }, - occasional: { number: "text-amber-600", chip: "bg-amber-50 text-amber-700" }, - "one-off": { - number: "text-neutral-500", - chip: "bg-neutral-100 text-neutral-600", - }, - none: { number: "text-neutral-400", chip: "bg-neutral-100 text-neutral-500" }, +const BAND_HEADLINE: Record = { + consistent: "Consistently on-topic", + frequent: "Frequently on-topic", + occasional: "Occasionally on-topic", + "one-off": "One-off mention", + none: "No recent matches", }; +const TOPIC_FIT_BAND_CHIP: Record = { + consistent: "bg-green-50 text-green-700", + frequent: "bg-blue-50 text-blue-700", + occasional: "bg-amber-50 text-amber-700", + "one-off": "bg-neutral-100 text-neutral-600", + none: "bg-neutral-100 text-neutral-500", +}; + +const MAX_QUERY_DISPLAY_LENGTH = 60; + +// The search box is natural-language, so a long query would wrap the headline. Trim +// to a tidy length for inline display — the full query still drives the search. +function truncateQuery(query: string) { + const trimmed = query.trim(); + return trimmed.length > MAX_QUERY_DISPLAY_LENGTH + ? `${trimmed.slice(0, MAX_QUERY_DISPLAY_LENGTH).trimEnd()}…` + : trimmed; +} + export function SearchFitPanel({ error, isLoading, isRefining = false, + query, summary: initialSummary, relevanceSummary, searchPartner, }: { error: unknown; isLoading: boolean; - // Scoped rerank in flight; when it lands, every row's relevance moves to one scale. isRefining?: boolean; + query?: string; summary?: PartnerContentSearchPartner["matchSummary"]; - // Scoped reranked summary, only to put per-row relevance on one scale. Null until it resolves. relevanceSummary?: PartnerContentSearchPartner["matchSummary"] | null; searchPartner?: PartnerContentSearchPartner; }) { const [visibleAllCount, setVisibleAllCount] = useState( - DETAIL_CONTENT_INITIAL_MATCH_COUNT, + DETAIL_CONTENT_INITIAL_DISPLAY_COUNT, ); const summary = initialSummary ?? searchPartner?.matchSummary; @@ -65,16 +77,14 @@ export function SearchFitPanel({ } const isLoadingRows = isLoading || isRefining; - // Per-item relevance on a single scale, from the scoped reranked summary. const unifiedRelevanceByItemId = buildUnifiedRelevanceMap(relevanceSummary); - // Hold row rendering until the scoped run finishes so snippets/order don't swap in. + // Hold rows until scoped rerank lands so scores/order don't swap in. const items = buildMatchedContentItems( summary, isLoadingRows ? [] : (searchPartner?.chunks ?? []), unifiedRelevanceByItemId, ); - // Top content: relevance + reach blend. All content: same set, newest-first. const topContent = [...items] .sort((a, b) => b.blendedScore - a.blendedScore) .slice(0, PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount); @@ -89,11 +99,14 @@ export function SearchFitPanel({ allContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; const band = summary?.band ?? "none"; - const bandStyles = TOPIC_FIT_BAND_STYLES[band]; - const topPlatforms = summary?.topPlatforms?.length - ? summary.topPlatforms - : summary?.platforms ?? []; const lastOnTopic = lastPostedLabel(summary?.lastOnTopicAt ?? null); + const metaParts = [ + summary?.followers ? `${nFormatter(summary.followers)} followers` : null, + summary?.medianViews + ? `${nFormatter(summary.medianViews)} median views` + : null, + lastOnTopic ? `last post ${lastOnTopic}` : null, + ].filter((part): part is string => Boolean(part)); const rankWindowPhrase = formatRankWindowPhrase(summary); const topContentCaption = rankWindowPhrase ? `Ranked by relevance + reach across ${rankWindowPhrase}.` @@ -101,77 +114,44 @@ export function SearchFitPanel({ return (
- {/* Topic fit summary row — layout adapted from the Partner Search hi-fi ref */} -
-
-
- Topic fit -
-
- - {summary?.topicFit ?? "—"} +
+
+
+ + {BAND_HEADLINE[band]} - {BAND_LABELS[band]} + Topic fit {summary?.topicFit ?? "—"}
-
- -
- -
-
- {summary?.followers ? ( - {nFormatter(summary.followers)} followers - ) : null} - {summary?.followers && summary?.medianViews ? ( - · - ) : null} - {summary?.medianViews ? ( - {nFormatter(summary.medianViews)} median views - ) : null} -
- {summary && ( -
- - {summary.matchedContentCount} of {summary.recentContentCount}{" "} - matching - - {topPlatforms.length > 0 && ( - <> - · - matched on - - {topPlatforms.map((platform) => ( - - ))} - - - )} - {lastOnTopic && ( - <> - · - last post {lastOnTopic} - +

+ {summary.matchedContentCount} of {summary.recentContentCount} recent + posts related to{" "} + {query ? ( + + “{truncateQuery(query)}” + + ) : ( + "your search" )} -

+ . +

)}
+ + + + {metaParts.length > 0 && ( +
+ {metaParts.join(" · ")} +
+ )}
@@ -233,11 +213,11 @@ export function SearchFitPanel({ variant="secondary" text={`Show ${Math.min( hiddenAllCount, - DETAIL_CONTENT_MATCH_INCREMENT, + DETAIL_CONTENT_PAGE_COUNT, )} more`} onClick={() => setVisibleAllCount( - (count) => count + DETAIL_CONTENT_MATCH_INCREMENT, + (count) => count + DETAIL_CONTENT_PAGE_COUNT, ) } className="h-9 rounded-lg px-4" @@ -254,30 +234,16 @@ export function SearchFitPanel({ export function SearchFitPanelSkeleton() { return (
-
-
-
-
-
-
-
-
- -
- -
-
-
- {[...Array(18)].map((_, index) => ( -
- ))} +
+
+
+
+
-
+
+
+
diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts index 06bcebb48a9..dc2f5d7605c 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts @@ -7,26 +7,18 @@ import { type PartnerContentSearchPartner, } from "@/lib/swr/use-partner-content-search"; -export const DETAIL_CONTENT_INITIAL_MATCH_COUNT = 8; -export const DETAIL_CONTENT_MATCH_INCREMENT = 8; +export const DETAIL_CONTENT_INITIAL_DISPLAY_COUNT = 8; +export const DETAIL_CONTENT_PAGE_COUNT = 8; -// A matched post for the detail lists: from the cached summary's bars, enriched -// with a loaded chunk (snippet, timed transcript, thumbnail) when available. export type MatchedContentItem = { contentItemId: string; platform: string; - platformContentId: string; - title: string | null; - url: string | null; - durationMs: number | null; publishedAt: string | null; viewCount: number | null; - // The displayed relevance rating (0-1); also feeds the blend. relevance: number; - // Relevance + reach blend, used only to order the Top content list. blendedScore: number; matchEvidence: PartnerContentMatchEvidence; - chunk?: PartnerContentSearchPartner["chunks"][number]; + chunk: PartnerContentSearchPartner["chunks"][number]; }; export function getEvidenceDisplayScore( @@ -40,8 +32,7 @@ export function getEvidenceDisplayScore( ); } -// Per-item relevance from the scoped reranked summary (one scale; the cached global -// summary mixes rerank + cosine). Includes any item with evidence, not just matched. +// Per-item relevance on one scale from the scoped rerank (cached global summary mixes scales). export function buildUnifiedRelevanceMap( relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, ) { @@ -60,25 +51,26 @@ export function buildMatchedContentItems( ): MatchedContentItem[] { const bars = summary?.contentBars ?? []; - // Best loaded chunk per content item, for snippet/thumbnail enrichment. - const chunkByContentItemId = new Map< + const bestChunkByContentItemId = new Map< string, PartnerContentSearchPartner["chunks"][number] >(); for (const chunk of chunks) { - const current = chunkByContentItemId.get(chunk.partnerContentItemId); - if (!current || chunk.score > current.score) { - chunkByContentItemId.set(chunk.partnerContentItemId, chunk); + const current = bestChunkByContentItemId.get(chunk.partnerContentItemId); + if (!current || isBetterDisplayChunk(chunk, current)) { + bestChunkByContentItemId.set(chunk.partnerContentItemId, chunk); } } - // Per-creator engagement baseline: median recent views (matched + unmatched). const baselineViews = getViewBaseline(bars.map((bar) => bar.viewCount)); return bars .filter((bar) => bar.matched) - .map((bar) => { - // Prefer the unified single-scale relevance once the rerank lands; else cached. + .map((bar): MatchedContentItem | null => { + // Skip rows we can't render; matched count in the header still comes from all matched bars. + const chunk = bestChunkByContentItemId.get(bar.partnerContentItemId); + if (!chunk) return null; + const relevance = unifiedRelevanceByItemId?.get(bar.partnerContentItemId) ?? getEvidenceDisplayScore(bar.matchEvidence) ?? @@ -88,10 +80,6 @@ export function buildMatchedContentItems( return { contentItemId: bar.partnerContentItemId, platform: bar.platform, - platformContentId: bar.platformContentId, - title: bar.title, - url: bar.url, - durationMs: bar.durationMs, publishedAt: bar.publishedAt, viewCount: bar.viewCount, relevance, @@ -101,9 +89,21 @@ export function buildMatchedContentItems( baselineViews, }), matchEvidence: bar.matchEvidence, - chunk: chunkByContentItemId.get(bar.partnerContentItemId), + chunk, }; - }); + }) + .filter((item): item is MatchedContentItem => item !== null); +} + +// Prefer transcript chunks for display enrichment (excerpt + timed link). +function isBetterDisplayChunk( + candidate: PartnerContentSearchPartner["chunks"][number], + current: PartnerContentSearchPartner["chunks"][number], +) { + const candidateIsTranscript = candidate.chunk.source !== "metadata"; + const currentIsTranscript = current.chunk.source !== "metadata"; + if (candidateIsTranscript !== currentIsTranscript) return candidateIsTranscript; + return candidate.score > current.score; } export function publishedAtMs(iso: string | null) { diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index 541b00be4c0..1faeb7a3eb5 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -443,14 +443,11 @@ function getContentEngagementMetrics( bar: ContentSearchBar | undefined, ) { return [ - { label: "Views", value: chunk.content.viewCount ?? bar?.viewCount }, - { label: "Likes", value: chunk.content.likeCount ?? bar?.likeCount }, - { - label: "Comments", - value: chunk.content.commentCount ?? bar?.commentCount, - }, - { label: "Shares", value: chunk.content.shareCount ?? bar?.shareCount }, - { label: "Saves", value: chunk.content.saveCount ?? bar?.saveCount }, + { label: "Views", value: chunk.content.viewCount ?? bar?.viewCount }, // views may still fall back to slim bar + { label: "Likes", value: chunk.content.likeCount }, + { label: "Comments", value: chunk.content.commentCount }, + { label: "Shares", value: chunk.content.shareCount }, + { label: "Saves", value: chunk.content.saveCount }, ] .filter((metric): metric is { label: string; value: number } => { return metric.value != null && metric.value > 0; diff --git a/apps/web/lib/partner-content-search/content-match-bars.ts b/apps/web/lib/partner-content-search/content-match-bars.ts index d48b324f7b6..4b6eb05d227 100644 --- a/apps/web/lib/partner-content-search/content-match-bars.ts +++ b/apps/web/lib/partner-content-search/content-match-bars.ts @@ -19,16 +19,8 @@ export type PartnerContentItemBestMatch = { export type PartnerContentMatchBar = { partnerContentItemId: string; platform: string; - platformContentId: string; - title: string | null; - url: string | null; - durationMs: number | null; publishedAt: string | null; viewCount: number | null; - likeCount: number | null; - commentCount: number | null; - shareCount: number | null; - saveCount: number | null; matched: boolean; matchScore: number | null; matchEvidence: PartnerContentMatchEvidence; @@ -55,8 +47,6 @@ export function getBestMatchByContentItemId(rows: PartnerContentSearchRow[]) { >(); for (const row of rows) { - // Use the effective score (rerank when present) so the content-match bars - // stay consistent with the reranked partner ordering. const score = row.rerankScore ?? toScore(Number(row.distance)); const evidenceSource = getEvidenceSource(row.chunkSource); const existing = bestMatchByContentItemId.get(row.partnerContentItemId); @@ -90,9 +80,7 @@ export function getQueryContentMatch({ row: PartnerRecentContentBarRow; context: QueryContentMatchContext; }) { - // On-topic gate: prefer the reranker's calibrated relevance; fall back to the - // cosine cutoff only for items the reranker didn't score. Both sources rely - // solely on embedding retrieval + reranking — no lexical title/description boost. + // On-topic gate: rerank score when present, else cosine cutoff (no lexical boost). const transcriptScore = getMatchedSourceScore({ rerankScore: getSourceScore( context.rerankByItemSource, @@ -168,26 +156,25 @@ export function toContentMatchBar({ return { partnerContentItemId: row.partnerContentItemId, platform: row.platformType, - platformContentId: row.platformContentId, - title: row.contentTitle, - url: row.contentUrl, - durationMs: - row.contentDurationMs != null ? Number(row.contentDurationMs) : null, publishedAt: row.publishedAt?.toISOString() ?? null, viewCount: row.viewCount != null ? Number(row.viewCount) : null, - likeCount: row.likeCount != null ? Number(row.likeCount) : null, - commentCount: row.commentCount != null ? Number(row.commentCount) : null, - shareCount: row.shareCount != null ? Number(row.shareCount) : null, - saveCount: row.saveCount != null ? Number(row.saveCount) : null, matched, matchScore, matchEvidence, }; } +// Strong vs partial split for the coverage bar (calibrated score ≥ 0.7). +const STRONG_MATCH_DISPLAY_THRESHOLD = 0.7; + export function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { const matchedBars = contentBars.filter((bar) => bar.matched); const matchedContentCount = matchedBars.length; + const strongMatchedContentCount = matchedBars.filter( + (bar) => (bar.matchScore ?? 0) >= STRONG_MATCH_DISPLAY_THRESHOLD, + ).length; + const partialMatchedContentCount = + matchedContentCount - strongMatchedContentCount; const transcriptMatchedContentCount = contentBars.filter( ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), ).length; @@ -216,6 +203,8 @@ export function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { return { matchedBars, matchedContentCount, + strongMatchedContentCount, + partialMatchedContentCount, transcriptMatchedContentCount, creatorTextMatchedContentCount, creatorTextOnlyContentCount, diff --git a/apps/web/lib/partner-content-search/match-summaries.ts b/apps/web/lib/partner-content-search/match-summaries.ts index 43f03661700..f1c65103976 100644 --- a/apps/web/lib/partner-content-search/match-summaries.ts +++ b/apps/web/lib/partner-content-search/match-summaries.ts @@ -42,12 +42,8 @@ export async function getPartnerMatchSummaries({ rows: PartnerContentSearchRow[]; partnerIds: string[]; platforms?: PlatformType[]; - // Query mode only: gates a partner's recent posts as matched when at least as - // relevant as the weakest candidate (cutoffDistance). Null in list mode. queryVector?: string | null; cutoffDistance?: number | null; - // Best distance per item+source from retrieval, reused to gate matched recent - // content without a second per-item DISTANCE pass. itemSourceBestDistance?: SourceScoreByContentItemId; logTiming?: PartnerContentSearchTimingLogger; }) { @@ -75,14 +71,11 @@ export async function getPartnerMatchSummaries({ ({ partnerId }) => partnerId, ); - // Query mode: reuse itemSourceBestDistance to gate matched recent posts (a post - // absent from it is provably beyond the cutoff — see retrieval's producer). const recentItemSourceBestDistance = queryVector && itemSourceBestDistance ? itemSourceBestDistance : new Map>(); - // Per-item+source reranker scores from the candidate pool, to gate on-topic posts. const rerankByItemSource: SourceScoreByContentItemId = new Map(); for (const row of rows) { if (row.rerankScore != null) { @@ -126,6 +119,8 @@ export async function getPartnerMatchSummaries({ const { matchedBars, matchedContentCount, + strongMatchedContentCount, + partialMatchedContentCount, transcriptMatchedContentCount, creatorTextMatchedContentCount, creatorTextOnlyContentCount, @@ -139,7 +134,6 @@ export async function getPartnerMatchSummaries({ weightedMatchedContentScore, recentContentCount, }); - // Brand-facing signals over on-topic posts: median views + last on-topic date. const medianViews = median( matchedBars .map((bar) => bar.viewCount) @@ -154,7 +148,6 @@ export async function getPartnerMatchSummaries({ ? new Date(Math.max(...matchedTimestamps)).toISOString() : null; const followers = followersByPartner.get(partnerId) ?? null; - // Top platforms by matched-post frequency (falls back to all recent when none matched). const platformFrequency = new Map(); for (const bar of matchedBars.length ? matchedBars : contentBars) { platformFrequency.set( @@ -183,6 +176,8 @@ export async function getPartnerMatchSummaries({ partnerId, { matchedContentCount, + strongMatchedContentCount, + partialMatchedContentCount, transcriptMatchedContentCount, creatorTextMatchedContentCount, creatorTextOnlyContentCount, diff --git a/apps/web/lib/partner-content-search/match-summary-queries.ts b/apps/web/lib/partner-content-search/match-summary-queries.ts index ee015beaccc..8c982a66e30 100644 --- a/apps/web/lib/partner-content-search/match-summary-queries.ts +++ b/apps/web/lib/partner-content-search/match-summary-queries.ts @@ -6,21 +6,14 @@ import { } from "./constants"; import type { PartnerContentSearchTimingLogger } from "./timing"; +// Slim bar row for aggregates only; display fields come from loaded chunks. export type PartnerRecentContentBarRow = { partnerId: string; partnerContentItemId: string; platformType: string; - platformContentId: string; contentType: string; - contentTitle: string | null; - contentUrl: string | null; - contentDurationMs: number | null; publishedAt: Date | null; viewCount: bigint | number | null; - likeCount: bigint | number | null; - commentCount: bigint | number | null; - shareCount: bigint | number | null; - saveCount: bigint | number | null; rowNumber: bigint | number; }; @@ -45,7 +38,6 @@ export async function fetchMatchSummaryBaseData({ PARTNER_CONTENT_SEARCH_LIMITS.recencyWindowMonths, ); - // Independent reads — issued as one parallel wave (fan out under Promise.all). const contentCountsPromise = prisma.partnerContentItem .groupBy({ by: ["partnerId"], @@ -87,34 +79,18 @@ export async function fetchMatchSummaryBaseData({ partnerId, partnerContentItemId, platformType, - platformContentId, contentType, - contentTitle, - contentUrl, - contentDurationMs, publishedAt, viewCount, - likeCount, - commentCount, - shareCount, - saveCount, rowNumber FROM ( SELECT pci.partnerId, pci.id AS partnerContentItemId, pp.type AS platformType, - pci.platformContentId, pci.contentType, - pci.title AS contentTitle, - pci.url AS contentUrl, - pci.durationMs AS contentDurationMs, pci.publishedAt, pci.viewCount, - pci.likeCount, - pci.commentCount, - pci.shareCount, - pci.saveCount, ROW_NUMBER() OVER ( PARTITION BY pci.partnerId ORDER BY pci.publishedAt DESC, pci.createdAt DESC, pci.id ASC @@ -141,7 +117,6 @@ export async function fetchMatchSummaryBaseData({ return recentContentRows; }); - // Reach: total followers across the partner's platforms (or the filtered platform). const followerRowsPromise = prisma.partnerPlatform .groupBy({ by: ["partnerId"], diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index 68a9506d200..a4f80e6cf56 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -207,16 +207,8 @@ const partnerContentSearchChunkSchema = z.object({ const partnerContentSearchContentBarSchema = z.object({ partnerContentItemId: z.string(), platform: z.string(), - platformContentId: z.string(), - title: z.string().nullable(), - url: z.string().nullable(), - durationMs: z.number().nullable(), publishedAt: z.string().nullable(), viewCount: z.number().nullable(), - likeCount: z.number().nullable(), - commentCount: z.number().nullable(), - shareCount: z.number().nullable(), - saveCount: z.number().nullable(), matched: z.boolean(), matchScore: z.number().nullable(), matchEvidence: partnerContentMatchEvidenceSchema, @@ -232,6 +224,8 @@ const partnerContentTopicFitBandSchema = z.enum([ const partnerContentSearchMatchSummarySchema = z.object({ matchedContentCount: z.number(), + strongMatchedContentCount: z.number(), + partialMatchedContentCount: z.number(), transcriptMatchedContentCount: z.number(), creatorTextMatchedContentCount: z.number(), creatorTextOnlyContentCount: z.number(), diff --git a/packages/ui/src/tooltip.tsx b/packages/ui/src/tooltip.tsx index bcccae00b95..d7b3a327b9e 100644 --- a/packages/ui/src/tooltip.tsx +++ b/packages/ui/src/tooltip.tsx @@ -72,11 +72,6 @@ export interface TooltipProps disabled?: boolean; disableHoverableContent?: TooltipPrimitive.TooltipProps["disableHoverableContent"]; delayDuration?: TooltipPrimitive.TooltipProps["delayDuration"]; - /** Controlled open state — pass with `onOpenChange`, or omit both to self-manage. */ - open?: boolean; - onOpenChange?: (open: boolean) => void; - /** Drop the entrance animation so the tooltip closes instantly without lingering. */ - disableAnimation?: boolean; } export function Tooltip({ @@ -87,16 +82,9 @@ export function Tooltip({ side = "top", disableHoverableContent, delayDuration = 0, - open: openProp, - onOpenChange, - disableAnimation, ...rest }: TooltipProps) { - const [uncontrolledOpen, setUncontrolledOpen] = useState(false); - const isControlled = openProp !== undefined; - const open = isControlled ? openProp : uncontrolledOpen; - const setOpen = (next: boolean) => - isControlled ? onOpenChange?.(next) : setUncontrolledOpen(next); + const [open, setOpen] = useState(false); return ( From b9b075b2458dc3d4089c35319b0fcec8d75c3e77 Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 13:54:49 -0400 Subject: [PATCH 20/25] partner content server fetch enhancements --- .../network/[partnerId]/page-client.tsx | 9 ++- .../lib/partner-content-search/retrieval.ts | 66 ++++++++++++++----- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index b7344f70e77..19784e69be1 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -68,10 +68,12 @@ export function NetworkPartnerDetailContent({ ? `/${workspaceSlug}/program/network${getBackQueryString(searchParams)}` : "#"; + // Skip standalone partner fetch on cold deep-link only — otherwise we'd re-run ranking SQL. + const hasInitialPartner = Boolean(initialSearchPartner?.partner); const { data: partners, isLoading: isLoadingPartner } = useSWR< NetworkPartnerProps[] >( - workspaceId + workspaceId && !hasInitialPartner ? getPartnerApiPath({ workspaceId, partnerStatus, partnerId, country }) : null, fetcher, @@ -113,7 +115,10 @@ export function NetworkPartnerDetailContent({ const fetchedPartner = searchResults?.partners?.[0]; const partner = - loadedPartner ?? initialSearchPartner?.partner ?? fetchedPartner?.partner; + initialSearchPartner?.partner ?? + loadedPartner ?? + fetchedPartner?.partner ?? + recentPartner?.partner; // Chunks prefer the fuller fetched set; the summary keeps the cached one so scores don't drift. const searchPartner = fetchedPartner ?? initialSearchPartner; const searchSummary = diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts index f6c1caa6609..6c7c0599f3d 100644 --- a/apps/web/lib/partner-content-search/retrieval.ts +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -1,6 +1,8 @@ import { DubApiError } from "@/lib/api/errors"; import { DISCOVERABLE_NETWORK_STATUSES } from "@/lib/api/network/partner-network-listing-where"; import { prisma } from "@/lib/prisma"; +import { redis } from "@/lib/upstash"; +import { createHash } from "crypto"; import { PlatformType, Prisma } from "@prisma/client"; import { PARTNER_CONTENT_CHUNK_VECTOR_DISTANCE, @@ -28,6 +30,30 @@ import { VoyageTimeoutError, } from "./voyage"; +// Cache query embeddings globally (list search + detail rerank share the same query). +const QUERY_EMBEDDING_CACHE_TTL_SECONDS = 60 * 60 * 24; + +function queryEmbeddingCacheKey(query: string) { + const digest = createHash("sha256").update(query).digest("hex").slice(0, 32); + return `pcs:query-embedding:${PARTNER_CONTENT_SEARCH_MODELS.embedding.id}:${digest}`; +} + +async function readCachedQueryEmbedding(key: string) { + try { + const cached = await redis.get(key); + return Array.isArray(cached) && cached.length ? cached : null; + } catch { + return null; + } +} + +function cacheQueryEmbedding(key: string, embedding: number[]) { + // Fire-and-forget — never block the search on the cache write. + redis + .set(key, embedding, { ex: QUERY_EMBEDDING_CACHE_TTL_SECONDS }) + .catch(() => {}); +} + type PartnerContentSearchCandidateRow = Pick< PartnerContentSearchRow, "chunkId" | "partnerContentItemId" | "chunkSource" | "distance" @@ -59,26 +85,36 @@ export async function searchPartnerNetworkContent({ rerank: boolean; logTiming?: PartnerContentSearchTimingLogger; }) { - let queryEmbedding: number[]; + const embeddingCacheKey = queryEmbeddingCacheKey(query); logTiming?.("query-embedding-start"); - try { - [queryEmbedding] = await embedPartnerContentTexts({ - input: [query], - inputType: "query", - timeoutMs: PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, + let queryEmbedding: number[]; + const cachedEmbedding = await readCachedQueryEmbedding(embeddingCacheKey); + if (cachedEmbedding) { + queryEmbedding = cachedEmbedding; + logTiming?.("query-embedding-cache-hit", { + embeddingDimensions: queryEmbedding.length, }); - } catch (error) { - if (error instanceof VoyageTimeoutError) { - throw new DubApiError({ - code: "internal_server_error", - message: "Partner content search timed out. Please try again.", + } else { + try { + [queryEmbedding] = await embedPartnerContentTexts({ + input: [query], + inputType: "query", + timeoutMs: PARTNER_CONTENT_SEARCH_VOYAGE_QUERY_TIMEOUT_MS, }); + } catch (error) { + if (error instanceof VoyageTimeoutError) { + throw new DubApiError({ + code: "internal_server_error", + message: "Partner content search timed out. Please try again.", + }); + } + throw error; } - throw error; + cacheQueryEmbedding(embeddingCacheKey, queryEmbedding); + logTiming?.("query-embedding-complete", { + embeddingDimensions: queryEmbedding.length, + }); } - logTiming?.("query-embedding-complete", { - embeddingDimensions: queryEmbedding.length, - }); const queryVector = serializeEmbeddingForVector(queryEmbedding); // Eligibility filters inline in the ANN query; post-filtering would starve the candidate pool. From 44dae44131d1f652c1c8fb2db9067617ab48646a Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 14:10:33 -0400 Subject: [PATCH 21/25] rename code to reflect content summary instead of individual content bars on detail page --- ...-fit-bars.tsx => coverage-summary-bar.tsx} | 4 +- .../network/[partnerId]/search-fit-panel.tsx | 4 +- .../network/[partnerId]/search-fit-utils.ts | 38 ++++++++--------- .../network-content-search-results.tsx | 24 +++++------ .../lib/partner-content-search/constants.ts | 2 +- ...ntent-match-bars.ts => content-matches.ts} | 41 ++++++++++--------- .../partner-content-search/match-summaries.ts | 36 ++++++++-------- .../match-summary-queries.ts | 6 +-- .../top-content-ranking.ts | 2 +- apps/web/lib/zod/schemas/partner-network.ts | 4 +- 10 files changed, 82 insertions(+), 79 deletions(-) rename apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/{search-fit-bars.tsx => coverage-summary-bar.tsx} (93%) rename apps/web/lib/partner-content-search/{content-match-bars.ts => content-matches.ts} (83%) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/coverage-summary-bar.tsx similarity index 93% rename from apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx rename to apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/coverage-summary-bar.tsx index 2e676b85a90..c5c6ba94ec9 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-bars.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/coverage-summary-bar.tsx @@ -3,14 +3,14 @@ import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; import { cn } from "@dub/utils"; -// Coverage bar from summary counts (strong / partial / no match). +// Single coverage summary bar from summary counts (strong / partial / no match). const SEGMENTS = [ { key: "strong", label: "Strong", color: "bg-[#1D9E75]" }, { key: "partial", label: "Partial", color: "bg-[#EF9F27]" }, { key: "none", label: "No match", color: "bg-neutral-300" }, ] as const; -export function ContentMatchBars({ +export function CoverageSummaryBar({ summary, }: { summary: PartnerContentSearchPartner["matchSummary"] | undefined; diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx index 4772f0d7bf1..491d8378ebe 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx @@ -14,7 +14,7 @@ import { ContentMatchSkeletons, formatRankWindowPhrase, } from "./content-match-row"; -import { ContentMatchBars } from "./search-fit-bars"; +import { CoverageSummaryBar } from "./coverage-summary-bar"; import { buildMatchedContentItems, buildUnifiedRelevanceMap, @@ -145,7 +145,7 @@ export function SearchFitPanel({ )}
- + {metaParts.length > 0 && (
diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts index dc2f5d7605c..adc7c7351aa 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts @@ -37,9 +37,9 @@ export function buildUnifiedRelevanceMap( relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, ) { const map = new Map(); - for (const bar of relevanceSummary?.contentBars ?? []) { - const score = getEvidenceDisplayScore(bar.matchEvidence); - if (score != null) map.set(bar.partnerContentItemId, score); + for (const match of relevanceSummary?.contentMatches ?? []) { + const score = getEvidenceDisplayScore(match.matchEvidence); + if (score != null) map.set(match.partnerContentItemId, score); } return map; } @@ -49,7 +49,7 @@ export function buildMatchedContentItems( chunks: PartnerContentSearchPartner["chunks"], unifiedRelevanceByItemId?: Map, ): MatchedContentItem[] { - const bars = summary?.contentBars ?? []; + const matches = summary?.contentMatches ?? []; const bestChunkByContentItemId = new Map< string, @@ -62,33 +62,33 @@ export function buildMatchedContentItems( } } - const baselineViews = getViewBaseline(bars.map((bar) => bar.viewCount)); + const baselineViews = getViewBaseline(matches.map((match) => match.viewCount)); - return bars - .filter((bar) => bar.matched) - .map((bar): MatchedContentItem | null => { - // Skip rows we can't render; matched count in the header still comes from all matched bars. - const chunk = bestChunkByContentItemId.get(bar.partnerContentItemId); + return matches + .filter((match) => match.matched) + .map((match): MatchedContentItem | null => { + // Skip rows we can't render; matched count in the header still comes from all matched content. + const chunk = bestChunkByContentItemId.get(match.partnerContentItemId); if (!chunk) return null; const relevance = - unifiedRelevanceByItemId?.get(bar.partnerContentItemId) ?? - getEvidenceDisplayScore(bar.matchEvidence) ?? - bar.matchScore ?? + unifiedRelevanceByItemId?.get(match.partnerContentItemId) ?? + getEvidenceDisplayScore(match.matchEvidence) ?? + match.matchScore ?? 0; return { - contentItemId: bar.partnerContentItemId, - platform: bar.platform, - publishedAt: bar.publishedAt, - viewCount: bar.viewCount, + contentItemId: match.partnerContentItemId, + platform: match.platform, + publishedAt: match.publishedAt, + viewCount: match.viewCount, relevance, blendedScore: getBlendedTopContentScore({ relevance, - views: bar.viewCount, + views: match.viewCount, baselineViews, }), - matchEvidence: bar.matchEvidence, + matchEvidence: match.matchEvidence, chunk, }; }) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx index 1faeb7a3eb5..9bd2b9e4b25 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/network-content-search-results.tsx @@ -38,12 +38,12 @@ function contentLabel(platform?: PlatformType) { const TOP_CONTENT_PREVIEW_COUNT = 2; -type ContentSearchBar = NonNullable< +type ContentMatch = NonNullable< PartnerContentSearchPartner["matchSummary"] ->["contentBars"][number]; +>["contentMatches"][number]; type ContentPreviewItem = { chunk: ContentSearchChunk; - bar?: ContentSearchBar; + match?: ContentMatch; }; export function NetworkContentSearchResults({ @@ -253,10 +253,10 @@ function NetworkPartnerContentMatch({ function getContentPreviewItems( partner: PartnerContentSearchPartner, ): ContentPreviewItem[] { - const barsByContentItemId = new Map( - partner.matchSummary?.contentBars.map((bar) => [ - bar.partnerContentItemId, - bar, + const matchByContentItemId = new Map( + partner.matchSummary?.contentMatches.map((match) => [ + match.partnerContentItemId, + match, ]) ?? [], ); const seenContentItems = new Set(); @@ -268,7 +268,7 @@ function getContentPreviewItems( seenContentItems.add(chunk.partnerContentItemId); items.push({ chunk, - bar: barsByContentItemId.get(chunk.partnerContentItemId), + match: matchByContentItemId.get(chunk.partnerContentItemId), }); if (items.length === TOP_CONTENT_PREVIEW_COUNT) break; @@ -373,7 +373,7 @@ function ContentPreviewImage({ } function ContentPreviewTooltip({ item }: { item: ContentPreviewItem }) { - const { chunk, bar } = item; + const { chunk, match } = item; const title = getContentTitle(chunk); const metadata = [ formatDuration(chunk.content.durationMs), @@ -381,7 +381,7 @@ function ContentPreviewTooltip({ item }: { item: ContentPreviewItem }) { ] .filter(Boolean) .join(" · "); - const engagementMetrics = getContentEngagementMetrics(chunk, bar); + const engagementMetrics = getContentEngagementMetrics(chunk, match); const matchLocation = formatChunkMatchLocation(chunk); const matchScore = formatMatchPercent(chunk.rerankScore ?? chunk.score); @@ -440,10 +440,10 @@ function ContentPreviewTooltip({ item }: { item: ContentPreviewItem }) { function getContentEngagementMetrics( chunk: ContentSearchChunk, - bar: ContentSearchBar | undefined, + match: ContentMatch | undefined, ) { return [ - { label: "Views", value: chunk.content.viewCount ?? bar?.viewCount }, // views may still fall back to slim bar + { label: "Views", value: chunk.content.viewCount ?? match?.viewCount }, // views may still fall back to the slim content match { label: "Likes", value: chunk.content.likeCount }, { label: "Comments", value: chunk.content.commentCount }, { label: "Shares", value: chunk.content.shareCount }, diff --git a/apps/web/lib/partner-content-search/constants.ts b/apps/web/lib/partner-content-search/constants.ts index ace2ac10422..6ca671cbe57 100644 --- a/apps/web/lib/partner-content-search/constants.ts +++ b/apps/web/lib/partner-content-search/constants.ts @@ -57,7 +57,7 @@ export const PARTNER_CONTENT_SEARCH_TOPIC_FIT = { } as const; // Tuning for the detail-pane "Top content" list only — never feeds Topic Fit, the -// bars, or partner ordering. +// coverage summary, or partner ordering. export const PARTNER_CONTENT_SEARCH_TOP_CONTENT = { topContentCount: 5, relevanceWeight: 0.7, diff --git a/apps/web/lib/partner-content-search/content-match-bars.ts b/apps/web/lib/partner-content-search/content-matches.ts similarity index 83% rename from apps/web/lib/partner-content-search/content-match-bars.ts rename to apps/web/lib/partner-content-search/content-matches.ts index 4b6eb05d227..c92874c93da 100644 --- a/apps/web/lib/partner-content-search/content-match-bars.ts +++ b/apps/web/lib/partner-content-search/content-matches.ts @@ -9,14 +9,14 @@ import { type SourceScoreByContentItemId, } from "./ranking"; import { toScore, type PartnerContentSearchRow } from "./search-utils"; -import type { PartnerRecentContentBarRow } from "./match-summary-queries"; +import type { PartnerRecentContentRow } from "./match-summary-queries"; export type PartnerContentItemBestMatch = { transcriptScore: number | null; creatorTextScore: number | null; }; -export type PartnerContentMatchBar = { +export type PartnerContentMatch = { partnerContentItemId: string; platform: string; publishedAt: string | null; @@ -77,7 +77,7 @@ export function getQueryContentMatch({ row, context, }: { - row: PartnerRecentContentBarRow; + row: PartnerRecentContentRow; context: QueryContentMatchContext; }) { // On-topic gate: rerank score when present, else cosine cutoff (no lexical boost). @@ -124,7 +124,7 @@ export function getListContentMatch({ row, context, }: { - row: PartnerRecentContentBarRow; + row: PartnerRecentContentRow; context: ListContentMatchContext; }) { const match = context.bestMatchByContentItemId.get(row.partnerContentItemId); @@ -140,13 +140,13 @@ export function getListContentMatch({ }; } -export function toContentMatchBar({ +export function toContentMatch({ row, context, }: { - row: PartnerRecentContentBarRow; + row: PartnerRecentContentRow; context: ContentMatchContext; -}): PartnerContentMatchBar { +}): PartnerContentMatch { const { matchEvidence, matchScore } = "bestMatchByContentItemId" in context ? getListContentMatch({ row, context }) @@ -167,41 +167,42 @@ export function toContentMatchBar({ // Strong vs partial split for the coverage bar (calibrated score ≥ 0.7). const STRONG_MATCH_DISPLAY_THRESHOLD = 0.7; -export function getContentBarMatchStats(contentBars: PartnerContentMatchBar[]) { - const matchedBars = contentBars.filter((bar) => bar.matched); - const matchedContentCount = matchedBars.length; - const strongMatchedContentCount = matchedBars.filter( - (bar) => (bar.matchScore ?? 0) >= STRONG_MATCH_DISPLAY_THRESHOLD, +export function getContentMatchStats(contentMatches: PartnerContentMatch[]) { + const matchedContent = contentMatches.filter((match) => match.matched); + const matchedContentCount = matchedContent.length; + const strongMatchedContentCount = matchedContent.filter( + (match) => (match.matchScore ?? 0) >= STRONG_MATCH_DISPLAY_THRESHOLD, ).length; const partialMatchedContentCount = matchedContentCount - strongMatchedContentCount; - const transcriptMatchedContentCount = contentBars.filter( + const transcriptMatchedContentCount = contentMatches.filter( ({ matchEvidence }) => matchEvidence.sources.includes("transcript"), ).length; - const creatorTextMatchedContentCount = contentBars.filter( + const creatorTextMatchedContentCount = contentMatches.filter( ({ matchEvidence }) => matchEvidence.sources.includes("creatorText"), ).length; - const creatorTextOnlyContentCount = contentBars.filter( + const creatorTextOnlyContentCount = contentMatches.filter( ({ matchEvidence }) => matchEvidence.primarySource === "creatorText" && matchEvidence.sources.length === 1, ).length; const weightedMatchedContentCount = Number( - contentBars - .reduce((total, bar) => total + bar.matchEvidence.weight, 0) + contentMatches + .reduce((total, match) => total + match.matchEvidence.weight, 0) .toFixed(3), ); const weightedMatchedContentScore = Number( - contentBars + contentMatches .reduce( - (total, bar) => total + (getEvidenceTopicScore(bar.matchEvidence) ?? 0), + (total, match) => + total + (getEvidenceTopicScore(match.matchEvidence) ?? 0), 0, ) .toFixed(3), ); return { - matchedBars, + matchedContent, matchedContentCount, strongMatchedContentCount, partialMatchedContentCount, diff --git a/apps/web/lib/partner-content-search/match-summaries.ts b/apps/web/lib/partner-content-search/match-summaries.ts index f1c65103976..ce45c6f80af 100644 --- a/apps/web/lib/partner-content-search/match-summaries.ts +++ b/apps/web/lib/partner-content-search/match-summaries.ts @@ -2,10 +2,10 @@ import { PlatformType } from "@prisma/client"; import { ContentMatchContext, getBestMatchByContentItemId, - getContentBarMatchStats, + getContentMatchStats, QueryContentMatchContext, - toContentMatchBar, -} from "./content-match-bars"; + toContentMatch, +} from "./content-matches"; import { fetchMatchSummaryBaseData } from "./match-summary-queries"; import { deriveTopicFit, @@ -110,14 +110,14 @@ export async function getPartnerMatchSummaries({ const contentMatchContext: ContentMatchContext = queryMatchContext ?? { bestMatchByContentItemId, }; - const contentBars = recentRows.map((row) => - toContentMatchBar({ + const contentMatches = recentRows.map((row) => + toContentMatch({ row, context: contentMatchContext, }), ); const { - matchedBars, + matchedContent, matchedContentCount, strongMatchedContentCount, partialMatchedContentCount, @@ -126,8 +126,8 @@ export async function getPartnerMatchSummaries({ creatorTextOnlyContentCount, weightedMatchedContentCount, weightedMatchedContentScore, - } = getContentBarMatchStats(contentBars); - const recentContentCount = contentBars.length; + } = getContentMatchStats(contentMatches); + const recentContentCount = contentMatches.length; const { topicFit, band } = deriveTopicFit({ matchedContentCount, weightedMatchedContentCount, @@ -135,13 +135,13 @@ export async function getPartnerMatchSummaries({ recentContentCount, }); const medianViews = median( - matchedBars - .map((bar) => bar.viewCount) + matchedContent + .map((match) => match.viewCount) .filter((views): views is number => views != null && views > 0), { round: true }, ); - const matchedTimestamps = matchedBars - .map((bar) => bar.publishedAt) + const matchedTimestamps = matchedContent + .map((match) => match.publishedAt) .filter((publishedAt): publishedAt is string => Boolean(publishedAt)) .map((publishedAt) => new Date(publishedAt).getTime()); const lastOnTopicAt = matchedTimestamps.length @@ -149,16 +149,18 @@ export async function getPartnerMatchSummaries({ : null; const followers = followersByPartner.get(partnerId) ?? null; const platformFrequency = new Map(); - for (const bar of matchedBars.length ? matchedBars : contentBars) { + for (const match of matchedContent.length + ? matchedContent + : contentMatches) { platformFrequency.set( - bar.platform, - (platformFrequency.get(bar.platform) ?? 0) + 1, + match.platform, + (platformFrequency.get(match.platform) ?? 0) + 1, ); } const topPlatforms = Array.from(platformFrequency.entries()) .sort((a, b) => b[1] - a[1]) .map(([platform]) => platform); - const publishedDates = contentBars + const publishedDates = contentMatches .map(({ publishedAt }) => (publishedAt ? new Date(publishedAt) : null)) .filter((date): date is Date => Boolean(date)); const oldestPublishedAt = publishedDates.length @@ -208,7 +210,7 @@ export async function getPartnerMatchSummaries({ ).sort(), oldestPublishedAt, newestPublishedAt, - contentBars, + contentMatches, }, ] as const; }), diff --git a/apps/web/lib/partner-content-search/match-summary-queries.ts b/apps/web/lib/partner-content-search/match-summary-queries.ts index 8c982a66e30..1cb7b22458f 100644 --- a/apps/web/lib/partner-content-search/match-summary-queries.ts +++ b/apps/web/lib/partner-content-search/match-summary-queries.ts @@ -6,8 +6,8 @@ import { } from "./constants"; import type { PartnerContentSearchTimingLogger } from "./timing"; -// Slim bar row for aggregates only; display fields come from loaded chunks. -export type PartnerRecentContentBarRow = { +// Slim content row for aggregates only; display fields come from loaded chunks. +export type PartnerRecentContentRow = { partnerId: string; partnerContentItemId: string; platformType: string; @@ -73,7 +73,7 @@ export async function fetchMatchSummaryBaseData({ }); const recentContentRowsPromise = prisma - .$queryRaw( + .$queryRaw( Prisma.sql` SELECT partnerId, diff --git a/apps/web/lib/partner-content-search/top-content-ranking.ts b/apps/web/lib/partner-content-search/top-content-ranking.ts index 3237503b80d..9b48e529310 100644 --- a/apps/web/lib/partner-content-search/top-content-ranking.ts +++ b/apps/web/lib/partner-content-search/top-content-ranking.ts @@ -1,6 +1,6 @@ // Pure ranking helpers for the partner detail-pane "Top content" list. Dependency- // free (constants only) so it stays client-safe — no server-only code (voyage, db) -// in the bundle. Orders that list only; never feeds Topic Fit, the bars, or counts. +// in the bundle. Orders that list only; never feeds Topic Fit, the coverage summary, or counts. import { PARTNER_CONTENT_SEARCH_TOP_CONTENT } from "./constants"; import { median } from "./search-utils"; diff --git a/apps/web/lib/zod/schemas/partner-network.ts b/apps/web/lib/zod/schemas/partner-network.ts index a4f80e6cf56..bdf8c046057 100644 --- a/apps/web/lib/zod/schemas/partner-network.ts +++ b/apps/web/lib/zod/schemas/partner-network.ts @@ -204,7 +204,7 @@ const partnerContentSearchChunkSchema = z.object({ matchEvidence: partnerContentMatchEvidenceSchema.optional(), }); -const partnerContentSearchContentBarSchema = z.object({ +const partnerContentSearchContentMatchSchema = z.object({ partnerContentItemId: z.string(), platform: z.string(), publishedAt: z.string().nullable(), @@ -243,7 +243,7 @@ const partnerContentSearchMatchSummarySchema = z.object({ sources: z.array(z.string()), oldestPublishedAt: z.string().nullable(), newestPublishedAt: z.string().nullable(), - contentBars: z.array(partnerContentSearchContentBarSchema), + contentMatches: z.array(partnerContentSearchContentMatchSchema), }); const partnerContentSearchResponsePartnerSchema = z.object({ From 10a342a87f44cc1d181b604895c7e8edbbbc5eb0 Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 14:38:22 -0400 Subject: [PATCH 22/25] minor language changes on partner detail content page --- .../network/[partnerId]/content-match-row.tsx | 16 ++++----- .../network/[partnerId]/page-client.tsx | 2 +- .../[partnerId]/recent-content-panel.tsx | 2 +- .../network/[partnerId]/search-fit-panel.tsx | 30 +++++++++-------- .../program/network/content-display-utils.ts | 33 +++++++++++++++++++ 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx index e593ecab215..f826af73afd 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx @@ -4,6 +4,7 @@ import { PARTNER_CONTENT_SEARCH_LIMITS } from "@/lib/partner-content-search/cons import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; import { cn, nFormatter } from "@dub/utils"; import { + contentNoun, formatDuration, formatMatchPercent, formatPublishedDate, @@ -157,6 +158,7 @@ export function formatRankWindowPhrase( if (!summary || !summary.recentContentCount) return null; const { recentContentCount, oldestPublishedAt, newestPublishedAt } = summary; + const noun = contentNoun(summary.platforms ?? [], recentContentCount); const oldest = formatMonthYear(oldestPublishedAt); const newest = formatMonthYear(newestPublishedAt); const countCapped = @@ -164,21 +166,17 @@ export function formatRankWindowPhrase( if (countCapped) { return oldest - ? `the ${recentContentCount} most recent posts, back to ${oldest}` - : `the ${recentContentCount} most recent posts`; + ? `the ${recentContentCount} most recent ${noun}, back to ${oldest}` + : `the ${recentContentCount} most recent ${noun}`; } if (oldest && newest) { return oldest === newest - ? `${recentContentCount} ${ - recentContentCount === 1 ? "post" : "posts" - } from ${oldest}` - : `${recentContentCount} posts, ${oldest} – ${newest}`; + ? `${recentContentCount} ${noun} from ${oldest}` + : `${recentContentCount} ${noun}, ${oldest} – ${newest}`; } - return `${recentContentCount} recent ${ - recentContentCount === 1 ? "post" : "posts" - }`; + return `${recentContentCount} recent ${noun}`; } function formatMonthYear(iso: string | null) { diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 19784e69be1..e02d5f63e84 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -186,7 +186,7 @@ export function NetworkPartnerDetailContent({ partnerId={partnerId} currentTabId={currentTabId} setCurrentTabId={setCurrentTabId} - aboutLabel="Content matches" + aboutLabel="Content" />
{currentTabId === "about" && ( diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx index e96d67fbd4a..fac8e039aef 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/recent-content-panel.tsx @@ -72,7 +72,7 @@ export function RecentContentPanel({ const metaParts = [ followers ? `${nFormatter(followers)} followers` : null, medianViews ? `${nFormatter(medianViews)} median views` : null, - lastPublished ? `last post ${lastPublished}` : null, + lastPublished ? `last published ${lastPublished}` : null, ].filter((part): part is string => Boolean(part)); return ( diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx index 491d8378ebe..8c0abd232cd 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx @@ -8,7 +8,7 @@ import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content- import { Button } from "@dub/ui"; import { cn, nFormatter } from "@dub/utils"; import { useState } from "react"; -import { lastPostedLabel } from "../content-display-utils"; +import { contentNoun, lastPostedLabel } from "../content-display-utils"; import { ContentMatchRow, ContentMatchSkeletons, @@ -24,11 +24,11 @@ import { } from "./search-fit-utils"; const BAND_HEADLINE: Record = { - consistent: "Consistently on-topic", - frequent: "Frequently on-topic", - occasional: "Occasionally on-topic", - "one-off": "One-off mention", - none: "No recent matches", + consistent: "Consistently posts about this", + frequent: "Frequently posts about this", + occasional: "Occasionally posts about this", + "one-off": "Posted about this once", + none: "No recent posts about this", }; const TOPIC_FIT_BAND_CHIP: Record = { @@ -105,7 +105,7 @@ export function SearchFitPanel({ summary?.medianViews ? `${nFormatter(summary.medianViews)} median views` : null, - lastOnTopic ? `last post ${lastOnTopic}` : null, + lastOnTopic ? `last published ${lastOnTopic}` : null, ].filter((part): part is string => Boolean(part)); const rankWindowPhrase = formatRankWindowPhrase(summary); const topContentCaption = rankWindowPhrase @@ -131,8 +131,12 @@ export function SearchFitPanel({
{summary && (

- {summary.matchedContentCount} of {summary.recentContentCount} recent - posts related to{" "} + {summary.matchedContentCount} of {summary.recentContentCount} recent{" "} + {contentNoun( + summary.platforms ?? [], + summary.recentContentCount, + )}{" "} + related to{" "} {query ? ( “{truncateQuery(query)}” @@ -155,10 +159,10 @@ export function SearchFitPanel({

- {/* Top content — ranked by the relevance + reach blend */} + {/* Top matches — ranked by the relevance + reach blend */}

- Top content + Most relevant content

{topContentCaption} @@ -188,12 +192,12 @@ export function SearchFitPanel({ )}

- {/* All content — same matched set, simply most-recent-first */} + {/* All matches — same matched set, simply most-recent-first */} {showAllSection && (

- All content + All matching content

Most recent first diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts index 7451d1bf888..8409e53a190 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts @@ -70,6 +70,39 @@ export function lastPostedLabel(iso: string | null | undefined) { return `${Math.floor(days / 365)}y ago`; } +// Platform-aware noun for a partner's content so labels read naturally per type, and +// only fall back to a generic term when the set spans types: YouTube/TikTok → "videos", +// Instagram/X/LinkedIn → "posts", website → "pages", mixed/unknown → "content items". +const PLATFORM_CONTENT_NOUN: Record = { + youtube: { one: "video", many: "videos" }, + tiktok: { one: "video", many: "videos" }, + instagram: { one: "post", many: "posts" }, + twitter: { one: "post", many: "posts" }, + x: { one: "post", many: "posts" }, + linkedin: { one: "post", many: "posts" }, + website: { one: "page", many: "pages" }, +}; + +export function contentNoun(platforms: string[], count: number) { + const plural = count !== 1; + const manyForms = new Set( + platforms + .map((platform) => PLATFORM_CONTENT_NOUN[platform]?.many) + .filter((noun): noun is string => Boolean(noun)), + ); + + if (manyForms.size === 1) { + const many = [...manyForms][0]; + if (plural) return many; + const entry = Object.values(PLATFORM_CONTENT_NOUN).find( + (noun) => noun.many === many, + ); + return entry?.one ?? "content item"; + } + + return plural ? "content items" : "content item"; +} + export function getContentThumbnail(chunk: ContentSearchChunk) { if (chunk.content.thumbnailUrl) { return getPartnerContentThumbnailUrl(chunk.content.thumbnailUrl); From 4685eac86d91df18ea3f2d7e4aa5468969c923d9 Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 14:41:15 -0400 Subject: [PATCH 23/25] extend redis embedding cache to 60 days; extend on read. estimated cost $0.26 per 1K cached queries per TTL --- apps/web/lib/partner-content-search/retrieval.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/web/lib/partner-content-search/retrieval.ts b/apps/web/lib/partner-content-search/retrieval.ts index 6c7c0599f3d..a9b5f418dc3 100644 --- a/apps/web/lib/partner-content-search/retrieval.ts +++ b/apps/web/lib/partner-content-search/retrieval.ts @@ -30,8 +30,8 @@ import { VoyageTimeoutError, } from "./voyage"; -// Cache query embeddings globally (list search + detail rerank share the same query). -const QUERY_EMBEDDING_CACHE_TTL_SECONDS = 60 * 60 * 24; +// cache query embeddings for 60 days (query, embedding model). refresh on read +const QUERY_EMBEDDING_CACHE_TTL_SECONDS = 60 * 60 * 24 * 60; // 60 days function queryEmbeddingCacheKey(query: string) { const digest = createHash("sha256").update(query).digest("hex").slice(0, 32); @@ -40,7 +40,10 @@ function queryEmbeddingCacheKey(query: string) { async function readCachedQueryEmbedding(key: string) { try { - const cached = await redis.get(key); + // GETEX re-arms the TTL on every hit so frequently searched queries never expire. + const cached = await redis.getex(key, { + ex: QUERY_EMBEDDING_CACHE_TTL_SECONDS, + }); return Array.isArray(cached) && cached.length ? cached : null; } catch { return null; From 91b4500b600229f9ab72f930ed47f7c3e2c4b755 Mon Sep 17 00:00:00 2001 From: David Clark Date: Wed, 24 Jun 2026 16:10:58 -0400 Subject: [PATCH 24/25] improve ranking performance to clip to reranker results only; do not use cosine ANN for tail results + drop verified platform requirement for content-search hydration --- .../network/[partnerId]/page-client.tsx | 8 ++-- .../network/[partnerId]/search-fit-panel.tsx | 17 ++++---- .../network/[partnerId]/search-fit-utils.ts | 41 ++++++++----------- .../network-partners.ts | 6 ++- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index e02d5f63e84..99e7cefd0d2 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -123,11 +123,13 @@ export function NetworkPartnerDetailContent({ const searchPartner = fetchedPartner ?? initialSearchPartner; const searchSummary = initialSearchPartner?.matchSummary ?? fetchedPartner?.matchSummary; - const relevanceSummary = fetchedPartner?.matchSummary ?? null; + // Relevance is sourced from the scoped fetch's chunks on a single scale (rerank, + // or cosine when the reranker failed) — never blended. + const reranked = searchResults?.reranked ?? false; // Deep-link with nothing cached → full skeleton; otherwise a silent refinement. const isFallbackLoading = isLoadingSearch && !initialSearchPartner; const isRefiningRelevance = - isLoadingSearch && Boolean(initialSearchPartner) && !relevanceSummary; + isLoadingSearch && Boolean(initialSearchPartner) && !fetchedPartner; if (!isLoadingPartner && !partner) { return ( @@ -204,7 +206,7 @@ export function NetworkPartnerDetailContent({ isRefining={isRefiningRelevance} query={search} summary={searchSummary} - relevanceSummary={relevanceSummary} + reranked={reranked} searchPartner={searchPartner} /> ) : ( diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx index 8c0abd232cd..7a2b69c17ab 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx @@ -16,8 +16,8 @@ import { } from "./content-match-row"; import { CoverageSummaryBar } from "./coverage-summary-bar"; import { + buildContentRelevanceMap, buildMatchedContentItems, - buildUnifiedRelevanceMap, DETAIL_CONTENT_INITIAL_DISPLAY_COUNT, DETAIL_CONTENT_PAGE_COUNT, publishedAtMs, @@ -56,7 +56,7 @@ export function SearchFitPanel({ isRefining = false, query, summary: initialSummary, - relevanceSummary, + reranked = false, searchPartner, }: { error: unknown; @@ -64,7 +64,7 @@ export function SearchFitPanel({ isRefining?: boolean; query?: string; summary?: PartnerContentSearchPartner["matchSummary"]; - relevanceSummary?: PartnerContentSearchPartner["matchSummary"] | null; + reranked?: boolean; searchPartner?: PartnerContentSearchPartner; }) { const [visibleAllCount, setVisibleAllCount] = useState( @@ -77,13 +77,10 @@ export function SearchFitPanel({ } const isLoadingRows = isLoading || isRefining; - const unifiedRelevanceByItemId = buildUnifiedRelevanceMap(relevanceSummary); - // Hold rows until scoped rerank lands so scores/order don't swap in. - const items = buildMatchedContentItems( - summary, - isLoadingRows ? [] : (searchPartner?.chunks ?? []), - unifiedRelevanceByItemId, - ); + // Hold rows until the scoped rerank lands so scores/order don't swap in. + const chunks = isLoadingRows ? [] : (searchPartner?.chunks ?? []); + const relevanceByItemId = buildContentRelevanceMap(chunks, reranked); + const items = buildMatchedContentItems(summary, chunks, relevanceByItemId); const topContent = [...items] .sort((a, b) => b.blendedScore - a.blendedScore) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts index adc7c7351aa..88b79f50933 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-utils.ts @@ -21,25 +21,23 @@ export type MatchedContentItem = { chunk: PartnerContentSearchPartner["chunks"][number]; }; -export function getEvidenceDisplayScore( - evidence: PartnerContentMatchEvidence | undefined, -) { - if (!evidence || evidence.sources.length === 0) return null; - - return Math.max( - evidence.transcriptScore ?? 0, - evidence.creatorTextScore ?? 0, - ); -} - -// Per-item relevance on one scale from the scoped rerank (cached global summary mixes scales). -export function buildUnifiedRelevanceMap( - relevanceSummary: PartnerContentSearchPartner["matchSummary"] | null | undefined, +// Per-item relevance straight from the served chunks, on a single scale: rerank +// scores when the search reranked, cosine otherwise. The reranker already clips to +// its window and drops the cosine tail, so the two scales are never blended — and if +// the reranker failed, every row falls back to cosine together. +export function buildContentRelevanceMap( + chunks: PartnerContentSearchPartner["chunks"], + reranked: boolean, ) { const map = new Map(); - for (const match of relevanceSummary?.contentMatches ?? []) { - const score = getEvidenceDisplayScore(match.matchEvidence); - if (score != null) map.set(match.partnerContentItemId, score); + for (const chunk of chunks) { + const score = reranked ? chunk.rerankScore : chunk.cosineScore; + if (score == null) continue; + + const current = map.get(chunk.partnerContentItemId); + if (current == null || score > current) { + map.set(chunk.partnerContentItemId, score); + } } return map; } @@ -47,7 +45,7 @@ export function buildUnifiedRelevanceMap( export function buildMatchedContentItems( summary: PartnerContentSearchPartner["matchSummary"] | undefined, chunks: PartnerContentSearchPartner["chunks"], - unifiedRelevanceByItemId?: Map, + relevanceByItemId: Map, ): MatchedContentItem[] { const matches = summary?.contentMatches ?? []; @@ -71,11 +69,8 @@ export function buildMatchedContentItems( const chunk = bestChunkByContentItemId.get(match.partnerContentItemId); if (!chunk) return null; - const relevance = - unifiedRelevanceByItemId?.get(match.partnerContentItemId) ?? - getEvidenceDisplayScore(match.matchEvidence) ?? - match.matchScore ?? - 0; + // Single-scale relevance from the served chunks; no cross-scale fallback. + const relevance = relevanceByItemId.get(match.partnerContentItemId) ?? 0; return { contentItemId: match.partnerContentItemId, diff --git a/apps/web/lib/partner-content-search/network-partners.ts b/apps/web/lib/partner-content-search/network-partners.ts index 6c123e1efe8..0bb23b6bbb5 100644 --- a/apps/web/lib/partner-content-search/network-partners.ts +++ b/apps/web/lib/partner-content-search/network-partners.ts @@ -29,7 +29,11 @@ export async function getNetworkPartnersById({ const partners = await prisma.partner.findMany({ where: { AND: [ - partnerNetworkListingWhere({ partnerIds, country, platform: platforms }), + partnerNetworkListingWhere({ partnerIds, country }), + // do not require verified platforms for content search; move this out of partnerNetworkListingWhere + ...(platforms?.length + ? [{ platforms: { some: { type: { in: platforms } } } }] + : []), partnerReachWhere({ reach, platform: platforms }), ], }, From 42c259826480cd29e4b2f8a39931aa5169cdb463 Mon Sep 17 00:00:00 2001 From: David Clark Date: Sun, 28 Jun 2026 10:39:58 -0400 Subject: [PATCH 25/25] show full recent content feed on partner detail + share the recent-content list --- .../network/[partnerId]/content-match-row.tsx | 18 +- .../network/[partnerId]/page-client.tsx | 7 +- .../[partnerId]/recent-content-panel.tsx | 167 +++++++++--------- .../network/[partnerId]/search-fit-panel.tsx | 105 +++-------- .../program/network/content-display-utils.ts | 15 ++ .../(ee)/program/network/page-client.tsx | 4 +- 6 files changed, 137 insertions(+), 179 deletions(-) diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx index f826af73afd..da35c25288b 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/content-match-row.tsx @@ -2,10 +2,11 @@ import { PARTNER_CONTENT_SEARCH_LIMITS } from "@/lib/partner-content-search/constants"; import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; -import { cn, nFormatter } from "@dub/utils"; +import { cn } from "@dub/utils"; import { contentNoun, formatDuration, + formatEngagement, formatMatchPercent, formatPublishedDate, formatTimestamp, @@ -131,21 +132,6 @@ function formatMatchLocation({ return `Matched transcript ${formatTimestamp(startMs ?? endMs ?? 0)}`; } -// Reach + engagement we have for the post, in descending prominence. -function formatEngagement( - content: PartnerContentSearchPartner["chunks"][number]["content"], -) { - return [ - content.viewCount ? `${nFormatter(content.viewCount)} views` : null, - content.likeCount ? `${nFormatter(content.likeCount)} likes` : null, - content.commentCount ? `${nFormatter(content.commentCount)} comments` : null, - content.shareCount ? `${nFormatter(content.shareCount)} shares` : null, - content.saveCount ? `${nFormatter(content.saveCount)} saves` : null, - ] - .filter(Boolean) - .join(" · "); -} - function getMatchSnippet(chunk: PartnerContentSearchPartner["chunks"][number]) { return (chunk.chunk.text ?? "").trim() || null; } diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx index 99e7cefd0d2..8426292739a 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/page-client.tsx @@ -103,7 +103,9 @@ export function NetworkPartnerDetailContent({ error: recentError, isLoading: isLoadingRecent, } = usePartnerContentSearch({ - enabled: Boolean(workspaceId && !hasContentSearch), + // Always loaded: drives the no-search panel and the search panel's "All recent + // content" section. No-query path → a DB read, not a billed Voyage call. + enabled: Boolean(workspaceId), query: "", country, starred: false, @@ -208,6 +210,9 @@ export function NetworkPartnerDetailContent({ summary={searchSummary} reranked={reranked} searchPartner={searchPartner} + recentChunks={recentPartner?.chunks} + recentLoading={isLoadingRecent} + recentError={recentError} /> ) : ( (b.viewCount ?? 0) - (a.viewCount ?? 0)) .slice(0, PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount); - const recentContent = [...items].sort( - (a, b) => publishedAtMs(b.publishedAt) - publishedAtMs(a.publishedAt), - ); - const visibleRecent = recentContent.slice(0, visibleRecentCount); - const hiddenRecentCount = Math.max(0, recentContent.length - visibleRecent.length); // "Recent content" only earns its own section when it adds rows beyond the top set. const showRecentSection = !isLoading && - recentContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; + items.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; const categories = (partner?.categories ?? []).slice(0, MAX_NICHE_CHIPS); - const followers = sumFollowers(partner); - const medianViews = medianViewCount(items); - const lastPublished = lastPostedLabel( - recentContent.find((item) => item.publishedAt)?.publishedAt, - ); - const metaParts = [ - followers ? `${nFormatter(followers)} followers` : null, - medianViews ? `${nFormatter(medianViews)} median views` : null, - lastPublished ? `last published ${lastPublished}` : null, - ].filter((part): part is string => Boolean(part)); return (

@@ -90,11 +72,6 @@ export function RecentContentPanel({ ))}
)} - {metaParts.length > 0 && ( -
- {metaParts.join(" · ")} -
- )}
@@ -129,37 +106,83 @@ export function RecentContentPanel({ {showRecentSection && (
-
-

- Recent content -

-

- Most recent first -

-
+ +
+ )} +
+
+ ); +} -
- {visibleRecent.map((item) => ( - - ))} -
+// Newest-first content list with a "Show more" control. Shared by the no-search +// panel's "Recent content" and the search panel's "All recent content". +export function RecentContentList({ + chunks, + isLoading, + error, + title, + caption, + skeletonCount = PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount, +}: { + chunks: PartnerContentSearchPartner["chunks"]; + isLoading: boolean; + error: unknown; + title: string; + caption: string; + skeletonCount?: number; +}) { + const [visibleCount, setVisibleCount] = useState(RECENT_INITIAL_COUNT); + + const items = [...buildRecentItems(chunks)].sort( + (a, b) => publishedAtMs(b.publishedAt) - publishedAtMs(a.publishedAt), + ); + const visible = items.slice(0, visibleCount); + const hiddenCount = Math.max(0, items.length - visible.length); - {hiddenRecentCount > 0 ? ( -
-
- ) : null} + return ( +
+
+

+ {title} +

+

{caption}

+
+ +
+ {error ? ( +
+ Failed to load recent content +
+ ) : isLoading ? ( + + ) : visible.length ? ( + visible.map((item) => ( + + )) + ) : ( +
+ No indexed content found for this partner.
)}
+ + {hiddenCount > 0 ? ( +
+
+ ) : null}
); } @@ -184,15 +207,16 @@ function RecentContentRow({ item }: { item: RecentContentItem }) { {item.title}
- {meta && ( -
{meta}
- )} +
+ {meta && {meta}} + {item.engagement && ( + <> + {meta && ·} + {item.engagement} + + )} +
- {item.viewCount != null && item.viewCount > 0 && ( - - {nFormatter(item.viewCount)} views - - )} ); } @@ -215,30 +239,9 @@ function buildRecentItems( publishedAt: chunk.content.publishedAt, durationMs: chunk.content.durationMs, viewCount: chunk.content.viewCount, + engagement: formatEngagement(chunk.content), }); } return [...byContentItemId.values()]; } - -function sumFollowers(partner?: NetworkPartnerProps) { - if (!partner?.platforms?.length) return null; - const total = partner.platforms.reduce( - (sum, platform) => sum + Number(platform.subscribers ?? 0), - 0, - ); - return total > 0 ? total : null; -} - -function medianViewCount(items: RecentContentItem[]) { - const views = items - .map((item) => item.viewCount) - .filter((value): value is number => value != null && value > 0) - .sort((a, b) => a - b); - if (!views.length) return null; - - const mid = Math.floor(views.length / 2); - return views.length % 2 === 0 - ? Math.round((views[mid - 1] + views[mid]) / 2) - : views[mid]; -} diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx index 7a2b69c17ab..61124eb1e66 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/[partnerId]/search-fit-panel.tsx @@ -5,30 +5,26 @@ import { type PartnerContentTopicFitBand, } from "@/lib/partner-content-search/constants"; import { type PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; -import { Button } from "@dub/ui"; -import { cn, nFormatter } from "@dub/utils"; -import { useState } from "react"; -import { contentNoun, lastPostedLabel } from "../content-display-utils"; +import { cn } from "@dub/utils"; +import { contentNoun } from "../content-display-utils"; import { ContentMatchRow, ContentMatchSkeletons, formatRankWindowPhrase, } from "./content-match-row"; import { CoverageSummaryBar } from "./coverage-summary-bar"; +import { RecentContentList } from "./recent-content-panel"; import { buildContentRelevanceMap, buildMatchedContentItems, - DETAIL_CONTENT_INITIAL_DISPLAY_COUNT, - DETAIL_CONTENT_PAGE_COUNT, - publishedAtMs, } from "./search-fit-utils"; const BAND_HEADLINE: Record = { - consistent: "Consistently posts about this", - frequent: "Frequently posts about this", - occasional: "Occasionally posts about this", - "one-off": "Posted about this once", - none: "No recent posts about this", + consistent: "Consistently posts about this topic", + frequent: "Frequently posts about this topic", + occasional: "Occasionally posts about this topic", + "one-off": "Posted about this topic once", + none: "No recent posts about this topic", }; const TOPIC_FIT_BAND_CHIP: Record = { @@ -58,6 +54,9 @@ export function SearchFitPanel({ summary: initialSummary, reranked = false, searchPartner, + recentChunks, + recentLoading, + recentError, }: { error: unknown; isLoading: boolean; @@ -66,10 +65,12 @@ export function SearchFitPanel({ summary?: PartnerContentSearchPartner["matchSummary"]; reranked?: boolean; searchPartner?: PartnerContentSearchPartner; + // The partner's full recent content (matched + unmatched) for the "All recent + // content" section — fetched separately via the no-query path. + recentChunks?: PartnerContentSearchPartner["chunks"]; + recentLoading: boolean; + recentError: unknown; }) { - const [visibleAllCount, setVisibleAllCount] = useState( - DETAIL_CONTENT_INITIAL_DISPLAY_COUNT, - ); const summary = initialSummary ?? searchPartner?.matchSummary; if (isLoading && !summary) { @@ -85,25 +86,8 @@ export function SearchFitPanel({ const topContent = [...items] .sort((a, b) => b.blendedScore - a.blendedScore) .slice(0, PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount); - const allContent = [...items].sort( - (a, b) => publishedAtMs(b.publishedAt) - publishedAtMs(a.publishedAt), - ); - const visibleAll = allContent.slice(0, visibleAllCount); - const hiddenAllCount = Math.max(0, allContent.length - visibleAll.length); - // "All content" only earns its place when it adds rows beyond the top set. - const showAllSection = - !isLoadingRows && - allContent.length > PARTNER_CONTENT_SEARCH_TOP_CONTENT.topContentCount; const band = summary?.band ?? "none"; - const lastOnTopic = lastPostedLabel(summary?.lastOnTopicAt ?? null); - const metaParts = [ - summary?.followers ? `${nFormatter(summary.followers)} followers` : null, - summary?.medianViews - ? `${nFormatter(summary.medianViews)} median views` - : null, - lastOnTopic ? `last published ${lastOnTopic}` : null, - ].filter((part): part is string => Boolean(part)); const rankWindowPhrase = formatRankWindowPhrase(summary); const topContentCaption = rankWindowPhrase ? `Ranked by relevance + reach across ${rankWindowPhrase}.` @@ -147,19 +131,13 @@ export function SearchFitPanel({
- - {metaParts.length > 0 && ( -
- {metaParts.join(" · ")} -
- )}
{/* Top matches — ranked by the relevance + reach blend */}

- Most relevant content + Top relevant content

{topContentCaption} @@ -189,44 +167,17 @@ export function SearchFitPanel({ )}

- {/* All matches — same matched set, simply most-recent-first */} - {showAllSection && ( -
-
-

- All matching content -

-

- Most recent first -

-
- -
- {visibleAll.map((item) => ( - - ))} -
- - {hiddenAllCount > 0 ? ( -
-
- ) : null} -
- )} + {/* All recent content — the partner's full feed (matched + unmatched), + newest-first, so a brand can judge their overall output, not just matches */} +
+ +
); diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts index 8409e53a190..7f12f1e2853 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/content-display-utils.ts @@ -1,6 +1,7 @@ import type { PartnerContentTopicFitBand } from "@/lib/partner-content-search/constants"; import { getPartnerContentThumbnailUrl } from "@/lib/partner-content-search/thumbnail-url"; import type { PartnerContentSearchPartner } from "@/lib/swr/use-partner-content-search"; +import { nFormatter } from "@dub/utils"; // Shared display helpers for the network search surfaces (detail page + results // list): formatters, link/thumbnail resolvers, and the band label map. @@ -83,6 +84,20 @@ const PLATFORM_CONTENT_NOUN: Record = { website: { one: "page", many: "pages" }, }; +// Reach + engagement we have for a post, in descending prominence. Shared by the +// matched-content rows and the no-search content rows. +export function formatEngagement(content: ContentSearchChunk["content"]) { + return [ + content.viewCount ? `${nFormatter(content.viewCount)} views` : null, + content.likeCount ? `${nFormatter(content.likeCount)} likes` : null, + content.commentCount ? `${nFormatter(content.commentCount)} comments` : null, + content.shareCount ? `${nFormatter(content.shareCount)} shares` : null, + content.saveCount ? `${nFormatter(content.saveCount)} saves` : null, + ] + .filter(Boolean) + .join(" · "); +} + export function contentNoun(platforms: string[], count: number) { const plural = count !== 1; const manyForms = new Set( diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx index 9a94973880c..a65ddf60abe 100644 --- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx +++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/network/page-client.tsx @@ -10,7 +10,6 @@ import { AnimatedSizeContainer, Button, Filter, - InfoTooltip, PaginationControls, usePagination, useRouterStuff, @@ -273,10 +272,9 @@ export function ProgramPartnerNetworkPageClient({ />
-