Skip to content

Commit 1e24fff

Browse files
committed
fix: 批次任务随批次保留清理,未完成批次始终可见,默认读取不加载整页图片
- 已保存任务的 7 天清理只针对单独上传的任务;批次中的题随批次保留,全部入库超过保留期后连同批次一起清理 - 批次列表分开查询:未完成的批次全部返回,已完成的只列最近 10 个,较新的已完成批次不会挤掉更早的草稿 - 批次读取默认不从数据库读出整页图片,只有预览页明确取图或后台分题时读取
1 parent d51023b commit 1e24fff

3 files changed

Lines changed: 104 additions & 25 deletions

File tree

‎src/__tests__/integration/import-batches.test.ts‎

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ import { POST as CONFIRM_TASK } from '@/app/api/import-tasks/[id]/confirm/route'
5858
import { waitForImportBatchTasksForTest } from '@/lib/import-batches';
5959
import { simulateRestartForTest, waitForImportTaskForTest } from '@/lib/import-tasks';
6060

61-
let database: PrismaClient;
61+
let database: PrismaClient<{ log: Array<{ emit: 'event'; level: 'query' }> }>;
62+
const queries: string[] = [];
6263
let directory: string;
6364
let sequence = 0;
6465
const PAGE = 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAA==';
@@ -109,7 +110,9 @@ beforeAll(async () => {
109110
execFileSync(process.execPath, [resolve('node_modules/prisma/build/index.js'), 'migrate', 'deploy'], {
110111
env: { ...process.env, DATABASE_URL: url }, stdio: 'pipe',
111112
});
112-
database = new PrismaClient({ datasources: { db: { url } } });
113+
database = new PrismaClient({ datasources: { db: { url } }, log: [{ emit: 'event', level: 'query' }] });
114+
// 记录执行的 SQL,用于检查默认读取是否带出整页图片
115+
database.$on('query', (event) => { queries.push(event.query); });
113116
await database.$queryRaw`PRAGMA journal_mode = WAL`;
114117
await database.$queryRaw`PRAGMA busy_timeout = 30000`;
115118
mocks.prisma = database;
@@ -437,6 +440,68 @@ describe('照片自动分题批次', () => {
437440
});
438441
});
439442

443+
describe('批次生命周期', () => {
444+
const DAY = 24 * 60 * 60 * 1000;
445+
446+
it('批次中 7 天前已保存的题在批次未完成时不被清理,错题不受影响;独立任务按自身保留期清理', async () => {
447+
const item = await database.errorItem.create({ data: { userId: 'user-1', subjectId: 'math', questionText: '很早入库的题', originalImageUrl: '' } });
448+
const old = new Date(Date.now() - 10 * DAY);
449+
await database.importBatch.create({ data: { id: 'batch-long-draft', userId: 'user-1', source: 'text', splitter: 'local', status: 'started', mode: 'review', startedAt: old, createdAt: old } });
450+
await database.importTask.createMany({ data: [
451+
{ id: 'old-saved-in-batch', userId: 'user-1', mode: 'review', status: 'saved', imageData: '', inputText: 'a', errorItemId: item.id, batchId: 'batch-long-draft', batchItemKey: 'qa', finishedAt: old, createdAt: old },
452+
{ id: 'pending-in-batch', userId: 'user-1', mode: 'review', status: 'review', imageData: '', inputText: 'b', result: null, batchId: 'batch-long-draft', batchItemKey: 'qb', createdAt: old },
453+
{ id: 'old-standalone', userId: 'user-1', mode: 'auto', status: 'saved', imageData: '', finishedAt: old, createdAt: old },
454+
] });
455+
const list = await (await LIST()).json();
456+
expect(list.batches.find((batch: { id: string }) => batch.id === 'batch-long-draft')).toMatchObject({ counts: { total: 2, saved: 1, review: 1 } });
457+
expect(await database.importTask.count({ where: { id: 'old-saved-in-batch' } })).toBe(1);
458+
expect(await database.importTask.count({ where: { id: 'old-standalone' } })).toBe(0);
459+
expect(await database.errorItem.count({ where: { id: item.id } })).toBe(1);
460+
461+
// 全部入库且超过保留期后,批次连同其任务一起清理,错题保留
462+
await database.importTask.update({ where: { id: 'pending-in-batch' }, data: { status: 'saved', finishedAt: old } });
463+
await LIST();
464+
expect(await database.importBatch.count({ where: { id: 'batch-long-draft' } })).toBe(0);
465+
expect(await database.importTask.count({ where: { batchId: 'batch-long-draft' } })).toBe(0);
466+
expect(await database.errorItem.count({ where: { id: item.id } })).toBe(1);
467+
});
468+
469+
it('100 个较新的已完成批次不会挤掉更早的草稿,已完成的只列最近 10 个', async () => {
470+
const base = Date.now() - DAY;
471+
await database.importBatch.create({ data: { id: 'very-old-draft', userId: 'user-1', source: 'text', splitter: 'local', status: 'draft', createdAt: new Date(base - 30 * DAY) } });
472+
await database.importBatch.createMany({ data: Array.from({ length: 100 }, (_, index) => ({
473+
id: `done-batch-${String(index).padStart(3, '0')}`, userId: 'user-1', source: 'text', splitter: 'local', status: 'started',
474+
startedAt: new Date(base + index * 1000), createdAt: new Date(base + index * 1000),
475+
})) });
476+
const list = await (await LIST()).json();
477+
expect(list.batches.map((batch: { id: string }) => batch.id)).toContain('very-old-draft');
478+
expect(list.batches.filter((batch: { phase: string }) => batch.phase === 'done')).toHaveLength(10);
479+
expect(list.batches[0].id).toBe('done-batch-099');
480+
});
481+
482+
it('状态查询、列表与草稿保存不从数据库读出整页图片,只有明确取图时读取', async () => {
483+
const bigImage = `data:image/jpeg;base64,${'A'.repeat(2 * 1024 * 1024)}`;
484+
const draft = { items: [{ key: 'qa', label: '1', text: '题', material: null, section: null, selected: true, regions: [{ x: 0, y: 0, width: 1, height: 0.5 }], hasFigure: false, flags: [] }], ignored: [], warnings: [] };
485+
await database.importBatch.create({ data: { id: 'batch-big-image', userId: 'user-1', source: 'image', splitter: 'ai', status: 'draft', imageData: bigImage, draft: JSON.stringify(draft) } });
486+
const selectsImage = () => queries.some((query) => /[`"]ImportBatch[`"]\.[`"]imageData[`"]/.test(query) && /^SELECT/i.test(query));
487+
488+
queries.length = 0;
489+
const detail = await GET_BATCH(new Request('http://localhost/api/import-batches/batch-big-image'), params('batch-big-image'));
490+
const detailText = await detail.text();
491+
const listText = await (await LIST()).text();
492+
const patched = await PATCH(json({ version: 1, draft }, 'PATCH'), params('batch-big-image'));
493+
expect(patched.status).toBe(200);
494+
const patchText = await patched.text();
495+
expect(selectsImage()).toBe(false);
496+
for (const text of [detailText, listText, patchText]) expect(text.length).toBeLessThan(50_000);
497+
498+
queries.length = 0;
499+
const { batch } = await getBatch('batch-big-image', true);
500+
expect(batch.imageData).toBe(bigImage);
501+
expect(selectsImage()).toBe(true);
502+
});
503+
});
504+
440505
describe('批次归属与列表', () => {
441506
it('其他用户看不到、改不了、开始不了、丢弃不了他人的批次', async () => {
442507
const { batch } = await create({ source: 'text', text: '1.计算 1 + 1。\n2.计算 2 + 2。' });

‎src/lib/import-batches.ts‎

Lines changed: 35 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
*/
1111
import { randomUUID } from "node:crypto";
1212
import { NextResponse } from "next/server";
13-
import type { ImportBatch, Prisma, User } from "@prisma/client";
13+
import type { Prisma, User } from "@prisma/client";
1414
import { z } from "zod";
1515
import { consumeAiQuota } from "@/lib/ai-quota";
1616
import { MAX_SPLIT_LINES } from "@/lib/ai/question-split";
@@ -87,7 +87,18 @@ const startSchema = z.object({
8787
})).max(60).optional(),
8888
});
8989

90-
type BatchWithTasks = ImportBatch & { tasks: Array<Prisma.ImportTaskGetPayload<{ select: typeof summarySelect }>> };
90+
/**
91+
* 批次读取默认不含整页图片(可能有几 MB):状态轮询、列表与草稿保存都不从数据库读出原图,
92+
* 只有预览页明确取图(?include=image)或后台分题时才读取
93+
*/
94+
const batchSelect = {
95+
id: true, userId: true, clientRequestId: true, source: true, splitter: true, status: true, mode: true, language: true,
96+
subjectId: true, targetSubjectId: true, paperLevel: true, sourceText: true, imageInfo: true, draft: true, version: true,
97+
error: true, attempts: true, runnerId: true, heartbeatAt: true, readAt: true, startedAt: true, finishedAt: true,
98+
createdAt: true, updatedAt: true,
99+
tasks: { select: summarySelect },
100+
} satisfies Prisma.ImportBatchSelect;
101+
type BatchWithTasks = Prisma.ImportBatchGetPayload<{ select: typeof batchSelect }> & { imageData?: string };
91102

92103
function parseDraft(raw: string | null): BatchDraft | null {
93104
if (!raw) return null;
@@ -154,7 +165,6 @@ function toDetail(batch: BatchWithTasks, options: { image?: boolean } = {}): Imp
154165
};
155166
}
156167

157-
const batchInclude = { tasks: { select: summarySelect } } satisfies Prisma.ImportBatchInclude;
158168

159169
/** 分题中但执行它的进程已不在:标记失败,可重试 */
160170
async function reconcileInterrupted(userId: string, batchId?: string): Promise<void> {
@@ -183,10 +193,7 @@ async function findOwnBatch(user: Pick<User, 'id'>, id: string, image = false):
183193
await reconcileInterrupted(user.id, id);
184194
// 批次内正在解析或保存的题若因服务重启中断,同样标记为失败,批次页不会一直显示进行中
185195
await reconcileInterruptedTasks(user.id);
186-
const batch = await prisma.importBatch.findFirst({ where: { id, userId: user.id }, include: batchInclude });
187-
if (!batch) return null;
188-
if (!image) batch.imageData = '';
189-
return batch;
196+
return prisma.importBatch.findFirst({ where: { id, userId: user.id }, select: image ? { ...batchSelect, imageData: true } : batchSelect });
190197
}
191198

192199
async function ownsNotebook(userId: string, subjectId: string | null | undefined): Promise<boolean> {
@@ -274,7 +281,7 @@ export async function createImportBatch(user: User, raw: unknown): Promise<Creat
274281

275282
const existing = await prisma.importBatch.findUnique({
276283
where: { userId_clientRequestId: { userId: user.id, clientRequestId: input.clientRequestId } },
277-
include: batchInclude,
284+
select: batchSelect,
278285
});
279286
if (existing) return { batch: toDetail(existing), created: false };
280287
if (!await ownsNotebook(user.id, input.subjectId) || !await ownsNotebook(user.id, input.targetSubjectId)) {
@@ -315,14 +322,14 @@ export async function createImportBatch(user: User, raw: unknown): Promise<Creat
315322
readAt: draft ? new Date() : null,
316323
finishedAt: draft ? new Date() : null,
317324
},
318-
include: batchInclude,
325+
select: batchSelect,
319326
});
320327
} catch (error) {
321328
release();
322329
if ((error as { code?: string }).code === 'P2002') {
323330
const winner = await prisma.importBatch.findUnique({
324331
where: { userId_clientRequestId: { userId: user.id, clientRequestId: input.clientRequestId } },
325-
include: batchInclude,
332+
select: batchSelect,
326333
});
327334
if (winner) return { batch: toDetail(winner), created: false };
328335
}
@@ -340,7 +347,6 @@ export async function createImportBatch(user: User, raw: unknown): Promise<Creat
340347
scheduleWork(id, release, () => runSplit(id));
341348
}
342349
logger.info({ batchId: id, source: input.source, splitter, questions: draft?.items.length ?? null }, 'Import batch created');
343-
batch.imageData = '';
344350
return { batch: toDetail(batch), created: true };
345351
}
346352

@@ -548,16 +554,24 @@ export async function markImportBatchRead(user: Pick<User, 'id'>, id: string): P
548554
export async function listImportBatches(user: Pick<User, 'id'>): Promise<ImportBatchSummary[]> {
549555
await reconcileInterrupted(user.id);
550556
await pruneDone(user.id);
551-
const batches = await prisma.importBatch.findMany({
552-
where: { userId: user.id },
553-
include: batchInclude,
554-
orderBy: { createdAt: 'desc' },
555-
take: 100,
556-
});
557-
let done = 0;
558-
return batches
559-
.map((batch) => toBatchSummary(batch).summary)
560-
.filter((summary) => summary.phase !== 'done' || done++ < MAX_LISTED_DONE);
557+
// 未完成的批次(草稿、分题中/失败、仍有题未入库)全部返回,不会被较新的已完成批次挤出列表;
558+
// 全部入库的批次只返回最近若干个
559+
const [unfinished, done] = await Promise.all([
560+
prisma.importBatch.findMany({
561+
where: { userId: user.id, OR: [{ status: { not: 'started' } }, { tasks: { some: { status: { not: 'saved' } } } }] },
562+
select: batchSelect,
563+
orderBy: { createdAt: 'desc' },
564+
}),
565+
prisma.importBatch.findMany({
566+
where: { userId: user.id, status: 'started', tasks: { every: { status: 'saved' } } },
567+
select: batchSelect,
568+
orderBy: { createdAt: 'desc' },
569+
take: MAX_LISTED_DONE,
570+
}),
571+
]);
572+
return [...unfinished, ...done]
573+
.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime())
574+
.map((batch) => toBatchSummary(batch).summary);
561575
}
562576

563577
/** 识别任务中心:单独上传的任务与多题批次,执行中数与未读数都包含批次 */

‎src/lib/import-tasks.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,10 @@ export async function reconcileInterrupted(userId: string, taskId?: string): Pro
199199
logger.info({ userId, count: interrupted.length }, 'Marked interrupted import tasks as failed');
200200
}
201201

202-
/** 删除超过保留期的已保存任务 */
202+
/** 删除超过保留期的已保存独立任务;批次中的任务随批次一起保留或清理,不单独删除 */
203203
async function pruneSaved(userId: string): Promise<void> {
204204
const cutoff = new Date(Date.now() - SAVED_IMPORT_TASK_RETENTION_DAYS * 24 * 60 * 60 * 1000);
205-
await prisma.importTask.deleteMany({ where: { userId, status: 'saved', finishedAt: { lt: cutoff } } });
205+
await prisma.importTask.deleteMany({ where: { userId, batchId: null, status: 'saved', finishedAt: { lt: cutoff } } });
206206
}
207207

208208
async function findOwnTask(user: Pick<User, 'id'>, id: string): Promise<TaskWithItem | null> {

0 commit comments

Comments
 (0)