Skip to content

Commit 1aebb2c

Browse files
massimoalbarellomax-agent
andauthored
fix(openclaw): preserve learning progress across reauthorization (#235)
## Why Reauthorizing or reinstalling the OpenClaw memory plugin discarded its learning fingerprints along with credentials and pending evidence. Replaying an existing conversation then repeated curation and could recreate assets or rewrite pages. ## How Keep completion fingerprints and saved-upload receipts in private local storage scoped to the Context Use instance and OpenClaw agent. Reauthorization and removal preserve those receipts while clearing credentials and temporary evidence, so later installations skip completed messages and reuse saved attachments. First-time learning and retries of unfinished work continue as before. Copy existing progress before deleting a connection queue, and commit completion receipts atomically with clearing the learned evidence. Retained receipts contain no conversation text, attachment bytes, or credentials. Co-authored-by: max-agent <max-agent@Massimos-MacBook-Air-2.local>
1 parent 440ad3e commit 1aebb2c

7 files changed

Lines changed: 337 additions & 32 deletions

File tree

‎packages/openclaw-memory/scripts/e2e.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url';
88
import { ownerBrowser } from '@repo/browser-testing/owner-browser';
99
import metadata from '../package.json';
1010
import { CALLBACK_URL, PLUGIN_ID } from '../src/contract';
11+
import { evidenceFromMessages } from '../src/learning-evidence';
1112
import { LearningStore } from '../src/learning-store';
1213
import { OPENCLAW_INSTALL_COMMAND, openclawSetupPrompt } from '../src/setup-prompt';
1314
import { reservePort, startApp } from './e2e-app';
@@ -102,6 +103,35 @@ async function configuration() {
102103
);
103104
}
104105

106+
const completedEvidence = evidenceFromMessages([
107+
{ role: 'user', content: 'This conversation has already been learned.', timestamp: 1 },
108+
]);
109+
async function verifyLearningProgress(input: { seed?: boolean } = {}) {
110+
const state = await Bun.file(connectionFile).json();
111+
const store = new LearningStore({
112+
directory: dirname(connectionFile),
113+
config: state.config,
114+
connectionId: state.learningId,
115+
});
116+
try {
117+
const pending = store.status().pending;
118+
store.capture({
119+
source: 'completed-conversation',
120+
evidence: completedEvidence,
121+
now: Date.now(),
122+
});
123+
if (input.seed) {
124+
const job = store.next({ agentId: 'main', now: Date.now() })!;
125+
assert.equal(job.source, 'completed-conversation');
126+
store.acknowledge({ sessionKey: job.sessionKey });
127+
store.finish({ id: job.id, now: Date.now() });
128+
}
129+
assert.equal(store.status().pending, pending, 'Completed conversation was queued again');
130+
} finally {
131+
store.close();
132+
}
133+
}
134+
105135
await mkdir(join(directory, 'bin'), { recursive: true });
106136
await writeFile(
107137
join(directory, 'bin/openclaw'),
@@ -251,6 +281,7 @@ config.models={providers:{fixture:{baseUrl:${JSON.stringify(`${model.origin}/v1`
251281
assert(connected.plugins.entries['active-memory'].config.allowedChatTypes.includes('group'));
252282
assert((await command(['openclaw', 'context-use', 'status'])).includes('"connected": true'));
253283
console.log('Native installation and pasted callback authorization passed.');
284+
await verifyLearningProgress({ seed: true });
254285

255286
await command(['npx', '--yes', packageSpec, 'connect', app.origin]);
256287
assert((await Bun.file(connectionFile).json()).oauth.tokens);
@@ -285,6 +316,7 @@ config.models={providers:{fixture:{baseUrl:${JSON.stringify(`${model.origin}/v1`
285316
console.log(
286317
'Explicit reconnect registered and authorized a fresh client while retaining restoration history.',
287318
);
319+
await verifyLearningProgress();
288320
await mkdir(join(directory, 'moved'));
289321
const movedApp = await startApp({ repo, directory: join(directory, 'moved'), node });
290322
let movedOwner: Awaited<ReturnType<typeof ownerBrowser>> | undefined;
@@ -321,6 +353,7 @@ config.models={providers:{fixture:{baseUrl:${JSON.stringify(`${model.origin}/v1`
321353
}),
322354
);
323355
assert.deepEqual((await Bun.file(connectionFile).json()).changes, originalJournal);
356+
await verifyLearningProgress();
324357
const reauthorizedConfig = await configuration();
325358
assert.equal(
326359
reauthorizedConfig.plugins.entries['active-memory'].config.timeoutMs,
@@ -478,6 +511,7 @@ config.models={providers:{fixture:{baseUrl:${JSON.stringify(`${model.origin}/v1`
478511
clientName: 'OpenClaw reinstallation',
479512
});
480513
await authorize(newRedirect);
514+
await verifyLearningProgress();
481515
// This was the original timeout before reinstallation, so it must not look like a failed commit.
482516
await configure("config.plugins.entries['active-memory'].config.timeoutMs=45000;");
483517
await command(['npx', '--yes', packageSpec, 'connect', app.origin]);

‎packages/openclaw-memory/src/connection.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,13 @@ import {
1515
prepareConfiguration,
1616
restoreConfiguration,
1717
} from './configuration';
18-
import { AUTHORIZATION_SCOPE, PLUGIN_ID, REQUEST_TIMEOUT_MS, serverUrl } from './contract';
18+
import {
19+
AUTHORIZATION_SCOPE,
20+
PLUGIN_ID,
21+
type PluginConfig,
22+
REQUEST_TIMEOUT_MS,
23+
serverUrl,
24+
} from './contract';
1925
import { ConnectionError } from './error';
2026
import { attachmentDirectory, LearningStore, learningDatabase } from './learning-store';
2127
import { authorizationResponse, oauthProvider } from './oauth';
@@ -62,6 +68,18 @@ async function removeLearningData(input: {
6268
}
6369
}
6470

71+
function preserveLearningProgress(input: {
72+
directory: string;
73+
config: PluginConfig;
74+
connectionId?: string;
75+
}): void {
76+
const { connectionId } = input;
77+
if (connectionId && existsSync(join(input.directory, learningDatabase(connectionId)))) {
78+
// Export receipts from a queue created before durable progress was introduced.
79+
new LearningStore({ ...input, connectionId }).close();
80+
}
81+
}
82+
6583
export async function connect(input: {
6684
directory: string;
6785
instance: string;
@@ -86,6 +104,11 @@ export async function connect(input: {
86104
await installPackage();
87105
if (input.reauthorize || state.config.serverUrl !== config.serverUrl) {
88106
const previousLearningId = state.learningId;
107+
preserveLearningProgress({
108+
directory: input.directory,
109+
config: state.config,
110+
connectionId: previousLearningId,
111+
});
89112
state.config = config;
90113
state.oauth = {};
91114
state.tools = [];
@@ -147,6 +170,11 @@ export async function finishAuthorization(input: {
147170
delete state.oauth.verifier;
148171
// A new authorization can belong to another person even when the OAuth client is reused.
149172
const previousLearningId = state.learningId;
173+
preserveLearningProgress({
174+
directory: input.directory,
175+
config: state.config,
176+
connectionId: previousLearningId,
177+
});
150178
state.learningId = randomUUID();
151179
await writeState({ directory: input.directory, state });
152180
if (previousLearningId) {
@@ -187,6 +215,13 @@ export async function disconnect(directory: string): Promise<{ preserved: string
187215
}
188216
},
189217
});
218+
if (state?.learningId) {
219+
preserveLearningProgress({
220+
directory,
221+
config: state.config,
222+
connectionId: state.learningId,
223+
});
224+
}
190225
await rm(directory, { recursive: true, force: true });
191226
return { preserved };
192227
},

‎packages/openclaw-memory/src/learning-store.ts‎

Lines changed: 82 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { randomUUID } from 'node:crypto';
1+
import { createHash, randomUUID } from 'node:crypto';
22
import { chmodSync, mkdirSync } from 'node:fs';
33
import { join } from 'node:path';
44
import { DatabaseSync } from 'node:sqlite';
@@ -19,6 +19,14 @@ export const attachmentDirectory = (connectionId: string) =>
1919
export const DREAM_INTERVAL_MS = 21_600_000;
2020
export const learningDatabase = (connectionId: string) =>
2121
`learning-${encodeURIComponent(connectionId)}.sqlite`;
22+
const learningProgressDirectory = (directory: string) => `${directory}.progress`;
23+
24+
export function learningProgressDatabase(input: { directory: string; config: PluginConfig }) {
25+
const scope = createHash('sha256')
26+
.update(JSON.stringify([input.config.serverUrl, input.config.agentId]))
27+
.digest('hex');
28+
return join(learningProgressDirectory(input.directory), `${scope}.sqlite`);
29+
}
2230

2331
export type LearningJob = {
2432
id: string;
@@ -32,7 +40,7 @@ export type LearningJob = {
3240
startedAt: number;
3341
};
3442

35-
/** A temporary delivery queue, not an alternative personal memory store. */
43+
/** Temporary evidence with durable completion receipts, not an alternative personal memory store. */
3644
export class LearningStore {
3745
private readonly db: DatabaseSync;
3846

@@ -73,19 +81,53 @@ export class LearningStore {
7381
this.db.close();
7482
throw new Error('Context Use learning queue belongs to a different connection.');
7583
}
84+
// Receipts belong to the destination and agent, independently of OAuth credentials
85+
// and the private connection directory removed during uninstall.
86+
try {
87+
mkdirSync(learningProgressDirectory(input.directory), {
88+
recursive: true,
89+
mode: PRIVATE_DIRECTORY_MODE,
90+
});
91+
const progressPath = learningProgressDatabase(input);
92+
this.db.prepare('ATTACH DATABASE ? AS progress').run(progressPath);
93+
chmodSync(progressPath, PRIVATE_FILE_MODE);
94+
this.db.exec(`
95+
CREATE TABLE IF NOT EXISTS progress.completed (
96+
source TEXT NOT NULL, fingerprint TEXT NOT NULL, PRIMARY KEY (source, fingerprint)
97+
);
98+
CREATE TABLE IF NOT EXISTS progress.attachments (
99+
source TEXT NOT NULL, key TEXT NOT NULL, asset TEXT, uploadName TEXT,
100+
PRIMARY KEY (source, key)
101+
);
102+
INSERT OR IGNORE INTO progress.completed
103+
SELECT source, fingerprint FROM evidence WHERE text IS NULL;
104+
INSERT OR IGNORE INTO progress.attachments
105+
SELECT source, key, asset, uploadName FROM attachments
106+
WHERE asset IS NOT NULL OR uploadName IS NOT NULL;
107+
`);
108+
} catch (error) {
109+
this.db.close();
110+
throw error;
111+
}
76112
}
77113

78114
capture(input: { source: string; evidence: LearningEvidence[]; now: number }): void {
79115
this.transaction(() => {
80116
const insert = this.db.prepare(
81-
'INSERT OR IGNORE INTO evidence(source, fingerprint, text, capturedAt) VALUES (?, ?, ?, ?)',
117+
`INSERT OR IGNORE INTO evidence(source, fingerprint, text, capturedAt)
118+
SELECT ?, ?, ?, ? WHERE NOT EXISTS (
119+
SELECT 1 FROM progress.completed WHERE source = ? AND fingerprint = ?
120+
)`,
82121
);
83122
for (const item of input.evidence) {
84-
insert.run(input.source, item.key, item.text, input.now);
123+
insert.run(input.source, item.key, item.text, input.now, input.source, item.key);
85124
if (item.attachment) {
86125
this.db
87-
.prepare(`INSERT OR IGNORE INTO attachments(source, key, evidenceId, data)
88-
SELECT source, fingerprint, id, ? FROM evidence WHERE source = ? AND fingerprint = ?`)
126+
.prepare(`INSERT OR IGNORE INTO attachments(source, key, evidenceId, data, asset, uploadName)
127+
SELECT e.source, e.fingerprint, e.id, ?, p.asset, p.uploadName
128+
FROM evidence e LEFT JOIN progress.attachments p
129+
ON p.source = e.source AND p.key = e.fingerprint
130+
WHERE e.source = ? AND e.fingerprint = ? AND e.text IS NOT NULL`)
89131
.run(JSON.stringify(item.attachment), input.source, item.key);
90132
}
91133
}
@@ -209,30 +251,46 @@ export class LearningStore {
209251

210252
prepareAttachmentUpload(input: { jobId: string; key: string; name: string }): string {
211253
const source = this.attachmentSource(input);
212-
this.db
213-
.prepare(
214-
'UPDATE attachments SET uploadName = COALESCE(uploadName, ?) WHERE source = ? AND key = ?',
215-
)
216-
.run(input.name, source, input.key);
217-
return (
254+
return this.transaction(() => {
218255
this.db
219-
.prepare('SELECT uploadName FROM attachments WHERE source = ? AND key = ?')
220-
.get(source, input.key) as { uploadName: string }
221-
).uploadName;
256+
.prepare(
257+
'UPDATE attachments SET uploadName = COALESCE(uploadName, ?) WHERE source = ? AND key = ?',
258+
)
259+
.run(input.name, source, input.key);
260+
this.preserveAttachment({ source, key: input.key });
261+
return (
262+
this.db
263+
.prepare('SELECT uploadName FROM attachments WHERE source = ? AND key = ?')
264+
.get(source, input.key) as { uploadName: string }
265+
).uploadName;
266+
});
222267
}
223268

224269
clearAttachmentUploadName(input: { jobId: string; key: string }): void {
225270
const source = this.attachmentSource(input);
226-
this.db
227-
.prepare('UPDATE attachments SET uploadName = NULL WHERE source = ? AND key = ?')
228-
.run(source, input.key);
271+
this.transaction(() => {
272+
this.db
273+
.prepare('UPDATE attachments SET uploadName = NULL WHERE source = ? AND key = ?')
274+
.run(source, input.key);
275+
this.preserveAttachment({ source, key: input.key });
276+
});
229277
}
230278

231279
savedAttachment(input: { jobId: string; key: string; asset: string }): void {
232280
const source = this.attachmentSource(input);
281+
this.transaction(() => {
282+
this.db
283+
.prepare('UPDATE attachments SET asset = ? WHERE source = ? AND key = ?')
284+
.run(input.asset, source, input.key);
285+
this.preserveAttachment({ source, key: input.key });
286+
});
287+
}
288+
289+
private preserveAttachment(input: { source: string; key: string }): void {
233290
this.db
234-
.prepare('UPDATE attachments SET asset = ? WHERE source = ? AND key = ?')
235-
.run(input.asset, source, input.key);
291+
.prepare(`INSERT OR REPLACE INTO progress.attachments
292+
SELECT source, key, asset, uploadName FROM attachments WHERE source = ? AND key = ?`)
293+
.run(input.source, input.key);
236294
}
237295

238296
acknowledge(input: { sessionKey: string; omittedAttachments?: string[] }): void {
@@ -272,6 +330,10 @@ export class LearningStore {
272330
throw new Error('Learning must be acknowledged before removing its evidence.');
273331
}
274332
if (job.kind === 'learn') {
333+
this.db
334+
.prepare(`INSERT OR IGNORE INTO progress.completed
335+
SELECT source, fingerprint FROM evidence WHERE source = ? AND id <= ?`)
336+
.run(job.source, job.through);
275337
this.db
276338
.prepare('UPDATE evidence SET text = NULL WHERE source = ? AND id <= ?')
277339
.run(job.source, job.through);

‎packages/openclaw-memory/src/setup-skill.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ Run ${OPENCLAW_INSTALL_COMMAND} remove. It restores the plugin's configuration
4343
changes, clears local credentials, cancels unfinished learning, discards pending
4444
local evidence, and uninstalls the package. User edits, remote memories, and
4545
conversation history are preserved. It also works when setup was unfinished or disabled.
46+
Completed-learning fingerprints and saved-attachment references remain locally so reinstalling
47+
for the same instance and agent does not repeat completed work. These receipts contain no
48+
conversation text, attachment bytes, or credentials.
4649
4750
Removal runs separately so the requesting turn can finish. End this turn once
4851
removal is queued. Do not poll in this turn, schedule retries, stop the gateway,

‎packages/openclaw-memory/test/learning-attachments.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ import { attachmentsFromMessages } from '../src/learning-evidence';
1212
import { attachmentDirectory, LearningStore } from '../src/learning-store';
1313

1414
test('staged original bytes survive loss of the host cache and are removed after learning', async () => {
15-
const directory = await mkdtemp(join(tmpdir(), 'learning-attachments-'));
15+
const root = await mkdtemp(join(tmpdir(), 'learning-attachments-'));
16+
const directory = join(root, 'connection');
1617
const connectionId = 'owner';
1718
const config = { agentId: 'main', serverUrl: 'https://example.com/mcp' };
1819
const db = new LearningStore({ directory, config, connectionId });
@@ -53,6 +54,6 @@ test('staged original bytes survive loss of the host cache and are removed after
5354
await expect(read()).rejects.toThrow();
5455
} finally {
5556
db.close();
56-
await rm(directory, { recursive: true, force: true });
57+
await rm(root, { recursive: true, force: true });
5758
}
5859
});

0 commit comments

Comments
 (0)