From 07e04781429da0eeede81838c0c77db82bfd057b Mon Sep 17 00:00:00 2001 From: Naz Date: Sat, 5 Sep 2026 19:43:06 +0800 Subject: [PATCH 1/2] kvstore: stamp checkpoints with a behavioral tokenizer fingerprint Persisted KV checkpoints encode their conversation as a token history whose bytes, on future requests, are re-tokenized by whatever tokenizer the running engine has. When tokenizer code or data changes upstream (the JoyAI pre-tokenizer has seen several rule fixes), existing checkpoint token histories become unreproducible: every later request diverges from the restored session at the first changed merge boundary, reloads the same stale file by text key, and re-imports the stale tokenization. The visible symptom is a permanent per-turn live-miss + disk-reload + partial-re-prefill loop for every session resumed with old checkpoints (observed in production: 29 consecutive turns, ~38k tokens re-prefilled each turn, session permanently pinned to a stale frontier). A text-equality check at load cannot detect this class without also rejecting legitimate preserved-reasoning continuations, whose stored sampled tokens intentionally differ from a whole-text re-tokenization of their bytes - the bridge's premise. The correct discriminator is the engine identity that produced the tokens. This commit adds a behavioral fingerprint: an append-only set of probe strings (apostrophe-punctuation seams, digit runs, indentation, newline joins, UTF-8 letters, special-token edges) is run through the real tokenizer once, and the resulting ids hashed together with the token table. It covers tokenizer data AND code changes with no human-maintained version constant. Checkpoint trailers now begin with a TOKFP section carrying the fingerprint; the loader pre-checks it by a 16-byte seek to the trailer, before touching the payload (session untouched), and mismatches fall back to the normal cold path, which rewrites a fresh stamped file under the same key. Unstamped legacy files keep today's trusted behavior. Verified in a live agent session: stamped files restore at full depth (451k tokens, 38-token re-prefill); a synthetic probe change (simulated upstream tokenizer edit) makes every old stamped file reject in microseconds with a clear warning and the session self-heals on the next store. --- ds4.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ ds4.h | 7 +++++++ ds4_kvstore.c | 41 ++++++++++++++++++++++++++++++++++++++ ds4_kvstore.h | 3 +++ ds4_server.c | 51 ++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 154 insertions(+), 3 deletions(-) diff --git a/ds4.c b/ds4.c index 7509296bab..021ce03f97 100644 --- a/ds4.c +++ b/ds4.c @@ -38775,6 +38775,9 @@ typedef enum { } ds4_vision_kind; struct ds4_engine { + uint64_t tokenizer_fp; + int tokenizer_fp_ready; + pthread_mutex_t tokenizer_fp_mu; ds4_model model; ds4_model mtp_model; ds4_model vision_model; @@ -56338,6 +56341,57 @@ int ds4_session_load_layer_payload(ds4_session *s, FILE *fp, #endif } +/* Tokenizer behavioral fingerprint probes. APPEND-ONLY: never edit or + * reorder existing entries, or every checkpoint stamp changes globally. + * The set deliberately exercises JoyAI pre-tokenizer seams: apostrophe + * before punctuation runs, digits, spaced indentation, newline joins, + * UTF-8 letters, and special-token boundaries. */ +static const char *const DS4_TOKENIZER_FP_PROBES[] = { + "TABLE.*mailbox|\x27.*CREATE", + "box|\x27.*username\x22", + "The user\x27s int 123456789 x", + " int main() {\n return 0;\n}", + "caf\xc3\xa9 na\xc3\xafve \xe4\xb8\xad", + "a b\tc\n\nd", + "<\xe2\x80\x82User\xe2\x80\x82>hello", + "\x27.\x27+, *-[]{}|\\/`~!@#$%^&*()", + "yes) -> \n navigation 0%->33%->67%", + "1 22 333 4444 55555 \x22quoted\x22 \x27single\x27", +}; + + +static uint64_t tokenizer_fp_hash_bytes(uint64_t h, const void *p, size_t n) { + const uint8_t *b = (const uint8_t *)p; + for (size_t i = 0; i < n; i++) { + h ^= b[i]; + h *= 1099511628211ull; + } + return h; +} + +uint64_t ds4_engine_tokenizer_fingerprint(ds4_engine *e) { + if (!e) return 0; + pthread_mutex_lock(&e->tokenizer_fp_mu); + if (!e->tokenizer_fp_ready) { + uint64_t h = 14695981039346656037ull; + h = tokenizer_fp_hash_bytes(h, e->vocab.token, + (size_t)e->vocab.n_vocab * sizeof(ds4_str)); + for (size_t pi = 0; pi < sizeof(DS4_TOKENIZER_FP_PROBES) / + sizeof(DS4_TOKENIZER_FP_PROBES[0]); pi++) { + ds4_tokens t = {0}; + tokenize_rendered_chat_vocab(&e->vocab, + DS4_TOKENIZER_FP_PROBES[pi], &t); + h = tokenizer_fp_hash_bytes(h, t.v, (size_t)t.len * sizeof(t.v[0])); + ds4_tokens_free(&t); + } + e->tokenizer_fp = h; + e->tokenizer_fp_ready = 1; + } + uint64_t fp = e->tokenizer_fp; + pthread_mutex_unlock(&e->tokenizer_fp_mu); + return fp; +} + int ds4_engine_routed_quant_bits(ds4_engine *e) { if (!e) return 0; for (uint32_t il = 0; il < DS4_N_LAYER; il++) { @@ -63036,6 +63090,7 @@ static int ds4_engine_open_internal(ds4_engine **out, const ds4_engine_options *opt, const ds4_gpu_config *gpu_cfg) { ds4_engine *e = xcalloc(1, sizeof(*e)); + pthread_mutex_init(&e->tokenizer_fp_mu, NULL); #if defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU) g_glm_rocm_guard_available_baseline = 0; (void)ds4_linux_nonmovable_memory(&g_glm_rocm_guard_available_baseline); diff --git a/ds4.h b/ds4.h index 0921d4b68a..abf45eb435 100644 --- a/ds4.h +++ b/ds4.h @@ -548,6 +548,13 @@ void ds4_session_rewind(ds4_session *s, int pos); int ds4_session_pos(ds4_session *s); int ds4_session_ctx(ds4_session *s); int ds4_session_prefill_cap(ds4_session *s); +/* Behavioral tokenizer fingerprint: a fixed probe string set run through the + * real tokenizer, hashed. Covers data (vocab/merges) AND code (pre-tokenizer + * rules) changes, so persisted KV checkpoints can detect engine builds whose + * tokenization the file's token history can no longer be reproduced by. + * Stable for a given build; 0 only before the vocab is loaded. */ +uint64_t ds4_engine_tokenizer_fingerprint(ds4_engine *e); + int ds4_engine_routed_quant_bits(ds4_engine *e); bool ds4_engine_has_output_head(ds4_engine *e); bool ds4_engine_has_mtp(ds4_engine *e); diff --git a/ds4_kvstore.c b/ds4_kvstore.c index 5b73de7b65..3e48991578 100644 --- a/ds4_kvstore.c +++ b/ds4_kvstore.c @@ -1271,6 +1271,47 @@ int ds4_kvstore_try_load_text(ds4_kvstore *kc, } } } + /* Tokenizer fingerprint pre-check. Stamped files carry a behavioral + * fingerprint of the engine that produced their token history as the + * first trailer section. A mismatch means this engine can no longer + * reproduce those token ids from the stored text (tokenizer code or + * data changed since the checkpoint was written), so restoring it + * would re-import a tokenization that every future request mismatches + * at the drift point — a permanent reload loop through text-keyed + * tiers. Reject before touching the payload; the ordinary prefill + * path rewrites a fresh stamped file under the same text key. + * Unstamped legacy files keep today's trusted behavior. */ + if (header_ok && (hdr.ext_flags & DS4_KVSTORE_EXT_TOKFP)) { + long payload_start = ftell(fp); + uint64_t tokfp = 0; + bool have_fp = false; + if (payload_start >= 0 && + fseek(fp, payload_start + (long)hdr.payload_bytes, SEEK_SET) == 0) { + uint8_t sh[8]; + uint8_t fb[8]; + if (fread(sh, 1, sizeof(sh), fp) == (long)sizeof(sh) && + sh[0] == 'T' && sh[1] == 'K' && sh[2] == 'F' && + sh[3] == 1 && ds4_kvstore_le_get32(sh + 4) == 8 && + fread(fb, 1, sizeof(fb), fp) == (long)sizeof(fb)) { + for (int i = 0; i < 8; i++) tokfp |= (uint64_t)fb[i] << (8 * i); + have_fp = true; + } + fseek(fp, payload_start, SEEK_SET); + } + if (have_fp && tokfp != ds4_engine_tokenizer_fingerprint(engine)) { + kv_logf(kc, DS4_KVSTORE_LOG_WARNING, + "%s: kv cache tokenizer fingerprint mismatch, refusing stale tokenization%s%s %s", + kv_log_name(kc), + responses_protocol ? " " : "", + responses_protocol ? "RESPPROTO" : "", + path); + fclose(fp); + free(cached_text); + free(path); + return 0; + } + } + char err[160] = {0}; int loaded = 0; if (header_ok && diff --git a/ds4_kvstore.h b/ds4_kvstore.h index 28ccdb7eaf..c60f0c8b3e 100644 --- a/ds4_kvstore.h +++ b/ds4_kvstore.h @@ -16,6 +16,9 @@ #define DS4_KVSTORE_EXT_RESPONSES_VISIBLE (1u << 1) #define DS4_KVSTORE_EXT_THINKING_VISIBLE (1u << 2) #define DS4_KVSTORE_EXT_SESSION_TITLE (1u << 3) +/* Auxiliary (never a key-kind) bit: the trailer begins with a tokenizer + * fingerprint section the loader can reject on before touching the payload. */ +#define DS4_KVSTORE_EXT_TOKFP (1u << 5) typedef enum { DS4_KVSTORE_REASON_UNKNOWN = 0, diff --git a/ds4_server.c b/ds4_server.c index ee008a1637..48c4b8f53a 100644 --- a/ds4_server.c +++ b/ds4_server.c @@ -9954,6 +9954,17 @@ static void apply_anthropic_stream_tool_ids(tool_calls *calls, #define KV_CACHE_FIXED_HEADER DS4_KVSTORE_FIXED_HEADER #define KV_CACHE_HIT_HALF_LIFE_SECONDS DS4_KVSTORE_HIT_HALF_LIFE_SECONDS #define KV_EXT_TOOL_MAP DS4_KVSTORE_EXT_TOOL_MAP +#define KV_EXT_TOKFP DS4_KVSTORE_EXT_TOKFP + +/* Tokenizer fingerprint section: first trailer section of stamped files. + * 8-byte header (3 magic, 1 version, 4 payload length) + u64 fp. */ +#define KV_TOKFP_MAGIC0 'T' +#define KV_TOKFP_MAGIC1 'K' +#define KV_TOKFP_MAGIC2 'F' +#define KV_TOKFP_VERSION 1u +#define KV_TOKFP_HEADER 8u +#define KV_TOKFP_PAYLOAD 8u +#define KV_TOKFP_BYTES (KV_TOKFP_HEADER + KV_TOKFP_PAYLOAD) #define KV_EXT_RESPONSES_VISIBLE DS4_KVSTORE_EXT_RESPONSES_VISIBLE #define KV_EXT_THINKING_VISIBLE DS4_KVSTORE_EXT_THINKING_VISIBLE #define KV_TOOL_MAP_MAGIC0 'K' @@ -10362,15 +10373,49 @@ static bool kv_cache_file_size_fits(const kv_disk_cache *kc, static bool kv_cache_tool_map_size_cb(void *ud, const char *text, uint64_t *bytes_out) { - return kv_tool_map_serialized_size((server *)ud, text, bytes_out); + uint64_t tool_bytes = 0; + if (!kv_tool_map_serialized_size((server *)ud, text, &tool_bytes)) return false; + *bytes_out = KV_TOKFP_BYTES + tool_bytes; + return true; } static bool kv_cache_tool_map_write_cb(void *ud, FILE *fp, const char *text, uint64_t *written_bytes) { - return kv_tool_map_write((server *)ud, fp, text, written_bytes); + server *s = (server *)ud; + uint64_t written = 0; + uint8_t h[KV_TOKFP_HEADER]; + h[0] = KV_TOKFP_MAGIC0; + h[1] = KV_TOKFP_MAGIC1; + h[2] = KV_TOKFP_MAGIC2; + h[3] = KV_TOKFP_VERSION; + le_put32(h + 4, KV_TOKFP_PAYLOAD); + uint8_t fb[8]; + uint64_t tok_fp = ds4_engine_tokenizer_fingerprint(s ? s->engine : NULL); + for (int i = 0; i < 8; i++) fb[i] = (uint8_t)(tok_fp >> (8 * i)); + if (fwrite(h, 1, sizeof(h), fp) != sizeof(h)) return false; + if (fwrite(fb, 1, sizeof(fb), fp) != sizeof(fb)) return false; + written += KV_TOKFP_BYTES; + uint64_t tool_written = 0; + if (!kv_tool_map_write(s, fp, text, &tool_written)) return false; + written += tool_written; + if (written_bytes) *written_bytes = written; + return true; } static int kv_cache_tool_map_load_cb(void *ud, FILE *fp, const void *wanted) { + /* Skip the tokenizer fingerprint section when present; the loader has + * already rejected mismatches before the payload was touched, so the + * value is informational here. Older files start directly at the tool + * map; restore the position for them. */ + long start = ftell(fp); + uint8_t h[8]; + if (start >= 0 && fread(h, 1, sizeof(h), fp) == (long)sizeof(h) && + h[0] == KV_TOKFP_MAGIC0 && h[1] == KV_TOKFP_MAGIC1 && + h[2] == KV_TOKFP_MAGIC2 && h[3] == KV_TOKFP_VERSION) { + if (fseek(fp, (long)le_get32(h + 4), SEEK_CUR) != 0) return -1; + } else if (start >= 0) { + if (fseek(fp, start, SEEK_SET) != 0) return -1; + } return kv_tool_map_load_from_pos((server *)ud, fp, (const stop_list *)wanted); } @@ -10378,7 +10423,7 @@ static ds4_kvstore_trailer_hooks kv_cache_tool_map_hooks(server *s, const stop_list *wanted) { return (ds4_kvstore_trailer_hooks){ .ud = s, - .ext_flag = KV_EXT_TOOL_MAP, + .ext_flag = (uint8_t)(KV_EXT_TOKFP | KV_EXT_TOOL_MAP), .serialized_size = kv_cache_tool_map_size_cb, .write = kv_cache_tool_map_write_cb, .load = kv_cache_tool_map_load_cb, From aa0b4628eb777aa2eb94d46227f7c5adb07e0840 Mon Sep 17 00:00:00 2001 From: Naz Date: Sun, 6 Sep 2026 08:47:03 +0800 Subject: [PATCH 2/2] engine: hash tokenizer content, never struct bytes, in fingerprint ds4_str holds a pointer and a length; hashing the raw array therefore hashed process mapping state, not tokenizer data. Same-binary restarts can reproduce the allocator layout by luck, but any different mapping would silently change the fingerprint and reject valid stamped checkpoints after an ordinary restart - the opposite of the feature's purpose. Hash content instead: n_vocab, then every token's length + bytes in id order, then the merge-rank table (used/cap plus each used slot's key bytes and rank; open-addressed slot order is a deterministic function of the GGUF merge list). The behavioral probes still ride on top, so code changes are covered even when data is not. Validated live (Apple M5 Max): fingerprint e467e62ceeadf47e identical across separate process launches of the unchanged model, and a stamped checkpoint restored at full depth (32768-token grid, 69 ms load) across a stop/start cycle with zero mismatch lines. A single-byte token mutation on an APFS clone of the model (TABLE->TASLE, same length, outside every probe string) moves the fingerprint to 5f83c7a25a3b184e, proving data sensitivity independent of probe coverage. --- ds4.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/ds4.c b/ds4.c index 021ce03f97..3a4f40f9fc 100644 --- a/ds4.c +++ b/ds4.c @@ -56374,8 +56374,31 @@ uint64_t ds4_engine_tokenizer_fingerprint(ds4_engine *e) { pthread_mutex_lock(&e->tokenizer_fp_mu); if (!e->tokenizer_fp_ready) { uint64_t h = 14695981039346656037ull; - h = tokenizer_fp_hash_bytes(h, e->vocab.token, - (size_t)e->vocab.n_vocab * sizeof(ds4_str)); + /* Hash tokenizer CONTENT, never in-memory struct bytes: pointer + * fields change with every process mapping, which would make the + * fingerprint restart-unstable and reject valid checkpoints after + * an ordinary server restart. */ + h = tokenizer_fp_hash_bytes(h, &e->vocab.n_vocab, sizeof(e->vocab.n_vocab)); + for (int ti = 0; ti < e->vocab.n_vocab; ti++) { + h = tokenizer_fp_hash_bytes(h, &e->vocab.token[ti].len, + sizeof(e->vocab.token[ti].len)); + h = tokenizer_fp_hash_bytes(h, e->vocab.token[ti].ptr, + (size_t)e->vocab.token[ti].len); + } + /* Merge ranks: the open-addressed slot order is a deterministic + * function of the GGUF merge list, so hashing slots in index order + * (key bytes + rank) is stable across processes. */ + h = tokenizer_fp_hash_bytes(h, &e->vocab.merge_rank.used, + sizeof(e->vocab.merge_rank.used)); + h = tokenizer_fp_hash_bytes(h, &e->vocab.merge_rank.cap, + sizeof(e->vocab.merge_rank.cap)); + for (uint64_t mi = 0; mi < e->vocab.merge_rank.cap; mi++) { + str_i32_entry *en = &e->vocab.merge_rank.entry[mi]; + if (!en->used) continue; + h = tokenizer_fp_hash_bytes(h, &en->key.len, sizeof(en->key.len)); + h = tokenizer_fp_hash_bytes(h, en->key.ptr, (size_t)en->key.len); + h = tokenizer_fp_hash_bytes(h, &en->value, sizeof(en->value)); + } for (size_t pi = 0; pi < sizeof(DS4_TOKENIZER_FP_PROBES) / sizeof(DS4_TOKENIZER_FP_PROBES[0]); pi++) { ds4_tokens t = {0};