kvstore: stamp checkpoints with a behavioral tokenizer fingerprint

ds4.c

@@ -38775,6 +38775,9 @@ typedef enum {
 } ds4_vision_kind;
 
 struct ds4_engine {
+    uint64_t tokenizer_fp;
+    int tokenizer_fp_ready;
+    pthread_mutex_t tokenizer_fp_mu;
     ds4_model model;
     ds4_model mtp_model;
     ds4_model vision_model;
@@ -56338,6 +56341,80 @@ int ds4_session_load_layer_payload(ds4_session *s, FILE *fp,
 #endif
 }
 
+/* Tokenizer behavioral fingerprint probes.  APPEND-ONLY: never edit or
+ * reorder existing entries, or every checkpoint stamp changes globally.
+ * The set deliberately exercises JoyAI pre-tokenizer seams: apostrophe
+ * before punctuation runs, digits, spaced indentation, newline joins,
+ * UTF-8 letters, and special-token boundaries. */
+static const char *const DS4_TOKENIZER_FP_PROBES[] = {
+    "TABLE.*mailbox|\x27.*CREATE",
+    "box|\x27.*username\x22",
+    "The user\x27s int 123456789 x",
+    "    int main() {\n        return 0;\n}",
+    "caf\xc3\xa9 na\xc3\xafve \xe4\xb8\xad",
+    "a  b\tc\n\nd",
+    "<\xe2\x80\x82User\xe2\x80\x82>hello</\xe2\x80\x82User\xe2\x80\x82>",
+    "\x27.\x27+, *-[]{}|\\/`~!@#$%^&*()",
+    "yes) -> \n navigation 0%->33%->67%",
+    "1 22 333 4444 55555 \x22quoted\x22 \x27single\x27",
+};
+
+
+static uint64_t tokenizer_fp_hash_bytes(uint64_t h, const void *p, size_t n) {
+    const uint8_t *b = (const uint8_t *)p;
+    for (size_t i = 0; i < n; i++) {
+        h ^= b[i];
+        h *= 1099511628211ull;
+    }
+    return h;
+}
+
+uint64_t ds4_engine_tokenizer_fingerprint(ds4_engine *e) {
+    if (!e) return 0;
+    pthread_mutex_lock(&e->tokenizer_fp_mu);
+    if (!e->tokenizer_fp_ready) {
+        uint64_t h = 14695981039346656037ull;
+        /* Hash tokenizer CONTENT, never in-memory struct bytes: pointer
+         * fields change with every process mapping, which would make the
+         * fingerprint restart-unstable and reject valid checkpoints after
+         * an ordinary server restart. */
+        h = tokenizer_fp_hash_bytes(h, &e->vocab.n_vocab, sizeof(e->vocab.n_vocab));
+        for (int ti = 0; ti < e->vocab.n_vocab; ti++) {
+            h = tokenizer_fp_hash_bytes(h, &e->vocab.token[ti].len,
+                                        sizeof(e->vocab.token[ti].len));
+            h = tokenizer_fp_hash_bytes(h, e->vocab.token[ti].ptr,
+                                        (size_t)e->vocab.token[ti].len);
+        }
+        /* Merge ranks: the open-addressed slot order is a deterministic
+         * function of the GGUF merge list, so hashing slots in index order
+         * (key bytes + rank) is stable across processes. */
+        h = tokenizer_fp_hash_bytes(h, &e->vocab.merge_rank.used,
+                                    sizeof(e->vocab.merge_rank.used));
+        h = tokenizer_fp_hash_bytes(h, &e->vocab.merge_rank.cap,
+                                    sizeof(e->vocab.merge_rank.cap));
+        for (uint64_t mi = 0; mi < e->vocab.merge_rank.cap; mi++) {
+            str_i32_entry *en = &e->vocab.merge_rank.entry[mi];
+            if (!en->used) continue;
+            h = tokenizer_fp_hash_bytes(h, &en->key.len, sizeof(en->key.len));
+            h = tokenizer_fp_hash_bytes(h, en->key.ptr, (size_t)en->key.len);
+            h = tokenizer_fp_hash_bytes(h, &en->value, sizeof(en->value));
+        }
+        for (size_t pi = 0; pi < sizeof(DS4_TOKENIZER_FP_PROBES) /
+                                   sizeof(DS4_TOKENIZER_FP_PROBES[0]); pi++) {
+            ds4_tokens t = {0};
+            tokenize_rendered_chat_vocab(&e->vocab,
+                                         DS4_TOKENIZER_FP_PROBES[pi], &t);
+            h = tokenizer_fp_hash_bytes(h, t.v, (size_t)t.len * sizeof(t.v[0]));
+            ds4_tokens_free(&t);
+        }
+        e->tokenizer_fp = h;
+        e->tokenizer_fp_ready = 1;
+    }
+    uint64_t fp = e->tokenizer_fp;
+    pthread_mutex_unlock(&e->tokenizer_fp_mu);
+    return fp;
+}
+
 int ds4_engine_routed_quant_bits(ds4_engine *e) {
     if (!e) return 0;
     for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
@@ -63036,6 +63113,7 @@ static int ds4_engine_open_internal(ds4_engine **out,
                                      const ds4_engine_options *opt,
                                      const ds4_gpu_config *gpu_cfg) {
     ds4_engine *e = xcalloc(1, sizeof(*e));
+    pthread_mutex_init(&e->tokenizer_fp_mu, NULL);
 #if defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
     g_glm_rocm_guard_available_baseline = 0;
     (void)ds4_linux_nonmovable_memory(&g_glm_rocm_guard_available_baseline);

ds4.h

@@ -548,6 +548,13 @@ void ds4_session_rewind(ds4_session *s, int pos);
 int ds4_session_pos(ds4_session *s);
 int ds4_session_ctx(ds4_session *s);
 int ds4_session_prefill_cap(ds4_session *s);
+/* Behavioral tokenizer fingerprint: a fixed probe string set run through the
+ * real tokenizer, hashed.  Covers data (vocab/merges) AND code (pre-tokenizer
+ * rules) changes, so persisted KV checkpoints can detect engine builds whose
+ * tokenization the file's token history can no longer be reproduced by.
+ * Stable for a given build; 0 only before the vocab is loaded. */
+uint64_t ds4_engine_tokenizer_fingerprint(ds4_engine *e);
+
 int ds4_engine_routed_quant_bits(ds4_engine *e);
 bool ds4_engine_has_output_head(ds4_engine *e);
 bool ds4_engine_has_mtp(ds4_engine *e);

ds4_kvstore.c

@@ -1271,6 +1271,47 @@ int ds4_kvstore_try_load_text(ds4_kvstore *kc,
             }
         }
     }
+    /* Tokenizer fingerprint pre-check.  Stamped files carry a behavioral
+     * fingerprint of the engine that produced their token history as the
+     * first trailer section.  A mismatch means this engine can no longer
+     * reproduce those token ids from the stored text (tokenizer code or
+     * data changed since the checkpoint was written), so restoring it
+     * would re-import a tokenization that every future request mismatches
+     * at the drift point — a permanent reload loop through text-keyed
+     * tiers.  Reject before touching the payload; the ordinary prefill
+     * path rewrites a fresh stamped file under the same text key.
+     * Unstamped legacy files keep today's trusted behavior. */
+    if (header_ok && (hdr.ext_flags & DS4_KVSTORE_EXT_TOKFP)) {
+        long payload_start = ftell(fp);
+        uint64_t tokfp = 0;
+        bool have_fp = false;
+        if (payload_start >= 0 &&
+            fseek(fp, payload_start + (long)hdr.payload_bytes, SEEK_SET) == 0) {
+            uint8_t sh[8];
+            uint8_t fb[8];
+            if (fread(sh, 1, sizeof(sh), fp) == (long)sizeof(sh) &&
+                sh[0] == 'T' && sh[1] == 'K' && sh[2] == 'F' &&
+                sh[3] == 1 && ds4_kvstore_le_get32(sh + 4) == 8 &&
+                fread(fb, 1, sizeof(fb), fp) == (long)sizeof(fb)) {
+                for (int i = 0; i < 8; i++) tokfp |= (uint64_t)fb[i] << (8 * i);
+                have_fp = true;
+            }
+            fseek(fp, payload_start, SEEK_SET);
+        }
+        if (have_fp && tokfp != ds4_engine_tokenizer_fingerprint(engine)) {
+            kv_logf(kc, DS4_KVSTORE_LOG_WARNING,
+                    "%s: kv cache tokenizer fingerprint mismatch, refusing stale tokenization%s%s %s",
+                    kv_log_name(kc),
+                    responses_protocol ? " " : "",
+                    responses_protocol ? "RESPPROTO" : "",
+                    path);
+            fclose(fp);
+            free(cached_text);
+            free(path);
+            return 0;
+        }
+    }
+
     char err[160] = {0};
     int loaded = 0;
     if (header_ok &&

ds4_kvstore.h

@@ -16,6 +16,9 @@
 #define DS4_KVSTORE_EXT_RESPONSES_VISIBLE (1u << 1)
 #define DS4_KVSTORE_EXT_THINKING_VISIBLE  (1u << 2)
 #define DS4_KVSTORE_EXT_SESSION_TITLE     (1u << 3)
+/* Auxiliary (never a key-kind) bit: the trailer begins with a tokenizer
+ * fingerprint section the loader can reject on before touching the payload. */
+#define DS4_KVSTORE_EXT_TOKFP             (1u << 5)
 
 typedef enum {
     DS4_KVSTORE_REASON_UNKNOWN   = 0,

ds4_server.c

@@ -9954,6 +9954,17 @@ static void apply_anthropic_stream_tool_ids(tool_calls *calls,
 #define KV_CACHE_FIXED_HEADER DS4_KVSTORE_FIXED_HEADER
 #define KV_CACHE_HIT_HALF_LIFE_SECONDS DS4_KVSTORE_HIT_HALF_LIFE_SECONDS
 #define KV_EXT_TOOL_MAP DS4_KVSTORE_EXT_TOOL_MAP
+#define KV_EXT_TOKFP DS4_KVSTORE_EXT_TOKFP
+
+/* Tokenizer fingerprint section: first trailer section of stamped files.
+ * 8-byte header (3 magic, 1 version, 4 payload length) + u64 fp. */
+#define KV_TOKFP_MAGIC0 'T'
+#define KV_TOKFP_MAGIC1 'K'
+#define KV_TOKFP_MAGIC2 'F'
+#define KV_TOKFP_VERSION 1u
+#define KV_TOKFP_HEADER 8u
+#define KV_TOKFP_PAYLOAD 8u
+#define KV_TOKFP_BYTES (KV_TOKFP_HEADER + KV_TOKFP_PAYLOAD)
 #define KV_EXT_RESPONSES_VISIBLE DS4_KVSTORE_EXT_RESPONSES_VISIBLE
 #define KV_EXT_THINKING_VISIBLE DS4_KVSTORE_EXT_THINKING_VISIBLE
 #define KV_TOOL_MAP_MAGIC0 'K'
@@ -10362,23 +10373,57 @@ static bool kv_cache_file_size_fits(const kv_disk_cache *kc,
 
 static bool kv_cache_tool_map_size_cb(void *ud, const char *text,
                                       uint64_t *bytes_out) {
-    return kv_tool_map_serialized_size((server *)ud, text, bytes_out);
+    uint64_t tool_bytes = 0;
+    if (!kv_tool_map_serialized_size((server *)ud, text, &tool_bytes)) return false;
+    *bytes_out = KV_TOKFP_BYTES + tool_bytes;
+    return true;
 }
 
 static bool kv_cache_tool_map_write_cb(void *ud, FILE *fp, const char *text,
                                        uint64_t *written_bytes) {
-    return kv_tool_map_write((server *)ud, fp, text, written_bytes);
+    server *s = (server *)ud;
+    uint64_t written = 0;
+    uint8_t h[KV_TOKFP_HEADER];
+    h[0] = KV_TOKFP_MAGIC0;
+    h[1] = KV_TOKFP_MAGIC1;
+    h[2] = KV_TOKFP_MAGIC2;
+    h[3] = KV_TOKFP_VERSION;
+    le_put32(h + 4, KV_TOKFP_PAYLOAD);
+    uint8_t fb[8];
+    uint64_t tok_fp = ds4_engine_tokenizer_fingerprint(s ? s->engine : NULL);
+    for (int i = 0; i < 8; i++) fb[i] = (uint8_t)(tok_fp >> (8 * i));
+    if (fwrite(h, 1, sizeof(h), fp) != sizeof(h)) return false;
+    if (fwrite(fb, 1, sizeof(fb), fp) != sizeof(fb)) return false;
+    written += KV_TOKFP_BYTES;
+    uint64_t tool_written = 0;
+    if (!kv_tool_map_write(s, fp, text, &tool_written)) return false;
+    written += tool_written;
+    if (written_bytes) *written_bytes = written;
+    return true;
 }
 
 static int kv_cache_tool_map_load_cb(void *ud, FILE *fp, const void *wanted) {
+    /* Skip the tokenizer fingerprint section when present; the loader has
+     * already rejected mismatches before the payload was touched, so the
+     * value is informational here.  Older files start directly at the tool
+     * map; restore the position for them. */
+    long start = ftell(fp);
+    uint8_t h[8];
+    if (start >= 0 && fread(h, 1, sizeof(h), fp) == (long)sizeof(h) &&
+        h[0] == KV_TOKFP_MAGIC0 && h[1] == KV_TOKFP_MAGIC1 &&
+        h[2] == KV_TOKFP_MAGIC2 && h[3] == KV_TOKFP_VERSION) {
+        if (fseek(fp, (long)le_get32(h + 4), SEEK_CUR) != 0) return -1;
+    } else if (start >= 0) {
+        if (fseek(fp, start, SEEK_SET) != 0) return -1;
+    }
     return kv_tool_map_load_from_pos((server *)ud, fp, (const stop_list *)wanted);
 }
 
 static ds4_kvstore_trailer_hooks kv_cache_tool_map_hooks(server *s,
                                                          const stop_list *wanted) {
     return (ds4_kvstore_trailer_hooks){
         .ud = s,
-        .ext_flag = KV_EXT_TOOL_MAP,
+        .ext_flag = (uint8_t)(KV_EXT_TOKFP | KV_EXT_TOOL_MAP),
         .serialized_size = kv_cache_tool_map_size_cb,
         .write = kv_cache_tool_map_write_cb,
         .load = kv_cache_tool_map_load_cb,