From 709aad077e96f132a2aa36b392eb232f130c434b Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Sat, 30 May 2026 17:04:52 -0700 Subject: [PATCH 1/3] restore unreviewed schemadrift admin command --- go/cmd/dolt/commands/admin/admin.go | 2 + .../dolt/commands/admin/schemadrift/check.go | 872 +++++++++++++++++ .../schemadrift/check_classification_test.go | 208 ++++ .../admin/schemadrift/check_safeempty_test.go | 161 ++++ .../commands/admin/schemadrift/classify.go | 248 +++++ .../admin/schemadrift/classify_test.go | 240 +++++ .../commands/admin/schemadrift/export_test.go | 61 ++ .../admin/schemadrift/migrate_adaptive.go | 905 ++++++++++++++++++ .../migrate_adaptive_export_test.go | 32 + ..._adaptive_ignored_inline_invariant_test.go | 91 ++ .../migrate_adaptive_readerror_test.go | 61 ++ .../admin/schemadrift/recover_rows.go | 813 ++++++++++++++++ .../recover_rows_integration_test.go | 488 ++++++++++ .../admin/schemadrift/recover_rows_test.go | 342 +++++++ .../dolt/commands/admin/schemadrift/repair.go | 742 ++++++++++++++ .../repair_20byte_inline_hole_test.go | 153 +++ .../repair_inline_heterogeneous_test.go | 332 +++++++ .../commands/admin/schemadrift/schemadrift.go | 52 + .../schemadrift_integration_test.go | 281 ++++++ 19 files changed, 6084 insertions(+) create mode 100644 go/cmd/dolt/commands/admin/schemadrift/check.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/check_classification_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/check_safeempty_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/classify.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/classify_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/export_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_export_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_ignored_inline_invariant_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_readerror_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/recover_rows.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/recover_rows_integration_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/recover_rows_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/repair.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/repair_20byte_inline_hole_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/repair_inline_heterogeneous_test.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/schemadrift.go create mode 100644 go/cmd/dolt/commands/admin/schemadrift/schemadrift_integration_test.go diff --git a/go/cmd/dolt/commands/admin/admin.go b/go/cmd/dolt/commands/admin/admin.go index 0852c87ad18..f18bc343a30 100644 --- a/go/cmd/dolt/commands/admin/admin.go +++ b/go/cmd/dolt/commands/admin/admin.go @@ -17,6 +17,7 @@ package admin import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/createchunk" + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" ) var Commands = cli.NewHiddenSubCommandHandler("admin", "Commands for directly working with Dolt storage for purposes of testing or database recovery", []cli.Command{ @@ -29,4 +30,5 @@ var Commands = cli.NewHiddenSubCommandHandler("admin", "Commands for directly wo ArchiveInspectCmd{}, JournalInspectCmd{}, createchunk.Commands, + schemadrift.Commands, }) diff --git a/go/cmd/dolt/commands/admin/schemadrift/check.go b/go/cmd/dolt/commands/admin/schemadrift/check.go new file mode 100644 index 00000000000..674af53565d --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/check.go @@ -0,0 +1,872 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + "encoding/json" + "fmt" + "io" + "path/filepath" + "sort" + "strings" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/commands" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/val" +) + +// hashByteLen is the address length used by every legacy AddrEnc family. We +// alias it locally so the scanner's hot path doesn't carry an extra import. +const hashByteLen = hash.ByteLen + +// newHashFromField wraps hash.New so the scanner can call it without +// pulling the hash import into its inner loop more than once. +func newHashFromField(b []byte) hash.Hash { return hash.New(b) } + +// CheckCmd is the read-only side of the schema-encoding-drift admin pair. It +// walks every table on the current working root, identifies columns whose +// persisted TypeInfo.Encoding() is an adaptive variant, and inspects the first +// non-NULL row's field bytes via val.Tuple.GetField (NOT via the adaptive +// decoder, which is exactly what panics on legacy data tagged as adaptive). +type CheckCmd struct{} + +var _ cli.Command = CheckCmd{} + +const ( + jsonFlag = "json" + allDatabasesFlag = "all-databases" + databaseFlag = "database" +) + +var checkDocs = cli.CommandDocumentationContent{ + ShortDesc: "Detect persisted-schema / on-disk-row encoding drift", + LongDesc: `Walks every table on the current working root and reports columns whose persisted ` + + `adaptive encoding tag does not match the on-disk byte layout of the row data. + +The intended drift signature: a column whose schema record says ` + "`StringAdaptiveEnc`" + ` / ` + + "`BytesAdaptiveEnc`" + ` / ` + "`JsonAdaptiveEnc`" + ` / ` + "`GeomAdaptiveEnc`" + ` / ` + + "`ExtendedAdaptiveEnc`" + ` but whose first row's field bytes are exactly 20 bytes with a non-zero leading byte — the legacy raw-hash address that 1.x ` + "`*AddrEnc`" + ` columns persisted, and which adaptive dispatch panics on with ` + "`invalid hash length: 19`" + `. + +This command never invokes adaptive dispatch and never modifies state. It is safe to run against a corrupted database. + +Exit codes: + 0 no drift detected, or only adaptive-formatted data on adaptive-tagged columns + 1 at least one column shows the legacy-raw-hash payload under an adaptive tag + +Use ` + "`--json`" + ` to emit a structured array suitable for piping to ` + "`schema-encoding-drift repair`" + `.`, + Synopsis: []string{"[--json]"}, +} + +// Name returns the subcommand name as registered in admin's command list. +func (cmd CheckCmd) Name() string { return "check" } + +// Description is the short summary shown in `dolt admin` help. +func (cmd CheckCmd) Description() string { + return "Detect persisted-schema / on-disk-row encoding drift" +} + +// RequiresRepo is true: we need an initialized dolt environment to read tables. +func (cmd CheckCmd) RequiresRepo() bool { return true } + +// Docs is the full command documentation surfaced by `dolt admin schema-encoding-drift check --help`. +func (cmd CheckCmd) Docs() *cli.CommandDocumentation { + return cli.NewCommandDocumentation(checkDocs, cmd.ArgParser()) +} + +// ArgParser declares the command's flag surface. +func (cmd CheckCmd) ArgParser() *argparser.ArgParser { + ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0) + ap.SupportsFlag(jsonFlag, "", "Emit JSON output instead of a human-readable table") + ap.SupportsFlag(allDatabasesFlag, "", "Walk every database under the dolt data directory (default behaviour; specify --database to target a single DB)") + ap.SupportsString(databaseFlag, "", "name", "Restrict the scan to a single database by name. Mutually exclusive with --all-databases.") + return ap +} + +// Hidden mirrors the rest of the admin commands. +func (cmd CheckCmd) Hidden() bool { return true } + +// DriftRow is one persisted-schema / on-disk-row mismatch — or, in the JSON +// case, one row of the report. It also carries the suggested legacy sibling +// encoding so a downstream `repair` invocation can be reconstructed without +// re-deriving it. +// +// The Severity field classifies the row into one of three buckets: +// +// - "drift" — strong evidence of legacy raw hashes on an adaptive +// schema tag. Repair will flip without flags. +// - "safe-empty" — no strong evidence either way, but the entire scanned +// payload was structurally consistent with the legacy +// homogeneous-empty pattern (rows are all NULL, all 0x00 +// inline empties, or ambiguous 20-byte 0x00-leading where +// the hash is the all-zero/empty-content address). Per +// a forensic verdict these columns are safe to +// flip; repair surfaces them only with --include-empty. +// - (omitted) — columns that cannot be classified into either bucket +// are dropped from the report (not surfaced). +// +// The SafeToRepair flag pairs with Severity for JSON consumers piping into +// the repair tool — "drift" rows always set true; "safe-empty" rows set +// true only when repair has been opted into for the empty bucket. +type DriftRow struct { + Database string `json:"database"` + Table string `json:"table"` + Column string `json:"column"` + DeclaredEncoding string `json:"declared_encoding"` + ObservedFormat string `json:"observed_format"` + SuggestedEncoding string `json:"suggested_encoding,omitempty"` + Severity string `json:"severity"` + SafeToRepair bool `json:"safe_to_repair"` + // Hint is the literal command an operator should run to remediate this + // row — e.g. `dolt admin schema-encoding-drift repair --table X --column Y` + // for a drift row, or the `recover-rows` analogue for a heterogeneous + // row. Surfaced in both JSON and the TTY table so the next action is + // always one copy-paste away. + Hint string `json:"hint,omitempty"` +} + +// SeverityDrift is the bucket label for strong-evidence drift rows. +const SeverityDrift = "drift" + +// SeveritySafeEmpty is the bucket label for the homogeneous-empty pattern — +// columns where every scanned row was either NULL or the canonical 20-byte +// all-zero shape that arises when a legacy raw hash points at an empty / +// sparse chunk. forensic analysis verified these are safe to flip; repair requires +// --include-empty to act on them. +const SeveritySafeEmpty = "safe-empty" + +// SeverityHeterogeneous is the bucket label for columns whose payload mixes +// legacy raw-hash rows and genuine adaptive-format rows. The schema record +// can't be flipped in either direction without corrupting one side; the only +// correct remediation is row-by-row migration. `repair` refuses these; +// `recover-rows` performs the migration. +// +// a later fix/the fix: this bucket exists because the earlier `scanTableForDrift` used to +// short-circuit on the first strong-evidence row and silently misclassify +// heterogeneous columns as either pure drift or pure clean. The empirical analysis +// empirical numbers (real-world TEXT columns all +// heterogeneous) made it clear that operators needed an explicit surface +// for "this column has both kinds of payload and `repair` won't touch it". +const SeverityHeterogeneous = "heterogeneous" + +// Exec is the CLI entry point. +func (cmd CheckCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, _ cli.CliContext) int { + ap := cmd.ArgParser() + usage, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, checkDocs, ap)) + apr := cli.ParseArgsOrDie(ap, args, usage) + + jsonOut := apr.Contains(jsonFlag) + singleDB, hasSingleDB := apr.GetValue(databaseFlag) + allDBs := apr.Contains(allDatabasesFlag) + + if hasSingleDB && allDBs { + verr := errhand.BuildDError("--all-databases and --database are mutually exclusive").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + // Default posture: walk all sibling databases under the current dEnv's + // data directory. The single-DB invocation is only used when the operator + // explicitly opts in via --database. + walkAll := !hasSingleDB + + drifts, err := scanForDriftMultiDB(ctx, dEnv, walkAll, singleDB) + if err != nil { + verr := errhand.BuildDError("failed to scan for schema-encoding drift").AddCause(err).Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + if jsonOut { + return emitDriftJSON(cli.OutStream, drifts) + } + return emitDriftTable(cli.OutStream, drifts) +} + +// scanForDriftMultiDB dispatches the drift scan across one or more databases +// rooted at the dEnv's data directory. When walkAll is true, every sibling +// database under the data directory is scanned; when false, only the database +// matching singleDBName is scanned (case-sensitive against the env name dolt +// derives from the directory). The result is the concatenated drift list +// across all visited databases, sorted by (database, table, column). +// +// Operationally, "data directory" is the parent of the dEnv's working +// directory — that's how the managed-dolt deployment lays out multiple +// databases as siblings: `/db1/.dolt`, `/db2/.dolt`, +// `/db3/.dolt`, etc. The default `MultiEnvForSingleEnv` would +// otherwise iterate inside the current database directory and find nothing +// (only `.dolt`); we explicitly walk the parent so sibling DBs are visible. +func scanForDriftMultiDB(ctx context.Context, dEnv *env.DoltEnv, walkAll bool, singleDBName string) ([]DriftRow, error) { + mrEnv, err := buildMultiEnvWalkingParent(ctx, dEnv) + if err != nil { + return nil, fmt.Errorf("build multi-database env: %w", err) + } + + var all []DriftRow + // MultiEnvForDirectory adds the originating dEnv into the iteration AND + // adds it again under its actual sibling dir name. Dedup by the canonical + // filesystem path of the `.dolt` directory rather than the database name: + // + // - DoltDB.GetDatabaseName() is path-derived, not symlink-resolved. + // If the datadir has `app2 -> db2/`, both surface with distinct + // db-name strings even though they're the same underlying physical + // database — without canonical-path dedup we'd report every drift + // row twice for the symlinked alias. + // - The dEnv-added-twice case (the original reason for dedup) is still + // caught because both copies resolve to the same canonical .dolt + // directory. + // + // `filepath.EvalSymlinks` failures (path doesn't exist, permission + // errors) fall back to the raw db-name key so the dedup degrades to the + // earlier behaviour rather than crashing the walk. + visited := map[string]bool{} + err = mrEnv.Iter(func(name string, sibling *env.DoltEnv) (stop bool, iterErr error) { + if !walkAll && name != singleDBName { + return false, nil + } + if sibling.DBLoadError != nil { + // Skip unloadable databases (e.g. a sibling dir without a + // .dolt subfolder; or a partially-initialised DB). Don't fail + // the whole walk for an unloadable sibling — record nothing + // and continue. + return false, nil + } + ddb := sibling.DoltDB(ctx) + if ddb == nil { + return false, nil + } + key := canonicalDBKey(sibling) + if visited[key] { + return false, nil + } + visited[key] = true + rows, scanErr := scanForDrift(ctx, sibling) + if scanErr != nil { + return true, fmt.Errorf("scan database %s: %w", name, scanErr) + } + all = append(all, rows...) + return false, nil + }) + if err != nil { + return nil, err + } + sort.Slice(all, func(i, j int) bool { + if all[i].Database != all[j].Database { + return all[i].Database < all[j].Database + } + if all[i].Table != all[j].Table { + return all[i].Table < all[j].Table + } + return all[i].Column < all[j].Column + }) + return all, nil +} + +// canonicalDBKey returns a symlink-resolved key for the given dEnv suitable +// for deduplicating two iteration entries that refer to the same underlying +// database. The preferred key is the absolute path of the `.dolt` directory +// after symlink resolution; if that path cannot be derived or resolved +// (e.g. the dEnv has no FS, the path doesn't exist, or EvalSymlinks errors), +// we fall back to `DoltDB.GetDatabaseName()` so the dedup degrades to the +// earlier path-name behaviour rather than crashing the walk. +func canonicalDBKey(dEnv *env.DoltEnv) string { + if dEnv == nil { + return "" + } + if doltDir := dEnv.GetDoltDir(); doltDir != "" { + if dEnv.FS != nil { + if abs, err := dEnv.FS.Abs(doltDir); err == nil { + if resolved, evalErr := filepath.EvalSymlinks(abs); evalErr == nil { + return resolved + } + return abs + } + } + } + if ddb := dEnv.DoltDB(context.Background()); ddb != nil { + return ddb.GetDatabaseName() + } + return "" +} + +// buildMultiEnvWalkingParent constructs a MultiRepoEnv whose iteration walks +// the *parent* of the current dEnv's FS. This is the layout the managed-dolt +// data dir uses: every database is a sibling subdir +// (`/db1/`, `/db2/`, ...). MultiEnvForSingleEnv iterates the +// CURRENT dir, which from inside one of these DBs only sees `.dolt`. We +// rebase the FS to the parent so all siblings are discovered. +// +// If the parent cannot be walked (e.g. running from a temp test dir, or +// from a dolt env not laid out as a managed-dolt sibling tree), fall back +// to the default single-env iteration so the command still works for +// developer one-offs. +func buildMultiEnvWalkingParent(ctx context.Context, dEnv *env.DoltEnv) (*env.MultiRepoEnv, error) { + if dEnv == nil || dEnv.FS == nil { + return env.MultiEnvForSingleEnv(ctx, dEnv) + } + parentFS, err := dEnv.FS.WithWorkingDir("..") + if err != nil { + return env.MultiEnvForSingleEnv(ctx, dEnv) + } + parentAbs, err := parentFS.Abs("") + if err != nil { + return env.MultiEnvForSingleEnv(ctx, dEnv) + } + currentAbs, err := dEnv.FS.Abs("") + if err != nil { + return env.MultiEnvForSingleEnv(ctx, dEnv) + } + // Refuse to walk if "parent" equals current (we're already at FS root) + // or if it doesn't actually exist. + if parentAbs == currentAbs { + return env.MultiEnvForSingleEnv(ctx, dEnv) + } + if exists, _ := parentFS.Exists("."); !exists { + return env.MultiEnvForSingleEnv(ctx, dEnv) + } + return env.MultiEnvForDirectory(ctx, parentFS, dEnv) +} + +// scanForDrift walks the dolt env's current working root and returns one +// DriftRow per detected (table, column) mismatch. A column is reported when +// either: +// +// - Its TypeInfo.Encoding() is an adaptive variant AND at least one row's +// field bytes carry strong evidence of legacy raw-hash format → severity +// "drift", SafeToRepair=true. +// - The same encoding gate plus a row payload that is uniformly +// consistent with the homogeneous-empty pattern (every scanned row is +// NULL, a one-byte `[0x00]` inline empty, or the structurally-ambiguous +// 20-byte all-zero shape that arises when a legacy hash points at an +// empty / sparse chunk) AND ZERO rows of definitively-adaptive evidence +// → severity "safe-empty", SafeToRepair=true. These are the safe-empty bucket +// for which row-level payload is uniformly the all-zero artefact the +// adaptive reader synthesises for an empty content blob. +// +// Columns with strong evidence AGAINST drift (any FieldAdaptiveAddressed +// witness) AND no legacy witness are internally consistent — silently +// skipped. Columns with both legacy and adaptive witnesses (heterogeneous +// payloads) are also reported as "drift" with SafeToRepair=true; repair's +// own safety gate independently catches heterogeneous payloads with a +// stricter refusal at write time, so check stays diagnostic and lenient. +// +// The traversal uses includeRootObjects=true so that working-set-only +// tables (those `dolt_ignore`'d or otherwise excluded from the normal +// public schema view) are not silently skipped on the read side. The +// rootValue impl in `go/libraries/doltcore/doltdb/root_val.go` currently +// ignores the flag, but other RootValue implementations honour it; we +// pass the most permissive value to keep the scanner future-proof. +func scanForDrift(ctx context.Context, dEnv *env.DoltEnv) ([]DriftRow, error) { + ddb := dEnv.DoltDB(ctx) + dbName := ddb.GetDatabaseName() + + root, err := dEnv.WorkingRoot(ctx) + if err != nil { + return nil, fmt.Errorf("read working root: %w", err) + } + + tblNames, err := root.GetAllTableNames(ctx, true /* includeRootObjects */) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + sort.Slice(tblNames, func(i, j int) bool { return tblNames[i].Name < tblNames[j].Name }) + + // Identify dolt_ignore'd tables so hints steer the operator to + // migrate-adaptive (the safe force-inline heal) rather than repair / + // recover-rows, both of which commit and would dangling-fault on a + // never-committed table. + ignoredSet := map[string]bool{} + if roots, rErr := dEnv.Roots(ctx); rErr == nil { + if ign, iErr := doltdb.IdentifyIgnoredTables(ctx, roots, tblNames); iErr == nil { + for _, t := range ign { + ignoredSet[t.Name] = true + } + } + } + + var out []DriftRow + for _, tn := range tblNames { + tbl, _, err := root.GetTable(ctx, tn) + if err != nil { + return nil, fmt.Errorf("get table %s: %w", tn.Name, err) + } + if tbl == nil { + continue + } + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, fmt.Errorf("get schema for %s: %w", tn.Name, err) + } + + // Identify the indices of every adaptive-encoded NON-virtual NON-PK + // column in the value tuple. We use schema_impl's value-descriptor + // ordering (NonPKCols.Iter, skipping virtual). PK columns receive + // adaptive-incompatible encodings (the schema impl forces VarChar / + // VarBinary / Cell for BLOB / TEXT / GEOMETRY in the key tuple), so + // they're not drift candidates. + candidates := collectAdaptiveValueColumns(sch) + if len(candidates) == 0 { + continue + } + + idx, err := tbl.GetRowData(ctx) + if err != nil { + return nil, fmt.Errorf("get row data for %s: %w", tn.Name, err) + } + if idx == nil { + continue + } + empty, err := idx.Empty() + if err != nil { + return nil, fmt.Errorf("check empty for %s: %w", tn.Name, err) + } + if empty { + // Empty table — no payload to compare against the declared + // encoding. Skip silently; this is not drift. + continue + } + + pm, err := durable.ProllyMapFromIndex(idx) + if err != nil { + return nil, fmt.Errorf("open prolly map for %s: %w", tn.Name, err) + } + + buckets, err := scanTableForDrift(ctx, pm, candidates, ddb) + if err != nil { + return nil, fmt.Errorf("scan rows of %s: %w", tn.Name, err) + } + + for _, c := range buckets.drift { + // On a dolt_ignore'd table, repair (which commits) would + // dangling-fault — steer to migrate-adaptive only, and don't mark + // it SafeToRepair. + hint := fmt.Sprintf("dolt admin schema-encoding-drift repair --table %s --column %s (restore legacy tag), or migrate-adaptive --table %s --column %s (forward to v2-native adaptive)", tn.Name, c.colName, tn.Name, c.colName) + safeToRepair := true + if ignoredSet[tn.Name] { + hint = fmt.Sprintf("dolt admin schema-encoding-drift migrate-adaptive --table %s --column %s (dolt_ignore'd table: repair/recover-rows would dangling-fault; force-inline heal only)", tn.Name, c.colName) + safeToRepair = false + } + out = append(out, DriftRow{ + Database: dbName, + Table: tn.Name, + Column: c.colName, + DeclaredEncoding: encodingName(c.declared), + ObservedFormat: FieldLegacyRawHash.String(), + SuggestedEncoding: encodingName(c.suggested), + Severity: SeverityDrift, + SafeToRepair: safeToRepair, + Hint: hint, + }) + } + for _, c := range buckets.heterogeneous { + out = append(out, DriftRow{ + Database: dbName, + Table: tn.Name, + Column: c.colName, + DeclaredEncoding: encodingName(c.declared), + ObservedFormat: "mixed-legacy-and-adaptive", + SuggestedEncoding: encodingName(c.suggested), + Severity: SeverityHeterogeneous, + // SafeToRepair stays FALSE for heterogeneous: `repair` would + // corrupt one side of the mix in either direction. The only + // correct remediation is row-level migration. On a + // dolt_ignore'd table, recover-rows (which commits) would + // dangling-fault — hint migrate-adaptive only. + SafeToRepair: false, + Hint: heterogeneousHint(ignoredSet[tn.Name], tn.Name, c.colName), + }) + } + for _, c := range buckets.safeEmpty { + out = append(out, DriftRow{ + Database: dbName, + Table: tn.Name, + Column: c.colName, + DeclaredEncoding: encodingName(c.declared), + ObservedFormat: "homogeneous-empty", + SuggestedEncoding: encodingName(c.suggested), + Severity: SeveritySafeEmpty, + // SafeToRepair stays true: forensic analysis verdict + the structural + // guarantee that an all-zero 20-byte hash resolves to the + // empty-chunk address means the flip is safe. Repair gates + // this bucket behind --include-empty for operator discretion. + SafeToRepair: true, + Hint: fmt.Sprintf("dolt admin schema-encoding-drift repair --table %s --column %s --include-empty", tn.Name, c.colName), + }) + } + } + + return out, nil +} + +// adaptiveCandidate is a single column flagged as worth inspecting on the +// scan side: it's an adaptive-encoded non-virtual non-PK column, with its +// position in the value tuple precomputed. +type adaptiveCandidate struct { + colName string + colTag uint64 + tupleIndex int + declared val.Encoding + suggested val.Encoding // pre-resolved via LegacySibling for the drift report +} + +// collectAdaptiveValueColumns returns one candidate per adaptive-encoded +// non-virtual non-PK column, in value-tuple order. Returns nil if none. +func collectAdaptiveValueColumns(sch schema.Schema) []adaptiveCandidate { + var cands []adaptiveCandidate + i := 0 + _ = sch.GetNonPKCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { + if col.Virtual { + return false, nil + } + enc := col.TypeInfo.Encoding() + if IsAdaptiveEncoding(enc) { + legacy, ok := LegacySibling(enc) + if ok { + cands = append(cands, adaptiveCandidate{ + colName: col.Name, + colTag: tag, + tupleIndex: i, + declared: enc, + suggested: legacy, + }) + } + } + i++ + return false, nil + }) + return cands +} + +// scanBuckets is the bucketed return of scanTableForDrift. +// +// - drift — columns with strong-evidence legacy-raw-hash witnesses +// AND no genuine-adaptive witnesses. Schema record disagrees +// with on-disk row payload; repair fixes it. +// - heterogeneous — columns with BOTH strong-evidence legacy AND strong- +// evidence adaptive witnesses. Schema and payload disagree +// for some rows, agree for others. Repair refuses; the +// correct remediation is row-by-row migration via +// `recover-rows`. +// - safeEmpty — no strong evidence either way, but every row was +// structurally consistent with the homogeneous-empty +// pattern (NULL or the 20-byte all-zero-bytes shape). +// Per forensic analysis verdict these are safe to flip; surfaced under +// SeveritySafeEmpty. +// +// NOTE: a later fix/the fix — single-byte `[0x00]` inline empties USED to be admitted +// into safeEmpty, but they crash the legacy reader post-flip (hash.New on a +// 1-byte slice panics with `invalid hash length: 1`). forensic analysis only validated the +// 20-byte all-zero shape. Single-byte `[0x00]` rows now disqualify safeEmpty. +type scanBuckets struct { + drift []adaptiveCandidate + heterogeneous []adaptiveCandidate + safeEmpty []adaptiveCandidate +} + +// scanTableForDrift walks the prolly map and classifies each candidate +// column into one of four exclusive outcomes (drift / heterogeneous / +// safeEmpty / clean-and-silently-dropped). +// +// NOTE: a later fix/the fix — the previous implementation short-circuited per column +// on the first strong-evidence row. That mislabelled heterogeneous columns +// (some legacy rows, some adaptive rows) as clean if the first strong row +// happened to be adaptive, or as pure drift if the first strong row happened +// to be legacy. empirical data (real-world TEXT columns (empirical sampling): +// showed every a real-world database +// heterogeneous column was silently misclassified that way. The scanner now +// walks every row for every column, tracks sawLegacy + sawAdaptive +// independently, and only computes the final verdict at end-of-table: +// +// - sawLegacy && !sawAdaptive → drift +// - sawLegacy && sawAdaptive → heterogeneous +// - !sawLegacy && sawAdaptive → clean (silently dropped) +// - !sawLegacy && !sawAdaptive && sawHomogeneousEmptyOnly → safeEmpty +// - otherwise → silently dropped +// +// Cost: ALWAYS walks the full table per scanned database. Acceptable: a large database at +// a large number of rows takes ~tens of seconds on cold cache, and the alternative +// (false-negative on heterogeneous columns) is the kind of "silently miss +// half the corruption" failure that motivated the postgres-tier directive. +func scanTableForDrift(ctx context.Context, pm prolly.Map, cands []adaptiveCandidate, cs ChunkPresenceChecker) (scanBuckets, error) { + // State per column — flags are independent and ALL set during iteration; + // the verdict is computed only at end-of-table from the combined flags. + type colState struct { + cand adaptiveCandidate + sawLegacy bool + sawAdaptive bool + sawHomogeneousEmptyOnly bool + } + states := make(map[int]*colState, len(cands)) + for _, c := range cands { + states[c.tupleIndex] = &colState{ + cand: c, + sawHomogeneousEmptyOnly: true, + } + } + + iter, err := pm.IterAll(ctx) + if err != nil { + return scanBuckets{}, err + } + for { + _, value, iterErr := iter.Next(ctx) + if iterErr == io.EOF { + break + } + if iterErr != nil { + return scanBuckets{}, iterErr + } + for _, st := range states { + // Per-column micro-skip: once a column is known to be BOTH + // legacy and adaptive (heterogeneous), additional witnesses + // can't change the verdict. Skip the structural classification + // to save the per-row cost on the long tail. + // + // We do NOT skip on "saw legacy only" or "saw adaptive only" — + // a subsequent row could promote the column to heterogeneous, + // and that's exactly the verdict this check exists to surface. + if st.sawLegacy && st.sawAdaptive { + continue + } + b := value.GetField(st.cand.tupleIndex) + structural := ClassifyFieldBytes(b) + switch structural { + case FieldLegacyRawHash: + st.sawLegacy = true + // Legacy raw hash is, by construction, not the homogeneous- + // empty shape; that pattern would have classified as + // FieldAdaptiveInline+all-zero-bytes-20-len. + st.sawHomogeneousEmptyOnly = false + case FieldAdaptiveAddressed: + st.sawAdaptive = true + st.sawHomogeneousEmptyOnly = false + case FieldNULL: + // NULL is consistent with homogeneous-empty; no flag flip. + case FieldAdaptiveInline: + if len(b) == hashByteLen && cs != nil { + // Try the chunkstore — if the hash IS present, it's + // a legacy raw address (0x00-leader coincidence) and + // the row is a definitive legacy witness. + h := newHashFromField(b) + present, hasErr := cs.Has(ctx, h) + if hasErr != nil { + return scanBuckets{}, fmt.Errorf("chunkstore presence check for column %s: %w", st.cand.colName, hasErr) + } + if present { + st.sawLegacy = true + st.sawHomogeneousEmptyOnly = false + continue + } + // Chunkstore miss: the 20-byte 0x00-leading shape. + // Stay homogeneous-empty only if the field is the + // canonical all-zero shape (matches the verified + // pattern). A non-zero content byte means it's an + // ambiguous row we can't classify either way. + if !isAllZeroBytes(b) { + st.sawHomogeneousEmptyOnly = false + } + continue + } + // len != 20: a 0x00-leading field that is NOT exactly 20 bytes + // cannot be a legacy raw hash (those are always exactly 20 + // bytes) — it is unambiguously adaptive-inline. Non-empty + // content (`[0x00]`, the documented "inline orphan" + // shape) is a DEFINITIVE adaptive witness, mirroring repair.go's + // scanColumnFullPayload. Without this, an inline-heterogeneous + // column (legacy + inline) sets only sawLegacy and is + // mis-bucketed as "drift / safe-to-repair" — and check then + // emits a `repair` hint that repair itself correctly REFUSES + // (check must not contradict repair). [review finding] + if len(b) != hashByteLen && len(b) > 1 { + st.sawAdaptive = true + } + // a later fix/the fix: short non-NULL inline payloads also disqualify + // safeEmpty. The 1-byte `[0x00]` (adaptive empty-inline) USED to + // be admitted as safe-empty in the earlier version, but the post-flip legacy + // reader does hash.New([1 byte]) → panic `invalid hash length: + // 1`. forensic analysis only validated the 20-byte all-zero shape. + st.sawHomogeneousEmptyOnly = false + case FieldUnknown: + // Unknown shape — disqualifies safeEmpty. No flag set on + // either legacy or adaptive, so a column with only + // FieldUnknown rows lands in "silently dropped". + st.sawHomogeneousEmptyOnly = false + } + } + } + + var buckets scanBuckets + for _, st := range states { + switch { + case st.sawLegacy && st.sawAdaptive: + buckets.heterogeneous = append(buckets.heterogeneous, st.cand) + case st.sawLegacy && !st.sawAdaptive: + buckets.drift = append(buckets.drift, st.cand) + case !st.sawLegacy && st.sawAdaptive: + // Clean: silently drop. + case !st.sawLegacy && !st.sawAdaptive && st.sawHomogeneousEmptyOnly: + buckets.safeEmpty = append(buckets.safeEmpty, st.cand) + } + } + sort.Slice(buckets.drift, func(i, j int) bool { return buckets.drift[i].tupleIndex < buckets.drift[j].tupleIndex }) + sort.Slice(buckets.heterogeneous, func(i, j int) bool { return buckets.heterogeneous[i].tupleIndex < buckets.heterogeneous[j].tupleIndex }) + sort.Slice(buckets.safeEmpty, func(i, j int) bool { return buckets.safeEmpty[i].tupleIndex < buckets.safeEmpty[j].tupleIndex }) + return buckets, nil +} + +// isAllZeroBytes reports whether every byte in b is 0x00. Used by the +// scanner to disambiguate the legacy-empty-content homogeneous shape from +// other 0x00-leading inline shapes. +func isAllZeroBytes(b []byte) bool { + for _, x := range b { + if x != 0 { + return false + } + } + return true +} + +// emitDriftTable renders a TTY-friendly table to w. Returns the appropriate +// process exit code (1 on any drift or safe-empty entries, 0 on clean). +func emitDriftTable(w io.Writer, drifts []DriftRow) int { + if len(drifts) == 0 { + _, _ = fmt.Fprintln(w, "no schema-encoding drift detected") + return 0 + } + + headers := []string{"database", "table", "column", "declared", "observed", "suggested", "severity"} + rows := make([][]string, 0, len(drifts)) + driftCount, heteroCount, safeEmptyCount := 0, 0, 0 + for _, d := range drifts { + rows = append(rows, []string{ + d.Database, d.Table, d.Column, + d.DeclaredEncoding, d.ObservedFormat, d.SuggestedEncoding, + d.Severity, + }) + switch d.Severity { + case SeverityDrift: + driftCount++ + case SeverityHeterogeneous: + heteroCount++ + case SeveritySafeEmpty: + safeEmptyCount++ + } + } + + widths := make([]int, len(headers)) + for i, h := range headers { + widths[i] = len(h) + } + for _, r := range rows { + for i, c := range r { + if len(c) > widths[i] { + widths[i] = len(c) + } + } + } + + writeRow := func(cols []string) { + parts := make([]string, len(cols)) + for i, c := range cols { + parts[i] = c + strings.Repeat(" ", widths[i]-len(c)) + } + _, _ = fmt.Fprintln(w, strings.Join(parts, " ")) + } + + writeRow(headers) + sep := make([]string, len(headers)) + for i := range sep { + sep[i] = strings.Repeat("-", widths[i]) + } + writeRow(sep) + for _, r := range rows { + writeRow(r) + } + _, _ = fmt.Fprintf(w, + "\n%d total — %d %s (repair by default), %d %s (recover-rows), %d %s (repair --include-empty)\n", + len(drifts), driftCount, SeverityDrift, heteroCount, SeverityHeterogeneous, safeEmptyCount, SeveritySafeEmpty) + return 1 +} + +// emitDriftJSON writes the drift list as a JSON array suitable for piping +// directly into `repair`. Returns the appropriate process exit code. +func emitDriftJSON(w io.Writer, drifts []DriftRow) int { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if drifts == nil { + // Encode an empty array (not null) so consumers can pipe unconditionally. + drifts = []DriftRow{} + } + if err := enc.Encode(drifts); err != nil { + // Fall back to plain output and signal error. + _, _ = fmt.Fprintf(w, "{\"error\":\"%s\"}\n", err.Error()) + return 1 + } + if len(drifts) == 0 { + return 0 + } + return 1 +} + +// encodingName returns a stable string label for a val.Encoding suitable for +// user-facing output. Falls back to %d for any encoding not in the +// adaptive/legacy pairing. +// heterogeneousHint returns the remediation hint for a heterogeneous column. +// On a dolt_ignore'd table only migrate-adaptive is safe: recover-rows commits +// and would dangling-fault on the table's never-committed content chunks. +func heterogeneousHint(ignored bool, table, col string) string { + if ignored { + return fmt.Sprintf("dolt admin schema-encoding-drift migrate-adaptive --table %s --column %s (dolt_ignore'd table: recover-rows would dangling-fault; force-inline heal only)", table, col) + } + return fmt.Sprintf("dolt admin schema-encoding-drift recover-rows --table %s --column %s (row-by-row to legacy), or migrate-adaptive --table %s --column %s (row-by-row to v2-native adaptive)", table, col, table, col) +} + +func encodingName(enc val.Encoding) string { + switch enc { + case val.StringAddrEnc: + return "StringAddrEnc" + case val.BytesAddrEnc: + return "BytesAddrEnc" + case val.JSONAddrEnc: + return "JSONAddrEnc" + case val.GeomAddrEnc: + return "GeomAddrEnc" + case val.ExtendedAddrEnc: + return "ExtendedAddrEnc" + case val.StringAdaptiveEnc: + return "StringAdaptiveEnc" + case val.BytesAdaptiveEnc: + return "BytesAdaptiveEnc" + case val.JsonAdaptiveEnc: + return "JsonAdaptiveEnc" + case val.GeomAdaptiveEnc: + return "GeomAdaptiveEnc" + case val.ExtendedAdaptiveEnc: + return "ExtendedAdaptiveEnc" + } + return fmt.Sprintf("Encoding(%d)", enc) +} + +// Compile-time check that doltdb.Table satisfies the row-access API we use. +// We don't actually need any methods from this var — it's a guard so a +// future doltdb.Table refactor that breaks GetRowData / GetSchema is caught +// at build time here, not at first invocation. +var _ = (*doltdb.Table)(nil) diff --git a/go/cmd/dolt/commands/admin/schemadrift/check_classification_test.go b/go/cmd/dolt/commands/admin/schemadrift/check_classification_test.go new file mode 100644 index 00000000000..0fe34fba44e --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/check_classification_test.go @@ -0,0 +1,208 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// TestHeterogeneousSurfacing is the regression guard. +// +// Pre-fix: `scanTableForDrift` short-circuited per column on the first +// strong-evidence row. If the first row was FieldAdaptiveAddressed, the +// column was marked clean and silently dropped — even if subsequent rows +// proved heterogeneous. the empirical real-world data (real-world TEXT columns, +// issues.description) all hit this path: their first non-NULL row happened +// to be adaptive and the legacy rows further down the prolly map were +// invisible to check. +// +// Post-fix: scanner walks every row, tracks sawLegacy + sawAdaptive +// independently, and surfaces the heterogeneous bucket separately from drift +// + clean. The hint string points at `recover-rows`. +// +// Fixture: create a TEXT column under UseAdaptiveEncoding=false, insert a +// row with a long content string (encoded as a 20-byte legacy raw hash) + +// flip to adaptive and insert a second row (encoded as a varint+hash +// adaptive-addressed value). The two rows in the same column are now of +// different shapes — the column is exactly heterogeneous. +func TestHeterogeneousSurfacing(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // 1) Insert a row under legacy encoding — produces a legacy raw-hash + // field for long content. + legacyBlob := makeBigString(1024) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, '`+legacyBlob+`');`) + require.NoError(t, err) + + // 2) Flip the schema to adaptive WITHOUT touching the existing row. + typeinfo.UseAdaptiveEncoding = true + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + // 3) Insert a second row under adaptive encoding — produces an + // adaptive-addressed field for long content. The column now mixes both + // shapes. + adaptiveBlob := makeBigString(2048) + _, err = sqle.ExecuteSql(ctx, dEnv, + `INSERT INTO issues VALUES (2, '`+adaptiveBlob+`');`) + require.NoError(t, err) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, drifts, 1, + "a heterogeneous column must surface as a single entry — not silently dropped (regression)") + + d := drifts[0] + require.Equal(t, "issues", d.Table) + require.Equal(t, "description", d.Column) + require.Equal(t, "StringAdaptiveEnc", d.DeclaredEncoding) + require.Equal(t, "StringAddrEnc", d.SuggestedEncoding) + require.Equal(t, schemadrift.SeverityHeterogeneous, d.Severity, + "heterogeneous column must use SeverityHeterogeneous, NOT drift or safe-empty") + require.False(t, d.SafeToRepair, + "heterogeneous rows must report SafeToRepair=false — repair would corrupt one side of the mix") + require.Equal(t, "mixed-legacy-and-adaptive", d.ObservedFormat, + "the observed-format label must explicitly say mixed so JSON consumers can branch") + require.Contains(t, d.Hint, "recover-rows", + "heterogeneous hint must explicitly point at the recover-rows command — that's the remediation") + require.Contains(t, d.Hint, "--table issues", + "hint must include the exact --table flag value so it's copy-pasteable") + require.Contains(t, d.Hint, "--column description", + "hint must include the exact --column flag value so it's copy-pasteable") +} + +// TestSingleByteInlineEmptyDisqualifiesSafeEmpty is the the fix regression +// guard for the most dangerous safe-empty edge case. +// +// Pre-fix: a 1-byte `[0x00]` field (adaptive empty-inline) was admitted into +// the safe-empty bucket. After `repair --include-empty`, the legacy reader +// did `hash.New([1 byte])` → panic `invalid hash length: 1`. forensic analysis only +// validated the 20-byte all-zero shape; an earlier version over-extended safe-empty by +// keeping `sawHomogeneousEmptyOnly=true` for 1-byte rows too. +// +// Post-fix: any non-NULL FieldAdaptiveInline row with len != hashByteLen +// disqualifies safe-empty unconditionally. Only NULL rows and 20-byte +// all-zero rows are admitted. +// +// Fixture: column under adaptive encoding, three empty-string rows +// (`empty-string` → `[0x00]` adaptive empty-inline). The column has all 1-byte +// payloads. Without the fix it would land in safe-empty; with the fix it must NOT. +func TestSingleByteInlineEmptyDisqualifiesSafeEmpty(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = true + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, ''), (2, ''), (3, '');`) + require.NoError(t, err) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, drifts, + "a column of 1-byte adaptive empty-inline rows must NOT surface in safe-empty — repair --include-empty on such a column crashes the legacy reader with `invalid hash length: 1` (regression)") +} + +// TestTwentyByteAllZeroStillSafeEmpty is the complement: confirm +// the canonical 20-byte all-zero shape (the forensically-validated pattern) still +// lands in safe-empty as before. This guards against an over-broad fix +// that would have made the entire safe-empty bucket inaccessible. +func TestTwentyByteAllZeroStillSafeEmpty(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // CREATE under legacy + INSERT empty strings → the prolly writer + // materialises each row's `description` as a 20-byte address of the + // empty-content chunk. The empty-content hash in dolt is the all-zero + // hash (verified by the forensic dig). + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, ''), (2, ''), (3, '');`) + require.NoError(t, err) + + typeinfo.UseAdaptiveEncoding = true + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, drifts, 1, + "the 20-byte all-zero shape (forensically-validated) must still surface as a single safe-empty entry") + require.Equal(t, schemadrift.SeveritySafeEmpty, drifts[0].Severity, + "the canonical 20-byte all-zero pattern stays in safe-empty (it's the bucket the forensic verdict applies to)") + require.Contains(t, drifts[0].Hint, "--include-empty", + "safe-empty hint must surface the --include-empty flag") +} + +// TestDriftStaysDriftWithoutAdaptiveWitness is the negative +// guard: a column with ONLY legacy witnesses (no adaptive rows at all) must +// still land in pure drift, not heterogeneous. This pairs with +// TestHeterogeneousSurfacing — together they prove the fix +// distinguishes the two cases correctly instead of conflating them. +func TestDriftStaysDriftWithoutAdaptiveWitness(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // Three legacy rows, no adaptive rows. + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, '`+makeBigString(1024)+`'), (2, '`+makeBigString(1024)+`'), (3, '`+makeBigString(1024)+`');`) + require.NoError(t, err) + + typeinfo.UseAdaptiveEncoding = true + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, drifts, 1) + require.Equal(t, schemadrift.SeverityDrift, drifts[0].Severity, + "pure-legacy column must stay in drift even though scanner now walks every row — only the presence of an adaptive witness should promote to heterogeneous") + require.True(t, drifts[0].SafeToRepair, "drift bucket is safe to repair by default") + require.Contains(t, drifts[0].Hint, "repair --table issues --column description", + "drift hint must point at the basic repair command without --include-empty") +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/check_safeempty_test.go b/go/cmd/dolt/commands/admin/schemadrift/check_safeempty_test.go new file mode 100644 index 00000000000..8a3bc52ff74 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/check_safeempty_test.go @@ -0,0 +1,161 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// TestCheckSurfacesSafeEmptyBucket is the load-bearing test for the +// homogeneous-empty bucket added earlier. The bug it exists to prevent: a +// column whose payload is uniformly the all-zero / empty pattern (every +// scanned row is NULL, `[0x00]` inline empty, or the 20-byte all-zero shape +// that arises when a legacy raw-hash address points at an empty/sparse +// chunk) gets silently dropped from the earlier check output. the forensic +// verdict says these ARE safe to flip; the updated version surfaces them under the new +// "safe-empty" severity so operators see them and can opt into repair via +// --include-empty. +// +// Fixture: create a TEXT column under UseAdaptiveEncoding=false, leave it +// empty (no inserts), then manually flip the persisted column TypeInfo to +// StringAdaptiveEnc. The row data side stays at its initial empty state; +// the only thing that's "drifted" is the schema record. With the earlier classifier +// this would be silently dropped because there's no row to witness either +// way. In the updated version it must surface under safe-empty. +func TestCheckSurfacesSafeEmptyBucket(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // Create a 2-column table and insert rows whose descriptions encode to + // the all-zero 20-byte shape — concretely, rows where description is an + // empty string. With StringAddrEnc the prolly writer materialises these + // rows as `[0x00]` inline empties; that's the safe-empty pattern. + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, ''), (2, ''), (3, '');`) + require.NoError(t, err) + + // No drift yet — schema is legacy, payload is consistent. + before, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, before) + + // Flip the global flag to the 2.0.7+ default and manually rewrite the + // persisted schema's column encoding from StringAddrEnc to + // StringAdaptiveEnc — exactly the v2.0.7 ALTER-MODIFY corruption shape, + // but on a column whose payload is uniformly empty. + typeinfo.UseAdaptiveEncoding = true + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, drifts, 1, + "a column with adaptive schema + homogeneous-empty payload must surface as a single safe-empty entry") + d := drifts[0] + require.Equal(t, "issues", d.Table) + require.Equal(t, "description", d.Column) + require.Equal(t, "StringAdaptiveEnc", d.DeclaredEncoding) + require.Equal(t, "StringAddrEnc", d.SuggestedEncoding) + require.Equal(t, schemadrift.SeveritySafeEmpty, d.Severity, + "safe-empty bucket must use the SeveritySafeEmpty label") + require.True(t, d.SafeToRepair, + "safe-empty rows must report SafeToRepair=true; the repair gate is at the --include-empty flag level") + require.Equal(t, "homogeneous-empty", d.ObservedFormat, + "safe-empty rows must report the homogeneous-empty observed-format label so JSON consumers can branch on it") +} + +// TestRepairRefusesSafeEmptyWithoutFlag confirms that the default +// repair posture stays conservative: a column in the safe-empty bucket is +// refused unless the operator explicitly opts in with --include-empty. +func TestRepairRefusesSafeEmptyWithoutFlag(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, ''), (2, '');`) + require.NoError(t, err) + + typeinfo.UseAdaptiveEncoding = true + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + res, err := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + require.Error(t, err, "default repair must refuse the safe-empty bucket without --include-empty") + require.Contains(t, err.Error(), "homogeneous-empty", + "the refusal must name the bucket so the operator knows which flag to pass next") + require.Contains(t, err.Error(), "--include-empty", + "the refusal must surface the opt-in flag name verbatim so it's discoverable from the error") + require.Equal(t, schemadrift.OutcomeGenuineAdaptive, res.Outcome, + "safe-empty refusal still uses OutcomeGenuineAdaptive — there's no separate outcome enum and the existing one signals 'refused, no flip applied'") + require.Empty(t, res.CommitHash, "refused repair must not touch the working set") +} + +// TestRepairAcceptsSafeEmptyWithFlag verifies the --include-empty +// opt-in path: with the flag, the safe-empty bucket is treated as drift, +// the schema flips to legacy, and a real dolt commit lands. +func TestRepairAcceptsSafeEmptyWithFlag(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, ''), (2, '');`) + require.NoError(t, err) + + typeinfo.UseAdaptiveEncoding = true + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + res, err := schemadrift.RepairColumnWithIncludeEmptyForTest(ctx, dEnv, "issues", "description", true) + require.NoError(t, err, "repair with --include-empty must accept the safe-empty bucket") + require.Equal(t, schemadrift.OutcomeFlipped, res.Outcome) + require.Equal(t, val.StringAdaptiveEnc, res.OldEncoding) + require.Equal(t, val.StringAddrEnc, res.NewEncoding) + require.NotEmpty(t, res.CommitHash, "include-empty repair must still produce a real dolt commit") + require.Contains(t, res.CommitMessage, "issues.description") + require.Contains(t, res.CommitMessage, "data unchanged", + "include-empty repair must still carry the data-unchanged guarantee in the audit message") + + // Subsequent check is now clean. + after, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, after, "after the flip the column is internally consistent on both sides") +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/classify.go b/go/cmd/dolt/commands/admin/schemadrift/classify.go new file mode 100644 index 00000000000..433911a8463 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/classify.go @@ -0,0 +1,248 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + + "github.com/mohae/uvarint" + + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/val" +) + +// ChunkPresenceChecker abstracts the "is this hash a real chunk?" lookup used +// to disambiguate the 0x00-leading-byte case (where a legacy raw-hash field +// and an adaptive-inline-with-empty-payload field have the same first byte). +// dEnv.DoltDB(ctx) satisfies this interface at runtime; tests substitute +// fakes. +type ChunkPresenceChecker interface { + Has(ctx context.Context, h hash.Hash) (bool, error) +} + +// FieldFormat classifies the on-disk byte layout of a single tuple field, +// without invoking any adaptive-encoding dispatch. We classify byte-by-byte so +// the classifier itself cannot panic on a corrupted field — the whole point of +// this command is to *find* the corruption that would panic the normal reader. +// +// The four mutually-exclusive categories are: +// +// - FieldNULL — empty slice (val.Tuple.GetField returned nil) +// - FieldAdaptiveInline — leading byte 0x00 (genuine adaptive inline tag) +// - FieldAdaptiveAddressed — non-zero leading byte AND total length = +// varint(declared_length) + hash.ByteLen, where the varint is the SQLite4 +// length prefix described by `val.AdaptiveValue` (always >= 21 bytes) +// - FieldLegacyRawHash — exactly hash.ByteLen (20) bytes, non-zero +// leading byte. This is what `StringAddrEnc`/`BytesAddrEnc`/`JSONAddrEnc`/ +// `GeomAddrEnc` wrote in 1.x — a bare 20-byte address with no length +// prefix. A 20-byte adaptive value would necessarily start with 0x00 +// (1-byte inline header + 19 inline bytes), so a 20-byte non-zero-leading +// field is unambiguously legacy. +// - FieldUnknown — anything else: 1-19 bytes, or 21+ bytes whose +// declared length doesn't match the trailer. We never auto-classify these +// as drift; they're surfaced for human triage. +type FieldFormat int + +const ( + FieldUnknown FieldFormat = iota + FieldNULL + FieldAdaptiveInline + FieldAdaptiveAddressed + FieldLegacyRawHash +) + +// String returns a stable, lowercased name suitable for table / JSON output. +func (f FieldFormat) String() string { + switch f { + case FieldNULL: + return "null" + case FieldAdaptiveInline: + return "adaptive-inline" + case FieldAdaptiveAddressed: + return "adaptive-addressed" + case FieldLegacyRawHash: + return "legacy-raw-hash" + default: + return "unknown" + } +} + +// ClassifyFieldBytes inspects a single field's persisted bytes (as returned by +// val.Tuple.GetField for a non-virtual column) and returns its FieldFormat. +// +// Crucially, this routine NEVER calls into adaptive dispatch — that is the +// dispatch that panics with `invalid hash length: 19` on legacy raw-hash data +// loaded under an adaptive-tagged schema. It also doesn't depend on the +// declared schema at all; the schema is consulted by the caller to decide +// whether the result represents drift (schema says adaptive, payload says +// legacy) vs. an internally consistent column. +func ClassifyFieldBytes(b []byte) FieldFormat { + if len(b) == 0 { + return FieldNULL + } + if b[0] == 0 { + // Adaptive inline format: [0x00][inline bytes...]. The inline payload + // can be any length, including the degenerate 0-byte case. + return FieldAdaptiveInline + } + // Leading byte is non-zero. Two possibilities: + // 1. Genuine adaptive-addressed value: varint(declared_length, >=1 byte) + + // hash.ByteLen address bytes. Total >= 21. + // 2. Legacy raw 20-byte hash address (StringAddrEnc et al.). Total = 20, + // leading byte != 0. + if len(b) == hash.ByteLen { + // 20 bytes, non-zero leader. Cannot be adaptive — adaptive needs at + // minimum 1 varint byte + 20 hash bytes = 21. So this is unambiguously + // legacy. + return FieldLegacyRawHash + } + if len(b) > hash.ByteLen { + // Try to read it as an adaptive-addressed value. Parse the varint + // prefix and check the trailer length matches. + declaredLen, varintSize := uvarint.Uvarint(b) + if varintSize > 0 && varintSize+hash.ByteLen == len(b) { + // The trailer is exactly hash.ByteLen after a clean varint — + // matches the adaptive-addressed shape spelled out in + // val.AdaptiveValue's doc comment. The declared length is a + // reasonableness signal only (we don't deref the address to + // confirm it because that would need a chunkstore round-trip and + // would defeat the no-side-effects contract of check). + _ = declaredLen + return FieldAdaptiveAddressed + } + } + return FieldUnknown +} + +// IsLegacyEncoding reports whether the given val.Encoding corresponds to the +// pre-adaptive (1.x) family for any TEXT / BLOB / JSON / GEOMETRY column. The +// repair path uses this to derive the target encoding from the drifted +// adaptive sibling, and the check path uses it to suppress noise from columns +// already at the legacy tag (which by definition cannot be drifted, because +// drift means "schema says adaptive, payload says legacy"). +func IsLegacyEncoding(enc val.Encoding) bool { + switch enc { + case val.StringAddrEnc, + val.BytesAddrEnc, + val.JSONAddrEnc, + val.GeomAddrEnc, + val.ExtendedAddrEnc: + return true + } + return false +} + +// IsAdaptiveEncoding reports whether the given val.Encoding corresponds to +// the v2.0.7+ adaptive family for any TEXT / BLOB / JSON / GEOMETRY column. +// These are the encodings whose schema records can drift under the regression +// the rest of this branch fixes — and whose on-disk row layouts the repair +// command may need to inspect for legacy-raw-hash bytes. +func IsAdaptiveEncoding(enc val.Encoding) bool { + switch enc { + case val.StringAdaptiveEnc, + val.BytesAdaptiveEnc, + val.JsonAdaptiveEnc, + val.GeomAdaptiveEnc, + val.ExtendedAdaptiveEnc: + return true + } + return false +} + +// ClassifyFieldWithChunkstore is the disambiguating counterpart to +// ClassifyFieldBytes for the load-bearing 0x00-leading-byte case. A field of +// length exactly hash.ByteLen whose first byte is 0x00 is structurally +// ambiguous: it could be an adaptive-inline value with one prefix byte and 19 +// inline bytes, OR a legacy raw-hash field that happens to begin with 0x00 +// (~1 in 256 hashes). The structural classifier alone can't tell them apart +// and consequently misses ~1/256 of legacy rows — exactly the kind of silent +// false-negative the operator pushback flagged as unacceptable. +// +// This routine adds a chunkstore presence check: if the field interpreted as +// hash.New(field) resolves to a stored chunk, the field is a legacy raw hash. +// Otherwise (and otherwise only) it's adaptive inline. The lookup is a single +// presence check, not a chunk fetch — it's fast and never triggers any +// adaptive dispatch. +// +// For NON-ambiguous shapes (length != 20, leading byte != 0, etc.) this +// routine delegates to ClassifyFieldBytes and never touches the chunkstore. +func ClassifyFieldWithChunkstore(ctx context.Context, cs ChunkPresenceChecker, b []byte) (FieldFormat, error) { + f := ClassifyFieldBytes(b) + if f != FieldAdaptiveInline { + return f, nil + } + // Adaptive-inline classification — is it possibly a legacy raw hash with + // a 0x00 leading byte instead? Only the 20-byte case is ambiguous; a 1- or + // 100-byte payload starting with 0x00 cannot be a legacy raw hash. + if len(b) != hash.ByteLen { + return f, nil + } + if cs == nil { + // No chunkstore available (e.g. structural-only test path). Trust + // the structural classification. + return f, nil + } + h := hash.New(b) + present, err := cs.Has(ctx, h) + if err != nil { + return f, err + } + if present { + return FieldLegacyRawHash, nil + } + return f, nil +} + +// LegacySibling returns the pre-adaptive sibling encoding for an adaptive +// encoding (the encoding `repair` will flip the persisted schema record to). +// For a non-adaptive input the call returns (0, false). The pairing is the +// single source of truth used by both `check`'s display column and `repair`'s +// schema mutation. +func LegacySibling(adaptive val.Encoding) (val.Encoding, bool) { + switch adaptive { + case val.StringAdaptiveEnc: + return val.StringAddrEnc, true + case val.BytesAdaptiveEnc: + return val.BytesAddrEnc, true + case val.JsonAdaptiveEnc: + return val.JSONAddrEnc, true + case val.GeomAdaptiveEnc: + return val.GeomAddrEnc, true + case val.ExtendedAdaptiveEnc: + return val.ExtendedAddrEnc, true + } + return 0, false +} + +// AdaptiveSibling returns the v2-native adaptive sibling encoding for a legacy +// (pre-adaptive) TEXT / BLOB / JSON / GEOMETRY encoding — the encoding the +// forward `migrate-adaptive` heal flips the persisted schema record to. For a +// non-legacy input the call returns (0, false). This is the exact inverse of +// LegacySibling and the single source of truth used by the forward migration. +func AdaptiveSibling(legacy val.Encoding) (val.Encoding, bool) { + switch legacy { + case val.StringAddrEnc: + return val.StringAdaptiveEnc, true + case val.BytesAddrEnc: + return val.BytesAdaptiveEnc, true + case val.JSONAddrEnc: + return val.JsonAdaptiveEnc, true + case val.GeomAddrEnc: + return val.GeomAdaptiveEnc, true + case val.ExtendedAddrEnc: + return val.ExtendedAdaptiveEnc, true + } + return 0, false +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/classify_test.go b/go/cmd/dolt/commands/admin/schemadrift/classify_test.go new file mode 100644 index 00000000000..32cd357f15b --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/classify_test.go @@ -0,0 +1,240 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "bytes" + "context" + "testing" + + "github.com/mohae/uvarint" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/val" +) + +// fakeChunkPresence implements ChunkPresenceChecker for tests, with explicit +// per-hash "present" answers. +type fakeChunkPresence struct{ present map[hash.Hash]bool } + +func (f *fakeChunkPresence) Has(ctx context.Context, h hash.Hash) (bool, error) { + return f.present[h], nil +} + +// TestClassifyFieldBytes_NULL covers the empty-slice case, which is what +// val.Tuple.GetField returns for a NULL field. +func TestClassifyFieldBytes_NULL(t *testing.T) { + require.Equal(t, FieldNULL, ClassifyFieldBytes(nil)) + require.Equal(t, FieldNULL, ClassifyFieldBytes([]byte{})) +} + +// TestClassifyFieldBytes_AdaptiveInline covers the genuine adaptive-inline +// case: [0x00][inline bytes...]. The inline payload can be anything. +func TestClassifyFieldBytes_AdaptiveInline(t *testing.T) { + cases := [][]byte{ + {0x00}, // empty inline string + {0x00, 'h', 'i'}, // short inline + append([]byte{0x00}, bytes.Repeat([]byte{'x'}, 100)...), // 100-byte inline + append([]byte{0x00}, bytes.Repeat([]byte{0xff}, 19)...), // 20-byte total with 0x00 leader — this is adaptive inline, NOT legacy + } + for _, b := range cases { + require.Equal(t, FieldAdaptiveInline, ClassifyFieldBytes(b), "bytes=% x", b) + } +} + +// TestClassifyFieldBytes_LegacyRawHash covers the smoking-gun case: a 1.x +// StringAddrEnc / BytesAddrEnc / JSONAddrEnc / GeomAddrEnc field is exactly +// 20 bytes of raw hash with a non-zero leading byte. This is what the drift +// detector keys on. +func TestClassifyFieldBytes_LegacyRawHash(t *testing.T) { + // Build a 20-byte hash with a non-zero leading byte. + var raw [hash.ByteLen]byte + for i := range raw { + raw[i] = byte(i + 1) + } + require.Equal(t, byte(1), raw[0]) + require.Equal(t, FieldLegacyRawHash, ClassifyFieldBytes(raw[:])) + + // Sanity: 20 bytes leading with 0x00 must NOT classify as legacy — that + // would be ambiguous with adaptive inline, and we deliberately route that + // to FieldAdaptiveInline. + zeroLed := make([]byte, hash.ByteLen) + zeroLed[0] = 0x00 + require.Equal(t, FieldAdaptiveInline, ClassifyFieldBytes(zeroLed)) +} + +// TestClassifyFieldBytes_AdaptiveAddressed exercises the well-formed adaptive +// addressed shape: varint(declared length) + hash.ByteLen. The varint can be +// 1-9 bytes long. +func TestClassifyFieldBytes_AdaptiveAddressed(t *testing.T) { + // 1-byte varint encoding a small length, then a 20-byte hash. + short := encodeAdaptiveAddressed(t, 42, byte(1)) + require.Equal(t, hash.ByteLen+1, len(short)) + require.Equal(t, FieldAdaptiveAddressed, ClassifyFieldBytes(short)) + + // Multi-byte varint encoding a larger length. + big := encodeAdaptiveAddressed(t, 1<<20 /*=1MiB*/, byte(2)) + require.Equal(t, FieldAdaptiveAddressed, ClassifyFieldBytes(big), + "adaptive-addressed value with multi-byte varint must classify correctly") +} + +// TestClassifyFieldBytes_Unknown covers shapes we deliberately refuse to +// auto-classify: anything outside the four valid layouts. +func TestClassifyFieldBytes_Unknown(t *testing.T) { + // Non-zero leader, 19 bytes total — too short for both legacy (20) + // and adaptive addressed (21+). + require.Equal(t, FieldUnknown, ClassifyFieldBytes(bytes.Repeat([]byte{0x01}, 19))) + + // Non-zero leader, 21+ bytes but varint length declares a trailer that + // doesn't match. Construct: [0x05] [10 zero bytes] — leader 0x05 is a + // single-byte varint (value 5), so the parser would expect total length = + // 1 + 20 = 21, but we give it 11 bytes. Should be FieldUnknown. + bad := append([]byte{0x05}, bytes.Repeat([]byte{0x00}, 10)...) + require.Equal(t, FieldUnknown, ClassifyFieldBytes(bad)) +} + +// TestIsLegacyEncoding_pairs and TestIsAdaptiveEncoding_pairs are the unit +// guards that LegacySibling is symmetric on the encoding pairs we support. +// LegacySibling is what `repair` consults to derive the new schema encoding; +// any drift here would silently mis-flip the persisted tag. +func TestLegacySibling_pairs(t *testing.T) { + pairs := []struct { + adaptive val.Encoding + legacy val.Encoding + }{ + {val.StringAdaptiveEnc, val.StringAddrEnc}, + {val.BytesAdaptiveEnc, val.BytesAddrEnc}, + {val.JsonAdaptiveEnc, val.JSONAddrEnc}, + {val.GeomAdaptiveEnc, val.GeomAddrEnc}, + {val.ExtendedAdaptiveEnc, val.ExtendedAddrEnc}, + } + for _, p := range pairs { + got, ok := LegacySibling(p.adaptive) + require.True(t, ok, "LegacySibling(%v) must succeed", p.adaptive) + require.Equal(t, p.legacy, got) + + require.True(t, IsAdaptiveEncoding(p.adaptive), "%v must be classified adaptive", p.adaptive) + require.True(t, IsLegacyEncoding(p.legacy), "%v must be classified legacy", p.legacy) + + require.False(t, IsLegacyEncoding(p.adaptive)) + require.False(t, IsAdaptiveEncoding(p.legacy)) + } + + // Non-pair fixed-encoding inputs must NOT have a legacy sibling. + _, ok := LegacySibling(val.StringAddrEnc) + require.False(t, ok, "legacy encoding should not have its own legacy sibling") + _, ok = LegacySibling(val.Encoding(0)) + require.False(t, ok, "unknown encoding should not have a legacy sibling") +} + +// TestClassifyFieldWithChunkstore_AmbiguousZeroLeadingPromotesOnHashHit is +// the load-bearing test for the chunkstore-aware classifier — the case the +// structural classifier alone cannot solve. A 20-byte field starting with +// 0x00 is structurally an adaptive-inline value with 19 inline bytes; the +// chunkstore lookup is the only way to tell whether it's actually a legacy +// raw hash whose first byte happens to be 0x00 (~1/256 of all hashes). +// +// When the chunkstore reports that hash.New(field) IS present, promote to +// FieldLegacyRawHash. When it is NOT present, trust the structural +// FieldAdaptiveInline classification. +func TestClassifyFieldWithChunkstore_AmbiguousZeroLeadingPromotesOnHashHit(t *testing.T) { + ctx := context.Background() + + // Build a 20-byte field with leading 0x00 — the structurally ambiguous + // shape. + var raw [hash.ByteLen]byte + raw[0] = 0x00 + for i := 1; i < hash.ByteLen; i++ { + raw[i] = byte(i) + } + field := raw[:] + h := hash.New(field) + + // Case A: chunkstore HAS this hash → field is a legacy raw hash whose + // first byte coincidentally happens to be 0x00. Must promote. + csHit := &fakeChunkPresence{present: map[hash.Hash]bool{h: true}} + f, err := ClassifyFieldWithChunkstore(ctx, csHit, field) + require.NoError(t, err) + require.Equal(t, FieldLegacyRawHash, f, + "a 20-byte 0x00-leading field whose hash IS present in the chunkstore must classify as legacy raw hash") + + // Case B: chunkstore does NOT have this hash → field is a genuine + // adaptive-inline value with 19 zero-content inline bytes (or whatever + // inline content). Stay at FieldAdaptiveInline. + csMiss := &fakeChunkPresence{present: map[hash.Hash]bool{}} + f, err = ClassifyFieldWithChunkstore(ctx, csMiss, field) + require.NoError(t, err) + require.Equal(t, FieldAdaptiveInline, f, + "a 20-byte 0x00-leading field whose hash is NOT present must classify as genuine adaptive inline") + + // Case C: no chunkstore (nil) → trust structural classification. + f, err = ClassifyFieldWithChunkstore(ctx, nil, field) + require.NoError(t, err) + require.Equal(t, FieldAdaptiveInline, f, + "with no chunkstore, ambiguous 20-byte 0x00-leading must default to the structural answer") +} + +// TestClassifyFieldWithChunkstore_NoAmbiguityShortcuts confirms the chunkstore +// is not consulted for unambiguous shapes. We use a "never called" fake to +// verify it. +func TestClassifyFieldWithChunkstore_NoAmbiguityShortcuts(t *testing.T) { + ctx := context.Background() + + // 20 bytes, leading 0x01 — unambiguously legacy. + legacy := make([]byte, hash.ByteLen) + legacy[0] = 0x01 + + cs := &neverCalledChunkPresence{t: t} + f, err := ClassifyFieldWithChunkstore(ctx, cs, legacy) + require.NoError(t, err) + require.Equal(t, FieldLegacyRawHash, f) + + // 21-byte adaptive addressed — unambiguous. + addressed := encodeAdaptiveAddressed(t, 99, byte(7)) + f, err = ClassifyFieldWithChunkstore(ctx, cs, addressed) + require.NoError(t, err) + require.Equal(t, FieldAdaptiveAddressed, f) + + // 5-byte 0x00-leading inline — too short to be ambiguous. + shortInline := []byte{0x00, 'h', 'i', '!', '!'} + f, err = ClassifyFieldWithChunkstore(ctx, cs, shortInline) + require.NoError(t, err) + require.Equal(t, FieldAdaptiveInline, f) +} + +type neverCalledChunkPresence struct{ t *testing.T } + +func (n *neverCalledChunkPresence) Has(ctx context.Context, h hash.Hash) (bool, error) { + n.t.Fatalf("chunkstore must not be consulted for unambiguous classifications, but Has was called on %s", h) + return false, nil +} + +// encodeAdaptiveAddressed constructs a synthetic adaptive-addressed value +// with the given declared length and a deterministic hash whose leading byte +// is |hashLeader|. Used by the classifier tests to keep them independent of +// any real chunkstore. +func encodeAdaptiveAddressed(t *testing.T, length uint64, hashLeader byte) []byte { + t.Helper() + buf := make([]byte, 9) // max varint length + n := uvarint.Encode(buf, length) + out := append([]byte{}, buf[:n]...) + var h [hash.ByteLen]byte + h[0] = hashLeader + for i := 1; i < hash.ByteLen; i++ { + h[i] = byte(i) + } + return append(out, h[:]...) +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/export_test.go b/go/cmd/dolt/commands/admin/schemadrift/export_test.go new file mode 100644 index 00000000000..67c62caef2e --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/export_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/store/val" +) + +// ScanForDriftForTest is an exported test helper that drives the same +// drift-scan pipeline the CLI uses. It exists only in the test binary so +// cross-package integration tests can exercise scanForDrift without going +// through Exec(). +func ScanForDriftForTest(ctx context.Context, dEnv *env.DoltEnv) ([]DriftRow, error) { + return scanForDrift(ctx, dEnv) +} + +// RepairColumnForTest is the test-side hook into the repair pipeline. Same +// rationale as ScanForDriftForTest: it routes around the CLI wrapper so the +// integration test can assert on the structured RepairResult. +func RepairColumnForTest(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (RepairResult, error) { + return repairColumn(ctx, dEnv, tableName, colName) +} + +// RepairColumnWithIncludeEmptyForTest exposes the include-empty repair path +// added earlier. It accepts the homogeneous-empty bucket as a valid repair +// target (when the third argument is true) and refuses it otherwise. Tests +// use this to assert both sides of the default-conservative / opt-in posture. +func RepairColumnWithIncludeEmptyForTest(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string, includeEmpty bool) (RepairResult, error) { + return repairColumnWithOptions(ctx, dEnv, tableName, colName, includeEmpty) +} + +// RecoverRowsColumnForTest is the test-side hook into the recover-rows +// pipeline. Routes around the CLI wrapper so the integration test can assert +// on the structured RecoverRowsResult. +func RecoverRowsColumnForTest(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (RecoverRowsResult, error) { + return recoverRowsColumn(ctx, dEnv, tableName, colName) +} + +// ResolveFieldToLegacyForTest exposes the per-row classifier+rewrite helper +// for unit tests. Returns the same tuple the inner pipeline uses: the new +// 20-byte legacy field bytes (or nil for NULL), whether the input was already +// canonical legacy, whether the field bytes changed, and any classification +// or chunkstore error. +func ResolveFieldToLegacyForTest(ctx context.Context, b []byte, vs val.ValueStore, cs ChunkPresenceChecker) (newField []byte, isCanonicalLegacy bool, changed bool, err error) { + return resolveFieldToLegacy(ctx, b, vs, cs) +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go new file mode 100644 index 00000000000..da542275dd8 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go @@ -0,0 +1,905 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/mohae/uvarint" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/commands" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/val" +) + +// MigrateAdaptiveCmd is the FORWARD (v2-native) counterpart of RecoverRowsCmd. +// Where recover-rows rewrites a heterogeneous column DOWN to canonical legacy +// (1.x) form, migrate-adaptive rewrites it UP to canonical adaptive (2.x) form: +// every legacy-raw-hash row is converted to a canonical adaptive value and the +// persisted column encoding is flipped to its adaptive sibling. Rows that are +// already adaptive (inline or addressed) are left byte-for-byte untouched. +// +// This is the heal for the operator's chosen "the forward-migration approach" — resolve the drift +// natively for the v2 format, with no reader-compat fallback. After this runs +// on every affected column, the column's schema record AND every row share one +// consistent adaptive format, so reads no longer panic and future writes (which +// already emit adaptive by default on 2.0.7+) stay consistent. +// +// CONTENT is preserved byte-for-byte. Only STORAGE LAYOUT changes for the +// migrated legacy rows: +// +// - NULL stays NULL. +// - A legacy-raw-hash row (a bare 20-byte content address, either non-zero +// leader or a 0x00 leader whose hash IS present in the chunkstore) is +// dereferenced to its content and re-encoded canonically: content ≤ 20 +// bytes is inlined as `[0x00][content]`; larger content stays out-of-band +// as `[varint(len)][20-byte hash]`, REUSING the existing chunk (the content +// hash is unchanged), so no large blob is rewritten. +// - An already-adaptive row — inline (`[0x00]…`) or addressed +// (`[varint][20-byte hash]`) — is canonical and copied verbatim. +// - A row whose field bytes cannot be classified is fatal; the migration +// aborts and reports the row's primary key. Partial migrations never commit. +// +// Atomicity matches recover-rows: a single dolt commit records both the +// rewritten prolly map and the schema-encoding flip. +type MigrateAdaptiveCmd struct{} + +var _ cli.Command = MigrateAdaptiveCmd{} + +var migrateAdaptiveDocs = cli.CommandDocumentationContent{ + ShortDesc: "Row-by-row forward migration of a drifted/legacy column into canonical adaptive (v2) form", + LongDesc: `Walks every row in ` + "`--table`" + `, reads the target ` + "`--column`" + `'s field bytes through the correct per-row dispatch (legacy raw-hash, adaptive inline, or adaptive addressed), and rewrites each legacy row into a canonical adaptive value. After every row is processed the persisted column encoding is flipped to its adaptive sibling (e.g. ` + "`StringAddrEnc` → `StringAdaptiveEnc`" + `). Row data and schema update inside a single dolt commit. + +This is the FORWARD counterpart to ` + "`recover-rows`" + ` (which migrates the other way, to legacy). Use ` + "`migrate-adaptive`" + ` to resolve encoding drift natively for the v2 format: after it runs, the column's schema record and every row share one consistent adaptive format, with no reader-compat fallback required. + +Content is unchanged — the bytes the user originally inserted are readable byte-for-byte before and after. Legacy rows whose content is larger than 20 bytes keep their existing out-of-band chunk (only a length prefix is added); content of 20 bytes or fewer is inlined. Already-adaptive rows are copied verbatim. + +The command aborts the entire migration and reports the offending row's primary-key bytes if any field cannot be classified into a known shape; partial migrations are never committed. + +Required flags: ` + "`--table`" + ` and ` + "`--column`" + ` — one column per invocation, to keep each commit a single auditable change. + +Exit codes: + 0 migration succeeded (or the column was already fully canonical adaptive — idempotent no-op) + 1 migration refused (unknown-shape rows, or the operation failed)`, + Synopsis: []string{"--table --column "}, +} + +func (cmd MigrateAdaptiveCmd) Name() string { return "migrate-adaptive" } + +func (cmd MigrateAdaptiveCmd) Description() string { + return "Row-by-row migration of a drifted/legacy payload into canonical adaptive (v2) form (atomic prolly rewrite + schema flip + dolt commit)" +} + +func (cmd MigrateAdaptiveCmd) RequiresRepo() bool { return true } + +func (cmd MigrateAdaptiveCmd) Docs() *cli.CommandDocumentation { + return cli.NewCommandDocumentation(migrateAdaptiveDocs, cmd.ArgParser()) +} + +const dryRunFlag = "dry-run" + +func (cmd MigrateAdaptiveCmd) ArgParser() *argparser.ArgParser { + ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0) + ap.SupportsString(tableFlag, "", "name", "the table to migrate") + ap.SupportsString(colFlag, "", "name", "the column whose payload should be rewritten to canonical adaptive form") + ap.SupportsFlag(dryRunFlag, "", "Report, per row class, what would change (legacy rows to rewrite / already-adaptive / empty) WITHOUT writing or committing. Classifies by field shape only — no content is dereferenced, so it is safe on a column whose secondary indexes or content chunks are damaged.") + return ap +} + +func (cmd MigrateAdaptiveCmd) Hidden() bool { return true } + +func (cmd MigrateAdaptiveCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, _ cli.CliContext) int { + ap := cmd.ArgParser() + usage, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, migrateAdaptiveDocs, ap)) + apr := cli.ParseArgsOrDie(ap, args, usage) + + tableName, ok := apr.GetValue(tableFlag) + if !ok || tableName == "" { + verr := errhand.BuildDError("--table is required").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + colName, ok := apr.GetValue(colFlag) + if !ok || colName == "" { + verr := errhand.BuildDError("--column is required").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + result, err := migrateAdaptiveColumnOpts(ctx, dEnv, tableName, colName, apr.Contains(dryRunFlag)) + if err != nil { + verr := errhand.BuildDError("migrate-adaptive failed").AddCause(err).Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + emitMigrateAdaptiveResult(cli.OutStream, result) + return 0 +} + +// MigrateAdaptiveResult is the structured outcome of a migrate-adaptive call. +type MigrateAdaptiveResult struct { + Outcome RecoverRowsOutcome + Database string + Table string + Column string + OldEncoding val.Encoding + NewEncoding val.Encoding + RowsScanned int + RowsRewritten int // legacy rows converted to adaptive + CollateralRewrites int // sibling AddrEnc fields normalized to satisfy the descriptor + CommitHash string + CommitMessage string + + // DryRun and the Class* tallies are populated only when --dry-run is set: + // a non-mutating preview of what the real run would do, classified by field + // shape (no content dereference). + DryRun bool + ClassLegacyToRewrite int // FieldLegacyRawHash rows that would be rewritten to adaptive + ClassAlreadyAdaptive int // adaptive inline/addressed rows that pass through untouched + ClassEmptyOrNull int // NULL or empty-inline rows (no-op) + ClassUnknown int // unclassifiable shapes (the real run would error on these) + + // MaxContentLen is the largest content size force-inlined (dolt_ignore'd + // inline path only); surfaced so the operator is warned if it nears the + // tuple size limit. + MaxContentLen int + + // InlineCount / AddressedCount are the post-migration storage report (the + // dolt_ignore'd inline path): how many migrated values ended up inline vs + // out-of-band-addressed. AddressedCount MUST be 0 for a dolt_ignore'd table + // — a deterministic deploy-verification + test signal. + InlineCount int + AddressedCount int +} + +// migrateAdaptiveColumn is the forward-migration pipeline. It mirrors +// recoverRowsColumn but drives the rewrite with resolveFieldToAdaptive and +// flips the schema to the adaptive sibling instead of the legacy one. +// migrateAdaptiveColumn performs the real (mutating) forward migration. It is +// the stable entry point used by the test hook (MigrateAdaptiveColumnForTest); +// Exec routes through migrateAdaptiveColumnOpts to thread --dry-run. +func migrateAdaptiveColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (MigrateAdaptiveResult, error) { + return migrateAdaptiveColumnOpts(ctx, dEnv, tableName, colName, false) +} + +func migrateAdaptiveColumnOpts(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string, dryRun bool) (MigrateAdaptiveResult, error) { + ddb := dEnv.DoltDB(ctx) + dbName := ddb.GetDatabaseName() + + rsr := dEnv.RepoStateReader() + headRef, err := rsr.CWBHeadRef(ctx) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("get current branch: %w", err) + } + + roots, err := dEnv.Roots(ctx) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("get roots: %w", err) + } + + tn := doltdb.TableName{Name: tableName, Schema: doltdb.DefaultSchemaName} + tbl, resolvedName, ok, err := doltdb.GetTableInsensitive(ctx, roots.Working, tn) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("get table %s: %w", tableName, err) + } + if !ok { + return MigrateAdaptiveResult{}, fmt.Errorf("table %q not found on current working root", tableName) + } + tn.Name = resolvedName + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("get schema for %s: %w", resolvedName, err) + } + + // Keyless tables prepend a cardinality column to the value tuple; their + // value-tuple ordinals don't line up with NonPKCols ordering. Refuse them, + // exactly as recover-rows does. + if schema.IsKeyless(sch) { + return MigrateAdaptiveResult{}, fmt.Errorf( + "table %s is keyless; migrate-adaptive does not support keyless tables", resolvedName) + } + + existingCol, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) + if !ok { + return MigrateAdaptiveResult{}, fmt.Errorf("column %q not found in table %s", colName, resolvedName) + } + + if existingCol.IsPartOfPK { + return MigrateAdaptiveResult{}, fmt.Errorf( + "column %s.%s is part of the primary key; PK columns do not carry adaptive encodings", + resolvedName, existingCol.Name) + } + + oldEnc := existingCol.TypeInfo.Encoding() + + // Determine the target adaptive encoding. + // - legacy tag (e.g. StringAddr=23): flip to the adaptive sibling. + // - already-adaptive tag (e.g. StringAdaptive=135): target == old; we may + // still need to rewrite legacy ROWS that drifted under an adaptive tag. + var newEnc val.Encoding + switch { + case IsAdaptiveEncoding(oldEnc): + newEnc = oldEnc + case IsLegacyEncoding(oldEnc): + sib, ok := AdaptiveSibling(oldEnc) + if !ok { + return MigrateAdaptiveResult{}, fmt.Errorf( + "column %s.%s has legacy encoding %s with no adaptive sibling", + resolvedName, existingCol.Name, encodingName(oldEnc)) + } + newEnc = sib + default: + return MigrateAdaptiveResult{}, fmt.Errorf( + "column %s.%s has encoding %s which is not a TEXT/BLOB/JSON/GEOMETRY variant; nothing to migrate", + resolvedName, existingCol.Name, encodingName(oldEnc)) + } + + tupleIndex, ok := valueTupleIndexForColumn(sch, existingCol.Name) + if !ok { + return MigrateAdaptiveResult{}, fmt.Errorf( + "column %s.%s does not appear in the value tuple (maybe virtual?)", + resolvedName, existingCol.Name) + } + + // Dry-run: classify every row's target field by SHAPE (no content + // dereference, so it is safe even when content chunks / secondary indexes + // are damaged) and report what the real run would do, without writing. + if dryRun { + counts, scanned, err := classifyColumnForMigration(ctx, tbl, tupleIndex, ddb) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("dry-run scan of %s.%s: %w", resolvedName, existingCol.Name, err) + } + outcome := RecoverRowsMigrated + if oldEnc == newEnc && counts.legacyToRewrite == 0 { + outcome = RecoverRowsAlreadyOK // already canonical adaptive; real run is a no-op + } + return MigrateAdaptiveResult{ + Outcome: outcome, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: newEnc, + RowsScanned: scanned, + RowsRewritten: counts.legacyToRewrite, // the planned rewrite count + DryRun: true, + ClassLegacyToRewrite: counts.legacyToRewrite, + ClassAlreadyAdaptive: counts.alreadyAdaptive, + ClassEmptyOrNull: counts.emptyOrNull, + ClassUnknown: counts.unknown, + }, nil + } + + // dolt_ignore'd tables (ignored_log, ignored_meta) live only in the working set and + // are NEVER committed, so their content chunks are not commit-rooted — + // out-of-band adaptive values would reference chunks the persist's refCheck + // cannot root (and dolt gc, which roots from commits, can reclaim them). The + // correct storage policy for these tables is force-INLINE: migrate EVERY + // text/blob column to adaptive-inline in one atomic pass so the rewritten map + // references zero content chunks and retains no legacy AddrEnc columns. + ignored, ignErr := doltdb.IdentifyIgnoredTables(ctx, roots, []doltdb.TableName{tn}) + if ignErr != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("check dolt_ignore status for %s: %w", resolvedName, ignErr) + } + if len(ignored) > 0 { + return migrateIgnoredTableInline(ctx, dEnv, tbl, sch, dbName, resolvedName, tn, roots, ddb) + } + + // Build the patched schema with the target column flipped to its adaptive + // sibling. If oldEnc is already adaptive this is a no-op schema change, but + // we still rebuild so the rewrite's value descriptor is derived consistently. + newCol := existingCol + newCol.TypeInfo = existingCol.TypeInfo.WithEncoding(newEnc) + newCol.Kind = newCol.TypeInfo.NomsKind() + + newSch, err := replaceColumnByTag(sch, newCol) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("rebuild schema for %s: %w", resolvedName, err) + } + + // Rewrite every legacy row's target field forward to canonical adaptive, + // leaving already-adaptive rows verbatim. Sibling AddrEnc fields with + // non-canonical bytes are healed to canonical legacy in the same pass (the + // direction-agnostic collateral heal), so the new tuple serializes cleanly. + rewritten, scanned, _, err := rewriteColumn(ctx, tbl, tupleIndex, newSch, ddb, resolveFieldToAdaptive) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("rewrite rows of %s.%s: %w", resolvedName, existingCol.Name, err) + } + + // No-op detection: if the schema tag is already adaptive AND not a single + // row changed, the column is already fully canonical adaptive — return a + // successful idempotent no-op without a commit. + if oldEnc == newEnc && rewritten.RowsRewritten == 0 { + return MigrateAdaptiveResult{ + Outcome: RecoverRowsAlreadyOK, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: newEnc, + RowsScanned: scanned, + }, nil + } + + updatedTbl, err := tbl.UpdateSchema(ctx, newSch) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("apply schema change to %s: %w", resolvedName, err) + } + updatedTbl, err = updatedTbl.UpdateRows(ctx, durable.IndexFromProllyMap(rewritten.Map)) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("apply rewritten rows to %s: %w", resolvedName, err) + } + + newWorking, err := roots.Working.PutTable(ctx, tn, updatedTbl) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("put updated table %s: %w", resolvedName, err) + } + roots.Working = newWorking + + rowsRewritten := rewritten.RowsRewritten + collateral := rewritten.CollateralRewrites + var commitMsg string + if collateral > 0 { + commitMsg = fmt.Sprintf( + "admin: schema-encoding-drift migrate-adaptive on %s.%s (%d legacy rows rewritten to adaptive format + schema flipped to %s; %d rows had non-target AddrEnc fields healed as collateral; content unchanged)", + resolvedName, existingCol.Name, rowsRewritten, encodingName(newEnc), collateral) + } else { + commitMsg = fmt.Sprintf( + "admin: schema-encoding-drift migrate-adaptive on %s.%s (%d legacy rows rewritten to adaptive format + schema flipped to %s; content unchanged)", + resolvedName, existingCol.Name, rowsRewritten, encodingName(newEnc)) + } + + // Non-ignored (committed) table: commit as usual so the heal lands in + // history. (dolt_ignore'd tables took the force-inline path above and + // returned before reaching here.) + roots.Staged = newWorking + commitHash, err := commitWorkingRoot(ctx, dEnv, headRef, roots, commitMsg) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("commit migrate-adaptive: %w", err) + } + + return MigrateAdaptiveResult{ + Outcome: RecoverRowsMigrated, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: newEnc, + RowsScanned: scanned, + RowsRewritten: rowsRewritten, + CollateralRewrites: collateral, + CommitHash: commitHash.String(), + CommitMessage: commitMsg, + }, nil +} + +// persistWorkingSetRoot updates the current working set's working root to +// |working| WITHOUT creating a dolt commit. This is the persistence path for +// dolt_ignore'd tables, whose data lives only in the working set. +func persistWorkingSetRoot(ctx context.Context, dEnv *env.DoltEnv, working doltdb.RootValue) error { + ws, err := dEnv.WorkingSet(ctx) + if err != nil { + return fmt.Errorf("get working set: %w", err) + } + return dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(working)) +} + +// inlineSizeCapBytes is a defensive ceiling on the content we force-inline on a +// dolt_ignore'd table. The census shows ignored_*/ignored_meta content is ≤~200B, so this +// should never trip; if it does we still inline (one large row beats a dangling +// out-of-band ref) but record the max so the operator is warned. +const inlineSizeCapBytes = 16 * 1024 + +// readContentAuthoritative reads the content addressed by |h| using the +// all-generation read path (vs.ReadBytes sees oldgen+newgen+memtables; ddb.Has +// can false-negative per a review finding, so it must NOT be used for a +// present/absent decision). It distinguishes THREE outcomes — critically, a read +// ERROR is NOT the same as an absence: +// - (content, true, nil) — the chunk exists; content returned. +// - (nil, false, nil) — the chunk is GENUINELY ABSENT: vs.ReadBytes hit the +// NodeStore's "empty chunk" assertion (a missing chunk), which we recover so +// we never crash. Safe for callers to treat as "not a legacy hash". +// - (nil, false, err) — a READ ERROR (transient or otherwise). The caller +// MUST fail loud and must NOT treat this as an absence — doing so could +// misclassify a present legacy row as adaptive-inline and corrupt it. +func readContentAuthoritative(ctx context.Context, vs val.ValueStore, h hash.Hash) (content []byte, present bool, err error) { + defer func() { + if r := recover(); r != nil { + // Recovered the empty-chunk assertion ⇒ GENUINE absence, not an error. + content, present, err = nil, false, nil + } + }() + b, rErr := vs.ReadBytes(ctx, h) + if rErr != nil { + // A read error is NOT an absence — propagate it so the caller fails loud. + return nil, false, rErr + } + return b, true, nil +} + +// underlyingContent extracts the raw content bytes a value-tuple field +// represents, regardless of its on-disk shape, using authoritative reads: +// - legacy raw 20-byte hash → ReadBytes the chunk. +// - adaptive addressed [varint][20-byte hash] → ReadBytes the chunk. +// - adaptive inline [0x00][content] → content = bytes[1:]. +// - 0x00-leading 20-byte (ambiguous) → ReadBytes(hash): present ⇒ legacy hash +// (use its content); absent ⇒ adaptive-inline (content = bytes[1:]). +// +// Returns present=false with err=nil ONLY when a legacy/addressed hash's content +// chunk is GENUINELY absent (recovered empty-chunk assertion) — the caller FAILS +// LOUD on this (no tolerate-missing; the loss-census confirmed zero absent +// chunks). Returns a non-nil err for a structurally unknown shape OR a chunkstore +// READ ERROR (never silently treated as an absence, which could corrupt a +// present legacy row). +func underlyingContent(ctx context.Context, vs val.ValueStore, b []byte) (content []byte, present bool, err error) { + if len(b) == 0 { + return nil, true, nil // NULL — caller maps to NULL + } + if b[0] != 0 { + switch { + case len(b) == hash.ByteLen: + // Legacy raw hash: a read error propagates (fail loud); a genuine + // absence returns present=false (caller fails loud — no + // tolerate-missing). + return readContentAuthoritative(ctx, vs, hash.New(b)) + case len(b) > hash.ByteLen: + _, varintSize := uvarint.Uvarint(b) + if varintSize > 0 && varintSize+hash.ByteLen == len(b) { + return readContentAuthoritative(ctx, vs, hash.New(b[varintSize:])) + } + return nil, false, fmt.Errorf("field has unknown shape: %d bytes, non-zero leader, malformed adaptive-addressed trailer", len(b)) + default: + return nil, false, fmt.Errorf("field has unknown shape: %d bytes, non-zero leader, too short", len(b)) + } + } + // Leading byte 0x00. + if len(b) == hash.ByteLen { + // Ambiguous: a legacy raw hash whose first byte is coincidentally 0x00, + // or an adaptive-inline value with 19 content bytes. Resolve via an + // authoritative read — never ddb.Has (a review finding). A READ ERROR + // must fail loud (not be mistaken for an absence); only a GENUINE absence + // (recovered empty-chunk assertion) means "adaptive inline". + c, ok, rErr := readContentAuthoritative(ctx, vs, hash.New(b)) + if rErr != nil { + return nil, false, fmt.Errorf("authoritative read for 0x00-leading 20-byte field: %w", rErr) + } + if ok { + return c, true, nil // legacy raw hash + } + return b[1:], true, nil // adaptive inline (19 content bytes) + } + // Adaptive inline: [0x00] (empty) or [0x00][content]. + return b[1:], true, nil +} + +// migrateIgnoredTableInline force-inlines EVERY out-of-band-capable column +// (text/blob/json/geometry, legacy or adaptive) of a dolt_ignore'd table to +// adaptive-inline form in ONE atomic pass, then persists the working set. +// +// Why inline-only for ignored tables: their content chunks are never committed +// (bd never `dolt add`s them), so an out-of-band adaptive value would reference +// a chunk the working-set persist's refCheck cannot root — and dolt gc, which +// roots from commits, can reclaim those unrooted chunks. Inlining puts every +// value's content in the tuple itself: the rewritten map references ZERO content +// chunks and retains no legacy AddrEnc columns, so refCheck has nothing to +// dangle on and the data is immune to the gc hazard. +func migrateIgnoredTableInline(ctx context.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, sch schema.Schema, dbName, resolvedName string, tn doltdb.TableName, roots doltdb.Roots, ddb *doltdb.DoltDB) (MigrateAdaptiveResult, error) { + if schema.IsKeyless(sch) { + return MigrateAdaptiveResult{}, fmt.Errorf("table %s is keyless; migrate-adaptive does not support keyless tables", resolvedName) + } + + // Identify every non-PK, non-virtual out-of-band-capable column and its + // target adaptive encoding; build the patched schema flipping them all. + targetIdx := map[int]bool{} + var colNames []string + newSch := sch + var buildErr error + _ = sch.GetNonPKCols().Iter(func(tag uint64, col schema.Column) (bool, error) { + if col.Virtual { + return false, nil + } + enc := col.TypeInfo.Encoding() + var newEnc val.Encoding + switch { + case IsAdaptiveEncoding(enc): + newEnc = enc // already adaptive; rows still force-inlined for gc-safety + case IsLegacyEncoding(enc): + sib, ok := AdaptiveSibling(enc) + if !ok { + return false, nil + } + newEnc = sib + default: + return false, nil // not out-of-band-capable + } + idx, ok := valueTupleIndexForColumn(sch, col.Name) + if !ok { + return false, nil + } + targetIdx[idx] = true + colNames = append(colNames, col.Name) + nc := col + nc.TypeInfo = col.TypeInfo.WithEncoding(newEnc) + nc.Kind = nc.TypeInfo.NomsKind() + newSch, buildErr = replaceColumnByTag(newSch, nc) + if buildErr != nil { + return true, buildErr + } + return false, nil + }) + if buildErr != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("rebuild schema for %s: %w", resolvedName, buildErr) + } + if len(targetIdx) == 0 { + // No out-of-band columns — nothing to inline. + return MigrateAdaptiveResult{Outcome: RecoverRowsAlreadyOK, Database: dbName, Table: resolvedName}, nil + } + + durableIdx, err := tbl.GetRowData(ctx) + if err != nil { + return MigrateAdaptiveResult{}, err + } + pm, err := durable.ProllyMapFromIndex(durableIdx) + if err != nil { + return MigrateAdaptiveResult{}, err + } + ns := pm.NodeStore() + pool := pm.Pool() + newValDesc := newSch.GetValueDescriptor(ns) + mut := pm.Rewriter(pm.KeyDesc(), newValDesc) + keyDesc := pm.KeyDesc() + + empty, err := durableIdx.Empty() + if err != nil { + return MigrateAdaptiveResult{}, err + } + scanned, rewritten, maxLen, inlineCount := 0, 0, 0, 0 + if !empty { + iter, err := pm.IterAll(ctx) + if err != nil { + return MigrateAdaptiveResult{}, err + } + for { + key, value, iterErr := iter.Next(ctx) + if iterErr == io.EOF { + break + } + if iterErr != nil { + return MigrateAdaptiveResult{}, iterErr + } + scanned++ + fieldCount := value.Count() + if dc := newValDesc.Count(); dc > fieldCount { + fieldCount = dc + } else if value.Count() > dc { + return MigrateAdaptiveResult{}, fmt.Errorf("schema-row width mismatch at %s: tuple has %d fields, descriptor expects %d (concurrent schema change?)", formatKeyForError(key, keyDesc), value.Count(), dc) + } + fields := make([][]byte, fieldCount) + rowChanged := false + for i := 0; i < fieldCount; i++ { + orig := value.GetField(i) + if !targetIdx[i] { + fields[i] = orig + continue + } + if len(orig) == 0 { + fields[i] = nil // NULL stays NULL + continue + } + content, ePresent, cErr := underlyingContent(ctx, ns, orig) + if cErr != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("row %s, field %d: %w", formatKeyForError(key, keyDesc), i, cErr) + } + if !ePresent { + // A legacy/addressed hash whose content chunk is genuinely + // absent — the authoritative all-generation read failed (the + // NodeStore empty-chunk assertion, recovered in + // readContentAuthoritative, so we do NOT crash). This is real + // data loss: FAIL LOUD with the row's primary key rather than + // silently emptying it. We do not tolerate-missing — the + // loss-census confirmed zero genuinely-absent chunks, so this + // must never trigger; if it does, the operator investigates + // before healing. + return MigrateAdaptiveResult{}, fmt.Errorf( + "row %s, field %d: content chunk is genuinely absent (authoritative read failed); refusing to silently empty it — investigate before healing %s", + formatKeyForError(key, keyDesc), i, resolvedName) + } + if len(content) > maxLen { + maxLen = len(content) + } + inlined := val.AdaptiveValueInlineBytes(content) + // [invariant — arch-detective] force-inline MUST produce an + // inline adaptive value (first byte 0x00) — never an out-of-line + // ref — so the rewritten map references zero content chunks and + // cannot dangle on the ignored-table persist. Assert it at the + // source (fail-fast) rather than relying only on a test. + if len(inlined) == 0 || inlined[0] != 0x00 { + return MigrateAdaptiveResult{}, fmt.Errorf("internal invariant violated: force-inline produced a non-inline value (len=%d) at row %s field %d", len(inlined), formatKeyForError(key, keyDesc), i) + } + fields[i] = inlined + inlineCount++ + if !bytes.Equal(inlined, orig) { + rowChanged = true + } + } + if rowChanged { + rewritten++ + } + if putErr := mut.Put(ctx, key, val.NewTuple(pool, fields...)); putErr != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("put row %s: %w", formatKeyForError(key, keyDesc), putErr) + } + } + } + + newMap, err := mut.Map(ctx) + if err != nil { + return MigrateAdaptiveResult{}, err + } + updatedTbl, err := tbl.UpdateSchema(ctx, newSch) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("apply schema to %s: %w", resolvedName, err) + } + updatedTbl, err = updatedTbl.UpdateRows(ctx, durable.IndexFromProllyMap(newMap)) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("apply rows to %s: %w", resolvedName, err) + } + newWorking, err := roots.Working.PutTable(ctx, tn, updatedTbl) + if err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("put table %s: %w", resolvedName, err) + } + roots.Working = newWorking + // dolt_ignore'd table: persist the working set (no commit; bd reads here). + if err := persistWorkingSetRoot(ctx, dEnv, roots.Working); err != nil { + return MigrateAdaptiveResult{}, fmt.Errorf("persist working set for dolt_ignore'd table %s: %w", resolvedName, err) + } + + msg := fmt.Sprintf("admin: schema-encoding-drift migrate-adaptive (inline) on dolt_ignore'd %s [%d cols]: %d/%d rows force-inlined to adaptive; content unchanged", resolvedName, len(colNames), rewritten, scanned) + return MigrateAdaptiveResult{ + Outcome: RecoverRowsMigrated, + Database: dbName, + Table: resolvedName, + Column: fmt.Sprintf("%v (force-inline, dolt_ignore'd table)", colNames), + RowsScanned: scanned, + RowsRewritten: rewritten, + CommitMessage: msg, + MaxContentLen: maxLen, + InlineCount: inlineCount, + AddressedCount: 0, // force-inline guarantees zero out-of-band values + }, nil +} + +// migrationClassCounts tallies a column's value-tuple fields by migration class +// for the --dry-run report. +type migrationClassCounts struct { + legacyToRewrite int // FieldLegacyRawHash — rewritten forward to adaptive + alreadyAdaptive int // adaptive inline (non-empty) / addressed — pass through + emptyOrNull int // NULL or empty-inline [0x00] — no-op + unknown int // unclassifiable — the real run would error +} + +// classifyColumnForMigration walks every row and classifies the target field by +// SHAPE only (via the chunkstore-aware classifier; it never dereferences +// content), so it is safe to run on a column whose content chunks or secondary +// indexes are damaged. It powers --dry-run. +func classifyColumnForMigration(ctx context.Context, tbl *doltdb.Table, tupleIndex int, cs ChunkPresenceChecker) (counts migrationClassCounts, scanned int, err error) { + durableIdx, err := tbl.GetRowData(ctx) + if err != nil { + return counts, 0, err + } + if durableIdx == nil { + return counts, 0, nil + } + pm, err := durable.ProllyMapFromIndex(durableIdx) + if err != nil { + return counts, 0, err + } + iter, err := pm.IterAll(ctx) + if err != nil { + return counts, 0, err + } + for { + _, value, iterErr := iter.Next(ctx) + if iterErr == io.EOF { + break + } + if iterErr != nil { + return counts, scanned, iterErr + } + scanned++ + b := value.GetField(tupleIndex) + if len(b) == 0 { + counts.emptyOrNull++ + continue + } + f, cErr := ClassifyFieldWithChunkstore(ctx, cs, b) + if cErr != nil { + return counts, scanned, cErr + } + switch f { + case FieldLegacyRawHash: + counts.legacyToRewrite++ + case FieldAdaptiveInline: + if len(b) == 1 { // [0x00] — empty inline + counts.emptyOrNull++ + } else { + counts.alreadyAdaptive++ + } + case FieldAdaptiveAddressed: + counts.alreadyAdaptive++ + case FieldNULL: + counts.emptyOrNull++ + default: + counts.unknown++ + } + } + return counts, scanned, nil +} + +// resolveFieldToAdaptive is the per-row workhorse for the forward migration. It +// is the inverse of resolveFieldToLegacy: given a single field's bytes, it +// returns the canonical ADAPTIVE form of those bytes plus: +// - isLegacyRow: the input was a legacy-raw-hash value that this call +// converted to adaptive. The caller ORs these into sawAnyLegacy. +// - changed: the returned bytes are NOT byte-identical to the input. +// +// Errors mirror resolveFieldToLegacy: unknown shapes, and chunkstore read +// failures (a legacy hash whose chunk is missing). +func resolveFieldToAdaptive(ctx context.Context, b []byte, vs val.ValueStore, cs ChunkPresenceChecker) (newField []byte, isLegacyRow bool, changed bool, err error) { + if len(b) == 0 { + // NULL: same in both formats. + return nil, false, false, nil + } + + if b[0] != 0 { + // Non-zero leading byte: either legacy raw hash (exactly 20 bytes) or an + // already-canonical adaptive-addressed value (varint length + 20 hash + // bytes, total >= 21). + switch { + case len(b) == hash.ByteLen: + // Legacy raw hash. Dereference and re-encode as adaptive, reusing + // the existing content chunk (b is its 20-byte address). + content, rErr := vs.ReadBytes(ctx, hash.New(b)) + if rErr != nil { + return nil, false, false, fmt.Errorf("read legacy content for 20-byte hash: %w", rErr) + } + return encodeContentAsAdaptive(content, b), true, true, nil + case len(b) > hash.ByteLen: + declaredLen, varintSize := uvarint.Uvarint(b) + if varintSize > 0 && varintSize+hash.ByteLen == len(b) { + _ = declaredLen + // Already a canonical adaptive-addressed value. Leave verbatim. + return b, false, false, nil + } + return nil, false, false, fmt.Errorf("field has unknown shape: %d bytes, non-zero leader, malformed adaptive-addressed trailer", len(b)) + default: + return nil, false, false, fmt.Errorf("field has unknown shape: %d bytes, non-zero leader, too short for any known encoding", len(b)) + } + } + + // Leading byte is 0x00 — structurally adaptive inline, but a 20-byte field + // could be a legacy raw hash whose first byte is coincidentally 0x00 + // (~1/256). Disambiguate via an AUTHORITATIVE read (vs.ReadBytes, + // all-generation; recover-wrapped), NOT cs.Has — so the present/absent + // decision and the content read are consistent by construction and this + // path carries no dependence on ddb.Has completeness (the fragility class + // a review flagged, even though ddb.Has is correct at runtime). + if len(b) == hash.ByteLen { + content, ok, rErr := readContentAuthoritative(ctx, vs, hash.New(b)) + if rErr != nil { + // A READ ERROR is not an absence — fail loud rather than silently + // treat this as adaptive-inline (which would inline the 20 hash + // bytes as if they were content and corrupt a present legacy row). + return nil, false, false, fmt.Errorf("authoritative read for 0x00-leading 20-byte field: %w", rErr) + } + if ok { + // Legacy raw hash with a coincidental 0x00 leader (chunk present). + // Re-encode as adaptive, reusing the existing content chunk. + return encodeContentAsAdaptive(content, b), true, true, nil + } + // Chunk GENUINELY absent ⇒ a real adaptive-inline value with 19 content + // bytes, not a legacy hash. Fall through to keep it verbatim. + } + + // Genuine adaptive inline (`[0x00][content]`). Already canonical — verbatim. + return b, false, false, nil +} + +// maxAdaptiveVarIntLen is the maximum width of a SQLite4 variable-length int as +// produced by uvarint.Encode (mirrors the unexported val.maxVarIntLength). +const maxAdaptiveVarIntLen = 9 + +// encodeContentAsAdaptive encodes raw content (already stored out-of-band at +// |origHash|) into a canonical adaptive value, matching dolt's per-value +// inline-vs-out-of-band threshold and writing NO new chunk: +// - content of 20 bytes or fewer (including empty) inlines as `[0x00][content]` +// — inlining is never larger than the 20-byte address it would replace, and +// the (now-unreferenced) source chunk is reclaimed by a later `dolt gc`. +// - larger content stays out-of-band as `[varint(len)][origHash]`, REUSING the +// existing content chunk verbatim (its hash is unchanged). No blob is +// rewritten and no new chunk enters the commit's novelty set — which both +// preserves data exactly and avoids dangling-reference faults on tables +// whose content chunks are not independently rooted. +// +// Because every produced value is at most 1+20 (inline) or varint+20 (OOB) +// bytes — never materially larger than the legacy 20-byte address it replaces — +// the rebuilt value tuple can never overflow the prolly tuple size target, so no +// per-tuple out-of-band reshuffle is required. +func encodeContentAsAdaptive(content []byte, origHash []byte) []byte { + if len(content) <= hash.ByteLen { + // Inline. AdaptiveValueInlineBytes handles empty content as `[0x00]`. + return val.AdaptiveValueInlineBytes(content) + } + // Out-of-band, reusing the existing chunk: [varint(len)][origHash]. + buf := make([]byte, maxAdaptiveVarIntLen) + n := uvarint.Encode(buf, uint64(len(content))) + out := make([]byte, n+hash.ByteLen) + copy(out, buf[:n]) + copy(out[n:], origHash) + return out +} + +// emitMigrateAdaptiveResult writes a human-readable summary of the migration. +func emitMigrateAdaptiveResult(w io.Writer, r MigrateAdaptiveResult) { + if r.DryRun { + action := fmt.Sprintf("would rewrite %d legacy row(s) → adaptive and flip schema %s → %s", + r.ClassLegacyToRewrite, encodingName(r.OldEncoding), encodingName(r.NewEncoding)) + if r.Outcome == RecoverRowsAlreadyOK { + action = "no-op (already canonical adaptive; nothing to rewrite)" + } + _, _ = fmt.Fprintf(w, "[DRY RUN] %s.%s (%d rows scanned): %s\n", r.Table, r.Column, r.RowsScanned, action) + _, _ = fmt.Fprintf(w, " legacy→rewrite: %d already-adaptive: %d empty/null: %d unknown: %d\n", + r.ClassLegacyToRewrite, r.ClassAlreadyAdaptive, r.ClassEmptyOrNull, r.ClassUnknown) + if r.ClassUnknown > 0 { + _, _ = fmt.Fprintf(w, " WARNING: %d unclassifiable row(s); the real run would abort on these.\n", r.ClassUnknown) + } + return + } + switch r.Outcome { + case RecoverRowsMigrated: + if r.CommitHash == "" { + // dolt_ignore'd force-inline path: persisted to the working set, no commit. + _, _ = fmt.Fprintf(w, "migrate-adaptive (inline) %s: %d/%d rows force-inlined to adaptive (dolt_ignore'd table; working-set persist, no commit)\n", + r.Table, r.RowsRewritten, r.RowsScanned) + _, _ = fmt.Fprintf(w, " columns: %s\n", r.Column) + _, _ = fmt.Fprintf(w, " storage: %d values inline, %d addressed (addressed MUST be 0 for a dolt_ignore'd table)\n", r.InlineCount, r.AddressedCount) + if r.MaxContentLen > inlineSizeCapBytes { + _, _ = fmt.Fprintf(w, " WARNING: largest inlined content is %d bytes (> %d-byte soft cap)\n", r.MaxContentLen, inlineSizeCapBytes) + } + _, _ = fmt.Fprintf(w, " message: %s\n", r.CommitMessage) + return + } + _, _ = fmt.Fprintf(w, "migrate-adaptive %s.%s: %d/%d legacy rows rewritten, schema %s → %s\n", + r.Table, r.Column, r.RowsRewritten, r.RowsScanned, + encodingName(r.OldEncoding), encodingName(r.NewEncoding)) + if r.CollateralRewrites > 0 { + _, _ = fmt.Fprintf(w, " %d rows had non-target AddrEnc fields healed as collateral\n", r.CollateralRewrites) + } + _, _ = fmt.Fprintf(w, " commit: %s\n", r.CommitHash) + _, _ = fmt.Fprintf(w, " message: %s\n", r.CommitMessage) + case RecoverRowsAlreadyOK: + _, _ = fmt.Fprintf(w, "no-op for %s.%s: already canonical adaptive at %s (idempotent success)\n", + r.Table, r.Column, encodingName(r.OldEncoding)) + default: + } +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_export_test.go b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_export_test.go new file mode 100644 index 00000000000..d8054213a9d --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_export_test.go @@ -0,0 +1,32 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + + "github.com/dolthub/dolt/go/libraries/doltcore/env" +) + +// MigrateAdaptiveColumnForTest is the test-side hook into the forward +// migrate-adaptive pipeline . Mirrors RepairColumnForTest / +// RecoverRowsColumnForTest so cross-package tests can assert on the structured +// MigrateAdaptiveResult without going through Exec(). +// +// Part of this test suite (added in a dedicated file so it does not +// clash with later edits to export_test.go). +func MigrateAdaptiveColumnForTest(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (MigrateAdaptiveResult, error) { + return migrateAdaptiveColumn(ctx, dEnv, tableName, colName) +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_ignored_inline_invariant_test.go b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_ignored_inline_invariant_test.go new file mode 100644 index 00000000000..0865eca55f0 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_ignored_inline_invariant_test.go @@ -0,0 +1,91 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +// Guards the Option-C FORCE-INLINE INVARIANT for dolt_ignore'd tables. +// +// dolt_ignore'd tables (ignored_%, ignored_meta) live ONLY in the working set; their +// content chunks are never rooted in a commit. If migrate-adaptive emits an +// out-of-line (adaptive-ADDRESSED) value for such a table, persisting the +// working set dangling-faults on the un-rooted content chunk (the affected +// deploy-blocker). the force-inline mode's fix: force-INLINE every value for an ignored table +// so there are ZERO out-of-line chunk references to dangle on. +// +// This test does NOT reproduce the non-durable-chunk dangling-ref — +// that is state-dependent and only validated on real-world data (see validation notes / deploy notes). It DOES deterministically guard the fix's +// structural invariant: if anyone reverts ignored-table migration to emit +// out-of-line refs, the invariant breaks here. +// +// FAILS-WITHOUT the force-inline mode: a >2KB value migrates to adaptive-ADDRESSED (out-of- +// line) -> FieldAdaptiveAddressed present -> assertion fails. +// PASSES-WITH it: force-inlined -> every value FieldAdaptiveInline. +// Held uncommitted until the an earlier review force-inline fix lands. + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" +) + +func TestMigrateAdaptive_DoltIgnoredTable_ForceInlineInvariant(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false // seed legacy rows + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + t.Cleanup(func() { dEnv.Close() }) + + _, err := sqle.ExecuteSql(ctx, dEnv, `INSERT INTO dolt_ignore VALUES ('ignored_%', true);`) + require.NoError(t, err) + _, err = sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE ignored_log (id INT PRIMARY KEY, actor VARCHAR(255) NOT NULL, body LONGTEXT NOT NULL);`) + require.NoError(t, err) + + // A >2KB value: without force-inline this migrates to adaptive-ADDRESSED + // (out-of-line), the shape that dangles when persisting an ignored table. + big := strings.Repeat("a", 3000) + short := "example inline content payload" + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO ignored_log VALUES (1, 'x', '%s'), (2, 'y', '%s');`, big, short)) + require.NoError(t, err) + + typeinfo.UseAdaptiveEncoding = true // default binary behavior during migrate + + res, err := schemadrift.MigrateAdaptiveColumnForTest(ctx, dEnv, "ignored_log", "body") + require.NoError(t, err) + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + + // INVARIANT: every value must be adaptive-inline — ZERO out-of-line refs. + formats := collectFieldFormats(t, ctx, dEnv, "ignored_log", "body") + require.False(t, formats[schemadrift.FieldAdaptiveAddressed], + "Option-C force-inline: a dolt_ignore'd table must have NO out-of-line (addressed) values after migrate-adaptive") + require.False(t, formats[schemadrift.FieldLegacyRawHash], + "no legacy raw-hash references may remain after migrate-adaptive") + + // Content still round-trips byte-identical. + got := collectColumnContent(t, ctx, dEnv, "ignored_log", "body") + require.Equal(t, map[int]string{1: big, 2: short}, got, + "content must round-trip exactly through the force-inline heal") +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_readerror_test.go b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_readerror_test.go new file mode 100644 index 00000000000..14fef96d050 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive_readerror_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/store/hash" +) + +// TestResolveFieldToAdaptive_FailsLoudOnTransientReadError pins the read-error fix +// (the read-error fix): the migrate-path resolver must DISTINGUISH a genuine +// chunk-absence (recovered empty-chunk panic -> treat as inline) from a +// TRANSIENT read error (propagate / fail loud). +// +// A 20-byte 0x00-leading field is ambiguous: a legacy raw-hash address that +// coincidentally starts 0x00, OR an adaptive-inline value with 19 content +// bytes. resolveFieldToAdaptive performs an authoritative read to disambiguate. +// If that read returns a transient error, the resolver MUST return a non-nil +// error — NOT swallow it and silently inline the 20 hash bytes as "content" +// (which would corrupt a real legacy row by storing its address-bytes as data). +// +// This is the migrate-path analog of TestResolveFieldToLegacy_ChunkstoreHasError. +// FAILS-WITHOUT the read-error fix (an earlier switch swallowed read errors as inline); +// PASSES-WITH it. +func TestResolveFieldToAdaptive_FailsLoudOnTransientReadError(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + vs.readErr = errors.New("transient read failure") + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + // 20-byte field, 0x00 leader, non-zero remainder -> the ambiguous shape that + // forces an authoritative disambiguating read. + var raw [hash.ByteLen]byte + raw[0] = 0x00 + for i := 1; i < hash.ByteLen; i++ { + raw[i] = byte(i) + } + + _, _, _, err := resolveFieldToAdaptive(ctx, raw[:], vs, cs) + require.Error(t, err, + "a transient read error during 0x00-lead disambiguation must propagate (fail loud), not be swallowed as inline") + require.Contains(t, err.Error(), "transient read failure", + "the propagated error must carry the underlying transient read failure") +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go b/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go new file mode 100644 index 00000000000..5afc8c807bb --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go @@ -0,0 +1,813 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + + "github.com/mohae/uvarint" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/commands" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/val" +) + +// RecoverRowsCmd is the heterogeneous-payload sibling of RepairCmd. Where +// repair refuses (correctly) to flip a schema whose rows are a mix of legacy- +// raw-hash and adaptive-formatted bytes, recover-rows rewrites every +// non-canonical row's target field into its canonical legacy form (a bare +// 20-byte content-hash address), then flips the schema record exactly the way +// repair would. The row data and the schema record both update inside a single +// dolt commit so a partial migration can never be observed. +// +// The migration preserves CONTENT — the bytes the user originally inserted are +// readable byte-for-byte before and after — but it changes STORAGE LAYOUT for +// inline-adaptive rows. Specifically: +// +// - A legacy-raw-hash row (20-byte non-zero leader) is already canonical +// and is skipped without a chunkstore round-trip. +// - A 0x00-leading 20-byte row whose hash IS present in the chunkstore is a +// legacy raw hash with a coincidentally-zero leader (~1/256 hashes); it is +// canonical and is skipped. +// - An adaptive-addressed row (varint length + 20 hash bytes, total >= 21) +// has its trailing 20 bytes extracted — those bytes are already the +// content address — and replaces the field bytes verbatim. No chunkstore +// write is needed; the content is already stored out-of-band. +// - A 0x00-leading row whose hash is absent from the chunkstore, OR a +// 0x00-leading row whose length differs from 20, is a genuine inline +// adaptive value: its content bytes (v[1:]) are written to the chunkstore +// via vs.WriteBytes, and the resulting hash becomes the field bytes. +// - A row whose field bytes cannot be classified into any of the above +// shapes is fatal — recover-rows aborts the entire migration and reports +// the affected row's primary-key bytes so the operator can triage manually. +// +// Atomicity: a single dolt commit records both the rewritten prolly map and +// the schema-encoding flip. Readers either see the pre-migration state (all +// reads via adaptive dispatch, panics on legacy bytes) or the post-migration +// state (all reads via legacy dispatch, succeeds on every row). They never +// see an intermediate state where some rows have moved and the schema hasn't. +type RecoverRowsCmd struct{} + +var _ cli.Command = RecoverRowsCmd{} + +var recoverRowsDocs = cli.CommandDocumentationContent{ + ShortDesc: "Row-by-row migration of heterogeneous adaptive/legacy payload into canonical legacy form", + LongDesc: `Walks every row in ` + "`--table`" + `, reads the target ` + "`--column`" + `'s field bytes through the correct per-row dispatch (legacy raw-hash, adaptive inline, or adaptive addressed), and rewrites the field as a canonical 20-byte legacy content-hash address. After every row is processed the persisted column encoding is flipped from its adaptive variant back to the matching legacy sibling. Row data and schema update inside a single dolt commit. + +This command is the heterogeneous-payload counterpart to ` + "`schema-encoding-drift repair`" + `. Use ` + "`repair`" + ` when every row in the affected column is in the same legacy-raw-hash format. Use ` + "`recover-rows`" + ` when ` + "`check`" + ` reports a column as heterogeneous (a mix of legacy and adaptive rows) — that situation can arise on tables that survived the v2.0.7 ALTER MODIFY corruption AND received subsequent writes through the adaptive encoder. + +Content is unchanged — the bytes the user originally inserted are readable byte-for-byte before and after — but inline-adaptive values are promoted to out-of-band storage so every row's field is exactly the same 20-byte hash address shape. + +The command aborts the entire migration and reports the offending row's primary-key bytes if any field cannot be classified into a known shape; partial migrations are never committed. + +Required flags: ` + "`--table`" + ` and ` + "`--column`" + ` — one column per invocation, to keep each commit a single auditable change. + +Exit codes: + 0 migration succeeded (or the column was already at a legacy encoding — idempotent no-op) + 1 migration refused (the column is genuinely adaptive with no legacy rows, or has unknown-shape rows, or the operation failed)`, + Synopsis: []string{"--table --column "}, +} + +// Name returns the subcommand name as registered in admin's command list. +func (cmd RecoverRowsCmd) Name() string { return "recover-rows" } + +// Description is the short summary shown in `dolt admin` help. +func (cmd RecoverRowsCmd) Description() string { + return "Row-by-row migration of heterogeneous adaptive/legacy payload into canonical legacy form (atomic prolly rewrite + schema flip + dolt commit)" +} + +// RequiresRepo is true: we need an initialized dolt environment to write to. +func (cmd RecoverRowsCmd) RequiresRepo() bool { return true } + +// Docs is the full documentation surfaced by `dolt admin schema-encoding-drift recover-rows --help`. +func (cmd RecoverRowsCmd) Docs() *cli.CommandDocumentation { + return cli.NewCommandDocumentation(recoverRowsDocs, cmd.ArgParser()) +} + +// ArgParser declares the flag surface. +func (cmd RecoverRowsCmd) ArgParser() *argparser.ArgParser { + ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0) + ap.SupportsString(tableFlag, "", "name", "the table to migrate") + ap.SupportsString(colFlag, "", "name", "the column whose heterogeneous payload should be rewritten to canonical legacy form") + return ap +} + +// Hidden mirrors the rest of the admin commands. +func (cmd RecoverRowsCmd) Hidden() bool { return true } + +// Exec is the CLI entry point. +func (cmd RecoverRowsCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, _ cli.CliContext) int { + ap := cmd.ArgParser() + usage, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, recoverRowsDocs, ap)) + apr := cli.ParseArgsOrDie(ap, args, usage) + + tableName, ok := apr.GetValue(tableFlag) + if !ok || tableName == "" { + verr := errhand.BuildDError("--table is required").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + colName, ok := apr.GetValue(colFlag) + if !ok || colName == "" { + verr := errhand.BuildDError("--column is required").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + result, err := recoverRowsColumn(ctx, dEnv, tableName, colName) + if err != nil { + verr := errhand.BuildDError("recover-rows failed").AddCause(err).Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + emitRecoverRowsResult(cli.OutStream, result) + return 0 +} + +// RecoverRowsOutcome enumerates the three terminal states of a recover-rows +// call: +// +// - RecoverRowsMigrated — at least one row was rewritten and the schema +// was flipped to the matching legacy encoding. Includes the count of +// rows touched and the resulting dolt commit hash. +// - RecoverRowsAlreadyOK — the column is already at a legacy encoding; +// nothing to do. Returned as a successful no-op so scripted migration +// loops are idempotent. +// - RecoverRowsNoLegacyRows — the column has an adaptive schema tag and +// ONLY genuine adaptive rows (no legacy-raw-hash payload). recover-rows +// refuses this case for the same reason repair refuses it: the column is +// internally consistent and migrating would be wasted work that +// promotes nothing. +type RecoverRowsOutcome int + +const ( + RecoverRowsMigrated RecoverRowsOutcome = iota + RecoverRowsAlreadyOK + RecoverRowsNoLegacyRows +) + +// RecoverRowsResult is the structured outcome of a recover-rows call, used by +// the CLI emitter and by the integration tests. +type RecoverRowsResult struct { + Outcome RecoverRowsOutcome + Database string + Table string + Column string + OldEncoding val.Encoding + NewEncoding val.Encoding + RowsScanned int // total rows iterated + RowsRewritten int // subset of RowsScanned whose target field bytes changed + CollateralRewrites int // subset of RowsRewritten where one or more NON-target AddrEnc fields had to be normalized to satisfy the new descriptor — surfaces residual schema-row mismatch left by a prior `repair` invocation on a sibling column + CommitHash string // empty for RecoverRowsAlreadyOK and RecoverRowsNoLegacyRows + CommitMessage string // empty for RecoverRowsAlreadyOK and RecoverRowsNoLegacyRows +} + +// recoverRowsColumn is the entire migration pipeline. Broken out so unit / +// integration tests can call it without invoking the CLI wrapper. +func recoverRowsColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (RecoverRowsResult, error) { + ddb := dEnv.DoltDB(ctx) + dbName := ddb.GetDatabaseName() + + rsr := dEnv.RepoStateReader() + headRef, err := rsr.CWBHeadRef(ctx) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("get current branch: %w", err) + } + + roots, err := dEnv.Roots(ctx) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("get roots: %w", err) + } + + tn := doltdb.TableName{Name: tableName, Schema: doltdb.DefaultSchemaName} + tbl, resolvedName, ok, err := doltdb.GetTableInsensitive(ctx, roots.Working, tn) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("get table %s: %w", tableName, err) + } + if !ok { + return RecoverRowsResult{}, fmt.Errorf("table %q not found on current working root", tableName) + } + tn.Name = resolvedName + + // recover-rows commits unconditionally, which dangling-faults on a + // dolt_ignore'd table: those tables are never committed, so their content + // chunks are not commit-rooted and the rewritten rows' out-of-band refs + // cannot be rooted at commit time. Refuse such tables and point the operator + // at migrate-adaptive, whose force-inline path is the safe heal for them. + ignored, ignErr := doltdb.IdentifyIgnoredTables(ctx, roots, []doltdb.TableName{tn}) + if ignErr != nil { + return RecoverRowsResult{}, fmt.Errorf("check dolt_ignore status for %s: %w", resolvedName, ignErr) + } + if len(ignored) > 0 { + return RecoverRowsResult{}, fmt.Errorf( + "table %s is dolt_ignore'd; recover-rows commits unconditionally and would dangling-fault on its never-committed content chunks. Use `dolt admin schema-encoding-drift migrate-adaptive --table %s --column ` instead — it force-inlines dolt_ignore'd tables (no out-of-band refs) and persists the working set", + resolvedName, resolvedName, + ) + } + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("get schema for %s: %w", resolvedName, err) + } + + // Keyless tables prepend a cardinality column to the value tuple. Their + // value-tuple ordinals don't line up with NonPKCols ordering, so the + // simple value-tuple index we compute would point at the wrong column. + // recover-rows refuses keyless tables to keep that bug from biting; the database's + // affected tables (`issues`, `events`) are both keyed. + if schema.IsKeyless(sch) { + return RecoverRowsResult{}, fmt.Errorf( + "table %s is keyless; recover-rows does not yet support keyless tables (real-world tables are all keyed)", + resolvedName, + ) + } + + existingCol, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) + if !ok { + return RecoverRowsResult{}, fmt.Errorf("column %q not found in table %s", colName, resolvedName) + } + + oldEnc := existingCol.TypeInfo.Encoding() + + // Fast-path: already at a legacy encoding. Nothing to do — return success + // as an idempotent no-op. This matches repair's contract for the same + // shape so scripted loops can call either command without branching. + if IsLegacyEncoding(oldEnc) { + return RecoverRowsResult{ + Outcome: RecoverRowsAlreadyOK, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: oldEnc, + }, nil + } + + newEnc, hasLegacy := LegacySibling(oldEnc) + if !IsAdaptiveEncoding(oldEnc) || !hasLegacy { + return RecoverRowsResult{ + Outcome: RecoverRowsNoLegacyRows, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + }, fmt.Errorf( + "column %s.%s has encoding %s which is not an adaptive TEXT/BLOB/JSON/GEOMETRY variant; "+ + "there is no legacy sibling to migrate to", + resolvedName, existingCol.Name, encodingName(oldEnc), + ) + } + + if existingCol.IsPartOfPK { + return RecoverRowsResult{}, fmt.Errorf( + "column %s.%s is part of the primary key; PK columns do not carry adaptive encodings", + resolvedName, existingCol.Name, + ) + } + + tupleIndex, ok := valueTupleIndexForColumn(sch, existingCol.Name) + if !ok { + return RecoverRowsResult{}, fmt.Errorf( + "column %s.%s does not appear in the value tuple (maybe virtual?)", + resolvedName, existingCol.Name, + ) + } + + // Build the patched schema with the target column flipped to its legacy + // sibling encoding. We use the same replaceColumnByTag helper repair.go + // uses so the index/check/constraint carryover is identical. + newCol := existingCol + newCol.TypeInfo = existingCol.TypeInfo.WithEncoding(newEnc) + newCol.Kind = newCol.TypeInfo.NomsKind() + + newSch, err := replaceColumnByTag(sch, newCol) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("rebuild schema for %s: %w", resolvedName, err) + } + + // Walk every row, classify the target field, and rewrite the row into the + // canonical legacy form. We refuse the migration up front if the column + // has no legacy-raw-hash witnesses (it's purely adaptive — repair would + // also refuse). Otherwise we proceed even with mixed rows; the entire + // point of recover-rows is to handle that case. + rewritten, scanned, sawAnyLegacy, err := rewriteColumn(ctx, tbl, tupleIndex, newSch, ddb, resolveFieldToLegacy) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("rewrite rows of %s.%s: %w", resolvedName, existingCol.Name, err) + } + + // If we walked every row and saw no canonical-legacy / inline / addressed + // content at all, the column was empty (no value rows). Then we still need + // to flip the schema because the encoding tag is the load-bearing thing. + // If we saw NO legacy witnesses but DID see adaptive rows, refuse: that's + // repair's "genuine adaptive" case, not recover-rows territory. + if scanned > 0 && !sawAnyLegacy { + // Pure-adaptive column. recover-rows should NOT flip — that would + // corrupt the adaptive payload exactly the way repair guards against. + return RecoverRowsResult{ + Outcome: RecoverRowsNoLegacyRows, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + RowsScanned: scanned, + }, fmt.Errorf( + "column %s.%s has only genuine adaptive rows (no legacy-raw-hash witnesses); recover-rows refused — the column is internally consistent and a migration would be a no-op flip best handled by checking column health (run `check`) first", + resolvedName, existingCol.Name, + ) + } + + // Wrap the rewritten map as a durable index and apply both changes (rows + // + schema) to the table. Order matters: SetSchema first so subsequent + // reads use the legacy descriptors; SetTableRows second so the new bytes + // are correctly interpreted. + updatedTbl, err := tbl.UpdateSchema(ctx, newSch) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("apply schema change to %s: %w", resolvedName, err) + } + updatedTbl, err = updatedTbl.UpdateRows(ctx, durable.IndexFromProllyMap(rewritten.Map)) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("apply rewritten rows to %s: %w", resolvedName, err) + } + + newWorking, err := roots.Working.PutTable(ctx, tn, updatedTbl) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("put updated table %s: %w", resolvedName, err) + } + roots.Working = newWorking + roots.Staged = newWorking + + // Count rows rewritten — the unchanged subset is the difference between + // scanned and rewritten. We surface both so the operator can audit the + // migration's blast radius. Collateral rewrites (non-target AddrEnc + // fields healed in the same row) are reported separately so the operator + // can tell when a prior `repair` left a sibling column with a schema-row + // mismatch that this migration repaired as a side-effect. + rowsRewritten := rewritten.RowsRewritten + collateral := rewritten.CollateralRewrites + var commitMsg string + if collateral > 0 { + commitMsg = fmt.Sprintf( + "admin: schema-encoding-drift recover-rows on %s.%s (%d rows rewritten to legacy format + schema flipped to %s; %d rows had non-target AddrEnc fields healed as collateral; content unchanged)", + resolvedName, existingCol.Name, rowsRewritten, encodingName(newEnc), collateral, + ) + } else { + commitMsg = fmt.Sprintf( + "admin: schema-encoding-drift recover-rows on %s.%s (%d rows rewritten to legacy format + schema flipped to %s; content unchanged)", + resolvedName, existingCol.Name, rowsRewritten, encodingName(newEnc), + ) + } + commitHash, err := commitWorkingRoot(ctx, dEnv, headRef, roots, commitMsg) + if err != nil { + return RecoverRowsResult{}, fmt.Errorf("commit recover-rows: %w", err) + } + + return RecoverRowsResult{ + Outcome: RecoverRowsMigrated, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: newEnc, + RowsScanned: scanned, + RowsRewritten: rowsRewritten, + CollateralRewrites: collateral, + CommitHash: commitHash.String(), + CommitMessage: commitMsg, + }, nil +} + +// rewrittenMap is the output of rewriteColumnToLegacy. It carries the new +// prolly.Map (suitable for wrapping as a durable Index) and the count of rows +// whose target field bytes were actually changed by the migration. We use a +// named struct rather than a multi-return so the row-count travels with the +// map all the way back to the commit-message builder. +// +// CollateralRewrites counts rows where one or more NON-TARGET fields had to be +// normalized to satisfy the new value descriptor. The case this surfaces: +// a previous `repair` invocation on a sibling column may have flipped its +// schema to legacy without scanning every row — leaving non-canonical +// adaptive bytes in that column under a legacy encoding tag. When THIS +// migration constructs a new tuple, the serializer's countAddresses walks +// every AddrEnc field and calls hash.New on its bytes, which panics on +// any field whose length is not exactly 20. Normalizing those sibling fields +// in-tuple unblocks the rewrite AND repairs the latent mismatch in one shot +// (content-preserving: inline bytes are written to the chunkstore, addressed +// trailers are reused verbatim). +type rewrittenMap struct { + Map prolly.Map + RowsRewritten int + CollateralRewrites int +} + +// targetFieldResolver resolves a single value-tuple field's raw bytes into the +// canonical form for a migration's TARGET encoding. The two implementations are +// resolveFieldToLegacy (adaptive/legacy → canonical legacy 20-byte hash) and +// resolveFieldToAdaptive (legacy/adaptive → canonical adaptive inline/addressed). +// Both share the same signature so rewriteColumn can drive either direction +// while keeping the (direction-agnostic) sibling-AddrEnc collateral heal fixed +// on resolveFieldToLegacy. Return contract: +// - newField: the resolved bytes for the target position. +// - wasSourceFormat: the input was in the migration's SOURCE format (a legacy +// witness for the forward heal; a legacy witness for the legacy heal too) — +// the caller ORs these into sawAnySource to gate the schema flip. +// - changed: newField is not byte-identical to the input (drives Put/skip). +type targetFieldResolver func(ctx context.Context, b []byte, vs val.ValueStore, cs ChunkPresenceChecker) (newField []byte, wasSourceFormat bool, changed bool, err error) + +// rewriteColumnToLegacy iterates every row of |tbl|'s primary index, resolves +// the field bytes at |tupleIndex| into canonical legacy form, and produces a +// new prolly.Map whose value tuples carry the canonical bytes. The resulting +// map's value descriptor is derived from |newSch|, so downstream readers (with +// the legacy schema applied) interpret every field correctly. +// +// Returns: +// - rewritten: the new prolly.Map (always non-empty unless the original was +// empty), wrapped with the rewritten-row count. +// - scanned: total rows iterated from the source map (NOT the same as the +// resulting map's row count when an error aborts the loop early). +// - sawAnyLegacy: true if at least one row had a legacy-raw-hash field, OR +// the table had no row data at all (vacuous truth: a 0-row migration is +// always safe). Used by the caller to decide whether the schema flip is +// justified. +// - err: non-nil if a row's bytes cannot be classified (unknown shape) or a +// vs.WriteBytes / vs.ReadBytes call fails. Aborts mid-iteration; partial +// mutations are discarded. +func rewriteColumn( + ctx context.Context, + tbl *doltdb.Table, + tupleIndex int, + newSch schema.Schema, + cs ChunkPresenceChecker, + resolveTarget targetFieldResolver, +) (rewritten rewrittenMap, scanned int, sawAnyLegacy bool, err error) { + durableIdx, err := tbl.GetRowData(ctx) + if err != nil { + return rewrittenMap{}, 0, false, err + } + if durableIdx == nil { + return rewrittenMap{}, 0, true, nil + } + empty, err := durableIdx.Empty() + if err != nil { + return rewrittenMap{}, 0, false, err + } + pm, err := durable.ProllyMapFromIndex(durableIdx) + if err != nil { + return rewrittenMap{}, 0, false, err + } + ns := pm.NodeStore() + pool := pm.Pool() + + // Derive the post-migration value descriptor from the patched schema. We + // seed the MutableMap rewriter with this descriptor so any caller that + // later reads from the rewritten map sees the canonical legacy view. + newValDesc := newSch.GetValueDescriptor(ns) + mut := pm.Rewriter(pm.KeyDesc(), newValDesc) + + // Pre-compute which positions in the new value descriptor are legacy + // AddrEnc — i.e., positions where the field bytes MUST be either NULL or + // exactly 20 bytes (a legacy raw content-hash address). Any other shape + // would trip ProllyMapSerializer.countAddresses' hash.New call on the + // serialization path (panic: "invalid hash length: N"). We normalize + // non-canonical bytes in these positions into proper 20-byte hashes + // before assembling the new tuple — see the sibling-mismatch case-study comment on + // rewrittenMap.CollateralRewrites. + addrEncPositions := make([]bool, newValDesc.Count()) + val.IterAddressFields(newValDesc, func(i int, _ val.Type) { + addrEncPositions[i] = true + }) + + if empty { + // No rows to migrate. The rewriter is still a no-op MutableMap; we + // materialize it to get a properly-typed (key/val descriptors) empty + // Map back. sawAnyLegacy=true (vacuous truth) so the caller flips the + // schema for the (also-empty) row payload. + nm, mErr := mut.Map(ctx) + if mErr != nil { + return rewrittenMap{}, 0, false, mErr + } + return rewrittenMap{Map: nm}, 0, true, nil + } + + iter, err := pm.IterAll(ctx) + if err != nil { + return rewrittenMap{}, 0, false, err + } + + rowsRewritten := 0 + collateralRewrites := 0 + keyDesc := pm.KeyDesc() + for { + key, value, iterErr := iter.Next(ctx) + if errors.Is(iterErr, io.EOF) { + break + } + if iterErr != nil { + return rewrittenMap{}, scanned, sawAnyLegacy, iterErr + } + scanned++ + + oldField := value.GetField(tupleIndex) + newField, isLegacy, targetChanged, resolveErr := resolveTarget(ctx, oldField, ns, cs) + if resolveErr != nil { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "row with primary key %s: %w", pkHex, resolveErr, + ) + } + if isLegacy { + sawAnyLegacy = true + } + + // a later fix/ defensive width check on BOTH sides. The encoding flip + // performed by this command MUST NOT change column count — in-place + // schema mutations preserve the value-tuple shape. If we ever observe + // a source tuple WIDER than the new value descriptor (e.g. a + // concurrent ALTER raced ahead and dropped a column we still see), + // silently growing `values` to descCount would produce a tuple that + // disagrees with the persisted schema. Refuse rather than swallow it. + fieldCount := value.Count() + descCount := newValDesc.Count() + if fieldCount > descCount { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "schema-row width mismatch at primary key %s: source tuple has %d fields, new value descriptor expects %d — refusing rewrite (likely concurrent schema change)", + pkHex, fieldCount, descCount, + ) + } + if descCount > fieldCount { + fieldCount = descCount + } + + // Pre-scan: do any NON-TARGET AddrEnc fields need normalization? If + // every AddrEnc field is already canonical (20-byte hash or NULL) AND + // the target field is unchanged, we can skip this row entirely. + needCollateral := false + for i := 0; i < fieldCount; i++ { + if i == tupleIndex { + continue + } + if i >= len(addrEncPositions) || !addrEncPositions[i] { + continue + } + b := value.GetField(i) + if len(b) == 0 || len(b) == hashByteLen { + // NULL or canonical 20-byte hash — already serializer-safe. + // 20-byte 0x00-leading bytes that are actually adaptive + // inline payloads get caught by the chunkstore-aware + // resolveFieldToLegacy if we DO normalize, but the structural + // fast-path here treats them as fine; mis-classification at + // this stage only forces an unnecessary normalization, not a + // silent corruption. + continue + } + needCollateral = true + break + } + + if !targetChanged && !needCollateral { + continue + } + + // Rebuild the value tuple with the rewritten target field. We + // construct it via val.NewTuple directly rather than via TupleBuilder + // — the TupleBuilder.Build path can re-shuffle adaptive columns based + // on a size heuristic, and that's exactly what we need to avoid when + // the source tuple may contain another not-yet-migrated adaptive + // column with non-canonical bytes. + // + // Normalization model for each position: + // - target column: use newField (already 20-byte canonical hash) + // - non-target legacy AddrEnc position w/ non-canonical bytes: resolve + // to canonical 20-byte hash via resolveFieldToLegacy. This is the + // content-preserving heal for the documented case where a prior `repair` + // left a sibling column with adaptive bytes under a legacy schema. + // - non-target legacy AddrEnc position w/ canonical bytes (NULL or + // 20-byte hash): copy verbatim — already serializer-safe. + // - non-target adaptive / non-address position: copy verbatim — the + // serializer expects variable-length bytes here. + values := make([][]byte, fieldCount) + rowHadCollateral := false + for i := 0; i < fieldCount; i++ { + if i == tupleIndex { + values[i] = newField + continue + } + b := value.GetField(i) + if i < len(addrEncPositions) && addrEncPositions[i] && len(b) != 0 && len(b) != hashByteLen { + normalized, _, normChanged, normErr := resolveFieldToLegacy(ctx, b, ns, cs) + if normErr != nil { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "row with primary key %s, field %d (non-target AddrEnc heal): %w", + pkHex, i, normErr, + ) + } + values[i] = normalized + if normChanged { + rowHadCollateral = true + } + continue + } + values[i] = b + } + newValue := val.NewTuple(pool, values...) + + if putErr := mut.Put(ctx, key, newValue); putErr != nil { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "put rewritten row with primary key %s: %w", pkHex, putErr, + ) + } + rowsRewritten++ + if rowHadCollateral { + collateralRewrites++ + } + } + + newMap, err := mut.Map(ctx) + if err != nil { + return rewrittenMap{}, scanned, sawAnyLegacy, err + } + return rewrittenMap{ + Map: newMap, + RowsRewritten: rowsRewritten, + CollateralRewrites: collateralRewrites, + }, scanned, sawAnyLegacy, nil +} + +// resolveFieldToLegacy is the per-row workhorse. Given a single field's bytes +// from a (possibly drifted) adaptive-tagged value tuple, it returns the +// canonical legacy form of those bytes plus two flags: +// +// - isCanonicalLegacy: true if the input bytes were already in the legacy +// shape (NULL, or non-zero-leading 20 bytes, or 0x00-leading 20 bytes +// whose hash IS present in the chunkstore). The caller uses this to +// compute the "sawAnyLegacy" signal that guards the schema flip. +// - changed: true if the returned bytes are NOT byte-identical to the input. +// The caller uses this to decide whether to Put the row into the rewriter +// (saves a skip-list slot per unchanged row). +// +// The classification logic mirrors classify.go's structural classifier but +// promotes the ambiguous 0x00-leading 20-byte case using the chunkstore in the +// same way ClassifyFieldWithChunkstore does. We don't call that function +// directly because we also need the resolved CONTENT for inline-to-OOB +// promotion, and that's not part of the classifier's contract. +// +// Errors are returned for: +// - FieldUnknown shapes (1-19 bytes, or 21+ bytes with malformed varint +// trailer). The caller adds the row's primary-key bytes to the error. +// - vs.ReadBytes / vs.WriteBytes failures (e.g., missing NBS chunk for an +// adaptive-addressed row whose hash is no longer present). +func resolveFieldToLegacy(ctx context.Context, b []byte, vs val.ValueStore, cs ChunkPresenceChecker) (newField []byte, isCanonicalLegacy bool, changed bool, err error) { + if len(b) == 0 { + // NULL: same in both formats. + return nil, true, false, nil + } + + if b[0] != 0 { + // Non-zero leading byte: either legacy raw hash (20 bytes) or + // adaptive addressed (varint length + 20 hash bytes, total >= 21). + switch { + case len(b) == hash.ByteLen: + // Legacy raw hash: already canonical. No rewrite. + return b, true, false, nil + case len(b) > hash.ByteLen: + // Try to interpret as adaptive addressed. + declaredLen, varintSize := uvarint.Uvarint(b) + if varintSize > 0 && varintSize+hash.ByteLen == len(b) { + _ = declaredLen + // Extract the trailing 20 bytes — that IS the canonical legacy + // hash. The content is already stored out-of-band at that + // hash; no chunkstore write needed. + addr := b[varintSize:] + out := make([]byte, hash.ByteLen) + copy(out, addr) + return out, false, true, nil + } + // 21+ bytes, non-zero leader, but the varint+hash shape doesn't + // add up. Unrecognized. + return nil, false, false, fmt.Errorf("field has unknown shape: %d bytes, non-zero leader, malformed adaptive-addressed trailer", len(b)) + default: + // 1-19 bytes, non-zero leader — too short for either legacy + // (needs 20) or adaptive addressed (needs >= 21). + return nil, false, false, fmt.Errorf("field has unknown shape: %d bytes, non-zero leader, too short for any known encoding", len(b)) + } + } + + // Leading byte is 0x00 — structurally adaptive inline. But this is also the + // disambiguous case for legacy raw hashes whose first byte happens to be + // 0x00 (~1/256 of all hashes). We disambiguate via chunkstore presence for + // the 20-byte case only. + // + // This deployed recover-rows path KEEPS cs.Has deliberately (the new + // forward-migrate path resolveFieldToAdaptive uses the authoritative + // vs.ReadBytes-attempt). The review code-proved cs.Has is correct at runtime in + // real-world: cs MUST be the table's own ddb (callers satisfy this), vs is + // the same table's NodeStore, and both resolve to ONE chunks.ChunkStore + // where Has⟺ReadBytes-success are consistent (GenerationalNBS ORs all + // generations) — no stranding false-negative is possible. Switching this + // call site to the ReadBytes-attempt is NOT behavior-equivalent here: cs.Has + // PROPAGATES a transient chunkstore error (a contract the + // ChunkstoreHasError regression pins), whereas a ReadBytes-attempt would + // swallow it as "absent ⇒ inline" and could misclassify a present legacy + // row. Per the a maintainer escape hatch we keep cs.Has + this comment. + // Defense in depth: recover-rows now REFUSES dolt_ignore'd tables up front, + // so the only case where chunk visibility could ever differ across stores + // (never-committed chunks) can't reach here. + if len(b) == hash.ByteLen && cs != nil { + h := hash.New(b) + present, hasErr := cs.Has(ctx, h) + if hasErr != nil { + return nil, false, false, fmt.Errorf("chunkstore presence check: %w", hasErr) + } + if present { + // Legacy raw hash with coincidental 0x00 leader. Canonical. + return b, true, false, nil + } + } + + // Genuine adaptive inline. Promote to out-of-band: write the inline + // content (b[1:]) to the chunkstore, then store the resulting hash as + // the 20-byte field bytes. + content := b[1:] + addr, writeErr := vs.WriteBytes(ctx, content) + if writeErr != nil { + return nil, false, false, fmt.Errorf("write inline content (%d bytes) to chunkstore: %w", len(content), writeErr) + } + out := make([]byte, hash.ByteLen) + copy(out, addr[:]) + return out, false, true, nil +} + +// formatKeyForError renders a primary-key tuple as a human-readable hex string +// for inclusion in error messages. We deliberately don't invoke the type +// descriptor's Format method here: that path can itself touch adaptive +// dispatch for index keys that contain adaptive fields, and we don't want +// error reporting to recurse into the very panic recover-rows exists to fix. +func formatKeyForError(key val.Tuple, keyDesc *val.TupleDesc) string { + // Show one field per ordinal, hex-encoded, comma-separated. For most + // real-world tables the PK is a small integer or short string so this stays + // readable. + if keyDesc == nil { + return hex.EncodeToString(key) + } + out := "[" + for i := 0; i < keyDesc.Count(); i++ { + if i > 0 { + out += ", " + } + f := key.GetField(i) + if f == nil { + out += "NULL" + continue + } + out += "0x" + hex.EncodeToString(f) + } + out += "]" + return out +} + +// emitRecoverRowsResult writes a human-readable summary of the migration to +// |w|. +func emitRecoverRowsResult(w io.Writer, r RecoverRowsResult) { + switch r.Outcome { + case RecoverRowsMigrated: + _, _ = fmt.Fprintf(w, "recover-rows %s.%s: %d/%d rows rewritten, schema %s → %s\n", + r.Table, r.Column, r.RowsRewritten, r.RowsScanned, + encodingName(r.OldEncoding), encodingName(r.NewEncoding)) + if r.CollateralRewrites > 0 { + _, _ = fmt.Fprintf(w, " %d rows had non-target AddrEnc fields healed as collateral (residual mismatch from prior repair on a sibling column)\n", + r.CollateralRewrites) + } + _, _ = fmt.Fprintf(w, " commit: %s\n", r.CommitHash) + _, _ = fmt.Fprintf(w, " message: %s\n", r.CommitMessage) + case RecoverRowsAlreadyOK: + _, _ = fmt.Fprintf(w, "no-op for %s.%s: already at %s (idempotent success)\n", + r.Table, r.Column, encodingName(r.OldEncoding)) + default: + // RecoverRowsNoLegacyRows surfaces via the returned error, so the CLI + // emitter never reaches this branch. + } +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/recover_rows_integration_test.go b/go/cmd/dolt/commands/admin/schemadrift/recover_rows_integration_test.go new file mode 100644 index 00000000000..bed30d11aef --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/recover_rows_integration_test.go @@ -0,0 +1,488 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// TestRecoverRows_End2End_HeterogeneousColumn drives the entire recover-rows +// pipeline against a hand-crafted heterogeneous fixture and asserts: +// +// 1. The pre-state has the column tagged adaptive with a mix of legacy and +// adaptive rows. +// 2. recover-rows succeeds with outcome=RecoverRowsMigrated and the count of +// rewritten rows matches the count of non-legacy rows in the pre-state. +// 3. The post-state has the column tagged legacy and every row readable +// without panic (i.e. content matches what was inserted). +// 4. The commit message names the table.column, the row count, and the +// target encoding. +// 5. Re-running recover-rows is a no-op (OutcomeAlreadyOK). +func TestRecoverRows_End2End_HeterogeneousColumn(t *testing.T) { + ctx := context.Background() + + // Step 1: CREATE TABLE under legacy encoding so the first batch of rows + // is written as raw 20-byte hashes — exactly the legacy-side payload of + // the heterogeneous mix we want to test recovery on. + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL);`) + require.NoError(t, err) + + // Insert two rows under legacy encoding. Use long payloads so they spill + // to the legacy raw-hash representation (i.e. the field bytes are exactly + // 20 bytes, addressing an out-of-band content chunk). + legacyPayload1 := strings.Repeat("L", 1500) + legacyPayload2 := strings.Repeat("M", 2500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (1, '%s'), (2, '%s');`, legacyPayload1, legacyPayload2)) + require.NoError(t, err) + + // Step 2: manually flip the column's encoding to adaptive — simulating + // the v2.0.7 ALTER MODIFY corruption. The schema record now says + // StringAdaptiveEnc while the existing rows are still in legacy raw-hash + // form. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + // Step 3: insert additional rows under the now-adaptive schema. These + // will be written via the adaptive encoder, producing either inline or + // out-of-band-addressed adaptive bytes. + typeinfo.UseAdaptiveEncoding = true + adaptiveShort := "short adaptive payload" + adaptiveLong := strings.Repeat("A", 1700) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (3, '%s'), (4, '%s');`, adaptiveShort, adaptiveLong)) + require.NoError(t, err) + + // Sanity: collect the field-format distribution to confirm we built a + // truly heterogeneous fixture (not all-legacy, not all-adaptive). + preStateFormats := collectFieldFormats(t, ctx, dEnv, "issues", "description") + require.Contains(t, preStateFormats, schemadrift.FieldLegacyRawHash, + "pre-state must contain at least one legacy row (otherwise it's not heterogeneous)") + require.True(t, len(preStateFormats) >= 2, + "pre-state must mix at least two distinct field formats (got %v)", preStateFormats) + + // We deliberately do NOT call SELECT on the corrupted pre-state — that's + // the very crash this whole admin command exists to mitigate. We know + // what we inserted, so we compare against the literal values after the + // migration completes. + expected := map[int]string{ + 1: legacyPayload1, + 2: legacyPayload2, + 3: adaptiveShort, + 4: adaptiveLong, + } + + // Step 4: run recover-rows. + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome, + "heterogeneous-with-legacy-witnesses case must migrate") + require.Equal(t, val.StringAdaptiveEnc, res.OldEncoding) + require.Equal(t, val.StringAddrEnc, res.NewEncoding) + require.Equal(t, 4, res.RowsScanned) + // At least the 2 adaptive rows must have been rewritten. Legacy rows are + // already canonical and might be skipped depending on classifier output. + require.GreaterOrEqual(t, res.RowsRewritten, 2, + "at least 2 adaptive rows must be rewritten") + require.NotEmpty(t, res.CommitHash, "successful migration must produce a real commit hash") + require.Contains(t, res.CommitMessage, "issues.description", + "commit message must name the specific table.column") + require.Contains(t, res.CommitMessage, "StringAddrEnc", + "commit message must name the post-migration encoding") + require.Contains(t, res.CommitMessage, "content unchanged", + "commit message must declare the content-unchanged contract for downstream consumers") + + // Step 5: verify post-state. Schema is legacy, every row readable, content + // matches the literal values we inserted in steps 1 and 3. + postContent := collectColumnContent(t, ctx, dEnv, "issues", "description") + require.Equal(t, expected, postContent, + "row content must equal the originally-inserted strings after recover-rows") + + postFormats := collectFieldFormats(t, ctx, dEnv, "issues", "description") + require.NotContains(t, postFormats, schemadrift.FieldAdaptiveAddressed, + "post-migration must contain no adaptive-addressed bytes") + require.NotContains(t, postFormats, schemadrift.FieldAdaptiveInline, + "post-migration must contain no adaptive-inline bytes (all promoted to OOB legacy)") + // Every row must now be legacy raw hash (or NULL, but we inserted NOT NULL). + for f := range postFormats { + require.Equal(t, schemadrift.FieldLegacyRawHash, f, + "every post-migration row must be in legacy raw hash format; saw %v", f) + } + + // Step 6: idempotency. Re-running recover-rows on the now-legacy column + // must be a no-op. + res2, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.RecoverRowsAlreadyOK, res2.Outcome, + "re-running on a now-legacy column must be an idempotent no-op") + require.Empty(t, res2.CommitHash, "no-op recover-rows must NOT produce a commit") +} + +// TestRecoverRows_RepairRefusesSameFixture is the empirical +// FAILS-WITHOUT/PASSES-WITH proof requested by the task brief: take the +// hand-crafted heterogeneous fixture, attempt `repair` first (which MUST +// refuse with the "heterogeneous payload" message), then attempt +// `recover-rows` (which MUST succeed). Same fixture, different outcomes — +// that's the load-bearing distinction between the two commands. +func TestRecoverRows_RepairRefusesSameFixture(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // Build the same heterogeneous fixture as the End2End test. + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL);`) + require.NoError(t, err) + legacyPayload := strings.Repeat("L", 1500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (1, '%s');`, legacyPayload)) + require.NoError(t, err) + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + typeinfo.UseAdaptiveEncoding = true + // Use a payload large enough (> DefaultTupleLengthTarget = 2048 bytes) to + // force the adaptive writer to spill to out-of-band storage. A spilled + // adaptive row classifies as FieldAdaptiveAddressed, which is the signal + // repair uses to detect "genuine adaptive payload" — exactly the + // heterogeneous condition we want repair to refuse. + adaptivePayload := strings.Repeat("A", 3500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (2, '%s');`, adaptivePayload)) + require.NoError(t, err) + + // FAILS-WITHOUT: repair must refuse because the column is heterogeneous. + repairRes, repairErr := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + require.Error(t, repairErr, + "repair MUST refuse a heterogeneous column — that's the prerequisite for recover-rows existing") + require.Contains(t, repairErr.Error(), "heterogeneous", + "refusal message must name the heterogeneous condition") + require.Empty(t, repairRes.CommitHash, + "refused repair must NOT create a commit") + + // PASSES-WITH: recover-rows must succeed on the exact same fixture. + recoverRes, recoverErr := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, recoverErr, + "recover-rows must accept what repair refused") + require.Equal(t, schemadrift.RecoverRowsMigrated, recoverRes.Outcome, + "recover-rows must migrate the heterogeneous fixture") + require.NotEmpty(t, recoverRes.CommitHash, + "successful recover-rows must produce a commit hash") + + // And after recover-rows, the post-state matches what we inserted. + postContent := collectColumnContent(t, ctx, dEnv, "issues", "description") + require.Equal(t, map[int]string{1: legacyPayload, 2: adaptivePayload}, postContent, + "content must round-trip exactly through the migration") +} + +// TestRecoverRows_RefusesPureAdaptiveColumn asserts the load-bearing safety +// guard: if a column's payload is purely adaptive (no legacy-raw-hash +// witnesses), recover-rows refuses with a clear error. The intent matches +// repair's "OutcomeGenuineAdaptive" path: a column with only adaptive rows is +// internally consistent and recover-rows must not flip its schema (that +// would corrupt the adaptive bytes the same way repair would). +func TestRecoverRows_RefusesPureAdaptiveColumn(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = true + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // Create + insert under genuine adaptive encoding. The column is + // internally consistent; recover-rows must refuse. + bigPayload := strings.Repeat("z", 2048) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +`+fmt.Sprintf(`INSERT INTO issues VALUES (1, '%s');`, bigPayload)) + require.NoError(t, err) + + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.Error(t, err, "recover-rows must refuse a purely-adaptive column") + require.Contains(t, err.Error(), "only genuine adaptive rows", + "refusal must explain why the migration is refused") + require.Equal(t, schemadrift.RecoverRowsNoLegacyRows, res.Outcome, + "refused recover-rows must return RecoverRowsNoLegacyRows so callers can branch on it") + require.Empty(t, res.CommitHash, "refused recover-rows must NOT touch the working set") +} + +// TestRecoverRows_IdempotentOnLegacyColumn asserts the fast-path: calling +// recover-rows on a column whose schema is already legacy returns +// OutcomeAlreadyOK without iterating rows. +func TestRecoverRows_IdempotentOnLegacyColumn(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + bigPayload := strings.Repeat("y", 2048) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +`+fmt.Sprintf(`INSERT INTO issues VALUES (1, '%s');`, bigPayload)) + require.NoError(t, err) + + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.RecoverRowsAlreadyOK, res.Outcome, + "legacy-tagged column must short-circuit with RecoverRowsAlreadyOK") + require.Empty(t, res.CommitHash, "no-op must NOT produce a commit") +} + +// TestRecoverRows_HealsSiblingAddrEncMismatch is a sibling-mismatch regression test. +// +// Scenario: an earlier `repair` invocation flipped a sibling column's schema +// from StringAdaptiveEnc to StringAddrEnc, but the column's row data still +// held heterogeneous bytes — some rows have legacy 20-byte hashes (canonical +// for the new schema), some have 1-byte `[0x00]` (adaptive-inline empty) or +// longer non-canonical adaptive shapes. The schema check passed because +// `repair`'s scanner only needs one legacy witness and short-circuits. +// +// Now the operator runs `recover-rows` on a DIFFERENT column. The serializer's +// countAddresses walks every AddrEnc field in the rebuilt tuple and calls +// hash.New(field) on each. For sibling-column fields that are NOT exactly 20 +// bytes, hash.New panics with "invalid hash length: N" — taking down the +// recover-rows command before it can commit anything. +// +// The sibling-mismatch fix is for recover-rows to NORMALIZE every legacy-AddrEnc field +// in the post-schema tuple — not just the target column. Non-canonical bytes +// in sibling AddrEnc positions get resolved through resolveFieldToLegacy and +// rewritten to canonical 20-byte hashes. CollateralRewrites counts how many +// rows had at least one sibling field healed this way. +// +// This test builds the exact pre-state: an `issues` table with one row whose +// description is legacy AND whose sibling close_reason is `[0x00]` (1-byte +// adaptive-inline empty under a schema that's been flipped to StringAddrEnc). +// Without the sibling-mismatch fix, recover-rows on description panics. With the fix it +// succeeds AND reports collateral healing. +func TestRecoverRows_HealsSiblingAddrEncMismatch(t *testing.T) { + ctx := context.Background() + + // Build the source schema under legacy encoding so the initial + // CREATE/INSERT writes legacy raw-hash bytes for both `description` + // (target) and `close_reason` (sibling). + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL, close_reason TEXT);`) + require.NoError(t, err) + // One legacy-bearing row so description has a legacy-raw-hash witness. + legacyPayload := strings.Repeat("L", 1500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (1, '%s', 'orig-close-reason');`, legacyPayload)) + require.NoError(t, err) + + // Flip BOTH columns' schemas to adaptive — simulating the v2.0.7 drift + // landing on both columns simultaneously. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "close_reason", val.StringAdaptiveEnc)) + + // Insert additional rows under the adaptive schema. One has an + // out-of-band-sized description (forcing FieldAdaptiveAddressed) so the + // drift detector sees heterogeneous payload on description. The + // close_reason for these rows is short, so it goes into FieldAdaptiveInline + // shape (1-byte [0x00] for empty, [0x00][content] for non-empty). + typeinfo.UseAdaptiveEncoding = true + adaptivePayload := strings.Repeat("A", 3500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (2, '%s', ''), (3, '%s', 'short');`, + adaptivePayload, adaptivePayload)) + require.NoError(t, err) + + // Now silently flip close_reason's schema BACK to legacy without + // rewriting rows — the exact mistake a buggy `repair` would make. Row + // data for close_reason still has adaptive-inline bytes (`[0x00]` for + // row 2, `[0x00]short` for row 3), but the schema says legacy. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "close_reason", val.StringAddrEnc)) + + // recover-rows on `description` must succeed AND heal the sibling + // close_reason in the process. Without the sibling-mismatch fix the serializer + // would panic on the 1-byte close_reason bytes for row 2. + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err, "recover-rows must succeed even when a sibling AddrEnc column has heterogeneous bytes") + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.GreaterOrEqual(t, res.CollateralRewrites, 2, + "both adaptive-inserted rows (id=2, id=3) had non-canonical close_reason bytes; both must be reported as collateral healings") + require.Contains(t, res.CommitMessage, "non-target AddrEnc fields healed as collateral", + "commit message must surface the collateral healing for the audit trail") + + // Post-migration, both description and close_reason must read back to + // their pre-migration content via SQL — the heal is content-preserving. + descContent := collectColumnContent(t, ctx, dEnv, "issues", "description") + require.Equal(t, map[int]string{1: legacyPayload, 2: adaptivePayload, 3: adaptivePayload}, descContent) + + closeContent := collectColumnContent(t, ctx, dEnv, "issues", "close_reason") + require.Equal(t, map[int]string{1: "orig-close-reason", 2: "", 3: "short"}, closeContent, + "sibling close_reason content must round-trip — empty inline becomes empty legacy, short inline becomes legacy hash addressing the original short bytes") +} + +// TestRecoverRows_NonExistentColumn asserts the resolver surfaces a clear +// error for unknown columns rather than silently no-op. +func TestRecoverRows_NonExistentColumn(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = true + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +INSERT INTO issues VALUES (1, 'hello');`) + require.NoError(t, err) + + _, err = schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "no_such_column") + require.Error(t, err) + require.Contains(t, err.Error(), "no_such_column") +} + +// collectFieldFormats reads every row of |tableName| via the prolly map +// directly and classifies the bytes at |colName|'s value-tuple position. +// Returns the SET of distinct formats observed (not a per-row list — the +// test only needs presence/absence). +func collectFieldFormats(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) map[schemadrift.FieldFormat]bool { + t.Helper() + root, err := dEnv.WorkingRoot(ctx) + require.NoError(t, err) + tbl, _, ok, err := doltdb.GetTableInsensitive(ctx, root, doltdb.TableName{Name: tableName}) + require.NoError(t, err) + require.True(t, ok) + sch, err := tbl.GetSchema(ctx) + require.NoError(t, err) + col, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) + require.True(t, ok) + + tupleIdx := -1 + i := 0 + _ = sch.GetNonPKCols().Iter(func(tag uint64, c schema.Column) (stop bool, err error) { + if c.Virtual { + return false, nil + } + if strings.EqualFold(c.Name, col.Name) { + tupleIdx = i + return true, nil + } + i++ + return false, nil + }) + require.GreaterOrEqual(t, tupleIdx, 0) + + idx, err := tbl.GetRowData(ctx) + require.NoError(t, err) + pm, err := durable.ProllyMapFromIndex(idx) + require.NoError(t, err) + + iter, err := pm.IterAll(ctx) + require.NoError(t, err) + out := make(map[schemadrift.FieldFormat]bool) + for { + _, value, err := iter.Next(ctx) + if err == io.EOF { + break + } + require.NoError(t, err) + b := value.GetField(tupleIdx) + out[schemadrift.ClassifyFieldBytes(b)] = true + } + return out +} + +// collectColumnContent reads every row of |tableName| via SQL SELECT and +// returns the value of |colName| keyed by primary key. This lets the test +// assert post-migration content equality without going through low-level +// prolly access. +func collectColumnContent(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) map[int]string { + t.Helper() + root, err := dEnv.WorkingRoot(ctx) + require.NoError(t, err) + rows, err := sqle.ExecuteSelect(ctx, dEnv, root, fmt.Sprintf(`SELECT id, %s FROM %s ORDER BY id;`, colName, tableName)) + require.NoError(t, err) + out := make(map[int]string) + for _, r := range rows { + require.Equal(t, 2, len(r)) + var id int + switch v := r[0].(type) { + case int32: + id = int(v) + case int64: + id = int(v) + case int: + id = v + default: + t.Fatalf("expected id column to be integer type, got %T", r[0]) + } + var s string + switch v := r[1].(type) { + case string: + s = v + case []byte: + s = string(v) + case sql.StringWrapper: + // Out-of-band TEXT storage — the engine returns a wrapper so it + // can defer the chunkstore round-trip until the value is actually + // observed. We force the unwrap here for the assertion. + unwrapped, uerr := v.Unwrap(ctx) + require.NoError(t, uerr) + s = unwrapped + case sql.AnyWrapper: + unwrapped, uerr := v.UnwrapAny(ctx) + require.NoError(t, uerr) + s = fmt.Sprintf("%v", unwrapped) + default: + t.Fatalf("expected description column to be string, []byte, or sql.StringWrapper; got %T", r[1]) + } + out[id] = s + } + return out +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/recover_rows_test.go b/go/cmd/dolt/commands/admin/schemadrift/recover_rows_test.go new file mode 100644 index 00000000000..ddb5bd64711 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/recover_rows_test.go @@ -0,0 +1,342 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "bytes" + "context" + "errors" + "testing" + + "github.com/mohae/uvarint" + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/store/hash" +) + +// stubValueStore is a minimal val.ValueStore implementation for unit tests on +// resolveFieldToLegacy. It records every Write and supplies a deterministic +// hash so tests can assert what got written. +type stubValueStore struct { + writes [][]byte + reads map[hash.Hash][]byte + // readErr is returned by ReadBytes if non-nil. Lets us simulate a missing + // NBS chunk without constructing one. + readErr error + // writeErr is returned by WriteBytes if non-nil. Simulates a chunkstore + // write failure. + writeErr error +} + +func newStubVS() *stubValueStore { + return &stubValueStore{reads: make(map[hash.Hash][]byte)} +} + +func (s *stubValueStore) ReadBytes(ctx context.Context, h hash.Hash) ([]byte, error) { + if s.readErr != nil { + return nil, s.readErr + } + if b, ok := s.reads[h]; ok { + return b, nil + } + return nil, errors.New("stubValueStore: hash not present") +} + +func (s *stubValueStore) WriteBytes(ctx context.Context, b []byte) (hash.Hash, error) { + if s.writeErr != nil { + return hash.Hash{}, s.writeErr + } + s.writes = append(s.writes, append([]byte(nil), b...)) + // Deterministic: synthesize a hash from a sequence number derived from + // the content + the write count so duplicates collide as they would in + // a real content-addressed store. + h := hash.Of(b) + s.reads[h] = append([]byte(nil), b...) + return h, nil +} + +// stubChunkPresence is the test counterpart to ChunkPresenceChecker. Per-hash +// truth values are supplied at construction. +type stubChunkPresence struct { + present map[hash.Hash]bool + hasErr error +} + +func (s *stubChunkPresence) Has(ctx context.Context, h hash.Hash) (bool, error) { + if s.hasErr != nil { + return false, s.hasErr + } + return s.present[h], nil +} + +// TestResolveFieldToLegacy_NULL covers the NULL (empty slice) case. NULL is +// represented identically in legacy and adaptive, so the result is unchanged. +func TestResolveFieldToLegacy_NULL(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, nil, vs, cs) + require.NoError(t, err) + require.True(t, isLegacy, "NULL is canonical legacy") + require.False(t, changed, "NULL bytes must not change") + require.Nil(t, out) + + out, isLegacy, changed, err = resolveFieldToLegacy(ctx, []byte{}, vs, cs) + require.NoError(t, err) + require.True(t, isLegacy) + require.False(t, changed) + require.Nil(t, out) + + require.Empty(t, vs.writes, "NULL must not write to chunkstore") +} + +// TestResolveFieldToLegacy_LegacyRawHash covers the unambiguously-legacy +// shape: 20 bytes with a non-zero leading byte. Already canonical, no rewrite. +func TestResolveFieldToLegacy_LegacyRawHash(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + // Build a 20-byte field with a non-zero leading byte. + var raw [hash.ByteLen]byte + for i := range raw { + raw[i] = byte(i + 1) + } + require.NotEqual(t, byte(0), raw[0]) + + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, raw[:], vs, cs) + require.NoError(t, err) + require.True(t, isLegacy, "legacy raw hash is canonical") + require.False(t, changed, "legacy raw hash bytes must not change") + require.Equal(t, raw[:], out, "output bytes must be the input bytes verbatim") + require.Empty(t, vs.writes, "legacy raw hash must not write to chunkstore") +} + +// TestResolveFieldToLegacy_AdaptiveAddressed covers the adaptive addressed +// shape: varint(length) + 20-byte hash. The migration extracts the trailing +// 20 bytes and returns them as the new field; no content is re-written. +func TestResolveFieldToLegacy_AdaptiveAddressed(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + // Build a synthetic adaptive-addressed value: 1-byte varint = 42, then 20 + // hash bytes. + buf := make([]byte, 9) + n := uvarint.Encode(buf, 42) + field := append([]byte{}, buf[:n]...) + var h [hash.ByteLen]byte + for i := range h { + h[i] = byte(i + 100) // distinct from legacy test + } + field = append(field, h[:]...) + require.Equal(t, n+hash.ByteLen, len(field)) + + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, field, vs, cs) + require.NoError(t, err) + require.False(t, isLegacy, "adaptive-addressed is not legacy") + require.True(t, changed, "adaptive-addressed must rewrite to legacy shape") + require.Equal(t, hash.ByteLen, len(out), "rewritten field is exactly 20 bytes") + require.Equal(t, h[:], out, "rewritten bytes are the trailing 20 from the input") + require.Empty(t, vs.writes, "adaptive-addressed must NOT write to chunkstore — content is already OOB") + + // Multi-byte varint case. + buf = make([]byte, 9) + n = uvarint.Encode(buf, 1<<20) + field2 := append([]byte{}, buf[:n]...) + field2 = append(field2, h[:]...) + out, isLegacy, changed, err = resolveFieldToLegacy(ctx, field2, vs, cs) + require.NoError(t, err) + require.False(t, isLegacy) + require.True(t, changed) + require.Equal(t, h[:], out) + require.Empty(t, vs.writes) +} + +// TestResolveFieldToLegacy_AdaptiveInline covers the inline shape: leading +// 0x00 byte + inline content. The migration writes the inline content to the +// chunkstore and stores the resulting hash as the new field bytes. +func TestResolveFieldToLegacy_AdaptiveInline(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + // Short inline content: leader + "hello" + field := append([]byte{0x00}, []byte("hello")...) + + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, field, vs, cs) + require.NoError(t, err) + require.False(t, isLegacy, "adaptive inline is not legacy") + require.True(t, changed, "adaptive inline must rewrite to legacy shape") + require.Equal(t, hash.ByteLen, len(out)) + require.Len(t, vs.writes, 1, "inline content must be written to chunkstore exactly once") + require.Equal(t, []byte("hello"), vs.writes[0], "chunkstore must receive the inline content (minus the leader byte)") + + // The new field bytes must equal hash.Of("hello") so a subsequent + // vs.ReadBytes round-trips. + expectedHash := hash.Of([]byte("hello")) + require.Equal(t, expectedHash[:], out) +} + +// TestResolveFieldToLegacy_ZeroLeading20Bytes_ChunkstoreHit covers the +// load-bearing disambiguation: a 20-byte field whose leading byte is 0x00 is +// structurally ambiguous between (a) legacy raw hash with coincidental 0x00 +// leader, and (b) adaptive inline with 19 bytes of content. The chunkstore +// presence check resolves it: a HIT promotes to legacy raw hash (canonical, +// no rewrite); a MISS treats it as inline and rewrites. +func TestResolveFieldToLegacy_ZeroLeading20Bytes_ChunkstoreHit(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + + var raw [hash.ByteLen]byte + raw[0] = 0x00 + for i := 1; i < hash.ByteLen; i++ { + raw[i] = byte(i) + } + h := hash.New(raw[:]) + + cs := &stubChunkPresence{present: map[hash.Hash]bool{h: true}} + + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, raw[:], vs, cs) + require.NoError(t, err) + require.True(t, isLegacy, "chunkstore hit promotes 0x00-leading 20-byte to canonical legacy") + require.False(t, changed, "chunkstore-hit case must not change bytes") + require.Equal(t, raw[:], out) + require.Empty(t, vs.writes, "chunkstore hit must not trigger inline-to-OOB promotion") +} + +func TestResolveFieldToLegacy_ZeroLeading20Bytes_ChunkstoreMiss(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + + var raw [hash.ByteLen]byte + raw[0] = 0x00 + for i := 1; i < hash.ByteLen; i++ { + raw[i] = byte(i + 50) + } + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, raw[:], vs, cs) + require.NoError(t, err) + require.False(t, isLegacy, "chunkstore miss => adaptive inline => not legacy") + require.True(t, changed, "chunkstore miss must trigger inline-to-OOB promotion") + require.Len(t, vs.writes, 1, "chunkstore must receive the 19-byte inline content") + require.Equal(t, raw[1:], vs.writes[0], "chunkstore content must equal the inline payload (v[1:])") + // The new field is the hash of the inline content (the 19 bytes), not the + // hash of the original 20-byte field. + expectedHash := hash.Of(raw[1:]) + require.Equal(t, expectedHash[:], out) +} + +// TestResolveFieldToLegacy_UnknownShape covers the abort cases: bytes that +// don't match any of the four valid shapes must produce an error so the outer +// migration can stop and report the offending row's primary key. +func TestResolveFieldToLegacy_UnknownShape(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + // Non-zero leader, 19 bytes — too short for legacy (20) and adaptive + // addressed (21+). + tooShort := bytes.Repeat([]byte{0x01}, 19) + out, isLegacy, changed, err := resolveFieldToLegacy(ctx, tooShort, vs, cs) + require.Error(t, err) + require.False(t, isLegacy) + require.False(t, changed) + require.Nil(t, out) + require.Contains(t, err.Error(), "unknown shape") + + // Non-zero leader, 21+ bytes, but malformed adaptive trailer. Construct: + // leader 0x05 = 1-byte varint (declared length 5), then 10 bytes of + // zero. Parser expects total = 1 + 20 = 21 bytes; we give 11. Should + // abort. + malformed := append([]byte{0x05}, bytes.Repeat([]byte{0x00}, 10)...) + out, isLegacy, changed, err = resolveFieldToLegacy(ctx, malformed, vs, cs) + require.Error(t, err) + require.False(t, isLegacy) + require.False(t, changed) + require.Nil(t, out) +} + +// TestResolveFieldToLegacy_ChunkstoreWriteError verifies that a chunkstore +// failure during inline-to-OOB promotion propagates as an error rather than +// silently dropping the content. +func TestResolveFieldToLegacy_ChunkstoreWriteError(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + vs.writeErr = errors.New("simulated chunkstore failure") + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + field := append([]byte{0x00}, []byte("payload")...) + + _, _, _, err := resolveFieldToLegacy(ctx, field, vs, cs) + require.Error(t, err) + require.Contains(t, err.Error(), "simulated chunkstore failure") +} + +// TestResolveFieldToLegacy_ChunkstoreHasError verifies that a chunkstore +// presence-check failure propagates rather than silently treating it as a +// miss (which would corrupt a legacy-raw-hash row with a 0x00 leader). +func TestResolveFieldToLegacy_ChunkstoreHasError(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{hasErr: errors.New("simulated presence-check failure")} + + var raw [hash.ByteLen]byte + raw[0] = 0x00 + for i := 1; i < hash.ByteLen; i++ { + raw[i] = byte(i) + } + + _, _, _, err := resolveFieldToLegacy(ctx, raw[:], vs, cs) + require.Error(t, err) + require.Contains(t, err.Error(), "presence check") +} + +// TestResolveFieldToLegacy_RoundTripVerifiesContent is the integration-style +// unit test: write some inline content via resolveFieldToLegacy, then read +// it back via the resolver's reverse operation (vs.ReadBytes on the returned +// hash). The byte sequences must match. +func TestResolveFieldToLegacy_RoundTripVerifiesContent(t *testing.T) { + ctx := context.Background() + vs := newStubVS() + cs := &stubChunkPresence{present: map[hash.Hash]bool{}} + + // Three different inline payloads, including the degenerate empty case. + cases := [][]byte{ + nil, // not actually tested here (NULL path is separate) + {}, + []byte("short"), + bytes.Repeat([]byte{'x'}, 1024), + } + for i, payload := range cases { + if i == 0 { + continue // NULL handled by a different branch + } + inline := append([]byte{0x00}, payload...) + out, _, changed, err := resolveFieldToLegacy(ctx, inline, vs, cs) + require.NoError(t, err) + require.True(t, changed) + require.Equal(t, hash.ByteLen, len(out)) + h := hash.New(out) + got, readErr := vs.ReadBytes(ctx, h) + require.NoError(t, readErr) + // Compare byte-wise (treat nil and empty as equivalent — both + // represent "no content"). + require.True(t, bytes.Equal(payload, got), + "round-tripped content must match the original inline payload (i=%d): want=%q got=%q", i, payload, got) + } +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/repair.go b/go/cmd/dolt/commands/admin/schemadrift/repair.go new file mode 100644 index 00000000000..e75c10b6a86 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/repair.go @@ -0,0 +1,742 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/cmd/dolt/commands" + "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/store/datas" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/val" +) + +// RepairCmd is the write side of the schema-encoding-drift admin pair. It +// rewrites exactly one column's persisted encoding back to its pre-adaptive +// legacy sibling and records the change as a real dolt commit on the working +// branch — leaving the row data byte-identical. +type RepairCmd struct{} + +var _ cli.Command = RepairCmd{} + +const ( + tableFlag = "table" + colFlag = "column" + includeEmptyFlag = "include-empty" +) + +var repairDocs = cli.CommandDocumentationContent{ + ShortDesc: "Repair a single column's drifted persisted-encoding tag", + LongDesc: `Atomically flips the persisted encoding of one column from its drifted adaptive tag (e.g. ` + "`StringAdaptiveEnc`" + `) back to the matching legacy address tag (e.g. ` + "`StringAddrEnc`" + `) so that on-disk row data and schema-record agree. + +Row data is NOT modified. The bytes the column held before the repair are exactly the bytes it holds after — repair only changes how the schema TELLS the reader to interpret them. + +The repair is refused (with a clear error and a non-zero exit code) when the column's first non-NULL row is in adaptive format — that means the column is internally consistent and a "repair" would corrupt it. + +Each repair is committed as its own dolt commit with an auto-generated message of the form ` + "`admin: schema-encoding-drift repair on T.C (declared → restored ; data unchanged)`" + ` so the operation is recoverable via ` + "`dolt log`" + `. + +Required flags: ` + "`--table`" + ` and ` + "`--column`" + ` — one column per invocation, to keep each commit a single auditable change.`, + Synopsis: []string{"--table --column "}, +} + +// Name returns the subcommand name as registered in admin's command list. +func (cmd RepairCmd) Name() string { return "repair" } + +// Description is the short summary shown in `dolt admin` help. +func (cmd RepairCmd) Description() string { + return "Repair a single column's drifted persisted-encoding tag (atomic per-column flip + dolt commit)" +} + +// RequiresRepo is true: we need an initialized dolt environment to write to. +func (cmd RepairCmd) RequiresRepo() bool { return true } + +// Docs is the full documentation surfaced by `dolt admin schema-encoding-drift repair --help`. +func (cmd RepairCmd) Docs() *cli.CommandDocumentation { + return cli.NewCommandDocumentation(repairDocs, cmd.ArgParser()) +} + +// ArgParser declares the flag surface. +func (cmd RepairCmd) ArgParser() *argparser.ArgParser { + ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0) + ap.SupportsString(tableFlag, "", "name", "the table to repair") + ap.SupportsString(colFlag, "", "name", "the column whose drifted encoding tag should be restored to the legacy sibling") + ap.SupportsFlag(includeEmptyFlag, "", "Also accept the homogeneous-empty bucket as a valid repair target. By default repair refuses any column without strong legacy-raw-hash evidence; --include-empty trusts a forensic verdict that an all-zero payload on an adaptive schema is safe to flip back to legacy.") + return ap +} + +// Hidden mirrors the rest of the admin commands. +func (cmd RepairCmd) Hidden() bool { return true } + +// Exec is the CLI entry point. +func (cmd RepairCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, _ cli.CliContext) int { + ap := cmd.ArgParser() + usage, _ := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, repairDocs, ap)) + apr := cli.ParseArgsOrDie(ap, args, usage) + + tableName, ok := apr.GetValue(tableFlag) + if !ok || tableName == "" { + verr := errhand.BuildDError("--table is required").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + colName, ok := apr.GetValue(colFlag) + if !ok || colName == "" { + verr := errhand.BuildDError("--column is required").SetPrintUsage().Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + includeEmpty := apr.Contains(includeEmptyFlag) + result, err := repairColumnWithOptions(ctx, dEnv, tableName, colName, includeEmpty) + if err != nil { + verr := errhand.BuildDError("repair failed").AddCause(err).Build() + return commands.HandleVErrAndExitCode(verr, usage) + } + + emitRepairResult(cli.OutStream, result) + return 0 +} + +// RepairOutcome enumerates the four terminal states of a repair call: +// +// - OutcomeFlipped — the column's encoding was an adaptive tag with a +// legacy on-disk payload; we flipped the tag and committed. +// - OutcomeAlreadyOK — the column is already at a legacy encoding; nothing +// to do. Returned as a successful no-op so scripted repair loops are +// idempotent. +// - OutcomeNotAdaptive — the column has neither an adaptive nor a legacy +// encoding (e.g. an INT column was passed); refuse with a clear error +// rather than corrupting an unrelated schema record. +// - OutcomeGenuineAdaptive — the column's first non-NULL row is in adaptive +// format. The column is internally consistent. Refusing protects against +// the catastrophic "repair caused the corruption" case. +type RepairOutcome int + +const ( + OutcomeFlipped RepairOutcome = iota + OutcomeAlreadyOK + OutcomeNotAdaptive + OutcomeGenuineAdaptive +) + +// RepairResult is the structured outcome of a repair call, used by the CLI +// emitter and by the integration tests. +type RepairResult struct { + Outcome RepairOutcome + Database string + Table string + Column string + OldEncoding val.Encoding + NewEncoding val.Encoding + ObservedFormat FieldFormat + CommitHash string // empty for OutcomeAlreadyOK + CommitMessage string // empty for OutcomeAlreadyOK + WitnessRowSampled bool // true if at least one non-NULL row was inspected +} + +// repairColumn invokes repairColumnWithOptions with the default conservative +// posture (includeEmpty=false): refuse the homogeneous-empty bucket. Tests +// and callers that want to accept the empty bucket should call +// repairColumnWithOptions directly. +func repairColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (RepairResult, error) { + return repairColumnWithOptions(ctx, dEnv, tableName, colName, false) +} + +// repairColumnWithOptions is the entire repair pipeline. Broken out so unit +// and integration tests can call it without invoking the CLI wrapper. The +// includeEmpty flag toggles whether the homogeneous-empty bucket (the forensic analysis +// "safe-empty" case — every scanned row is NULL, `[0x00]` inline empty, or +// the 20-byte all-zero shape) counts as a valid repair target. +func repairColumnWithOptions(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string, includeEmpty bool) (RepairResult, error) { + ddb := dEnv.DoltDB(ctx) + dbName := ddb.GetDatabaseName() + + rsr := dEnv.RepoStateReader() + headRef, err := rsr.CWBHeadRef(ctx) + if err != nil { + return RepairResult{}, fmt.Errorf("get current branch: %w", err) + } + + roots, err := dEnv.Roots(ctx) + if err != nil { + return RepairResult{}, fmt.Errorf("get roots: %w", err) + } + + // Resolve the table case-insensitively against the working root so that + // `--table issues` works whether the persisted name is `issues`, `Issues`, + // or `ISSUES`. + tn := doltdb.TableName{Name: tableName, Schema: doltdb.DefaultSchemaName} + tbl, resolvedName, ok, err := doltdb.GetTableInsensitive(ctx, roots.Working, tn) + if err != nil { + return RepairResult{}, fmt.Errorf("get table %s: %w", tableName, err) + } + if !ok { + return RepairResult{}, fmt.Errorf("table %q not found on current working root", tableName) + } + tn.Name = resolvedName + + // repair commits its schema flip, which dangling-faults on a + // dolt_ignore'd table (never-committed content chunks are not commit-rooted). + // Refuse such tables and point the operator at the force-inline heal. + ignored, ignErr := doltdb.IdentifyIgnoredTables(ctx, roots, []doltdb.TableName{tn}) + if ignErr != nil { + return RepairResult{}, fmt.Errorf("check dolt_ignore status for %s: %w", resolvedName, ignErr) + } + if len(ignored) > 0 { + return RepairResult{}, fmt.Errorf( + "table %s is dolt_ignore'd; repair commits its schema flip and would dangling-fault on its never-committed content chunks. Use `dolt admin schema-encoding-drift migrate-adaptive --table %s --column ` (force-inline heal) instead", + resolvedName, resolvedName, + ) + } + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return RepairResult{}, fmt.Errorf("get schema for %s: %w", resolvedName, err) + } + + // Find the column by case-insensitive name match. + existingCol, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) + if !ok { + return RepairResult{}, fmt.Errorf("column %q not found in table %s", colName, resolvedName) + } + + oldEnc := existingCol.TypeInfo.Encoding() + + // Fast-path: already at a legacy encoding. Nothing to do — return success + // as an idempotent no-op. + if IsLegacyEncoding(oldEnc) { + return RepairResult{ + Outcome: OutcomeAlreadyOK, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: oldEnc, + }, nil + } + + // Refuse columns outside the adaptive family entirely. We never want to + // silently rewrite an INT or DECIMAL column's TypeInfo encoding — those + // have fixed encodings by contract and there is no "legacy sibling" to + // flip to. + newEnc, hasLegacy := LegacySibling(oldEnc) + if !IsAdaptiveEncoding(oldEnc) || !hasLegacy { + return RepairResult{ + Outcome: OutcomeNotAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + }, fmt.Errorf( + "column %s.%s has encoding %s which is not an adaptive TEXT/BLOB/JSON/GEOMETRY variant; "+ + "there is no legacy sibling to flip to", + resolvedName, existingCol.Name, encodingName(oldEnc), + ) + } + + // Locate the column's position in the value tuple so we can inspect the + // first non-NULL row's raw bytes. PK columns receive forced encodings + // (VarChar/VarBinary/Cell) on the key side and are not adaptive — so a + // column reaching this point should never be a PK. Guard defensively. + if existingCol.IsPartOfPK { + return RepairResult{ + Outcome: OutcomeNotAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + }, fmt.Errorf("column %s.%s is part of the primary key; PK columns do not carry adaptive encodings", resolvedName, existingCol.Name) + } + + tupleIndex, ok := valueTupleIndexForColumn(sch, existingCol.Name) + if !ok { + return RepairResult{}, fmt.Errorf("column %s.%s does not appear in the value tuple (maybe virtual?)", resolvedName, existingCol.Name) + } + + // The load-bearing safety check: walk every row and gather DEFINITIVE + // evidence for both directions plus the homogeneous-empty signal. The + // five outcomes are: + // + // - definitive-legacy only → drift confirmed; flip is correct. + // - definitive-adaptive only → genuine adaptive column; refuse. + // - both → heterogeneous; refuse (row-by-row + // migration needed, see the empirical + // `recover-rows` tool). + // - neither, but at least one ambiguous-non-empty row was seen + // → cannot prove drift; refuse. + // - neither, AND every row was NULL/empty-inline/all-zero-20-byte + // → homogeneous-empty; flip iff + // --include-empty (forensic analysis verdict). + findings, err := scanColumnFullPayload(ctx, tbl, tupleIndex, ddb) + if err != nil { + return RepairResult{}, fmt.Errorf("inspect row payload for %s.%s: %w", resolvedName, existingCol.Name, err) + } + switch { + case findings.sawDefinitiveAdaptive && !findings.sawDefinitiveLegacy: + return RepairResult{ + Outcome: OutcomeGenuineAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + ObservedFormat: FieldAdaptiveAddressed, + WitnessRowSampled: true, + }, fmt.Errorf( + "column %s.%s has genuine adaptive-format data (no legacy rows observed); repair would corrupt it (refusing)", + resolvedName, existingCol.Name, + ) + case findings.sawDefinitiveAdaptive && findings.sawDefinitiveLegacy: + return RepairResult{ + Outcome: OutcomeGenuineAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + ObservedFormat: FieldUnknown, // mixed; no single canonical + WitnessRowSampled: true, + }, fmt.Errorf( + "column %s.%s has BOTH legacy AND genuine adaptive rows (heterogeneous payload); repair cannot safely flip the schema in either direction without row-level migration (see `dolt admin schema-encoding-drift recover-rows`)", + resolvedName, existingCol.Name, + ) + case findings.sawDefinitiveLegacy && findings.sawNonEmptyAmbiguous: + // [review] Legacy witnesses coexisting with ambiguous non-empty rows (a + // 20-byte 0x00-leading value the chunkstore could not confirm as a + // legacy hash — it may be a genuine adaptive-inline value with 19 + // content bytes). A schema-only flip would strand those rows if they are + // adaptive. We cannot classify them without a row-level resolve, so + // repair REFUSES conservatively rather than risk orphaning content. + return RepairResult{ + Outcome: OutcomeGenuineAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + ObservedFormat: FieldUnknown, + WitnessRowSampled: true, + }, fmt.Errorf( + "column %s.%s has legacy witnesses AND ambiguous non-empty 0x00-leading 20-byte rows (the ambiguous rows may be adaptive-inline content a schema-only flip would strand); repair refuses — use `dolt admin schema-encoding-drift recover-rows` (to legacy) or `migrate-adaptive` (to v2 adaptive) for a row-level migration", + resolvedName, existingCol.Name, + ) + case !findings.sawDefinitiveLegacy && !findings.sawDefinitiveAdaptive && findings.sawNonEmptyAmbiguous: + return RepairResult{ + Outcome: OutcomeGenuineAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + ObservedFormat: FieldUnknown, + WitnessRowSampled: false, + }, fmt.Errorf( + "column %s.%s has ambiguous payload (no legacy-raw-hash witnesses, no adaptive-addressed witnesses, but at least one non-trivial non-empty row); cannot prove drift, refusing to repair", + resolvedName, existingCol.Name, + ) + case !findings.sawDefinitiveLegacy && !findings.sawDefinitiveAdaptive && !findings.sawNonEmptyAmbiguous: + // Homogeneous-empty: every scanned row was NULL, inline-empty, or + // the all-zero 20-byte shape. forensic analysis verified these are safe to flip. + if !includeEmpty { + return RepairResult{ + Outcome: OutcomeGenuineAdaptive, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + ObservedFormat: FieldNULL, + WitnessRowSampled: false, + }, fmt.Errorf( + "column %s.%s shows the homogeneous-empty payload pattern (safe-empty bucket); rerun with --include-empty to accept this bucket per the forensic verdict", + resolvedName, existingCol.Name, + ) + } + // fall through to perform the flip with the safe-empty witness. + } + // One of: + // - sawDefinitiveLegacy && !sawDefinitiveAdaptive → drift confirmed. + // - homogeneous-empty + --include-empty → operator opted in. + observed := FieldLegacyRawHash + if !findings.sawDefinitiveLegacy { + observed = FieldNULL // safe-empty branch + } + sampled := findings.sawDefinitiveLegacy + + // Build the patched schema. We replace the column in place by tag, so + // column order, indexes, checks, defaults, and constraints all carry over. + newCol := existingCol + newCol.TypeInfo = existingCol.TypeInfo.WithEncoding(newEnc) + newCol.Kind = newCol.TypeInfo.NomsKind() + + newSch, err := replaceColumnByTag(sch, newCol) + if err != nil { + return RepairResult{}, fmt.Errorf("rebuild schema for %s: %w", resolvedName, err) + } + + updatedTbl, err := tbl.UpdateSchema(ctx, newSch) + if err != nil { + return RepairResult{}, fmt.Errorf("apply schema change to %s: %w", resolvedName, err) + } + + newWorking, err := roots.Working.PutTable(ctx, tn, updatedTbl) + if err != nil { + return RepairResult{}, fmt.Errorf("put updated table %s: %w", resolvedName, err) + } + roots.Working = newWorking + roots.Staged = newWorking + + // Commit the change as a single audit-loggable dolt commit. We bypass the + // SQL queryist deliberately: SQL ALTER is the path that produced this + // corruption, so we're routing around it on principle. + commitMsg := fmt.Sprintf( + "admin: schema-encoding-drift repair on %s.%s (declared %s → restored %s; data unchanged)", + resolvedName, existingCol.Name, encodingName(oldEnc), encodingName(newEnc), + ) + commitHash, err := commitWorkingRoot(ctx, dEnv, headRef, roots, commitMsg) + if err != nil { + return RepairResult{}, fmt.Errorf("commit repair: %w", err) + } + + return RepairResult{ + Outcome: OutcomeFlipped, + Database: dbName, + Table: resolvedName, + Column: existingCol.Name, + OldEncoding: oldEnc, + NewEncoding: newEnc, + ObservedFormat: observed, + CommitHash: commitHash.String(), + CommitMessage: commitMsg, + WitnessRowSampled: sampled, + }, nil +} + +// valueTupleIndexForColumn returns the column's index in the value tuple, or +// (-1, false) if the column is a PK, virtual, or absent. Order matches +// schemaImpl.GetValueDescriptor: NonPKCols.Iter, skipping Virtual. +func valueTupleIndexForColumn(sch schema.Schema, colName string) (int, bool) { + idx := -1 + i := 0 + _ = sch.GetNonPKCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { + if col.Virtual { + return false, nil + } + if equalsCaseInsensitive(col.Name, colName) { + idx = i + return true, nil + } + i++ + return false, nil + }) + return idx, idx >= 0 +} + +// equalsCaseInsensitive matches the column-resolver semantics used elsewhere +// in dolt (schema lookups are case-insensitive). +func equalsCaseInsensitive(a, b string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + ca, cb := a[i], b[i] + if ca >= 'A' && ca <= 'Z' { + ca += 'a' - 'A' + } + if cb >= 'A' && cb <= 'Z' { + cb += 'a' - 'A' + } + if ca != cb { + return false + } + } + return true +} + +// payloadFindings captures the per-column verdict the repair safety guard +// derives from a full row-by-row walk. The four fields are independent +// flags that the call site composes into one of five repair outcomes: +// +// | legacy | adaptive | nonEmptyAmbiguous | verdict | +// |--------|----------|-------------------|------------------------| +// | true | false | n/a | drift → flip is safe | +// | false | true | n/a | clean → refuse | +// | true | true | n/a | heterogeneous → refuse | +// | false | false | true | unknown → refuse | +// | false | false | false | homogeneous-empty → | +// | | | | flip iff --include-empty| +// +// The "nonEmptyAmbiguous" signal is set when we see any row that's +// neither strong legacy nor strong adaptive AND not consistent with the +// homogeneous-empty pattern (i.e. it's a real non-trivial inline value, a +// 20-byte 0x00-leading hash with chunkstore miss but non-zero content +// bytes, or FieldUnknown). It exists to distinguish "we couldn't tell +// because the column was empty everywhere" (safe-empty) from "we couldn't +// tell because the rows had weird shapes" (refuse). +type payloadFindings struct { + sawDefinitiveLegacy bool + sawDefinitiveAdaptive bool + sawNonEmptyAmbiguous bool +} + +// scanColumnFullPayload walks every row in |tbl|'s row data and returns the +// payloadFindings for the column at |tupleIndex|. See payloadFindings' +// table for how the call site composes the flags into a repair verdict. +// +// This is intentionally more conservative than the check-side scanner, which +// only needs to surface "this column shows drift" as a diagnostic. Repair +// makes a destructive (well, audit-loggable but persistent) change, so it +// requires strictly stronger evidence. +func scanColumnFullPayload(ctx context.Context, tbl *doltdb.Table, tupleIndex int, cs ChunkPresenceChecker) (findings payloadFindings, err error) { + durableIdx, err := tbl.GetRowData(ctx) + if err != nil { + return findings, err + } + if durableIdx == nil { + return findings, nil + } + empty, err := durableIdx.Empty() + if err != nil { + return findings, err + } + if empty { + return findings, nil + } + pm, err := durable.ProllyMapFromIndex(durableIdx) + if err != nil { + return findings, err + } + iter, err := pm.IterAll(ctx) + if err != nil { + return findings, err + } + for { + _, value, iterErr := iter.Next(ctx) + if iterErr == io.EOF { + return findings, nil + } + if iterErr != nil { + return findings, iterErr + } + b := value.GetField(tupleIndex) + structural := ClassifyFieldBytes(b) + switch structural { + case FieldLegacyRawHash: + findings.sawDefinitiveLegacy = true + case FieldAdaptiveAddressed: + findings.sawDefinitiveAdaptive = true + case FieldAdaptiveInline: + if len(b) == hashByteLen { + if cs != nil { + // Try the chunkstore — present means it's a legacy raw + // hash whose leading byte happens to be 0x00. + h := newHashFromField(b) + present, hasErr := cs.Has(ctx, h) + if hasErr != nil { + return findings, hasErr + } + if present { + findings.sawDefinitiveLegacy = true + break + } + } + // 20-byte 0x00-leading field with no chunkstore hit (or no + // chunkstore available): genuinely ambiguous between an + // adaptive-inline value with 19 content bytes and a legacy + // raw hash that coincidentally starts 0x00. It is consistent + // with the homogeneous-empty pattern only if every content + // byte is also 0x00; any non-zero byte makes it non-trivial. + if !isAllZeroBytes(b) { + findings.sawNonEmptyAmbiguous = true + } + break + } + // len != 20: a 0x00-leading field that is NOT exactly 20 bytes + // CANNOT be a legacy raw hash (those are always exactly 20 bytes). + // It is unambiguously an adaptive-inline value. Non-empty content + // makes it a DEFINITIVE adaptive witness — this is the affected + // "inline orphan" shape (e.g. `[0x00]<58 bytes>`) that a prior + // first-witness repair stranded. Counting it definitively is what + // makes a legacy+inline heterogeneous column correctly refused. + if len(b) > 1 { + findings.sawDefinitiveAdaptive = true + } + // `[0x00]` (empty inline) is the safe-empty pattern; no flag. + case FieldNULL: + // NULL is consistent with safe-empty; no flag. + case FieldUnknown: + // Unknown shape is ambiguous and non-trivial. + findings.sawNonEmptyAmbiguous = true + } + // Bail out early if we've seen both — no need to scan further; the + // outcome is "refuse, heterogeneous column". + if findings.sawDefinitiveLegacy && findings.sawDefinitiveAdaptive { + return findings, nil + } + } +} + +// hashByteLen / newHashFromField are aliased in check.go to keep the scanner +// hot path lean. We use the same helpers here for symmetry. +var _ = hashByteLen // keep referenced when this file is read in isolation + +// replaceColumnByTag returns a new schema identical to |sch| except that the +// column with the same tag as |newCol| has been replaced. The replacement +// preserves column ORDER, all indexes, all checks, the collation, and the PK +// ordinals — everything except the column's own TypeInfo. +// +// This is the dolt-internal-only counterpart of replaceColumnInSchema in +// alterschema.go, but without the order/reorder ceremony and without going +// through any ALTER planning. We want exactly the same column collection, +// minus the bug. +func replaceColumnByTag(sch schema.Schema, newCol schema.Column) (schema.Schema, error) { + cols := sch.GetAllCols().GetColumns() + replaced := false + for i := range cols { + if cols[i].Tag == newCol.Tag { + cols[i] = newCol + replaced = true + break + } + } + if !replaced { + return nil, fmt.Errorf("column tag %d not found in schema", newCol.Tag) + } + newCC := schema.NewColCollection(cols...) + + pkCols := sch.GetPKCols().GetColumns() + for i := range pkCols { + if pkCols[i].Tag == newCol.Tag { + pkCols[i] = newCol + } + } + newPKCC := schema.NewColCollection(pkCols...) + + newSch, err := schema.NewSchema( + newCC, + sch.GetPkOrdinals(), + sch.GetCollation(), + schema.NewIndexCollection(newCC, newPKCC), + sch.Checks(), + ) + if err != nil { + return nil, err + } + + // Copy all existing indexes verbatim — the column we're patching is, by + // contract for this repair, a TEXT / BLOB / JSON / GEOMETRY adaptive + // column, and adaptive columns can't be part of an index that references + // them by length / hash directly. So no index-tag rewrite is needed: the + // tags are unchanged. Re-add the indexes for ColCollection coherence. + for _, ix := range sch.Indexes().AllIndexes() { + _, err := newSch.Indexes().AddIndexByColTags( + ix.Name(), + ix.IndexedColumnTags(), + ix.PrefixLengths(), + schema.IndexProperties{ + IsUnique: ix.IsUnique(), + IsSpatial: ix.IsSpatial(), + IsFullText: ix.IsFullText(), + IsVector: ix.IsVector(), + IsUserDefined: ix.IsUserDefined(), + Comment: ix.Comment(), + Predicate: ix.Predicate(), + FullTextProperties: ix.FullTextProperties(), + VectorProperties: ix.VectorProperties(), + }, + ) + if err != nil { + return nil, fmt.Errorf("re-add index %s: %w", ix.Name(), err) + } + } + + return newSch, nil +} + +// commitWorkingRoot stages the working root, creates a pending commit, and +// records it on |headRef| atomically with the working-set update. Returns the +// new commit's content hash. +func commitWorkingRoot(ctx context.Context, dEnv *env.DoltEnv, headRef ref.DoltRef, roots doltdb.Roots, message string) (hash.Hash, error) { + ddb := dEnv.DoltDB(ctx) + + name, email, err := env.GetNameAndEmail(dEnv.GetConfig()) + if err != nil { + // Fall back to a stable identity if the operator hasn't set + // user.name / user.email globally — admin-driven schema repairs + // shouldn't be blocked on git-style config. + name, email = "dolt-admin", "dolt-admin@localhost" + } + meta, err := datas.NewCommitMeta(name, email, message) + if err != nil { + return hash.Hash{}, fmt.Errorf("build commit metadata: %w", err) + } + + ws, err := dEnv.WorkingSet(ctx) + if errors.Is(err, doltdb.ErrWorkingSetNotFound) { + wsRef, refErr := ref.WorkingSetRefForHead(headRef) + if refErr != nil { + return hash.Hash{}, refErr + } + ws = doltdb.EmptyWorkingSet(wsRef) + } else if err != nil { + return hash.Hash{}, fmt.Errorf("get working set: %w", err) + } + + prevWSHash, err := ws.HashOf() + if err != nil { + return hash.Hash{}, fmt.Errorf("hash working set: %w", err) + } + + pending, err := ddb.NewPendingCommit(ctx, roots, nil, false, meta) + if err != nil { + return hash.Hash{}, fmt.Errorf("create pending commit: %w", err) + } + + // After commit, working == staged == HEAD's new root, so the working set + // itself has no "uncommitted" delta on top of the repair. + newWS := ws.WithWorkingRoot(roots.Working).WithStagedRoot(roots.Staged).ClearMerge() + + wsMeta := dEnv.NewWorkingSetMeta(message) + commit, err := ddb.CommitWithWorkingSet(ctx, headRef, ws.Ref(), pending, newWS, prevWSHash, wsMeta, nil) + if err != nil { + return hash.Hash{}, fmt.Errorf("commit + update working set: %w", err) + } + + commitHash, err := commit.HashOf() + if err != nil { + return hash.Hash{}, fmt.Errorf("hash new commit: %w", err) + } + return commitHash, nil +} + +// emitRepairResult writes a human-readable summary of the repair to |w|. +func emitRepairResult(w io.Writer, r RepairResult) { + switch r.Outcome { + case OutcomeFlipped: + _, _ = fmt.Fprintf(w, "repaired %s.%s: %s → %s\n", r.Table, r.Column, encodingName(r.OldEncoding), encodingName(r.NewEncoding)) + _, _ = fmt.Fprintf(w, " observed payload format: %s\n", r.ObservedFormat.String()) + _, _ = fmt.Fprintf(w, " commit: %s\n", r.CommitHash) + _, _ = fmt.Fprintf(w, " message: %s\n", r.CommitMessage) + case OutcomeAlreadyOK: + _, _ = fmt.Fprintf(w, "no-op for %s.%s: already at %s (idempotent success)\n", r.Table, r.Column, encodingName(r.OldEncoding)) + default: + // OutcomeNotAdaptive / OutcomeGenuineAdaptive surface via the + // returned error, so the CLI emitter never reaches them here. + } +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/repair_20byte_inline_hole_test.go b/go/cmd/dolt/commands/admin/schemadrift/repair_20byte_inline_hole_test.go new file mode 100644 index 00000000000..9a6958a944d --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/repair_20byte_inline_hole_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +// Regression for a review's repair hole: the 20-byte 0x00-LEADING +// adaptive-inline shape. +// +// scanColumnFullPayload's FieldAdaptiveInline branch treats a 20-byte 0x00-lead +// field as AMBIGUOUS (it could be a legacy raw hash that coincidentally starts +// 0x00, or an adaptive-inline value with 19 content bytes) and, on a chunkstore +// miss with non-zero content, only sets sawNonEmptyAmbiguous. With a legacy +// witness also present, repair's switch then falls through to FLIP — silently +// stranding that row: `[0x00]<19 bytes>` is NOT a valid 20-byte content-hash, so +// after the flip the StringAddr reader dereferences a bogus address (a QUIET +// misread, not a loud panic, since the field is exactly 20 bytes and hash.New +// accepts it). This is the same quiet-failure class the census flagged +// (BROKEN adaptive-inline ==20B) and is exactly why deploy verification must be +// content-byte-identical, not panic-based. +// +// repair MUST refuse this column. FAILS-WITHOUT the an earlier review fix (repair flips), +// PASSES-WITH it. Held uncommitted until the an earlier review repair fix lands. + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// twentyByteInlineContent is 19 bytes -> adaptive inline encodes it as +// [0x00]<19 bytes> = exactly 20 bytes (the ambiguous "20-byte 0x00-leading" +// shape). Non-all-zero so it is not the safe-empty bucket. +const twentyByteInlineContent = "ABCDEFGHIJKLMNOPQRS" // len 19 + +func TestRepair_Refuses20ByteInlineWithLegacyWitness_REGRESSION(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + t.Cleanup(func() { dEnv.Close() }) + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, actor VARCHAR(255) NOT NULL, description TEXT NOT NULL);`) + require.NoError(t, err) + + // Legacy witness row (long -> legacy raw 20-byte hash). + legacyPayload := strings.Repeat("L", 1500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (1, 'a-1', '%s');`, legacyPayload)) + require.NoError(t, err) + + // Drift to adaptive, then insert the 19-byte value -> 20-byte 0x00-lead inline. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + typeinfo.UseAdaptiveEncoding = true + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (2, 'a-2', '%s');`, twentyByteInlineContent)) + require.NoError(t, err) + + // The fixture must actually contain a 20-byte 0x00-leading inline value AND a + // legacy witness — otherwise it doesn't exercise the review's hole. + require.True(t, hasTwentyByteZeroLeadInline(t, ctx, dEnv, "issues", "description"), + "fixture must contain a 20-byte 0x00-leading adaptive-inline value (the ambiguous shape)") + formats := collectFieldFormats(t, ctx, dEnv, "issues", "description") + require.True(t, formats[schemadrift.FieldLegacyRawHash], "fixture must contain a legacy witness") + + // repair MUST refuse — flipping would silently strand the 20-byte inline row. + res, err := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + require.Error(t, err, + "repair MUST refuse a 20-byte 0x00-leading inline value with a legacy witness (flipping silently misreads it as a bogus hash)") + require.NotEqual(t, schemadrift.OutcomeFlipped, res.Outcome, + "repair MUST NOT flip — this is the quiet-corruption hole") + require.Empty(t, res.CommitHash, "a refused repair must not create a commit") +} + +// hasTwentyByteZeroLeadInline reports whether any row's |colName| field is +// exactly 20 bytes with a 0x00 leading byte and at least one non-zero content +// byte (i.e. a genuine adaptive-inline value of 19 content bytes, NOT the +// all-zero safe-empty sentinel). +func hasTwentyByteZeroLeadInline(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) bool { + t.Helper() + root, err := dEnv.WorkingRoot(ctx) + require.NoError(t, err) + tbl, _, ok, err := doltdb.GetTableInsensitive(ctx, root, doltdb.TableName{Name: tableName}) + require.NoError(t, err) + require.True(t, ok) + sch, err := tbl.GetSchema(ctx) + require.NoError(t, err) + + tupleIdx := -1 + i := 0 + _ = sch.GetNonPKCols().Iter(func(tag uint64, c schema.Column) (stop bool, err error) { + if c.Virtual { + return false, nil + } + if strings.EqualFold(c.Name, colName) { + tupleIdx = i + return true, nil + } + i++ + return false, nil + }) + require.GreaterOrEqual(t, tupleIdx, 0) + + idx, err := tbl.GetRowData(ctx) + require.NoError(t, err) + pm, err := durable.ProllyMapFromIndex(idx) + require.NoError(t, err) + iter, err := pm.IterAll(ctx) + require.NoError(t, err) + for { + _, value, iterErr := iter.Next(ctx) + if iterErr == io.EOF { + return false + } + require.NoError(t, iterErr) + b := value.GetField(tupleIdx) + if len(b) == 20 && b[0] == 0 { + for _, x := range b { + if x != 0 { + return true + } + } + } + } +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/repair_inline_heterogeneous_test.go b/go/cmd/dolt/commands/admin/schemadrift/repair_inline_heterogeneous_test.go new file mode 100644 index 00000000000..51a415a6865 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/repair_inline_heterogeneous_test.go @@ -0,0 +1,332 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +// Regression coverage for the drift scenario on ignored_log. +// +// `repair` flipped a column whose drifted rows were adaptive-INLINE (small +// [0x00] payloads such as "example inline content payload: ...", NOT spilled to +// addressed). The schema-only flip stranded every inline row under a StringAddr +// schema: reads panic `invalid hash length: N` and any insert's node-flush +// panics in countAddresses. +// +// The pre-existing TestRecoverRows_RepairRefusesSameFixture only exercises the +// ADDRESSED adaptive shape (payload > 2KB -> FieldAdaptiveAddressed -> +// sawDefinitiveAdaptive). The INLINE shape is the gap: scanColumnFullPayload +// classifies non-empty inline as sawNonEmptyAmbiguous (not sawDefinitiveAdaptive) +// so a column with legacy witnesses + inline content falls through repair.go's +// switch to the FLIP branch — exactly what stranded ignored_log. +// +// Tests assert at the command boundary so they survive later churn in +// repair.go / recover_rows.go. They reuse helpers from +// recover_rows_integration_test.go (same package): manuallyFlipColumnEncoding, +// collectColumnContent, collectFieldFormats. + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// inlineContent stays adaptive-INLINE (well under the ~2KB out-of-band spill +// threshold) and is real UTF-8 — the exact ignored_log.new_value shape +// (a 59-byte [0x00]"example inline ..." row). +const inlineContent = "example inline content payload" + +// buildInlineHeterogeneousFixture creates +// +// issues(id PK, actor VARCHAR(255) NOT NULL, description TEXT NOT NULL) +// +// with: one legacy raw-hash row (long payload) and two adaptive-INLINE-content +// rows (short payloads). The persisted schema ends tagged StringAdaptiveEnc(135) +// — the "drifted but internally readable" pre-repair state. +// +// The leading VARIABLE-width `actor` column faithfully mirrors ignored_log +// (new_value sits after VARCHAR/TEXT siblings): it pushes `description` out of +// the value tuple's fixed-width prefix so a StringAddr read uses intrinsic +// offsets and sees the FULL field bytes — reproducing the +// hash.New(len) panic. A lone leading StringAddr column would instead be read as +// a fixed 20-byte slice (silent bogus hash), masking the crash. +func buildInlineHeterogeneousFixture(t *testing.T, ctx context.Context) (*env.DoltEnv, map[int]string) { + t.Helper() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false // legacy witness row first + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + t.Cleanup(func() { dEnv.Close() }) + + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, actor VARCHAR(255) NOT NULL, description TEXT NOT NULL);`) + require.NoError(t, err) + + legacyPayload := strings.Repeat("L", 1500) // long -> legacy raw-hash (out-of-band) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (1, 'actor-1', '%s');`, legacyPayload)) + require.NoError(t, err) + + // Flip description to adaptive (simulating v2.0.7 ALTER drift), then insert + // SHORT rows under the adaptive encoder so they land as inline [0x00]. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + typeinfo.UseAdaptiveEncoding = true + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO issues VALUES (2, 'actor-2', '%s'), (3, 'actor-3', '%s');`, inlineContent, "short note")) + require.NoError(t, err) + + // The fixture must be heterogeneous AND contain inline (not addressed) + // adaptive content — otherwise it doesn't reproduce the gap. + formats := collectFieldFormats(t, ctx, dEnv, "issues", "description") + require.True(t, formats[schemadrift.FieldLegacyRawHash], "fixture needs a legacy witness") + require.True(t, formats[schemadrift.FieldAdaptiveInline], "fixture needs adaptive-INLINE content (the affected shape)") + require.False(t, formats[schemadrift.FieldAdaptiveAddressed], "fixture must NOT spill to addressed (already covered elsewhere)") + + return dEnv, map[int]string{1: legacyPayload, 2: inlineContent, 3: "short note"} +} + +// TestRepair_RefusesInlineHeterogeneous_REGRESSION is the load-bearing +// fail-without/pass-with test. On today's binary repair FLIPS the schema (the +// real-world bug). After the fix (non-empty inline counted as definitive +// adaptive, or refuse when ambiguous coincides with a legacy witness), repair +// must REFUSE and leave the schema untouched. +func TestRepair_RefusesInlineHeterogeneous_REGRESSION(t *testing.T) { + ctx := context.Background() + dEnv, _ := buildInlineHeterogeneousFixture(t, ctx) + + res, err := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + + // FAILS-WITHOUT: today repair returns OutcomeFlipped + nil error. + // PASSES-WITH: repair must refuse a legacy+adaptive-inline heterogeneous column. + require.Error(t, err, + "repair MUST refuse a legacy+adaptive-inline heterogeneous column (it would otherwise strand the inline rows)") + require.NotEqual(t, schemadrift.OutcomeFlipped, res.Outcome, + "repair MUST NOT flip the schema of an inline-heterogeneous column") + require.Empty(t, res.CommitHash, "a refused repair must not create a commit") + // Soft secondary: the refusal should point operators at recover-rows. Kept + // loose so it doesn't couple to exact wording. + require.Contains(t, strings.ToLower(err.Error()), "heterogeneous", + "refusal should name the heterogeneous condition") +} + +// TestRepair_InlineFlipStrandsRows_ReadPanics characterizes the consequence of +// the flip: it reproduces the exact stranded state (schema flipped to StringAddr +// while the inline bytes remain) and proves both the orphaning (schema/data +// mismatch) and the read crash (GetStringAddr -> hash.New -> +// "invalid hash length"). This pins the failure shape independent of the SQL +// engine's lazy materialization. +func TestRepair_InlineFlipStrandsRows_ReadPanics(t *testing.T) { + ctx := context.Background() + dEnv, _ := buildInlineHeterogeneousFixture(t, ctx) + + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAddrEnc)) + + // Orphan proof (deterministic): schema now says StringAddr, yet the rows + // still carry adaptive-inline bytes. + formats := collectFieldFormats(t, ctx, dEnv, "issues", "description") + require.True(t, formats[schemadrift.FieldAdaptiveInline], + "post-flip data must still hold adaptive-inline bytes under a StringAddr schema (the orphaning)") + + // Panic proof: reading those inline fields via the StringAddr descriptor + // calls hash.New on a non-20-byte field -> panic `invalid hash length: N`. + require.True(t, stringAddrReadPanics(t, ctx, dEnv, "issues", "description"), + "reading stranded adaptive-inline rows via the StringAddr descriptor MUST panic (invalid hash length)") +} + +// TestOrphanedInlineColumn_IsRecoverable proves the data is NOT lost. An +// already-stranded column (StringAddr schema + adaptive-inline data — the live +// ignored_log state) is recoverable: re-tag it adaptive so recover-rows +// recognizes it as drifted, migrate, and every row reads back byte-identical. +// +// NB: recover-rows keys off the SCHEMA tag, so it no-ops on a StringAddr-tagged +// column — which is why healing the stranded columns needs either the +// re-tag step shown here OR the the forward `recover-rows --to-adaptive` command +// (see wp-tests.md). This test guards content recoverability with primitives +// available today; the --to-adaptive command gets its own test once it lands. +func TestOrphanedInlineColumn_IsRecoverable(t *testing.T) { + ctx := context.Background() + dEnv, expected := buildInlineHeterogeneousFixture(t, ctx) + + // Strand it exactly as the buggy schema-only flip does. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAddrEnc)) + + // Re-tag to adaptive so the column is recognized as drifted (schema now + // matches the adaptive-inline data again), then migrate to canonical legacy. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err, "recover-rows must heal the (re-tagged) drifted inline column") + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.NotEmpty(t, res.CommitHash) + + // Every row reads back byte-identical to what we inserted — no content loss. + got := collectColumnContent(t, ctx, dEnv, "issues", "description") + require.Equal(t, expected, got, "content must round-trip exactly through the heal") +} + +// TestMigrateAdaptive_HealsOrphanedInlineColumn is the Approach-B (operator's +// preferred, v2-native) heal test. It takes the exact stranded real-world state +// (StringAddr schema + heterogeneous legacy/adaptive-inline data) and runs the +// new `migrate-adaptive` command: legacy rows are rewritten forward to canonical +// adaptive, already-adaptive inline rows are kept verbatim, and the schema is +// flipped to the adaptive sibling — leaving one internally-consistent adaptive +// column. Every row must read back byte-identical with no panic. +func TestMigrateAdaptive_HealsOrphanedInlineColumn(t *testing.T) { + ctx := context.Background() + dEnv, expected := buildInlineHeterogeneousFixture(t, ctx) + + // Strand the column exactly as the buggy schema-only repair flip does. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAddrEnc)) + + res, err := schemadrift.MigrateAdaptiveColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err, "migrate-adaptive must heal a stranded heterogeneous column") + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.Equal(t, val.StringAdaptiveEnc, res.NewEncoding, "schema must end on the adaptive sibling") + require.GreaterOrEqual(t, res.RowsRewritten, 1, "the legacy row(s) must be rewritten forward to adaptive") + require.NotEmpty(t, res.CommitHash) + + // Post-heal the column is consistent adaptive: every row reads back + // byte-identical to what we inserted, with no invalid-hash-length panic. + got := collectColumnContent(t, ctx, dEnv, "issues", "description") + require.Equal(t, expected, got, "content must round-trip exactly through the forward migration") + + // And the column is now homogeneous adaptive (no legacy raw-hash bytes left). + formats := collectFieldFormats(t, ctx, dEnv, "issues", "description") + require.False(t, formats[schemadrift.FieldLegacyRawHash], + "after migrate-adaptive no legacy raw-hash bytes may remain (column is canonical adaptive)") +} + +// TestMigrateAdaptive_DoltIgnoredTable_PersistsRewrites is an API-level guard +// that migrate-adaptive heals a dolt_ignore'd table's content byte-identically +// (the the affected columns ignored_log.{old,new}_value are in +// dolt_ignore'd tables matching `ignored_%`). +// +// IMPORTANT — this does NOT catch the deploy-blocker found by real-data +// validation: on a real ON-DISK repo, migrate-adaptive on ignored_log fails +// with "persist working set for dolt_ignore'd table ignored_log: dangling ref". +// The in-memory dtestutils env does NOT reproduce that on-disk working-set +// persist path, so this test PASSES both before and after the fix. The on-disk +// failure is covered by the bats suite +// (repair_orphans_content_compatibility.bats, "migrate-adaptive on a +// dolt_ignore'd table") and by the real-data validation in validation notes. +// This test still has value: it guards that migrate-adaptive does not start +// REFUSING ignored tables and that content round-trips at the API level. +func TestMigrateAdaptive_DoltIgnoredTable_PersistsRewrites(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + t.Cleanup(func() { dEnv.Close() }) + + // Mark the table ignored (mirrors a database's `ignored_%` pattern), then build the + // same legacy-witness + adaptive-inline fixture on it. + _, err := sqle.ExecuteSql(ctx, dEnv, `INSERT INTO dolt_ignore VALUES ('ignored_%', true);`) + require.NoError(t, err) + _, err = sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE ignored_log (id INT PRIMARY KEY, actor VARCHAR(255) NOT NULL, body TEXT NOT NULL);`) + require.NoError(t, err) + + legacyPayload := strings.Repeat("L", 1500) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO ignored_log VALUES (1, 'a-1', '%s');`, legacyPayload)) + require.NoError(t, err) + + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "ignored_log", "body", val.StringAdaptiveEnc)) + typeinfo.UseAdaptiveEncoding = true + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO ignored_log VALUES (2, 'a-2', '%s'), (3, 'a-3', '%s');`, inlineContent, "short note")) + require.NoError(t, err) + + // The legacy row (id=1) must be rewritten forward — that is what forces the + // ignored-table working-set persist of novel chunks. + res, err := schemadrift.MigrateAdaptiveColumnForTest(ctx, dEnv, "ignored_log", "body") + require.NoError(t, err, "migrate-adaptive must persist a dolt_ignore'd table whose rows it rewrote (no dangling ref)") + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.GreaterOrEqual(t, res.RowsRewritten, 1) + + got := collectColumnContent(t, ctx, dEnv, "ignored_log", "body") + require.Equal(t, map[int]string{1: legacyPayload, 2: inlineContent, 3: "short note"}, got, + "content must round-trip exactly through the heal of an ignored table") +} + +// stringAddrReadPanics reads every row's |colName| field through the table's +// CURRENT value descriptor and reports whether GetStringAddr panics — the +// lowest-level rereal-world of the read crash +// (val.TupleDesc.GetStringAddr -> GetAddr -> hash.New), independent of the SQL +// engine's lazy materialization / errguard recovery. +func stringAddrReadPanics(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, tableName, colName string) (panicked bool) { + t.Helper() + root, err := dEnv.WorkingRoot(ctx) + require.NoError(t, err) + tbl, _, ok, err := doltdb.GetTableInsensitive(ctx, root, doltdb.TableName{Name: tableName}) + require.NoError(t, err) + require.True(t, ok) + sch, err := tbl.GetSchema(ctx) + require.NoError(t, err) + + tupleIdx := -1 + i := 0 + _ = sch.GetNonPKCols().Iter(func(tag uint64, c schema.Column) (stop bool, err error) { + if c.Virtual { + return false, nil + } + if strings.EqualFold(c.Name, colName) { + tupleIdx = i + return true, nil + } + i++ + return false, nil + }) + require.GreaterOrEqual(t, tupleIdx, 0) + + idx, err := tbl.GetRowData(ctx) + require.NoError(t, err) + pm, err := durable.ProllyMapFromIndex(idx) + require.NoError(t, err) + vd := sch.GetValueDescriptor(pm.NodeStore()) + + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + iter, err := pm.IterAll(ctx) + require.NoError(t, err) + for { + _, value, iterErr := iter.Next(ctx) + if iterErr == io.EOF { + return panicked + } + require.NoError(t, iterErr) + // Mirrors the read path: a StringAddr field is dereferenced as + // a 20-byte hash address; a non-20-byte inline payload panics here. + _, _ = vd.GetStringAddr(tupleIdx, value) + } +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/schemadrift.go b/go/cmd/dolt/commands/admin/schemadrift/schemadrift.go new file mode 100644 index 00000000000..f209c650c57 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/schemadrift.go @@ -0,0 +1,52 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package schemadrift implements the `dolt admin schema-encoding-drift` admin +// commands — check (read-only diagnose) and repair (atomic per-column flip). +// +// The drift this package targets is the upstream Dolt 2.0.7 regression where +// AlterableDoltTable.ModifyColumn (and the rewrite-path schema rebuild) lost a +// column's persisted storage encoding while leaving the on-disk row data +// untouched. A 1.x-written TEXT / BLOB / JSON / GEOMETRY column survives an +// upgrade with `StringAddrEnc(23)` (or sibling) in the persisted schema, but +// after any ALTER on the same table the schema record silently flips to +// `StringAdaptiveEnc(135)` while the row payload is still in the legacy raw +// 20-byte-hash format. Adaptive dispatch then panics with `invalid hash length: +// 19` on TEXT/BLOB and silently misclassifies the persisted format on JSON / +// GEOMETRY. +// +// The schema-side fix (commit `4969194e2` and `09278d859`) prevents NEW ALTERs +// from extending the corruption — but any column that has already been ALTERed +// past 2.0.7 has the wrong tag in its persisted schema. This package is the +// explicit, atomic, audit-logged repair path the operator runs to bring those +// columns back into alignment with their on-disk rows. There is no read-path +// fallback; if you read a corrupted column before running `repair`, dolt +// panics. That is the deliberate postgres-tier contract. +package schemadrift + +import ( + "github.com/dolthub/dolt/go/cmd/dolt/cli" +) + +// Commands is the parent admin subcommand handler. Registered from +// `admin.go` as `Commands` for the `schema-encoding-drift` slot. +var Commands = cli.NewHiddenSubCommandHandler("schema-encoding-drift", + "Detect and repair persisted-schema / on-disk-row encoding drift left by the v2.0.7 ALTER-MODIFY regression", + []cli.Command{ + CheckCmd{}, + RepairCmd{}, + RecoverRowsCmd{}, + MigrateAdaptiveCmd{}, + }, +) diff --git a/go/cmd/dolt/commands/admin/schemadrift/schemadrift_integration_test.go b/go/cmd/dolt/commands/admin/schemadrift/schemadrift_integration_test.go new file mode 100644 index 00000000000..638c15d521a --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/schemadrift_integration_test.go @@ -0,0 +1,281 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// TestSchemaEncodingDrift_End2End_CheckReportsDriftAfterManualCorruption is +// the load-bearing integration test for `dolt admin schema-encoding-drift +// check`. We: +// +// 1. Create a TEXT-bearing table under UseAdaptiveEncoding=false so the +// persisted column encoding is legacy StringAddrEnc and the row data is +// adaptive-inline for short content (StringAddrEnc only writes a 20-byte +// hash on out-of-band values; short strings still encode as adaptive +// inline through the prolly writer). +// 2. Insert one row with long content so the encoded payload is a legacy +// raw 20-byte hash — the exact shape v2.0.7 readers crash on. +// 3. Manually flip the persisted column TypeInfo from StringAddrEnc to +// StringAdaptiveEnc (simulating the v2.0.7 ALTER MODIFY corruption). +// This is the smallest possible rereal-world of an real-world-like drift — we +// bypass the SQL ALTER path so the fix in commits 4969194e2 + 09278d859 +// doesn't paper over the corruption while we test the diagnostic. +// 4. Run scanForDrift via the package's CheckCmd integration entry point and +// assert it reports exactly one drift row for the corrupted column. +// +// This test asserts the entire check pipeline — schema walk, candidate +// collection, prolly iteration, byte classification, drift attribution — +// without ever going through the panicking adaptive reader. +func TestSchemaEncodingDrift_End2End_CheckReportsDriftAfterManualCorruption(t *testing.T) { + ctx := context.Background() + + // Step 1: legacy-encoded CREATE. + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // Step 2: a row big enough that it spills to addressed storage in TEXT. + // 1024 bytes is well over the 1.x inline threshold for TEXT. + bigPayload := makeBigString(1024) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +`+`INSERT INTO issues VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + + // Sanity: schema reports StringAddrEnc, no drift expected. + driftBefore, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, driftBefore, "before corruption: schema and payload agree (StringAddrEnc on both sides)") + + // Step 3: simulate the v2.0.7 ALTER MODIFY corruption by manually rewriting + // the persisted schema's `description` column encoding from StringAddrEnc + // to StringAdaptiveEnc — bypassing the schema-side fix (which would refuse + // to corrupt in the first place). + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + // Step 4: check should now report drift on issues.description. + driftAfter, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, driftAfter, 1, "exactly one drifted column expected") + d := driftAfter[0] + require.Equal(t, "issues", d.Table) + require.Equal(t, "description", d.Column) + require.Equal(t, "StringAdaptiveEnc", d.DeclaredEncoding, + "check must report the post-corruption declared encoding") + require.Equal(t, "legacy-raw-hash", d.ObservedFormat, + "observed payload must be classified as legacy-raw-hash since the data was never rewritten") + require.Equal(t, "StringAddrEnc", d.SuggestedEncoding, + "check must suggest StringAddrEnc as the legacy sibling to flip back to") +} + +// TestSchemaEncodingDrift_End2End_RepairFlipsTagAndCommits drives the full +// repair pipeline against the same corrupted-by-hand fixture and asserts: +// +// 1. The drift exists before repair. +// 2. RepairColumnForTest succeeds with outcome=OutcomeFlipped. +// 3. The new commit hash is non-empty and the commit message names the +// specific table.column with the encoding transition. +// 4. A subsequent check returns no drift. +// 5. A SECOND repair call is a no-op (idempotent OutcomeAlreadyOK). +func TestSchemaEncodingDrift_End2End_RepairFlipsTagAndCommits(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + bigPayload := makeBigString(1024) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +`+`INSERT INTO issues VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "issues", "description", val.StringAdaptiveEnc)) + + // Confirm we start with one drift. + before, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, before, 1) + + // Repair. + res, err := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.OutcomeFlipped, res.Outcome) + require.Equal(t, val.StringAdaptiveEnc, res.OldEncoding) + require.Equal(t, val.StringAddrEnc, res.NewEncoding) + require.NotEmpty(t, res.CommitHash, "repair must produce a real dolt commit hash") + require.Contains(t, res.CommitMessage, "issues.description", + "commit message must name the specific table.column being repaired") + require.Contains(t, res.CommitMessage, "StringAdaptiveEnc", + "commit message must name the encoding being repaired FROM") + require.Contains(t, res.CommitMessage, "StringAddrEnc", + "commit message must name the encoding being repaired TO") + require.Contains(t, res.CommitMessage, "data unchanged", + "commit message must declare the data-unchanged contract for downstream consumers") + + // Drift should now be clean. + after, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, after, "repair must remove the drift entry from check") + + // Idempotent: re-running repair returns OutcomeAlreadyOK with no new + // commit hash, because the column is already at the legacy encoding. + res2, err := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.OutcomeAlreadyOK, res2.Outcome, + "repair on a non-drifted column must be an idempotent no-op") + require.Empty(t, res2.CommitHash, "no-op repair must NOT produce a commit") +} + +// TestSchemaEncodingDrift_RefusesGenuineAdaptiveData verifies the load-bearing +// safety guard: if a column's schema record says adaptive AND its on-disk +// payload IS in adaptive format (a non-corrupted column), repair must refuse +// rather than corrupt the column in the opposite direction. +// +// We test this WITHOUT manually flipping anything — we just create the table +// under UseAdaptiveEncoding=true (the v2.0.7 default), insert a row, and +// attempt repair. The column is internally consistent; the payload is +// genuinely adaptive; repair must refuse. +func TestSchemaEncodingDrift_RefusesGenuineAdaptiveData(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = true + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // Long enough to spill to addressed storage; payload will be encoded as + // genuine adaptive-addressed under StringAdaptiveEnc. + bigPayload := makeBigString(2048) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE issues (id INT PRIMARY KEY, description TEXT NOT NULL); +`+`INSERT INTO issues VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + + // Confirm pre-state: column declares adaptive, payload is genuine adaptive. + // No drift expected. + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, drifts, "genuine adaptive column must not register as drift") + + // Repair attempt must refuse. We expect an error path with OutcomeGenuineAdaptive. + res, err := schemadrift.RepairColumnForTest(ctx, dEnv, "issues", "description") + require.Error(t, err, "repair must refuse genuinely-adaptive payloads") + require.Contains(t, err.Error(), "genuine", + "refusal error must explain that the column is genuinely adaptive") + require.Equal(t, schemadrift.OutcomeGenuineAdaptive, res.Outcome, + "refused repair must return OutcomeGenuineAdaptive so callers can branch on it") + require.Empty(t, res.CommitHash, "refused repair must NOT touch the working set") +} + +// makeBigString builds a deterministic, easily-recognized payload of |n| bytes +// that is safely inlinable as a SQL string literal (no quotes, no backslashes). +func makeBigString(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = byte('a' + (i % 26)) + } + return string(b) +} + +// manuallyFlipColumnEncoding rewrites the persisted schema of |tableName| so +// that the column |colName|'s TypeInfo encoding becomes |newEnc|. The row +// data is not touched. This simulates the v2.0.7 ALTER MODIFY corruption +// without going through the SQL ALTER path (which the schema-side fix in this +// branch now blocks). +func manuallyFlipColumnEncoding(ctx context.Context, dEnv *env.DoltEnv, tableName, colName string, newEnc val.Encoding) error { + root, err := dEnv.WorkingRoot(ctx) + if err != nil { + return err + } + tbl, _, ok, err := doltdb.GetTableInsensitive(ctx, root, doltdb.TableName{Name: tableName}) + if err != nil { + return err + } + if !ok { + return doltdb.ErrTableNotFound + } + sch, err := tbl.GetSchema(ctx) + if err != nil { + return err + } + existing, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) + if !ok { + return doltdb.ErrTableNotFound + } + patched := existing + patched.TypeInfo = existing.TypeInfo.WithEncoding(newEnc) + + cols := sch.GetAllCols().GetColumns() + for i := range cols { + if cols[i].Tag == patched.Tag { + cols[i] = patched + break + } + } + newCC := schema.NewColCollection(cols...) + pkCols := sch.GetPKCols().GetColumns() + newPKCC := schema.NewColCollection(pkCols...) + newSch, err := schema.NewSchema(newCC, sch.GetPkOrdinals(), sch.GetCollation(), schema.NewIndexCollection(newCC, newPKCC), sch.Checks()) + if err != nil { + return err + } + for _, ix := range sch.Indexes().AllIndexes() { + _, err := newSch.Indexes().AddIndexByColTags(ix.Name(), ix.IndexedColumnTags(), ix.PrefixLengths(), schema.IndexProperties{ + IsUnique: ix.IsUnique(), + IsSpatial: ix.IsSpatial(), + IsFullText: ix.IsFullText(), + IsVector: ix.IsVector(), + IsUserDefined: ix.IsUserDefined(), + Comment: ix.Comment(), + Predicate: ix.Predicate(), + FullTextProperties: ix.FullTextProperties(), + VectorProperties: ix.VectorProperties(), + }) + if err != nil { + return err + } + } + updated, err := tbl.UpdateSchema(ctx, newSch) + if err != nil { + return err + } + newRoot, err := root.PutTable(ctx, doltdb.TableName{Name: tableName, Schema: doltdb.DefaultSchemaName}, updated) + if err != nil { + return err + } + return dEnv.UpdateWorkingRoot(ctx, newRoot) +} From d262bd98fc15dd5f4b669b9d42214a50a8cfc535 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 12 Jun 2026 13:05:30 -0700 Subject: [PATCH 2/3] bug fix for keyless tables --- .../dolt/commands/admin/schemadrift/check.go | 8 + .../admin/schemadrift/check_keyless_test.go | 176 ++++++++++++++++++ .../dolt/commands/admin/schemadrift/repair.go | 9 +- 3 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 go/cmd/dolt/commands/admin/schemadrift/check_keyless_test.go diff --git a/go/cmd/dolt/commands/admin/schemadrift/check.go b/go/cmd/dolt/commands/admin/schemadrift/check.go index 674af53565d..1747ae6a149 100644 --- a/go/cmd/dolt/commands/admin/schemadrift/check.go +++ b/go/cmd/dolt/commands/admin/schemadrift/check.go @@ -532,6 +532,14 @@ type adaptiveCandidate struct { func collectAdaptiveValueColumns(sch schema.Schema) []adaptiveCandidate { var cands []adaptiveCandidate i := 0 + // Keyless tables prepend a cardinality field to the value tuple + // (schemaImpl.GetValueDescriptor inserts val.KeylessCardType at index 0), + // so every user column sits one position later than its NonPKCols + // ordinal. Without this offset the scanner classifies the wrong field's + // bytes and silently reports drifted keyless tables as clean. + if schema.IsKeyless(sch) { + i = 1 + } _ = sch.GetNonPKCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { if col.Virtual { return false, nil diff --git a/go/cmd/dolt/commands/admin/schemadrift/check_keyless_test.go b/go/cmd/dolt/commands/admin/schemadrift/check_keyless_test.go new file mode 100644 index 00000000000..e94a7e10b22 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/check_keyless_test.go @@ -0,0 +1,176 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// Keyless tables prepend a cardinality field to the value tuple +// (schemaImpl.GetValueDescriptor inserts val.KeylessCardType at index 0), so +// every user column lives at value-tuple index N+1 relative to the NonPKCols +// ordering. recover-rows and migrate-adaptive refuse keyless tables because +// of exactly that ordinal mismatch — but check is the diagnostic entry point +// and must DETECT drift on keyless tables, not silently scan the wrong field +// and report the database clean. These tests pin that behavior. + +// TestKeyless_CheckDetectsDrift reproduces the customer-reported false +// negative: a keyless table with a TEXT column whose persisted schema tag +// drifted to StringAdaptiveEnc while its rows still hold legacy 20-byte raw +// hashes. check must report exactly one drift row for it. +func TestKeyless_CheckDetectsDrift(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + // No PRIMARY KEY → keyless table. The 1024-byte payload spills to + // out-of-band storage, so under StringAddrEnc the row field is a legacy + // raw 20-byte hash. + bigPayload := makeBigString(1024) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE events (id INT, description TEXT NOT NULL); +`+`INSERT INTO events VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + + // Sanity: before the simulated corruption there is no drift. + before, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, before, "before corruption: schema and payload agree") + + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + + after, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, after, 1, "keyless drifted column must be detected (cardinality-prefix offset)") + d := after[0] + require.Equal(t, "events", d.Table) + require.Equal(t, "description", d.Column) + require.Equal(t, schemadrift.SeverityDrift, d.Severity) + require.Equal(t, "StringAdaptiveEnc", d.DeclaredEncoding) + require.Equal(t, "legacy-raw-hash", d.ObservedFormat) + require.Equal(t, "StringAddrEnc", d.SuggestedEncoding) + require.True(t, d.SafeToRepair) +} + +// TestKeyless_CheckCleanAdaptiveStaysClean guards the other direction: a +// genuinely adaptive keyless column must NOT surface as drift (i.e. the +// keyless offset is applied exactly once, not zero or two times). Without the +// offset the scanner would read the neighboring field — for this fixture the +// 4-byte INT — and misclassify. +func TestKeyless_CheckCleanAdaptiveStaysClean(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = true + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + bigPayload := makeBigString(2048) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE events (id INT, description TEXT NOT NULL); +`+`INSERT INTO events VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, drifts, "genuine adaptive keyless column must not register as drift") +} + +// TestKeyless_RepairFlipsTag verifies the write side: repair shares the +// value-tuple index computation with check (valueTupleIndexForColumn), so a +// keyless drifted column must be repairable once detected — the flip itself +// is a schema-only change and is as safe on a keyless table as on a keyed +// one. Before the fix, repair scanned the wrong field, saw an ambiguous +// payload, and refused. +func TestKeyless_RepairFlipsTag(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + bigPayload := makeBigString(1024) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE events (id INT, description TEXT NOT NULL); +`+`INSERT INTO events VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + + res, err := schemadrift.RepairColumnForTest(ctx, dEnv, "events", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.OutcomeFlipped, res.Outcome) + require.Equal(t, val.StringAdaptiveEnc, res.OldEncoding) + require.Equal(t, val.StringAddrEnc, res.NewEncoding) + + after, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, after, "repair must clear the keyless drift entry") +} + +// TestKeyless_CheckDetectsHeterogeneous covers the mixed-payload keyless +// case: legacy rows written before the corrupting ALTER plus adaptive rows +// written after it. check must classify the column heterogeneous (not clean, +// not pure drift). +func TestKeyless_CheckDetectsHeterogeneous(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + bigPayload := makeBigString(1024) + _, err := sqle.ExecuteSql(ctx, dEnv, + `CREATE TABLE events (id INT, description TEXT NOT NULL); +`+`INSERT INTO events VALUES (1, '`+bigPayload+`');`) + require.NoError(t, err) + + // Flip the schema tag, then write a second row THROUGH the adaptive + // encoding so the column payload mixes legacy and adaptive rows. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + + typeinfo.UseAdaptiveEncoding = true + bigPayload2 := makeBigString(2048) + _, err = sqle.ExecuteSql(ctx, dEnv, + `INSERT INTO events VALUES (2, '`+bigPayload2+`');`) + require.NoError(t, err) + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Len(t, drifts, 1, "heterogeneous keyless column must be surfaced") + require.Equal(t, schemadrift.SeverityHeterogeneous, drifts[0].Severity) + require.False(t, drifts[0].SafeToRepair) +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/repair.go b/go/cmd/dolt/commands/admin/schemadrift/repair.go index e75c10b6a86..8181676a28f 100644 --- a/go/cmd/dolt/commands/admin/schemadrift/repair.go +++ b/go/cmd/dolt/commands/admin/schemadrift/repair.go @@ -430,10 +430,17 @@ func repairColumnWithOptions(ctx context.Context, dEnv *env.DoltEnv, tableName, // valueTupleIndexForColumn returns the column's index in the value tuple, or // (-1, false) if the column is a PK, virtual, or absent. Order matches -// schemaImpl.GetValueDescriptor: NonPKCols.Iter, skipping Virtual. +// schemaImpl.GetValueDescriptor: NonPKCols.Iter, skipping Virtual, with the +// keyless cardinality field occupying index 0 on keyless tables. func valueTupleIndexForColumn(sch schema.Schema, colName string) (int, bool) { idx := -1 i := 0 + // Keyless tables prepend a cardinality field to the value tuple, so user + // columns start at index 1. Without this offset the payload scan reads a + // neighboring field and refuses an otherwise-valid repair as ambiguous. + if schema.IsKeyless(sch) { + i = 1 + } _ = sch.GetNonPKCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { if col.Virtual { return false, nil From d0a5bf3d966861a4762172b200afc75c9f4c420d Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Fri, 12 Jun 2026 13:39:41 -0700 Subject: [PATCH 3/3] bug fixes for keyless repair --- .../admin/schemadrift/keyless_migrate_test.go | 254 ++++++++++++++++++ .../admin/schemadrift/migrate_adaptive.go | 35 ++- .../admin/schemadrift/recover_rows.go | 161 ++++++++++- go/libraries/doltcore/sqle/testutil.go | 2 +- 4 files changed, 431 insertions(+), 21 deletions(-) create mode 100644 go/cmd/dolt/commands/admin/schemadrift/keyless_migrate_test.go diff --git a/go/cmd/dolt/commands/admin/schemadrift/keyless_migrate_test.go b/go/cmd/dolt/commands/admin/schemadrift/keyless_migrate_test.go new file mode 100644 index 00000000000..51ed5bb17f2 --- /dev/null +++ b/go/cmd/dolt/commands/admin/schemadrift/keyless_migrate_test.go @@ -0,0 +1,254 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schemadrift_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/cmd/dolt/commands/admin/schemadrift" + "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle" + "github.com/dolthub/dolt/go/store/val" +) + +// Keyless row-migration tests. Keyless tables key each row by +// xxh3.Hash128 of the value tuple's field bytes (val.HashTupleFromValue), so +// rewriting a row's field bytes invalidates its stored key, and the rewritten +// bytes must be exactly what the engine's TupleBuilder would produce for the +// same logical content — otherwise subsequent UPDATE/DELETE statements hash a +// freshly-built tuple, miss the stored key, and silently affect zero rows. +// These tests therefore verify migrations on keyless tables at the ENGINE +// level: after migrating, SQL DELETEs must actually find and remove rows. + +// keylessRowCount returns SELECT COUNT(*) for the given table. +func keylessRowCount(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, table string) int { + t.Helper() + root, err := dEnv.WorkingRoot(ctx) + require.NoError(t, err) + rows, err := sqle.ExecuteSelect(ctx, dEnv, root, fmt.Sprintf(`SELECT COUNT(*) FROM %s;`, table)) + require.NoError(t, err) + require.Len(t, rows, 1) + switch v := rows[0][0].(type) { + case int64: + return int(v) + case int32: + return int(v) + case int: + return v + default: + t.Fatalf("unexpected COUNT(*) type %T", rows[0][0]) + return -1 + } +} + +// TestKeyless_MigrateAdaptive_EndToEnd drives the forward migration on a +// drifted keyless column and proves engine-level integrity afterward. +// +// The 100-byte payload is the load-bearing case: its engine-canonical adaptive +// form is INLINE (tuple well under the 2048-byte target), so a migration that +// re-encodes by a fixed content-size rule (e.g. "inline iff <= 20 bytes") +// produces out-of-band bytes the engine would never build — the row's stored +// key then never matches an engine-computed hash and DELETE silently misses. +func TestKeyless_MigrateAdaptive_EndToEnd(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + smallPayload := strings.Repeat("s", 100) // canonical adaptive: inline + bigPayload := strings.Repeat("b", 5000) // canonical adaptive: out-of-band + _, err := sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `CREATE TABLE events (id INT, description TEXT NOT NULL); +INSERT INTO events VALUES (1, '%s'), (2, '%s');`, smallPayload, bigPayload)) + require.NoError(t, err) + + // Simulate the v2.0.7 drift: schema says adaptive, rows are legacy. + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + typeinfo.UseAdaptiveEncoding = true + + res, err := schemadrift.MigrateAdaptiveColumnForTest(ctx, dEnv, "events", "description") + require.NoError(t, err, "migrate-adaptive must support keyless tables") + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.Equal(t, val.StringAdaptiveEnc, res.NewEncoding) + require.Equal(t, 2, res.RowsScanned) + require.Equal(t, 2, res.RowsRewritten) + + // check must be clean afterward. + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, drifts, "post-migration keyless column must scan clean") + + // Content must be readable and byte-identical. + content := collectColumnContent(t, ctx, dEnv, "events", "description") + require.Equal(t, map[int]string{1: smallPayload, 2: bigPayload}, content) + + // The engine-level proof: DELETE locates keyless rows by hashing a + // freshly-built value tuple. Both deletes must actually remove their row. + require.Equal(t, 2, keylessRowCount(t, ctx, dEnv, "events")) + _, err = sqle.ExecuteSql(ctx, dEnv, `DELETE FROM events WHERE id = 1;`) + require.NoError(t, err) + require.Equal(t, 1, keylessRowCount(t, ctx, dEnv, "events"), + "DELETE of the inline-canonical row must find it by hash (engine-canonical bytes + rehashed key)") + _, err = sqle.ExecuteSql(ctx, dEnv, `DELETE FROM events WHERE id = 2;`) + require.NoError(t, err) + require.Equal(t, 0, keylessRowCount(t, ctx, dEnv, "events"), + "DELETE of the out-of-band-canonical row must find it by hash") +} + +// TestKeyless_RecoverRows_EndToEnd drives the backward (to-legacy) migration +// on a heterogeneous keyless column — legacy rows written before the +// corrupting ALTER, adaptive rows written after it — and proves engine-level +// integrity afterward: every row readable, every row deletable. +func TestKeyless_RecoverRows_EndToEnd(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + legacyPayload := strings.Repeat("L", 1500) + _, err := sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `CREATE TABLE events (id INT, description TEXT NOT NULL); +INSERT INTO events VALUES (1, '%s');`, legacyPayload)) + require.NoError(t, err) + + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + + // Rows written through the drifted adaptive tag: one inline, one addressed. + typeinfo.UseAdaptiveEncoding = true + adaptiveShort := "short adaptive payload" + adaptiveLong := strings.Repeat("A", 3000) + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO events VALUES (2, '%s'), (3, '%s');`, adaptiveShort, adaptiveLong)) + require.NoError(t, err) + + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "events", "description") + require.NoError(t, err, "recover-rows must support keyless tables") + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.Equal(t, val.StringAddrEnc, res.NewEncoding) + require.Equal(t, 3, res.RowsScanned) + require.GreaterOrEqual(t, res.RowsRewritten, 2, "both adaptive rows must be rewritten") + + drifts, err := schemadrift.ScanForDriftForTest(ctx, dEnv) + require.NoError(t, err) + require.Empty(t, drifts, "post-migration keyless column must scan clean") + + content := collectColumnContent(t, ctx, dEnv, "events", "description") + require.Equal(t, map[int]string{1: legacyPayload, 2: adaptiveShort, 3: adaptiveLong}, content) + + // Engine-level: every row must be deletable. Rows 2 and 3 were re-keyed + // by the migration; row 1 kept its original key. + for expected, id := 2, 1; id <= 3; id, expected = id+1, expected-1 { + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf(`DELETE FROM events WHERE id = %d;`, id)) + require.NoError(t, err) + require.Equal(t, expected, keylessRowCount(t, ctx, dEnv, "events"), + "DELETE of row %d must find it by hash after recover-rows", id) + } +} + +// TestKeyless_RecoverRows_DuplicateMergesCardinality covers the collision +// case: the SAME logical row stored twice — once in legacy form (pre-drift +// insert) and once in adaptive form (post-drift insert). In a keyless table +// those are two distinct map entries (different bytes → different hash ids), +// but after canonicalization they become byte-identical, so the migration +// must MERGE them into one entry with cardinality 2 — not let the second +// rewrite silently overwrite the first (which would lose a row). +func TestKeyless_RecoverRows_DuplicateMergesCardinality(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + payload := strings.Repeat("D", 1500) + _, err := sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `CREATE TABLE events (id INT, description TEXT NOT NULL); +INSERT INTO events VALUES (1, '%s');`, payload)) + require.NoError(t, err) + + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + + // Insert the IDENTICAL logical row through the drifted adaptive tag. + typeinfo.UseAdaptiveEncoding = true + _, err = sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `INSERT INTO events VALUES (1, '%s');`, payload)) + require.NoError(t, err) + + res, err := schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "events", "description") + require.NoError(t, err) + require.Equal(t, schemadrift.RecoverRowsMigrated, res.Outcome) + require.Equal(t, 2, res.RowsScanned) + require.Equal(t, 1, res.RowsMerged, + "the adaptive duplicate must merge into the legacy entry's key, not overwrite it") + + // Both logical rows must survive the merge (one entry, cardinality 2). + require.Equal(t, 2, keylessRowCount(t, ctx, dEnv, "events"), + "duplicate logical rows must be preserved via cardinality merge") + + // And both must be deletable in one statement. + _, err = sqle.ExecuteSql(ctx, dEnv, `DELETE FROM events WHERE id = 1;`) + require.NoError(t, err) + require.Equal(t, 0, keylessRowCount(t, ctx, dEnv, "events")) +} + +// TestKeyless_Migrations_RefuseSecondaryIndexes pins the honest boundary: +// keyless secondary index entries embed each row's hash id, and re-keying +// rows invalidates them. Until the migrations rebuild secondary indexes, they +// must refuse keyless tables that have any — with an error that tells the +// operator the workaround (drop, migrate, re-create). +func TestKeyless_Migrations_RefuseSecondaryIndexes(t *testing.T) { + ctx := context.Background() + + prev := typeinfo.UseAdaptiveEncoding + typeinfo.UseAdaptiveEncoding = false + t.Cleanup(func() { typeinfo.UseAdaptiveEncoding = prev }) + + dEnv := dtestutils.CreateTestEnv() + defer dEnv.Close() + + payload := strings.Repeat("X", 1500) + _, err := sqle.ExecuteSql(ctx, dEnv, fmt.Sprintf( + `CREATE TABLE events (id INT, description TEXT NOT NULL); +CREATE INDEX id_idx ON events (id); +INSERT INTO events VALUES (1, '%s');`, payload)) + require.NoError(t, err) + + require.NoError(t, manuallyFlipColumnEncoding(ctx, dEnv, "events", "description", val.StringAdaptiveEnc)) + + _, err = schemadrift.RecoverRowsColumnForTest(ctx, dEnv, "events", "description") + require.Error(t, err, "recover-rows must refuse a keyless table with secondary indexes") + require.Contains(t, err.Error(), "secondary index") + + _, err = schemadrift.MigrateAdaptiveColumnForTest(ctx, dEnv, "events", "description") + require.Error(t, err, "migrate-adaptive must refuse a keyless table with secondary indexes") + require.Contains(t, err.Error(), "secondary index") +} diff --git a/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go index da542275dd8..f0f18fa4562 100644 --- a/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go +++ b/go/cmd/dolt/commands/admin/schemadrift/migrate_adaptive.go @@ -147,6 +147,7 @@ type MigrateAdaptiveResult struct { RowsScanned int RowsRewritten int // legacy rows converted to adaptive CollateralRewrites int // sibling AddrEnc fields normalized to satisfy the descriptor + RowsMerged int // keyless only: rewritten rows merged into an existing entry by cardinality CommitHash string CommitMessage string @@ -212,14 +213,6 @@ func migrateAdaptiveColumnOpts(ctx context.Context, dEnv *env.DoltEnv, tableName return MigrateAdaptiveResult{}, fmt.Errorf("get schema for %s: %w", resolvedName, err) } - // Keyless tables prepend a cardinality column to the value tuple; their - // value-tuple ordinals don't line up with NonPKCols ordering. Refuse them, - // exactly as recover-rows does. - if schema.IsKeyless(sch) { - return MigrateAdaptiveResult{}, fmt.Errorf( - "table %s is keyless; migrate-adaptive does not support keyless tables", resolvedName) - } - existingCol, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) if !ok { return MigrateAdaptiveResult{}, fmt.Errorf("column %q not found in table %s", colName, resolvedName) @@ -291,6 +284,16 @@ func migrateAdaptiveColumnOpts(ctx context.Context, dEnv *env.DoltEnv, tableName }, nil } + // Keyless tables are supported on the mutating path below (rewriteColumn + // rehashes their content-derived row ids), EXCEPT when the table has + // secondary indexes: keyless index entries embed each row's hash id, and + // re-keying rewritten rows would strand those entries. The check sits + // after the --dry-run block deliberately — dry-run never mutates, so it + // stays available everywhere. + if err := refuseKeylessWithSecondaryIndexes(sch, resolvedName, "migrate-adaptive"); err != nil { + return MigrateAdaptiveResult{}, err + } + // dolt_ignore'd tables (ignored_log, ignored_meta) live only in the working set and // are NEVER committed, so their content chunks are not commit-rooted — // out-of-band adaptive values would reference chunks the persist's refCheck @@ -359,6 +362,7 @@ func migrateAdaptiveColumnOpts(ctx context.Context, dEnv *env.DoltEnv, tableName rowsRewritten := rewritten.RowsRewritten collateral := rewritten.CollateralRewrites + merged := rewritten.RowsMerged var commitMsg string if collateral > 0 { commitMsg = fmt.Sprintf( @@ -369,6 +373,9 @@ func migrateAdaptiveColumnOpts(ctx context.Context, dEnv *env.DoltEnv, tableName "admin: schema-encoding-drift migrate-adaptive on %s.%s (%d legacy rows rewritten to adaptive format + schema flipped to %s; content unchanged)", resolvedName, existingCol.Name, rowsRewritten, encodingName(newEnc)) } + if merged > 0 { + commitMsg += fmt.Sprintf(" [%d duplicate keyless rows merged by cardinality]", merged) + } // Non-ignored (committed) table: commit as usual so the heal lands in // history. (dolt_ignore'd tables took the force-inline path above and @@ -389,6 +396,7 @@ func migrateAdaptiveColumnOpts(ctx context.Context, dEnv *env.DoltEnv, tableName RowsScanned: scanned, RowsRewritten: rowsRewritten, CollateralRewrites: collateral, + RowsMerged: merged, CommitHash: commitHash.String(), CommitMessage: commitMsg, }, nil @@ -505,8 +513,14 @@ func underlyingContent(ctx context.Context, vs val.ValueStore, b []byte) (conten // chunks and retains no legacy AddrEnc columns, so refCheck has nothing to // dangle on and the data is immune to the gc hazard. func migrateIgnoredTableInline(ctx context.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, sch schema.Schema, dbName, resolvedName string, tn doltdb.TableName, roots doltdb.Roots, ddb *doltdb.DoltDB) (MigrateAdaptiveResult, error) { + // The force-inline heal deliberately inlines content of ANY size (gc-safety + // for never-committed chunks trumps the engine's spill heuristic). On a + // keyless table that would leave rows whose bytes differ from what the + // engine's TupleBuilder builds for the same content, so content-derived + // hash lookups (UPDATE/DELETE) would miss them. Keyless + dolt_ignore'd + // stays refused until the force-inline path can re-key engine-canonically. if schema.IsKeyless(sch) { - return MigrateAdaptiveResult{}, fmt.Errorf("table %s is keyless; migrate-adaptive does not support keyless tables", resolvedName) + return MigrateAdaptiveResult{}, fmt.Errorf("table %s is keyless and dolt_ignore'd; the force-inline heal cannot produce engine-canonical keyless rows (force-inlining diverges from the engine's spill heuristic, breaking content-derived hash lookups)", resolvedName) } // Identify every non-PK, non-virtual out-of-band-capable column and its @@ -895,6 +909,9 @@ func emitMigrateAdaptiveResult(w io.Writer, r MigrateAdaptiveResult) { if r.CollateralRewrites > 0 { _, _ = fmt.Fprintf(w, " %d rows had non-target AddrEnc fields healed as collateral\n", r.CollateralRewrites) } + if r.RowsMerged > 0 { + _, _ = fmt.Fprintf(w, " %d duplicate keyless rows merged by cardinality\n", r.RowsMerged) + } _, _ = fmt.Fprintf(w, " commit: %s\n", r.CommitHash) _, _ = fmt.Fprintf(w, " message: %s\n", r.CommitMessage) case RecoverRowsAlreadyOK: diff --git a/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go b/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go index 5afc8c807bb..c92a9561546 100644 --- a/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go +++ b/go/cmd/dolt/commands/admin/schemadrift/recover_rows.go @@ -15,6 +15,7 @@ package schemadrift import ( + "bytes" "context" "encoding/hex" "errors" @@ -32,6 +33,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/pool" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/val" ) @@ -180,6 +182,7 @@ type RecoverRowsResult struct { RowsScanned int // total rows iterated RowsRewritten int // subset of RowsScanned whose target field bytes changed CollateralRewrites int // subset of RowsRewritten where one or more NON-target AddrEnc fields had to be normalized to satisfy the new descriptor — surfaces residual schema-row mismatch left by a prior `repair` invocation on a sibling column + RowsMerged int // keyless only: rewritten rows whose canonical bytes collided with an existing entry and were merged by summing cardinality (duplicate logical rows stored once per format) CommitHash string // empty for RecoverRowsAlreadyOK and RecoverRowsNoLegacyRows CommitMessage string // empty for RecoverRowsAlreadyOK and RecoverRowsNoLegacyRows } @@ -232,16 +235,12 @@ func recoverRowsColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colNam return RecoverRowsResult{}, fmt.Errorf("get schema for %s: %w", resolvedName, err) } - // Keyless tables prepend a cardinality column to the value tuple. Their - // value-tuple ordinals don't line up with NonPKCols ordering, so the - // simple value-tuple index we compute would point at the wrong column. - // recover-rows refuses keyless tables to keep that bug from biting; the database's - // affected tables (`issues`, `events`) are both keyed. - if schema.IsKeyless(sch) { - return RecoverRowsResult{}, fmt.Errorf( - "table %s is keyless; recover-rows does not yet support keyless tables (real-world tables are all keyed)", - resolvedName, - ) + // Keyless tables are supported (rewriteColumn rehashes their content- + // derived row ids), EXCEPT when the table has secondary indexes: keyless + // index entries embed each row's hash id, and re-keying rewritten rows + // would strand those entries. Refuse with the workaround spelled out. + if err := refuseKeylessWithSecondaryIndexes(sch, resolvedName, "recover-rows"); err != nil { + return RecoverRowsResult{}, err } existingCol, ok := sch.GetAllCols().GetByNameCaseInsensitive(colName) @@ -366,6 +365,7 @@ func recoverRowsColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colNam // mismatch that this migration repaired as a side-effect. rowsRewritten := rewritten.RowsRewritten collateral := rewritten.CollateralRewrites + merged := rewritten.RowsMerged var commitMsg string if collateral > 0 { commitMsg = fmt.Sprintf( @@ -378,6 +378,9 @@ func recoverRowsColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colNam resolvedName, existingCol.Name, rowsRewritten, encodingName(newEnc), ) } + if merged > 0 { + commitMsg += fmt.Sprintf(" [%d duplicate keyless rows merged by cardinality]", merged) + } commitHash, err := commitWorkingRoot(ctx, dEnv, headRef, roots, commitMsg) if err != nil { return RecoverRowsResult{}, fmt.Errorf("commit recover-rows: %w", err) @@ -393,6 +396,7 @@ func recoverRowsColumn(ctx context.Context, dEnv *env.DoltEnv, tableName, colNam RowsScanned: scanned, RowsRewritten: rowsRewritten, CollateralRewrites: collateral, + RowsMerged: merged, CommitHash: commitHash.String(), CommitMessage: commitMsg, }, nil @@ -419,6 +423,12 @@ type rewrittenMap struct { Map prolly.Map RowsRewritten int CollateralRewrites int + // RowsMerged counts keyless rows whose canonical rewrite collided with an + // existing entry (the same logical row stored once in legacy form and once + // in adaptive form). The colliding entries are merged by summing their + // cardinalities; without the merge the second Put would silently overwrite + // the first and lose rows. + RowsMerged int } // targetFieldResolver resolves a single value-tuple field's raw bytes into the @@ -485,6 +495,15 @@ func rewriteColumn( newValDesc := newSch.GetValueDescriptor(ns) mut := pm.Rewriter(pm.KeyDesc(), newValDesc) + // Keyless tables key each row by xxh3.Hash128 of the value tuple's field + // bytes (val.HashTupleFromValue), so a rewritten row must (a) carry bytes + // the engine's own TupleBuilder would produce for the same logical content + // — future DML re-derives the hash from a freshly-built tuple and must hit + // the stored key — and (b) be re-keyed to the hash of those canonical + // bytes, deleting the entry under the stale key. + isKeyless := schema.IsKeyless(newSch) + keylessTarget := newSch.GetTargetRowSize() + // Pre-compute which positions in the new value descriptor are legacy // AddrEnc — i.e., positions where the field bytes MUST be either NULL or // exactly 20 bytes (a legacy raw content-hash address). Any other shape @@ -517,6 +536,7 @@ func rewriteColumn( rowsRewritten := 0 collateralRewrites := 0 + rowsMerged := 0 keyDesc := pm.KeyDesc() for { key, value, iterErr := iter.Next(ctx) @@ -633,8 +653,60 @@ func rewriteColumn( values[i] = b } newValue := val.NewTuple(pool, values...) + putKey := key + + if isKeyless { + // Rebuild the tuple through the engine's TupleBuilder so the + // stored bytes are exactly what a future engine-built tuple for + // the same logical content hashes to. In particular, adaptive + // fields are fed inline-first and spilled out-of-band only by the + // builder's whole-tuple size heuristic — NOT by any fixed + // per-content rule. + canonical, cErr := buildCanonicalKeylessValue(ctx, ns, pool, newValDesc, keylessTarget, values) + if cErr != nil { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "canonicalize keyless row with hash id %s: %w", pkHex, cErr, + ) + } + if bytes.Equal(canonical, value) { + // The canonical rebuild proved this row was already in engine- + // canonical form; nothing to rewrite, key stays valid. + continue + } + newValue = canonical + newKey := val.HashTupleFromValue(pool, newValue) + if !bytes.Equal(newKey, key) { + // The row moves to a new content-derived key. If the + // destination already holds an entry (the same logical row + // stored once per format — possible in a heterogeneous + // column), MERGE by summing cardinalities; a plain Put would + // silently overwrite the earlier entry and lose rows. + var existing val.Tuple + if gErr := mut.Get(ctx, newKey, func(_, v val.Tuple) error { + existing = v + return nil + }); gErr != nil { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "check destination key for keyless row %s: %w", pkHex, gErr, + ) + } + if existing != nil { + newValue, _ = val.ModifyKeylessCardinality(pool, newValue, int64(val.ReadKeylessCardinality(existing))) + rowsMerged++ + } + if dErr := mut.Delete(ctx, key); dErr != nil { + pkHex := formatKeyForError(key, keyDesc) + return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( + "delete stale keyless key %s: %w", pkHex, dErr, + ) + } + putKey = newKey + } + } - if putErr := mut.Put(ctx, key, newValue); putErr != nil { + if putErr := mut.Put(ctx, putKey, newValue); putErr != nil { pkHex := formatKeyForError(key, keyDesc) return rewrittenMap{}, scanned, sawAnyLegacy, fmt.Errorf( "put rewritten row with primary key %s: %w", pkHex, putErr, @@ -654,9 +726,73 @@ func rewriteColumn( Map: newMap, RowsRewritten: rowsRewritten, CollateralRewrites: collateralRewrites, + RowsMerged: rowsMerged, }, scanned, sawAnyLegacy, nil } +// refuseKeylessWithSecondaryIndexes returns an error when |sch| is a keyless +// schema with one or more secondary indexes. Keyless secondary index entries +// embed each row's content-derived hash id; the row migrations re-key every +// rewritten row, which would strand those entries. Until the migrations learn +// to rebuild secondary indexes, the safe posture is an explicit refusal with +// the workaround spelled out. Keyed tables are unaffected (their index entries +// reference primary keys, which the migrations never change). +func refuseKeylessWithSecondaryIndexes(sch schema.Schema, tableName, cmdName string) error { + if !schema.IsKeyless(sch) { + return nil + } + if n := len(sch.Indexes().AllIndexes()); n > 0 { + return fmt.Errorf( + "table %s is keyless and has %d secondary index(es); keyless index entries embed row hash ids that %s's re-keying would invalidate. Drop the index(es), re-run %s, then re-create them", + tableName, n, cmdName, cmdName, + ) + } + return nil +} + +// buildCanonicalKeylessValue rebuilds a keyless value tuple in the engine- +// canonical form for |desc|: every adaptive field is fed to the TupleBuilder +// as full inline content (exactly like the writer's tuplesFromRow → PutField +// path) and the builder's whole-tuple heuristic decides inline vs out-of-band +// placement; every other field — including the cardinality at position 0 — is +// copied verbatim. The result is byte-identical to what the engine would +// build for the same logical row, which is the invariant keyless DML depends +// on: UPDATE/DELETE locate rows by hashing a freshly-built tuple. +// +// |fields| holds the resolved per-position bytes (length >= desc.Count(); +// positions beyond the source tuple's width are nil → NULL). |target| is the +// schema's tuple-length target (schema.GetTargetRowSize), matching the +// writer's WithMaxRowSize configuration. +func buildCanonicalKeylessValue(ctx context.Context, vs val.ValueStore, bp pool.BuffPool, desc *val.TupleDesc, target uint16, fields [][]byte) (val.Tuple, error) { + tb := val.NewTupleBuilder(desc, vs).WithMaxRowSize(target) + for i := 0; i < desc.Count(); i++ { + var b []byte + if i < len(fields) { + b = fields[i] + } + if len(b) == 0 { + // NULL (zero-length and NULL are the same thing at tuple level — + // see val.Tuple.GetField). Leave the builder slot empty. + continue + } + if val.IsAdaptiveEncoding(desc.Types[i].Enc) { + content, present, cErr := underlyingContent(ctx, vs, b) + if cErr != nil { + return nil, fmt.Errorf("field %d: %w", i, cErr) + } + if !present { + return nil, fmt.Errorf("field %d: content chunk is genuinely absent; refusing to rewrite the row", i) + } + if pErr := tb.PutAdaptiveFromInline(ctx, i, content); pErr != nil { + return nil, fmt.Errorf("field %d: re-encode adaptive content (%d bytes): %w", i, len(content), pErr) + } + continue + } + tb.PutRaw(i, b) + } + return tb.BuildPermissive(ctx, bp) +} + // resolveFieldToLegacy is the per-row workhorse. Given a single field's bytes // from a (possibly drifted) adaptive-tagged value tuple, it returns the // canonical legacy form of those bytes plus two flags: @@ -801,6 +937,9 @@ func emitRecoverRowsResult(w io.Writer, r RecoverRowsResult) { _, _ = fmt.Fprintf(w, " %d rows had non-target AddrEnc fields healed as collateral (residual mismatch from prior repair on a sibling column)\n", r.CollateralRewrites) } + if r.RowsMerged > 0 { + _, _ = fmt.Fprintf(w, " %d duplicate keyless rows merged by cardinality\n", r.RowsMerged) + } _, _ = fmt.Fprintf(w, " commit: %s\n", r.CommitHash) _, _ = fmt.Fprintf(w, " message: %s\n", r.CommitMessage) case RecoverRowsAlreadyOK: diff --git a/go/libraries/doltcore/sqle/testutil.go b/go/libraries/doltcore/sqle/testutil.go index a9e3cd1d4fc..02c1b96c277 100644 --- a/go/libraries/doltcore/sqle/testutil.go +++ b/go/libraries/doltcore/sqle/testutil.go @@ -81,7 +81,7 @@ func ExecuteSqlOnEngine(ctx *sql.Context, engine *sqle.Engine, statements string switch sqlStatement.(type) { case *sqlparser.Select, *sqlparser.OtherRead: return errors.New("Select statements aren't handled") - case *sqlparser.Insert: + case *sqlparser.Insert, *sqlparser.Delete, *sqlparser.Update: var rowIter sql.RowIter _, rowIter, _, execErr = engine.Query(ctx, query) if execErr == nil {