diff --git a/internal/dbtest/msgpack_test.go b/internal/dbtest/msgpack_test.go new file mode 100644 index 000000000..233f86834 --- /dev/null +++ b/internal/dbtest/msgpack_test.go @@ -0,0 +1,48 @@ +package dbtest_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/uptrace/bun" +) + +type msgpackPayload struct { + Something int `msgpack:"something"` + Name string `msgpack:"name"` +} + +type msgpackModel struct { + bun.BaseModel `bun:"table:msgpack_models"` + + ID int64 `bun:",pk,autoincrement"` + Encoded msgpackPayload `bun:",msgpack"` +} + +// Regression test for https://github.com/uptrace/bun/issues/1219: a field tagged +// with `bun:",msgpack"` must round-trip on every dialect, not just PostgreSQL. +// The value was always written using PostgreSQL's '\x...' binary literal, which +// SQLite and MySQL store verbatim as text and then fail to decode on read back. +func TestMsgpackRoundTrip(t *testing.T) { + testEachDB(t, func(t *testing.T, dbName string, db *bun.DB) { + _, err := db.NewDropTable().Model((*msgpackModel)(nil)).IfExists().Exec(ctx) + require.NoError(t, err) + + _, err = db.NewCreateTable().Model((*msgpackModel)(nil)).Exec(ctx) + require.NoError(t, err) + defer func() { + _, err := db.NewDropTable().Model((*msgpackModel)(nil)).IfExists().Exec(ctx) + require.NoError(t, err) + }() + + in := &msgpackModel{Encoded: msgpackPayload{Something: 42, Name: "hello"}} + _, err = db.NewInsert().Model(in).Exec(ctx) + require.NoError(t, err) + + out := new(msgpackModel) + err = db.NewSelect().Model(out).Where("id = ?", in.ID).Scan(ctx) + require.NoError(t, err) + require.Equal(t, in.Encoded, out.Encoded) + }) +} diff --git a/internal/hex.go b/internal/hex.go deleted file mode 100644 index 6fae2bb78..000000000 --- a/internal/hex.go +++ /dev/null @@ -1,43 +0,0 @@ -package internal - -import ( - fasthex "github.com/tmthrgd/go-hex" -) - -type HexEncoder struct { - b []byte - written bool -} - -func NewHexEncoder(b []byte) *HexEncoder { - return &HexEncoder{ - b: b, - } -} - -func (enc *HexEncoder) Bytes() []byte { - return enc.b -} - -func (enc *HexEncoder) Write(b []byte) (int, error) { - if !enc.written { - enc.b = append(enc.b, '\'') - enc.b = append(enc.b, `\x`...) - enc.written = true - } - - i := len(enc.b) - enc.b = append(enc.b, make([]byte, fasthex.EncodedLen(len(b)))...) - fasthex.Encode(enc.b[i:], b) - - return len(b), nil -} - -func (enc *HexEncoder) Close() error { - if enc.written { - enc.b = append(enc.b, '\'') - } else { - enc.b = append(enc.b, "NULL"...) - } - return nil -} diff --git a/schema/append_value.go b/schema/append_value.go index e1dce123c..c6d863ed0 100644 --- a/schema/append_value.go +++ b/schema/append_value.go @@ -1,6 +1,7 @@ package schema import ( + "bytes" "database/sql/driver" "fmt" "net" @@ -290,21 +291,20 @@ func addrAppender(fn AppenderFunc) AppenderFunc { } func appendMsgpack(gen QueryGen, b []byte, v reflect.Value) []byte { - hexEnc := internal.NewHexEncoder(b) + var buf bytes.Buffer enc := msgpack.GetEncoder() defer msgpack.PutEncoder(enc) - enc.Reset(hexEnc) + enc.Reset(&buf) if err := enc.EncodeValue(v); err != nil { return dialect.AppendError(b, err) } - if err := hexEnc.Close(); err != nil { - return dialect.AppendError(b, err) - } - - return hexEnc.Bytes() + // Delegate to the dialect so the encoded payload is emitted using the + // correct binary literal syntax (e.g. '\x...' for PostgreSQL, X'...' for + // SQLite and MySQL) instead of always using the PostgreSQL format. + return gen.Dialect().AppendBytes(b, buf.Bytes()) } func AppendQueryAppender(gen QueryGen, b []byte, app QueryAppender) []byte { diff --git a/schema/appendmsgpack_test.go b/schema/appendmsgpack_test.go new file mode 100644 index 000000000..20c065b96 --- /dev/null +++ b/schema/appendmsgpack_test.go @@ -0,0 +1,63 @@ +package schema + +import ( + "encoding/hex" + "reflect" + "strings" + "testing" + + "github.com/vmihailenco/msgpack/v5" +) + +// xDialect emulates dialects such as SQLite and MySQL that expect binary +// literals in the X'...' form rather than PostgreSQL's '\x...' form. +type xDialect struct { + *nopDialect +} + +func (xDialect) AppendBytes(b, bs []byte) []byte { + b = append(b, `X'`...) + s := len(b) + b = append(b, make([]byte, hex.EncodedLen(len(bs)))...) + hex.Encode(b[s:], bs) + b = append(b, '\'') + return b +} + +// A msgpack field must be emitted using the binary literal syntax of the target +// dialect. Previously it was always encoded in PostgreSQL's '\x...' form, which +// other dialects store verbatim as text and then fail to decode on read back. +func TestAppendMsgpack_DialectSpecificLiteral(t *testing.T) { + value := map[string]int{"something": 42} + + encoded, err := msgpack.Marshal(value) + if err != nil { + t.Fatal(err) + } + rv := reflect.ValueOf(value) + + t.Run("postgres", func(t *testing.T) { + gen := NewQueryGen(newNopDialect()) + got := string(appendMsgpack(gen, nil, rv)) + want := string(BaseDialect{}.AppendBytes(nil, encoded)) + if got != want { + t.Fatalf("appendMsgpack = %q, want %q", got, want) + } + if !strings.HasPrefix(got, `'\x`) { + t.Fatalf("appendMsgpack = %q, want a '\\x...' literal", got) + } + }) + + t.Run("x-literal", func(t *testing.T) { + d := xDialect{newNopDialect()} + gen := NewQueryGen(d) + got := string(appendMsgpack(gen, nil, rv)) + want := string(d.AppendBytes(nil, encoded)) + if got != want { + t.Fatalf("appendMsgpack = %q, want %q", got, want) + } + if !strings.HasPrefix(got, "X'") { + t.Fatalf("appendMsgpack = %q, want an X'...' literal", got) + } + }) +} diff --git a/schema/table.go b/schema/table.go index e11f997b1..aca41ea1f 100644 --- a/schema/table.go +++ b/schema/table.go @@ -13,6 +13,7 @@ import ( "github.com/jinzhu/inflection" "github.com/uptrace/bun/dialect/feature" + "github.com/uptrace/bun/dialect/sqltype" "github.com/uptrace/bun/internal" "github.com/uptrace/bun/internal/tagparser" ) @@ -590,6 +591,11 @@ func (t *Table) newField(sf reflect.StructField, tag tagparser.Tag) *Field { field.UserSQLType = s } field.DiscoveredSQLType = DiscoverSQLType(field.IndirectType) + if tag.HasOption("msgpack") { + // msgpack payloads are binary, so store them in a binary column instead + // of the JSON/text type the field's Go type would otherwise map to. + field.DiscoveredSQLType = sqltype.Blob + } field.Append = FieldAppender(t.dialect, field) field.Scan = FieldScanner(t.dialect, field) field.IsZero = zeroChecker(field.StructField.Type)