fix(schema): encode msgpack fields with the dialect binary literal (#1219)

internal/dbtest/msgpack_test.go

@@ -0,0 +1,48 @@
+package dbtest_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/uptrace/bun"
+)
+
+type msgpackPayload struct {
+	Something int    `msgpack:"something"`
+	Name      string `msgpack:"name"`
+}
+
+type msgpackModel struct {
+	bun.BaseModel `bun:"table:msgpack_models"`
+
+	ID      int64          `bun:",pk,autoincrement"`
+	Encoded msgpackPayload `bun:",msgpack"`
+}
+
+// Regression test for https://github.com/uptrace/bun/issues/1219: a field tagged
+// with `bun:",msgpack"` must round-trip on every dialect, not just PostgreSQL.
+// The value was always written using PostgreSQL's '\x...' binary literal, which
+// SQLite and MySQL store verbatim as text and then fail to decode on read back.
+func TestMsgpackRoundTrip(t *testing.T) {
+	testEachDB(t, func(t *testing.T, dbName string, db *bun.DB) {
+		_, err := db.NewDropTable().Model((*msgpackModel)(nil)).IfExists().Exec(ctx)
+		require.NoError(t, err)
+
+		_, err = db.NewCreateTable().Model((*msgpackModel)(nil)).Exec(ctx)
+		require.NoError(t, err)
+		defer func() {
+			_, err := db.NewDropTable().Model((*msgpackModel)(nil)).IfExists().Exec(ctx)
+			require.NoError(t, err)
+		}()
+
+		in := &msgpackModel{Encoded: msgpackPayload{Something: 42, Name: "hello"}}
+		_, err = db.NewInsert().Model(in).Exec(ctx)
+		require.NoError(t, err)
+
+		out := new(msgpackModel)
+		err = db.NewSelect().Model(out).Where("id = ?", in.ID).Scan(ctx)
+		require.NoError(t, err)
+		require.Equal(t, in.Encoded, out.Encoded)
+	})
+}

schema/append_value.go

@@ -1,6 +1,7 @@
 package schema
 
 import (
+	"bytes"
 	"database/sql/driver"
 	"fmt"
 	"net"
@@ -290,21 +291,20 @@ func addrAppender(fn AppenderFunc) AppenderFunc {
 }
 
 func appendMsgpack(gen QueryGen, b []byte, v reflect.Value) []byte {
-	hexEnc := internal.NewHexEncoder(b)
+	var buf bytes.Buffer
 
 	enc := msgpack.GetEncoder()
 	defer msgpack.PutEncoder(enc)
 
-	enc.Reset(hexEnc)
+	enc.Reset(&buf)
 	if err := enc.EncodeValue(v); err != nil {
 		return dialect.AppendError(b, err)
 	}
 
-	if err := hexEnc.Close(); err != nil {
-		return dialect.AppendError(b, err)
-	}
-
-	return hexEnc.Bytes()
+	// Delegate to the dialect so the encoded payload is emitted using the
+	// correct binary literal syntax (e.g. '\x...' for PostgreSQL, X'...' for
+	// SQLite and MySQL) instead of always using the PostgreSQL format.
+	return gen.Dialect().AppendBytes(b, buf.Bytes())
 }
 
 func AppendQueryAppender(gen QueryGen, b []byte, app QueryAppender) []byte {

schema/appendmsgpack_test.go

@@ -0,0 +1,63 @@
+package schema
+
+import (
+	"encoding/hex"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+// xDialect emulates dialects such as SQLite and MySQL that expect binary
+// literals in the X'...' form rather than PostgreSQL's '\x...' form.
+type xDialect struct {
+	*nopDialect
+}
+
+func (xDialect) AppendBytes(b, bs []byte) []byte {
+	b = append(b, `X'`...)
+	s := len(b)
+	b = append(b, make([]byte, hex.EncodedLen(len(bs)))...)
+	hex.Encode(b[s:], bs)
+	b = append(b, '\'')
+	return b
+}
+
+// A msgpack field must be emitted using the binary literal syntax of the target
+// dialect. Previously it was always encoded in PostgreSQL's '\x...' form, which
+// other dialects store verbatim as text and then fail to decode on read back.
+func TestAppendMsgpack_DialectSpecificLiteral(t *testing.T) {
+	value := map[string]int{"something": 42}
+
+	encoded, err := msgpack.Marshal(value)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rv := reflect.ValueOf(value)
+
+	t.Run("postgres", func(t *testing.T) {
+		gen := NewQueryGen(newNopDialect())
+		got := string(appendMsgpack(gen, nil, rv))
+		want := string(BaseDialect{}.AppendBytes(nil, encoded))
+		if got != want {
+			t.Fatalf("appendMsgpack = %q, want %q", got, want)
+		}
+		if !strings.HasPrefix(got, `'\x`) {
+			t.Fatalf("appendMsgpack = %q, want a '\\x...' literal", got)
+		}
+	})
+
+	t.Run("x-literal", func(t *testing.T) {
+		d := xDialect{newNopDialect()}
+		gen := NewQueryGen(d)
+		got := string(appendMsgpack(gen, nil, rv))
+		want := string(d.AppendBytes(nil, encoded))
+		if got != want {
+			t.Fatalf("appendMsgpack = %q, want %q", got, want)
+		}
+		if !strings.HasPrefix(got, "X'") {
+			t.Fatalf("appendMsgpack = %q, want an X'...' literal", got)
+		}
+	})
+}

schema/table.go

@@ -13,6 +13,7 @@ import (
 	"github.com/jinzhu/inflection"
 
 	"github.com/uptrace/bun/dialect/feature"
+	"github.com/uptrace/bun/dialect/sqltype"
 	"github.com/uptrace/bun/internal"
 	"github.com/uptrace/bun/internal/tagparser"
 )
@@ -590,6 +591,11 @@ func (t *Table) newField(sf reflect.StructField, tag tagparser.Tag) *Field {
 		field.UserSQLType = s
 	}
 	field.DiscoveredSQLType = DiscoverSQLType(field.IndirectType)
+	if tag.HasOption("msgpack") {
+		// msgpack payloads are binary, so store them in a binary column instead
+		// of the JSON/text type the field's Go type would otherwise map to.
+		field.DiscoveredSQLType = sqltype.Blob
+	}
 	field.Append = FieldAppender(t.dialect, field)
 	field.Scan = FieldScanner(t.dialect, field)
 	field.IsZero = zeroChecker(field.StructField.Type)