From 7aacfe8ecb3edb66e314a6a909928627bffc68a9 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Tue, 22 Feb 2022 16:03:39 +0800 Subject: [PATCH 01/17] ci: add golangci-lint config file --- .golangci.yml | 90 +++++++++++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 1 - 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..a64ce75d9 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,90 @@ +run: + timeout: 5m + skip-dirs: [] + skip-files: [] + allow-parallel-runners: true + +output: + format: colored-line-number + print-issued-lines: true + print-linter-name: true + uniq-by-line: true + sort-results: true + +linters-settings: + depguard: + list-type: blacklist + packages: + # logging is allowed only by logutils.Log, logrus + # is allowed to use only in logutils package + - github.com/sirupsen/logrus + packages-with-error-message: + - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + goconst: + min-len: 2 + min-occurrences: 3 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - whyNoLint + - wrapperFunc + - unnecessaryBlock + gocyclo: + min-complexity: 15 + goimports: + local-prefixes: github.com/golangci/golangci-lint + + govet: + settings: + printf: + funcs: + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + misspell: + locale: US + nolintlint: + allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) + allow-unused: false # report any unused nolint directives + require-explanation: false # don't require an explanation for nolint directives + require-specific: false # don't require nolint directives to be specific about which linter is being skipped + +linters: + disable-all: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - errcheck + - exportloopref + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - goprintffuncname + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - noctx + - nolintlint + - staticcheck + - structcheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f92efa6c..6569fa52e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: go-fmt - id: go-imports - id: golangci-lint - args: [--timeout=5m] # - id: go-cyclo # args: [-over=20] - repo: local From ef8ac26375faa925a98c04d5ceb6bbba6a016362 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Tue, 22 Feb 2022 16:06:37 +0800 Subject: [PATCH 02/17] ci: add gosec --- .github/workflows/ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcc43dbb6..de56a42be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,18 @@ jobs: - name: Build run: make build + go-security-check: + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Checkout Source + uses: actions/checkout@v2 + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: ./... + lint: runs-on: ubuntu-latest env: From e263c9ccd275df78277f6aaa96ec429c13ee290d Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Tue, 22 Feb 2022 17:17:32 +0800 Subject: [PATCH 03/17] refactor: handled gosec reported warnings --- internal/database/dbutil/util.go | 16 ++++++++-------- internal/database/metadata/mysql/store.go | 2 +- internal/database/metadata/postgres/store.go | 2 +- internal/database/metadata/sqlite/store.go | 2 +- internal/database/offline/sqlutil/join.go | 2 +- internal/database/online/test_impl/open.go | 2 +- internal/database/online/test_impl/util.go | 10 +++++++--- oomagent/root.go | 8 ++++---- oomagent/test/client.go | 16 +++++++++++++--- oomcli/cmd/apply.go | 4 +++- oomcli/cmd/edit.go | 7 ++++--- oomcli/cmd/edit_entity.go | 4 +++- oomcli/cmd/edit_feature.go | 4 +++- oomcli/cmd/edit_group.go | 4 +++- oomcli/cmd/root.go | 2 +- pkg/oomstore/export.go | 4 +++- pkg/oomstore/import.go | 5 ++++- pkg/oomstore/join.go | 16 +++++++++++----- 18 files changed, 72 insertions(+), 38 deletions(-) diff --git a/internal/database/dbutil/util.go b/internal/database/dbutil/util.go index a03ca5a1e..c80b38c2b 100644 --- a/internal/database/dbutil/util.go +++ b/internal/database/dbutil/util.go @@ -1,18 +1,13 @@ package dbutil import ( + "crypto/rand" "fmt" - "math/rand" + "math/big" "strings" "time" ) -var random *rand.Rand - -func init() { - random = rand.New(rand.NewSource(time.Now().UnixNano())) -} - type RowMap = map[string]interface{} func TempTable(prefix string) string { @@ -21,10 +16,15 @@ func TempTable(prefix string) string { var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +func randInt(size int) int { + n, _ := rand.Int(rand.Reader, big.NewInt(int64(size))) + return int(n.Int64()) +} + func RandString(n int) string { b := make([]rune, n) for i := range b { - b[i] = letterRunes[random.Intn(len(letterRunes))] + b[i] = letterRunes[randInt(len(letterRunes))] } return string(b) } diff --git a/internal/database/metadata/mysql/store.go b/internal/database/metadata/mysql/store.go index e88354b7a..5abc47a2f 100644 --- a/internal/database/metadata/mysql/store.go +++ b/internal/database/metadata/mysql/store.go @@ -51,7 +51,7 @@ func Open(ctx context.Context, option *types.MySQLOpt) (*DB, error) { return sqlutil.ListMetadata(ctx, db) }) if err != nil { - db.Close() + _ = db.Close() return nil, err } return &DB{ diff --git a/internal/database/metadata/postgres/store.go b/internal/database/metadata/postgres/store.go index b142184e1..c9916ddfd 100644 --- a/internal/database/metadata/postgres/store.go +++ b/internal/database/metadata/postgres/store.go @@ -38,7 +38,7 @@ func Open(ctx context.Context, option *types.PostgresOpt) (*DB, error) { return sqlutil.ListMetadata(ctx, db) }) if err != nil { - db.Close() + _ = db.Close() return nil, err } return &DB{ diff --git a/internal/database/metadata/sqlite/store.go b/internal/database/metadata/sqlite/store.go index 8662245ee..5ad334e4b 100644 --- a/internal/database/metadata/sqlite/store.go +++ b/internal/database/metadata/sqlite/store.go @@ -37,7 +37,7 @@ func Open(ctx context.Context, opt *types.SQLiteOpt) (*DB, error) { return sqlutil.ListMetadata(ctx, db) }) if err != nil { - db.Close() + _ = db.Close() return nil, err } return &DB{ diff --git a/internal/database/offline/sqlutil/join.go b/internal/database/offline/sqlutil/join.go index 80acc0dca..82e678c62 100644 --- a/internal/database/offline/sqlutil/join.go +++ b/internal/database/offline/sqlutil/join.go @@ -316,7 +316,7 @@ func sqlxQueryResults(ctx context.Context, dbOpt dbutil.DBOpt, query string, hea } } - rows.Close() + _ = rows.Close() close(data) }() diff --git a/internal/database/online/test_impl/open.go b/internal/database/online/test_impl/open.go index 182849d1a..afe4a3eca 100644 --- a/internal/database/online/test_impl/open.go +++ b/internal/database/online/test_impl/open.go @@ -11,5 +11,5 @@ func TestOpen(t *testing.T, prepareStore PrepareStoreFn, destroystore DestroySto _, store := prepareStore(t) assert.NotNil(t, store) - store.Close() + assert.NoError(t, store.Close()) } diff --git a/internal/database/online/test_impl/util.go b/internal/database/online/test_impl/util.go index 3db208f77..0b60e1466 100644 --- a/internal/database/online/test_impl/util.go +++ b/internal/database/online/test_impl/util.go @@ -2,7 +2,8 @@ package test_impl import ( "context" - "math/rand" + "crypto/rand" + "math/big" "testing" "time" @@ -31,7 +32,6 @@ var SampleMedium Sample var SampleStream Sample func init() { - rand.Seed(time.Now().UnixNano()) entity := types.Entity{ID: 1, Name: "user"} group1 := types.Group{ID: 1, Category: types.CategoryBatch, Entity: &entity} group2 := types.Group{ID: 2, Category: types.CategoryBatch, Entity: &entity} @@ -97,7 +97,11 @@ func init() { var data []types.ExportRecord for i := 0; i < 100; i++ { - record := []interface{}{dbutil.RandString(10), rand.Float64()} + n, err := rand.Int(rand.Reader, big.NewInt(1000)) + if err != nil { + panic(err) + } + record := []interface{}{dbutil.RandString(10), float64(n.Int64())} data = append(data, types.ExportRecord{Record: record, Error: nil}) } SampleMedium = Sample{ diff --git a/oomagent/root.go b/oomagent/root.go index 65b1b03a3..5e6f73835 100644 --- a/oomagent/root.go +++ b/oomagent/root.go @@ -49,10 +49,10 @@ var rootCmd = &cobra.Command{ // write the listening address tmpdir := filepath.Join(os.TempDir(), "oomagent", strconv.Itoa(os.Getpid())) - if err := os.MkdirAll(tmpdir, 0755); err != nil { + if err := os.MkdirAll(tmpdir, 0750); err != nil { log.Fatalf("failed to create temp directory: %v", err) } - if err := os.WriteFile(filepath.Join(tmpdir, "address"), []byte(lis.Addr().String()), 0644); err != nil { + if err := os.WriteFile(filepath.Join(tmpdir, "address"), []byte(lis.Addr().String()), 0600); err != nil { log.Fatalf("failed to write listen address: %v", err) } @@ -64,7 +64,7 @@ var rootCmd = &cobra.Command{ go func() { <-exit grpcServer.GracefulStop() - oomstore.Close() + _ = oomstore.Close() os.Exit(0) }() @@ -102,7 +102,7 @@ func initConfig() { if envCfgFile := os.Getenv("OOMAGENT_CONFIG"); envCfgFile != "" { cfgFile = envCfgFile } - cfgContent, err := ioutil.ReadFile(cfgFile) + cfgContent, err := ioutil.ReadFile(filepath.Clean(cfgFile)) if err != nil { fmt.Fprintf(os.Stderr, "failed reading config file: %v\n", err) os.Exit(1) diff --git a/oomagent/test/client.go b/oomagent/test/client.go index 1c7f4cb8f..8088c06dc 100644 --- a/oomagent/test/client.go +++ b/oomagent/test/client.go @@ -8,8 +8,9 @@ import ( "os" "time" - "github.com/oom-ai/oomstore/oomagent/codegen" "google.golang.org/grpc" + + "github.com/oom-ai/oomstore/oomagent/codegen" "google.golang.org/grpc/credentials/insecure" ) @@ -66,7 +67,12 @@ func Import() { if err != nil { log.Fatal(err) } - defer file.Close() + defer func() { + if err = file.Close(); err != nil { + log.Fatal(err) + } + + }() fileScanner := bufio.NewScanner(file) groupName := "please input your group name" @@ -97,5 +103,9 @@ func prepareOomAgentClient(addr string) (c codegen.OomAgentClient, cancel func() if err != nil { log.Fatalf("did not connect: %v", err) } - return codegen.NewOomAgentClient(conn), func() { conn.Close() } + return codegen.NewOomAgentClient(conn), func() { + if err := conn.Close(); err != nil { + log.Fatal(err) + } + } } diff --git a/oomcli/cmd/apply.go b/oomcli/cmd/apply.go index 12da391c9..0ab6aa1c7 100644 --- a/oomcli/cmd/apply.go +++ b/oomcli/cmd/apply.go @@ -28,7 +28,9 @@ var applyCmd = &cobra.Command{ if err != nil { exit(err) } - defer file.Close() + defer func() { + _ = file.Close() + }() if err := oomStore.Apply(ctx, apply.ApplyOpt{R: file}); err != nil { exitf("apply failed: %+v", err) diff --git a/oomcli/cmd/edit.go b/oomcli/cmd/edit.go index 2c0cf0f61..b375681b1 100644 --- a/oomcli/cmd/edit.go +++ b/oomcli/cmd/edit.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "github.com/oom-ai/oomstore/pkg/errdefs" "github.com/spf13/cobra" @@ -59,13 +60,13 @@ func edit(ctx context.Context, oomStore *oomstore.OomStore, fileName string) err return err } - file, err := os.OpenFile(fileName, os.O_RDONLY, 0666) + file, err := os.OpenFile(filepath.Clean(fileName), os.O_RDONLY, 0600) if err != nil { return err } defer func() { - file.Close() - os.Remove(file.Name()) + _ = file.Close() + _ = os.Remove(file.Name()) }() if err := oomStore.Apply(ctx, apply.ApplyOpt{R: file}); err != nil { diff --git a/oomcli/cmd/edit_entity.go b/oomcli/cmd/edit_entity.go index c665ec6ee..133628f24 100644 --- a/oomcli/cmd/edit_entity.go +++ b/oomcli/cmd/edit_entity.go @@ -61,7 +61,9 @@ func writeEntitiesToTempFile(ctx context.Context, oomStore *oomstore.OomStore, e if err != nil { exit(err) } - defer tempFile.Close() + defer func() { + _ = tempFile.Close() + }() if err = outputEntity(ctx, entities, outputParams{ writer: tempFile, diff --git a/oomcli/cmd/edit_feature.go b/oomcli/cmd/edit_feature.go index 1ace89a55..93331f7e5 100644 --- a/oomcli/cmd/edit_feature.go +++ b/oomcli/cmd/edit_feature.go @@ -67,7 +67,9 @@ func writeFeaturesToTempFile(features types.FeatureList) (string, error) { if err != nil { return "", err } - defer tempFile.Close() + defer func() { + _ = tempFile.Close() + }() if err = outputFeature(features, outputParams{ writer: tempFile, diff --git a/oomcli/cmd/edit_group.go b/oomcli/cmd/edit_group.go index bd98ffb61..f08b1e9c7 100644 --- a/oomcli/cmd/edit_group.go +++ b/oomcli/cmd/edit_group.go @@ -71,7 +71,9 @@ func writeGroupsToTempFile(ctx context.Context, oomStore *oomstore.OomStore, gro if err != nil { return "", err } - defer tempFile.Close() + defer func() { + _ = tempFile.Close() + }() if err = outputGroup(ctx, groups, outputParams{ writer: tempFile, diff --git a/oomcli/cmd/root.go b/oomcli/cmd/root.go index 924524249..dc308d08b 100644 --- a/oomcli/cmd/root.go +++ b/oomcli/cmd/root.go @@ -52,7 +52,7 @@ func initConfig() { if envCfgFile := os.Getenv("OOMCLI_CONFIG"); envCfgFile != "" { cfgFile = envCfgFile } - cfgContent, err := ioutil.ReadFile(cfgFile) + cfgContent, err := ioutil.ReadFile(filepath.Clean(cfgFile)) if err != nil { exitf("failed reading config file: %v\n", err) } diff --git a/pkg/oomstore/export.go b/pkg/oomstore/export.go index 3bd725ad9..9998527f6 100644 --- a/pkg/oomstore/export.go +++ b/pkg/oomstore/export.go @@ -106,7 +106,9 @@ func (s *OomStore) Export(ctx context.Context, opt types.ExportOpt) error { if err != nil { return err } - defer file.Close() + defer func() { + _ = file.Close() + }() w := csv.NewWriter(file) defer w.Flush() diff --git a/pkg/oomstore/import.go b/pkg/oomstore/import.go index f21657ea0..2a4e99949 100644 --- a/pkg/oomstore/import.go +++ b/pkg/oomstore/import.go @@ -26,7 +26,10 @@ func (s *OomStore) Import(ctx context.Context, opt types.ImportOpt) (int, error) if err != nil { return 0, err } - defer file.Close() + defer func() { + _ = file.Close() + }() + source := &types.CsvReaderDataSource{ Reader: file, Delimiter: src.Delimiter, diff --git a/pkg/oomstore/join.go b/pkg/oomstore/join.go index c99f264b7..7f2ed2298 100644 --- a/pkg/oomstore/join.go +++ b/pkg/oomstore/join.go @@ -6,6 +6,7 @@ import ( "io" "math" "os" + "path/filepath" "sort" "strconv" @@ -142,7 +143,7 @@ func (s *OomStore) buildRevisionRanges(ctx context.Context, group *types.Group) } func GetEntityRowsFromInputFile(ctx context.Context, inputFilePath string) (<-chan types.EntityRow, []string, error) { - input, err := os.Open(inputFilePath) + input, err := os.Open(filepath.Clean(inputFilePath)) if err != nil { return nil, nil, errdefs.WithStack(err) } @@ -154,8 +155,10 @@ func GetEntityRowsFromInputFile(ctx context.Context, inputFilePath string) (<-ch entityRows := make(chan types.EntityRow) go func() { - defer close(entityRows) - defer input.Close() + defer func() { + _ = input.Close() + close(entityRows) + }() for i := 1; ; i++ { line, err := reader.Read() @@ -208,11 +211,14 @@ func GetEntityRowsFromInputFile(ctx context.Context, inputFilePath string) (<-ch } func writeJoinResultToFile(outputFilePath string, joinResult *types.JoinResult) error { - file, err := os.Create(outputFilePath) + file, err := os.Create(filepath.Clean(outputFilePath)) if err != nil { return err } - defer file.Close() + defer func() { + _ = file.Close() + }() + w := csv.NewWriter(file) defer w.Flush() From 4abf0c9cbf3acdc0b244e0d5ee7105e053c9e399 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Tue, 22 Feb 2022 18:01:52 +0800 Subject: [PATCH 04/17] fix: declaration of "err" shadows declaration --- internal/database/metadata/sqlutil/feature.go | 4 +-- internal/database/offline/postgres/util.go | 20 ++++++------- internal/database/offline/sqlutil/join.go | 3 +- .../database/offline/sqlutil/join_helper.go | 6 ++-- internal/database/online/dynamodb/push.go | 12 ++++---- internal/database/online/redis/get.go | 6 ++-- internal/database/online/redis/import.go | 12 ++++---- internal/database/online/tikv/get.go | 12 ++++---- internal/database/online/tikv/import.go | 10 +++---- internal/database/online/tikv/push.go | 12 ++++---- oomagent/server.go | 30 +++++++++++-------- oomagent/test/client.go | 8 ++--- pkg/oomstore/apply.go | 21 +++++++------ pkg/oomstore/sync.go | 6 ++-- 14 files changed, 84 insertions(+), 78 deletions(-) diff --git a/internal/database/metadata/sqlutil/feature.go b/internal/database/metadata/sqlutil/feature.go index a0f7f7e12..98a757f64 100644 --- a/internal/database/metadata/sqlutil/feature.go +++ b/internal/database/metadata/sqlutil/feature.go @@ -40,7 +40,7 @@ func GetFeature(ctx context.Context, sqlxCtx metadata.SqlxContext, id int) (*typ ) query := `SELECT * FROM feature WHERE id = ?` - if err := sqlxCtx.GetContext(ctx, &feature, sqlxCtx.Rebind(query), id); err != nil { + if err = sqlxCtx.GetContext(ctx, &feature, sqlxCtx.Rebind(query), id); err != nil { if err == sql.ErrNoRows { return nil, errdefs.NotFound(errdefs.Errorf("feature %d not found", id)) } @@ -93,7 +93,7 @@ func ListFeature(ctx context.Context, sqlxCtx metadata.SqlxContext, opt metadata query = fmt.Sprintf("%s WHERE %s", query, strings.Join(cond, " AND ")) } query = fmt.Sprintf("%s ORDER BY id ASC", query) - if err := sqlxCtx.SelectContext(ctx, &features, sqlxCtx.Rebind(query), args...); err != nil { + if err = sqlxCtx.SelectContext(ctx, &features, sqlxCtx.Rebind(query), args...); err != nil { return nil, errdefs.WithStack(err) } diff --git a/internal/database/offline/postgres/util.go b/internal/database/offline/postgres/util.go index ecb4c41b0..b341a042c 100644 --- a/internal/database/offline/postgres/util.go +++ b/internal/database/offline/postgres/util.go @@ -4,11 +4,11 @@ import ( "context" "io" - "github.com/oom-ai/oomstore/pkg/errdefs" - "github.com/jmoiron/sqlx" "github.com/lib/pq" + "github.com/oom-ai/oomstore/internal/database/dbutil" + "github.com/oom-ai/oomstore/pkg/errdefs" ) func loadDataFromSource(tx *sqlx.Tx, ctx context.Context, opt dbutil.LoadDataFromSourceOpt) error { @@ -19,23 +19,23 @@ func loadDataFromSource(tx *sqlx.Tx, ctx context.Context, opt dbutil.LoadDataFro defer stmt.Close() for { - record, err := dbutil.ReadLine(dbutil.ReadLineOpt{ + record, err2 := dbutil.ReadLine(dbutil.ReadLineOpt{ Source: opt.Source, EntityName: opt.EntityName, Header: opt.Header, Features: opt.Features, }) - if errdefs.Cause(err) == io.EOF { - break - } - if err != nil { - return err + if err2 != nil { + if errdefs.Cause(err2) == io.EOF { + break + } + return err2 } if len(record) != len(opt.Header) { continue } - if _, err := stmt.ExecContext(ctx, record...); err != nil { - return errdefs.WithStack(err) + if _, err2 := stmt.ExecContext(ctx, record...); err2 != nil { + return errdefs.WithStack(err2) } } diff --git a/internal/database/offline/sqlutil/join.go b/internal/database/offline/sqlutil/join.go index 82e678c62..4116434ba 100644 --- a/internal/database/offline/sqlutil/join.go +++ b/internal/database/offline/sqlutil/join.go @@ -148,7 +148,8 @@ func joinOneGroup(ctx context.Context, dbOpt dbutil.DBOpt, opt joinOneGroupOpt) if *opt.TimeRange.MaxUnixMilli < r.MinRevision || *opt.TimeRange.MinUnixMilli > r.MaxRevision { continue } - query, err := buildJoinQuery(joinQueryParams{ + var query string + query, err = buildJoinQuery(joinQueryParams{ TableName: snapshotJoinedTableName, EntityName: opt.EntityName, EntityKey: "entity_key", diff --git a/internal/database/offline/sqlutil/join_helper.go b/internal/database/offline/sqlutil/join_helper.go index ee97e28cf..c4f1dc50c 100644 --- a/internal/database/offline/sqlutil/join_helper.go +++ b/internal/database/offline/sqlutil/join_helper.go @@ -92,9 +92,9 @@ func prepareJoinedTable( return "", err } for _, f := range features { - dbValueType, err := dbutil.DBValueType(dbOpt.Backend, f.ValueType) - if err != nil { - return "", err + dbValueType, err2 := dbutil.DBValueType(dbOpt.Backend, f.ValueType) + if err2 != nil { + return "", err2 } columnDefs = append(columnDefs, fmt.Sprintf(columnFormat, f.Name, dbValueType)) diff --git a/internal/database/online/dynamodb/push.go b/internal/database/online/dynamodb/push.go index 55df61c5e..ab468e1ee 100644 --- a/internal/database/online/dynamodb/push.go +++ b/internal/database/online/dynamodb/push.go @@ -26,13 +26,13 @@ func (db *DB) Push(ctx context.Context, opt online.PushOpt) error { item[opt.EntityName] = entityKeyValue for i, feature := range opt.Features { - value, err := dbutil.SerializeByValueType(opt.FeatureValues[i], feature.ValueType, Backend) - if err != nil { - return err + value, err2 := dbutil.SerializeByValueType(opt.FeatureValues[i], feature.ValueType, Backend) + if err2 != nil { + return err2 } - attributeValue, err := attributevalue.Marshal(value) - if err != nil { - return errdefs.WithStack(err) + attributeValue, err2 := attributevalue.Marshal(value) + if err2 != nil { + return errdefs.WithStack(err2) } item[feature.Name] = attributeValue } diff --git a/internal/database/online/redis/get.go b/internal/database/online/redis/get.go index 55fee975d..5459f1eac 100644 --- a/internal/database/online/redis/get.go +++ b/internal/database/online/redis/get.go @@ -31,9 +31,9 @@ func (db *DB) Get(ctx context.Context, opt online.GetOpt) (dbutil.RowMap, error) featureIDs := []string{} for _, f := range opt.Features { - id, err := dbutil.SerializeByValue(f.ID, Backend) - if err != nil { - return nil, err + id, err2 := dbutil.SerializeByValue(f.ID, Backend) + if err2 != nil { + return nil, err2 } featureIDs = append(featureIDs, id) } diff --git a/internal/database/online/redis/import.go b/internal/database/online/redis/import.go index 5bfb91de9..4c5507c7b 100644 --- a/internal/database/online/redis/import.go +++ b/internal/database/online/redis/import.go @@ -41,14 +41,14 @@ func (db *DB) Import(ctx context.Context, opt online.ImportOpt) error { if values[i] == nil { continue } - featureValue, err := dbutil.SerializeByValueType(values[i], opt.Features[i].ValueType, Backend) - if err != nil { - return err + featureValue, err2 := dbutil.SerializeByValueType(values[i], opt.Features[i].ValueType, Backend) + if err2 != nil { + return err2 } - featureID, err := dbutil.SerializeByValue(opt.Features[i].ID, Backend) - if err != nil { - return err + featureID, err2 := dbutil.SerializeByValue(opt.Features[i].ID, Backend) + if err2 != nil { + return err2 } featureValues[featureID] = featureValue.(string) } diff --git a/internal/database/online/tikv/get.go b/internal/database/online/tikv/get.go index 8a5b52b57..85a07a25b 100644 --- a/internal/database/online/tikv/get.go +++ b/internal/database/online/tikv/get.go @@ -58,18 +58,18 @@ func (db *DB) MultiGet(ctx context.Context, opt online.MultiGetOpt) (map[string] var serializedEntityKeys []string for _, entityKey := range opt.EntityKeys { - serializedEntityKey, err := dbutil.SerializeByValue(entityKey, Backend) - if err != nil { - return nil, err + serializedEntityKey, err2 := dbutil.SerializeByValue(entityKey, Backend) + if err2 != nil { + return nil, err2 } serializedEntityKeys = append(serializedEntityKeys, serializedEntityKey) } var serializedFeatureIDs []string for _, feature := range opt.Features { - serializedFeatureID, err := dbutil.SerializeByValue(feature.ID, Backend) - if err != nil { - return nil, err + serializedFeatureID, err2 := dbutil.SerializeByValue(feature.ID, Backend) + if err2 != nil { + return nil, err2 } serializedFeatureIDs = append(serializedFeatureIDs, serializedFeatureID) } diff --git a/internal/database/online/tikv/import.go b/internal/database/online/tikv/import.go index 7c60c8dba..a16808a1c 100644 --- a/internal/database/online/tikv/import.go +++ b/internal/database/online/tikv/import.go @@ -29,9 +29,9 @@ func (db *DB) Import(ctx context.Context, opt online.ImportOpt) error { var serializedFeatureIDs []string for _, feature := range opt.Features { - serializedFeatureID, err := dbutil.SerializeByValue(feature.ID, Backend) - if err != nil { - return err + serializedFeatureID, err2 := dbutil.SerializeByValue(feature.ID, Backend) + if err2 != nil { + return err2 } serializedFeatureIDs = append(serializedFeatureIDs, serializedFeatureID) } @@ -62,8 +62,8 @@ func (db *DB) Import(ctx context.Context, opt online.ImportOpt) error { continue } - serializedFeatureValue, err := dbutil.SerializeByValueType(featureValues[i], opt.Features[i].ValueType, Backend) - if err != nil { + serializedFeatureValue, err2 := dbutil.SerializeByValueType(featureValues[i], opt.Features[i].ValueType, Backend) + if err2 != nil { return err } if opt.Group.Category == types.CategoryBatch { diff --git a/internal/database/online/tikv/push.go b/internal/database/online/tikv/push.go index 2d6744fba..ff1b2c854 100644 --- a/internal/database/online/tikv/push.go +++ b/internal/database/online/tikv/push.go @@ -28,14 +28,14 @@ func (db *DB) Push(ctx context.Context, opt online.PushOpt) error { continue } - serializedFeatureID, err := dbutil.SerializeByValue(opt.Features[i].ID, Backend) - if err != nil { - return err + serializedFeatureID, err2 := dbutil.SerializeByValue(opt.Features[i].ID, Backend) + if err2 != nil { + return err2 } - serializedFeatureValue, err := dbutil.SerializeByValueType(value, opt.Features[i].ValueType, Backend) - if err != nil { - return err + serializedFeatureValue, err2 := dbutil.SerializeByValueType(value, opt.Features[i].ValueType, Backend) + if err2 != nil { + return err2 } putKeys = append(putKeys, getKeyOfStreamFeature(serializedGroupID, serializedEntityKey, serializedFeatureID)) diff --git a/oomagent/server.go b/oomagent/server.go index 549b62b07..1f2f7dd42 100644 --- a/oomagent/server.go +++ b/oomagent/server.go @@ -110,20 +110,26 @@ func (s *server) ChannelImport(stream codegen.OomAgent_ChannelImportServer) erro _ = writer.Close() }() - if _, err := writer.Write(firstReq.Row); err != nil { + if _, err2 := writer.Write(firstReq.Row); err2 != nil { + if err2 != io.ErrClosedPipe { + log.Println(err2) + } return } for { - req, err := stream.Recv() - if err == io.EOF { - break - } - if err != nil { - log.Println(err) + req, err2 := stream.Recv() + if err2 != nil { + if err2 != io.EOF { + log.Println(err2) + } break } - if _, err := writer.Write(req.Row); err != nil { + + if _, err2 := writer.Write(req.Row); err2 != nil { + if err2 != io.ErrClosedPipe { + log.Println(err2) + } return } } @@ -224,13 +230,13 @@ func (s *server) ChannelJoin(stream codegen.OomAgent_ChannelJoinServer) error { Values: firstReq.EntityRow.Values, } for { - req, err := stream.Recv() - if err == io.EOF { + req, err2 := stream.Recv() + if err2 == io.EOF { return } - if err != nil { + if err2 != nil { select { - case entityRows <- types.EntityRow{Error: err}: + case entityRows <- types.EntityRow{Error: err2}: return case <-ctx.Done(): return diff --git a/oomagent/test/client.go b/oomagent/test/client.go index 8088c06dc..433d6cc77 100644 --- a/oomagent/test/client.go +++ b/oomagent/test/client.go @@ -78,16 +78,16 @@ func Import() { groupName := "please input your group name" description := "please input you description" for fileScanner.Scan() { - if err := importClient.Send(&codegen.ChannelImportRequest{ + if err2 := importClient.Send(&codegen.ChannelImportRequest{ Group: &groupName, Description: &description, Row: fileScanner.Bytes(), - }); err != nil { - log.Fatal(err) + }); err2 != nil { + log.Fatal(err2) } } - if err := fileScanner.Err(); err != nil { + if err = fileScanner.Err(); err != nil { log.Fatal(err) } diff --git a/pkg/oomstore/apply.go b/pkg/oomstore/apply.go index d58dba3ad..a01ae2295 100644 --- a/pkg/oomstore/apply.go +++ b/pkg/oomstore/apply.go @@ -4,12 +4,11 @@ import ( "context" "io" - "github.com/oom-ai/oomstore/internal/database/dbutil" - "github.com/mitchellh/mapstructure" "github.com/spf13/cast" "gopkg.in/yaml.v3" + "github.com/oom-ai/oomstore/internal/database/dbutil" "github.com/oom-ai/oomstore/internal/database/metadata" "github.com/oom-ai/oomstore/internal/database/online" "github.com/oom-ai/oomstore/pkg/errdefs" @@ -25,19 +24,19 @@ func (s *OomStore) Apply(ctx context.Context, opt apply.ApplyOpt) error { } onlineJobs := make([]func() error, 0) - if err := s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { + if err = s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { // apply entity for _, entity := range stage.NewEntities { - if err := s.applyEntity(c, tx, entity); err != nil { - return err + if err2 := s.applyEntity(c, tx, entity); err2 != nil { + return err2 } } // apply group for _, group := range stage.NewGroups { - onlineJob, err := s.applyGroup(c, tx, group) - if err != nil { - return err + onlineJob, err2 := s.applyGroup(c, tx, group) + if err2 != nil { + return err2 } if onlineJob != nil { onlineJobs = append(onlineJobs, onlineJob) @@ -46,9 +45,9 @@ func (s *OomStore) Apply(ctx context.Context, opt apply.ApplyOpt) error { // apply feature for _, feature := range stage.NewFeatures { - onlineJob, err := s.applyFeature(c, tx, feature) - if err != nil { - return err + onlineJob, err2 := s.applyFeature(c, tx, feature) + if err2 != nil { + return err2 } if onlineJob != nil { onlineJobs = append(onlineJobs, onlineJob) diff --git a/pkg/oomstore/sync.go b/pkg/oomstore/sync.go index a536754ce..973b5cd4c 100644 --- a/pkg/oomstore/sync.go +++ b/pkg/oomstore/sync.go @@ -66,11 +66,11 @@ func (s *OomStore) syncBatch(ctx context.Context, opt types.SyncOpt, group *type if err = s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { // Update the online revision id of the feature group upon sync success - if err := tx.UpdateGroup(c, metadata.UpdateGroupOpt{ + if err2 := tx.UpdateGroup(c, metadata.UpdateGroupOpt{ GroupID: group.ID, NewOnlineRevisionID: &revision.ID, - }); err != nil { - return err + }); err2 != nil { + return err2 } if !revision.Anchored { newRevision := time.Now().UnixMilli() From 89dc4374d168ece84987c76b3ff3a40fd708e1a2 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Tue, 22 Feb 2022 18:03:06 +0800 Subject: [PATCH 05/17] fix(offline/test): fix description typo --- internal/database/offline/test_impl/create_table.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/database/offline/test_impl/create_table.go b/internal/database/offline/test_impl/create_table.go index 2c77b3955..e3a53c352 100644 --- a/internal/database/offline/test_impl/create_table.go +++ b/internal/database/offline/test_impl/create_table.go @@ -28,12 +28,12 @@ func TestCreateTable(t *testing.T, prepareStore PrepareStoreFn, destroyStore Des } testCases := []struct { - describtion string + description string opt offline.CreateTableOpt expectedError error }{ { - describtion: "cdc table (with unix_milli)", + description: "cdc table (with unix_milli)", opt: offline.CreateTableOpt{ TableName: tableName + "_0", EntityName: entity.Name, @@ -42,7 +42,7 @@ func TestCreateTable(t *testing.T, prepareStore PrepareStoreFn, destroyStore Des }, }, { - describtion: "stream snapshot table (with unix_milli)", + description: "stream snapshot table (with unix_milli)", opt: offline.CreateTableOpt{ TableName: tableName + "_1", EntityName: entity.Name, @@ -51,7 +51,7 @@ func TestCreateTable(t *testing.T, prepareStore PrepareStoreFn, destroyStore Des }, }, { - describtion: "batch snapshot table (without unix_milli)", + description: "batch snapshot table (without unix_milli)", opt: offline.CreateTableOpt{ TableName: tableName + "_2", EntityName: entity.Name, @@ -62,7 +62,7 @@ func TestCreateTable(t *testing.T, prepareStore PrepareStoreFn, destroyStore Des } for _, testCase := range testCases { - t.Run(testCase.describtion, func(t *testing.T) { + t.Run(testCase.description, func(t *testing.T) { err := store.CreateTable(ctx, testCase.opt) require.NoError(t, err) }) From 9e1c040e3c608fe84fe8aaac37398b0eda18e5d3 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Tue, 22 Feb 2022 18:22:32 +0800 Subject: [PATCH 06/17] refactor: remove unnecessary trailing newline --- internal/database/metadata/sqlite/tx.go | 1 - internal/database/offline/bigquery/store_test.go | 1 - internal/database/offline/postgres/store.go | 1 - internal/database/offline/snowflake/store_test.go | 1 - internal/database/offline/sqlite/store_test.go | 1 - internal/database/offline/sqlutil/join_helper.go | 1 - internal/database/online/test_impl/get.go | 2 -- oomagent/test/client.go | 1 - oomcli/cmd/register_group.go | 1 - pkg/oomstore/apply.go | 1 - 10 files changed, 11 deletions(-) diff --git a/internal/database/metadata/sqlite/tx.go b/internal/database/metadata/sqlite/tx.go index 4d777198f..3ad61d7c2 100644 --- a/internal/database/metadata/sqlite/tx.go +++ b/internal/database/metadata/sqlite/tx.go @@ -70,7 +70,6 @@ func (tx *Tx) ListFeature(ctx context.Context, opt metadata.ListFeatureOpt) (typ func (tx *Tx) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, tx, groupName, featureName) - } func (tx *Tx) CreateRevision(ctx context.Context, opt metadata.CreateRevisionOpt) (int, error) { diff --git a/internal/database/offline/bigquery/store_test.go b/internal/database/offline/bigquery/store_test.go index cecd3a2c6..3fbcfa77e 100644 --- a/internal/database/offline/bigquery/store_test.go +++ b/internal/database/offline/bigquery/store_test.go @@ -64,7 +64,6 @@ func destroyStore(datasetID string) func() { if err := db.Dataset(DATASET_ID).DeleteWithContents(context.Background()); err != nil { panic(err) } - } } diff --git a/internal/database/offline/postgres/store.go b/internal/database/offline/postgres/store.go index 0aa502ff9..f7951e3bd 100644 --- a/internal/database/offline/postgres/store.go +++ b/internal/database/offline/postgres/store.go @@ -36,7 +36,6 @@ func (db *DB) Import(ctx context.Context, opt offline.ImportOpt) (int64, error) func (db *DB) Export(ctx context.Context, opt offline.ExportOpt) (*types.ExportResult, error) { return sqlutil.Export(ctx, db.DB, opt, Backend) - } func (db *DB) Join(ctx context.Context, opt offline.JoinOpt) (*types.JoinResult, error) { diff --git a/internal/database/offline/snowflake/store_test.go b/internal/database/offline/snowflake/store_test.go index 0855be67e..eba02a95f 100644 --- a/internal/database/offline/snowflake/store_test.go +++ b/internal/database/offline/snowflake/store_test.go @@ -120,7 +120,6 @@ func TestTableSchema(t *testing.T) { } require.ElementsMatch(t, expected.Fields, actual.Fields) require.Equal(t, expected.TimeRange, actual.TimeRange) - } func int64Ptr(i int64) *int64 { diff --git a/internal/database/offline/sqlite/store_test.go b/internal/database/offline/sqlite/store_test.go index 94af41560..6620e3f2d 100644 --- a/internal/database/offline/sqlite/store_test.go +++ b/internal/database/offline/sqlite/store_test.go @@ -33,7 +33,6 @@ func destroyStore() { if err := os.RemoveAll(DATABASE); err != nil { panic(err) } - } func TestPing(t *testing.T) { diff --git a/internal/database/offline/sqlutil/join_helper.go b/internal/database/offline/sqlutil/join_helper.go index c4f1dc50c..9c097058c 100644 --- a/internal/database/offline/sqlutil/join_helper.go +++ b/internal/database/offline/sqlutil/join_helper.go @@ -97,7 +97,6 @@ func prepareJoinedTable( return "", err2 } columnDefs = append(columnDefs, fmt.Sprintf(columnFormat, f.Name, dbValueType)) - } schema := ` CREATE TABLE %s ( diff --git a/internal/database/online/test_impl/get.go b/internal/database/online/test_impl/get.go index 338c625ea..a348468e1 100644 --- a/internal/database/online/test_impl/get.go +++ b/internal/database/online/test_impl/get.go @@ -62,9 +62,7 @@ func TestGetNoRevision(t *testing.T, prepareStore PrepareStoreFn, destroyStore D require.EqualError(t, err, "invalid MultiGetOpt: the revisionID of get batch feature cannot be null") require.Nil(t, rs) } - }) - } func TestGetNotExistedEntityKey(t *testing.T, prepareStore PrepareStoreFn, destroystore DestroyStoreFn) { diff --git a/oomagent/test/client.go b/oomagent/test/client.go index 433d6cc77..8bbc2a9a9 100644 --- a/oomagent/test/client.go +++ b/oomagent/test/client.go @@ -71,7 +71,6 @@ func Import() { if err = file.Close(); err != nil { log.Fatal(err) } - }() fileScanner := bufio.NewScanner(file) diff --git a/oomcli/cmd/register_group.go b/oomcli/cmd/register_group.go index 45daa396c..3194d8f53 100644 --- a/oomcli/cmd/register_group.go +++ b/oomcli/cmd/register_group.go @@ -59,5 +59,4 @@ func init() { flags.StringVarP(®isterGroupOpt.snapshotInterval, "snapshot_interval", "s", "24h", "stream group snapshot interval") flags.StringVarP(®isterGroupOpt.Description, "description", "d", "", "group description") - } diff --git a/pkg/oomstore/apply.go b/pkg/oomstore/apply.go index a01ae2295..47a0b9035 100644 --- a/pkg/oomstore/apply.go +++ b/pkg/oomstore/apply.go @@ -145,7 +145,6 @@ func (s *OomStore) applyGroup(ctx context.Context, tx metadata.DBStore, newGroup NewSnapshotInterval: opt.NewSnapshotInterval, NewDescription: &newGroup.Description, }) - } return nil, nil From 03499d15cdd9cfaf109cc3ee52bf321e749624d7 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Wed, 23 Feb 2022 10:15:40 +0800 Subject: [PATCH 07/17] refactor: giving a name to function results --- internal/database/dbutil/db_opt.go | 4 ++-- internal/database/dbutil/sql.go | 8 ++++---- internal/database/metadata/sqlutil/feature.go | 2 +- internal/database/metadata/sqlutil/group.go | 11 +++++------ internal/database/metadata/test_impl/feature.go | 4 ++-- internal/database/metadata/test_impl/revision.go | 9 +++++---- internal/database/offline/sqlutil/join_helper.go | 4 ++-- internal/database/offline/test_impl/join.go | 2 +- internal/database/online/cassandra/get.go | 8 +++----- pkg/oomstore/join.go | 4 ++-- pkg/oomstore/util/feature.go | 2 +- 11 files changed, 28 insertions(+), 30 deletions(-) diff --git a/internal/database/dbutil/db_opt.go b/internal/database/dbutil/db_opt.go index 86ba4c876..6658c7094 100644 --- a/internal/database/dbutil/db_opt.go +++ b/internal/database/dbutil/db_opt.go @@ -44,7 +44,7 @@ func (d *DBOpt) ExecContext(ctx context.Context, query string, args ...interface } } -func (d *DBOpt) BuildInsertQuery(tableName string, records []interface{}, columns []string) (string, []interface{}, error) { +func (d *DBOpt) BuildInsertQuery(tableName string, records []interface{}, columns []string) (query string, args []interface{}, err error) { if len(records) == 0 { return "", nil, nil } @@ -60,7 +60,7 @@ func (d *DBOpt) BuildInsertQuery(tableName string, records []interface{}, column if d.Backend == types.BackendBigQuery { tableName = fmt.Sprintf("%s.%s", *d.DatasetID, tableName) } - query, args, err := sqlx.In( + query, args, err = sqlx.In( fmt.Sprintf(`INSERT INTO %s (%s) VALUES %s`, tableName, columnStr, strings.Join(valueFlags, ",")), records...) return query, args, errdefs.WithStack(err) diff --git a/internal/database/dbutil/sql.go b/internal/database/dbutil/sql.go index ceffae9a8..7411ddbbe 100644 --- a/internal/database/dbutil/sql.go +++ b/internal/database/dbutil/sql.go @@ -6,14 +6,14 @@ import ( "strings" "github.com/jmoiron/sqlx" - "github.com/oom-ai/oomstore/pkg/errdefs" + "github.com/oom-ai/oomstore/pkg/errdefs" "github.com/oom-ai/oomstore/pkg/oomstore/types" ) -func BuildConditions(equal map[string]interface{}, in map[string]interface{}) ([]string, []interface{}, error) { - cond := make([]string, 0) - args := make([]interface{}, 0) +func BuildConditions(equal map[string]interface{}, in map[string]interface{}) (cond []string, args []interface{}, err error) { + cond = make([]string, 0) + args = make([]interface{}, 0) for key, value := range equal { cond = append(cond, fmt.Sprintf("%s = ?", key)) args = append(args, value) diff --git a/internal/database/metadata/sqlutil/feature.go b/internal/database/metadata/sqlutil/feature.go index 98a757f64..b5fd2f4bb 100644 --- a/internal/database/metadata/sqlutil/feature.go +++ b/internal/database/metadata/sqlutil/feature.go @@ -127,7 +127,7 @@ func ListFeature(ctx context.Context, sqlxCtx metadata.SqlxContext, opt metadata return features, nil } -func buildListFeatureCond(opt metadata.ListFeatureOpt) ([]string, []interface{}, error) { +func buildListFeatureCond(opt metadata.ListFeatureOpt) (cond []string, args []interface{}, err error) { in := make(map[string]interface{}) and := make(map[string]interface{}) diff --git a/internal/database/metadata/sqlutil/group.go b/internal/database/metadata/sqlutil/group.go index 734895bed..42b4019c9 100644 --- a/internal/database/metadata/sqlutil/group.go +++ b/internal/database/metadata/sqlutil/group.go @@ -107,9 +107,8 @@ func ListGroup(ctx context.Context, sqlxCtx metadata.SqlxContext, opt metadata.L return groups, nil } -func buildListGroupCond(opt metadata.ListGroupOpt) (string, []interface{}, error) { - args := make([]interface{}, 0) - cond := make([]string, 0) +func buildListGroupCond(opt metadata.ListGroupOpt) (cond string, args []interface{}, err error) { + conds := make([]string, 0) if opt.EntityIDs != nil { if len(*opt.EntityIDs) == 0 { @@ -119,7 +118,7 @@ func buildListGroupCond(opt metadata.ListGroupOpt) (string, []interface{}, error if err != nil { return "", nil, errdefs.WithStack(err) } - cond = append(cond, s) + conds = append(conds, s) args = append(args, inArgs...) } if opt.GroupIDs != nil { @@ -130,10 +129,10 @@ func buildListGroupCond(opt metadata.ListGroupOpt) (string, []interface{}, error if err != nil { return "", nil, errdefs.WithStack(err) } - cond = append(cond, s) + conds = append(conds, s) args = append(args, inArgs...) } - return strings.Join(cond, " AND "), args, nil + return strings.Join(conds, " AND "), args, nil } func enrichGroups(ctx context.Context, sqlxCtx metadata.SqlxContext, groups types.GroupList) error { diff --git a/internal/database/metadata/test_impl/feature.go b/internal/database/metadata/test_impl/feature.go index 1248f066b..44b50d649 100644 --- a/internal/database/metadata/test_impl/feature.go +++ b/internal/database/metadata/test_impl/feature.go @@ -12,7 +12,7 @@ import ( "github.com/oom-ai/oomstore/pkg/oomstore/types" ) -func prepareEntityAndGroup(t *testing.T, ctx context.Context, store metadata.Store) (int, int) { +func prepareEntityAndGroup(t *testing.T, ctx context.Context, store metadata.Store) (entityID int, groupID int) { entityID, err := store.CreateEntity(ctx, metadata.CreateEntityOpt{ CreateEntityOpt: types.CreateEntityOpt{ EntityName: "device", @@ -21,7 +21,7 @@ func prepareEntityAndGroup(t *testing.T, ctx context.Context, store metadata.Sto }) require.NoError(t, err) - groupID, err := store.CreateGroup(ctx, metadata.CreateGroupOpt{ + groupID, err = store.CreateGroup(ctx, metadata.CreateGroupOpt{ GroupName: "device_info", EntityID: entityID, Description: "description", diff --git a/internal/database/metadata/test_impl/revision.go b/internal/database/metadata/test_impl/revision.go index b0f2b1e32..39eb831fc 100644 --- a/internal/database/metadata/test_impl/revision.go +++ b/internal/database/metadata/test_impl/revision.go @@ -6,10 +6,11 @@ import ( "testing" "time" - "github.com/oom-ai/oomstore/internal/database/metadata" - "github.com/oom-ai/oomstore/pkg/oomstore/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/oom-ai/oomstore/internal/database/metadata" + "github.com/oom-ai/oomstore/pkg/oomstore/types" ) func TestCreateRevision(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyStoreFn) { @@ -373,7 +374,7 @@ func ignoreCreateAndModifyTime(revision *types.Revision) { revision.ModifyTime = time.Time{} } -func prepareRevisions(t *testing.T, ctx context.Context, store metadata.Store) (int, int, []int, types.RevisionList) { +func prepareRevisions(t *testing.T, ctx context.Context, store metadata.Store) (entityID int, groupID int, revisionIDs []int, revisions types.RevisionList) { entityID, err := store.CreateEntity(ctx, metadata.CreateEntityOpt{ CreateEntityOpt: types.CreateEntityOpt{ EntityName: "device", @@ -382,7 +383,7 @@ func prepareRevisions(t *testing.T, ctx context.Context, store metadata.Store) ( }) require.NoError(t, err) - groupID, err := store.CreateGroup(ctx, metadata.CreateGroupOpt{ + groupID, err = store.CreateGroup(ctx, metadata.CreateGroupOpt{ GroupName: "device_info", EntityID: entityID, Description: "description", diff --git a/internal/database/offline/sqlutil/join_helper.go b/internal/database/offline/sqlutil/join_helper.go index 9c097058c..f17382d67 100644 --- a/internal/database/offline/sqlutil/join_helper.go +++ b/internal/database/offline/sqlutil/join_helper.go @@ -180,7 +180,7 @@ func buildTableName(dbOpt dbutil.DBOpt, tableName string) string { } } -func prepareTableSchema(dbOpt dbutil.DBOpt, params prepareTableSchemaParams) (string, []string, error) { +func prepareTableSchema(dbOpt dbutil.DBOpt, params prepareTableSchemaParams) (tableName string, columnDefs []string, err error) { columnFormat, err := dbutil.GetColumnFormat(dbOpt.Backend) if err != nil { return "", nil, err @@ -199,7 +199,7 @@ func prepareTableSchema(dbOpt dbutil.DBOpt, params prepareTableSchemaParams) (st entityType = "VARCHAR(255)" } - columnDefs := []string{ + columnDefs = []string{ fmt.Sprintf(`%s %s NOT NULL`, qt(params.entityName), entityType), } if params.hasUnixMilli { diff --git a/internal/database/offline/test_impl/join.go b/internal/database/offline/test_impl/join.go index 4cc4fe9f3..f0fe2e7b1 100644 --- a/internal/database/offline/test_impl/join.go +++ b/internal/database/offline/test_impl/join.go @@ -148,7 +148,7 @@ func TestJoin(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroySto } } -func prepareFeatures(oneGroup bool) (types.FeatureList, map[string]types.FeatureList, []string) { +func prepareFeatures(oneGroup bool) (features types.FeatureList, featureMap map[string]types.FeatureList, groupNames []string) { groupBasic := &types.Group{ ID: 1, Name: "device_basic", diff --git a/internal/database/online/cassandra/get.go b/internal/database/online/cassandra/get.go index c881045de..2a71015a4 100644 --- a/internal/database/online/cassandra/get.go +++ b/internal/database/online/cassandra/get.go @@ -90,15 +90,13 @@ func (db *DB) MultiGet(ctx context.Context, opt online.MultiGetOpt) (map[string] return rs, nil } -func deserializeIntoRowMap(values map[string]interface{}, entityName string, features types.FeatureList) (string, dbutil.RowMap) { - entityKey := values[entityName].(string) - - rs := make(dbutil.RowMap) +func deserializeIntoRowMap(values map[string]interface{}, entityName string, features types.FeatureList) (entityKey string, rs dbutil.RowMap) { + rs = make(dbutil.RowMap) for _, feature := range features { deserializedValue, _ := dbutil.DeserializeByValueType(values[feature.Name], feature.ValueType, types.BackendCassandra) rs[feature.FullName()] = deserializedValue } - return entityKey, rs + return values[entityName].(string), rs } func isTableNotFoundError(err error, tableName string) bool { diff --git a/pkg/oomstore/join.go b/pkg/oomstore/join.go index 7f2ed2298..3b52354be 100644 --- a/pkg/oomstore/join.go +++ b/pkg/oomstore/join.go @@ -142,13 +142,13 @@ func (s *OomStore) buildRevisionRanges(ctx context.Context, group *types.Group) return ranges, nil } -func GetEntityRowsFromInputFile(ctx context.Context, inputFilePath string) (<-chan types.EntityRow, []string, error) { +func GetEntityRowsFromInputFile(ctx context.Context, inputFilePath string) (readOnleEntityRows <-chan types.EntityRow, header []string, err error) { input, err := os.Open(filepath.Clean(inputFilePath)) if err != nil { return nil, nil, errdefs.WithStack(err) } reader := csv.NewReader(input) - header, err := reader.Read() + header, err = reader.Read() if err != nil { return nil, nil, errdefs.WithStack(err) } diff --git a/pkg/oomstore/util/feature.go b/pkg/oomstore/util/feature.go index 85183ca81..c9c77cb53 100644 --- a/pkg/oomstore/util/feature.go +++ b/pkg/oomstore/util/feature.go @@ -12,7 +12,7 @@ func ComposeFullFeatureName(groupName string, featureName string) string { return groupName + SepFullFeatureName + featureName } -func SplitFullFeatureName(fullName string) (string, string, error) { +func SplitFullFeatureName(fullName string) (groupName string, featureName string, err error) { parts := strings.SplitN(fullName, SepFullFeatureName, 2) if len(parts) != 2 { return "", "", errdefs.Errorf("invalid full feature name: '%s'", fullName) From 80f7bbff13cd671b80b656a9ab665bd134acc78f Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Wed, 23 Feb 2022 10:48:21 +0800 Subject: [PATCH 08/17] refactor: function param Type Combine --- internal/database/dbutil/schema.go | 2 +- internal/database/dbutil/sql.go | 2 +- internal/database/dbutil/util.go | 2 +- internal/database/metadata/mysql/db.go | 2 +- internal/database/metadata/mysql/tx.go | 2 +- internal/database/metadata/postgres/db.go | 2 +- internal/database/metadata/postgres/tx.go | 2 +- internal/database/metadata/sqlite/db.go | 2 +- internal/database/metadata/sqlite/tx.go | 2 +- internal/database/metadata/sqlutil/feature.go | 2 +- internal/database/metadata/test_impl/feature.go | 2 +- internal/database/metadata/test_impl/revision.go | 2 +- internal/database/offline/test_impl/export.go | 2 +- internal/database/offline/test_impl/join.go | 6 +++--- internal/database/online/tikv/util.go | 6 +++--- pkg/oomstore/feature.go | 2 +- pkg/oomstore/push_processor.go | 2 +- pkg/oomstore/util/feature.go | 4 ++-- 18 files changed, 23 insertions(+), 23 deletions(-) diff --git a/internal/database/dbutil/schema.go b/internal/database/dbutil/schema.go index ce9124e92..c2fba54df 100644 --- a/internal/database/dbutil/schema.go +++ b/internal/database/dbutil/schema.go @@ -66,7 +66,7 @@ func createTableDDL(tableName string, columns ColumnList, pkFields []string, bac return fmt.Sprintf("CREATE TABLE %s (\n%s\n)", tableName, tableDef) } -func BuildIndexDDL(tableName string, indexName string, fields []string, backend types.BackendType) string { +func BuildIndexDDL(tableName, indexName string, fields []string, backend types.BackendType) string { qt := QuoteFn(backend) // Some db like postgres, sqlite index must be database unique, diff --git a/internal/database/dbutil/sql.go b/internal/database/dbutil/sql.go index 7411ddbbe..4819914e4 100644 --- a/internal/database/dbutil/sql.go +++ b/internal/database/dbutil/sql.go @@ -11,7 +11,7 @@ import ( "github.com/oom-ai/oomstore/pkg/oomstore/types" ) -func BuildConditions(equal map[string]interface{}, in map[string]interface{}) (cond []string, args []interface{}, err error) { +func BuildConditions(equal, in map[string]interface{}) (cond []string, args []interface{}, err error) { cond = make([]string, 0) args = make([]interface{}, 0) for key, value := range equal { diff --git a/internal/database/dbutil/util.go b/internal/database/dbutil/util.go index c80b38c2b..c8e01a490 100644 --- a/internal/database/dbutil/util.go +++ b/internal/database/dbutil/util.go @@ -29,7 +29,7 @@ func RandString(n int) string { return string(b) } -func Fill(size int, elem string, sep string) string { +func Fill(size int, elem, sep string) string { r := make([]string, 0, size) for i := 0; i < size; i++ { diff --git a/internal/database/metadata/mysql/db.go b/internal/database/metadata/mysql/db.go index 16e3c70c1..46f52c0ae 100644 --- a/internal/database/metadata/mysql/db.go +++ b/internal/database/metadata/mysql/db.go @@ -76,7 +76,7 @@ func (db *DB) GetFeature(ctx context.Context, id int) (*types.Feature, error) { return sqlutil.GetFeature(ctx, db, id) } -func (db *DB) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (db *DB) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, db, groupName, featureName) } diff --git a/internal/database/metadata/mysql/tx.go b/internal/database/metadata/mysql/tx.go index 3d2a73e31..d6b3536a1 100644 --- a/internal/database/metadata/mysql/tx.go +++ b/internal/database/metadata/mysql/tx.go @@ -68,7 +68,7 @@ func (tx *Tx) ListFeature(ctx context.Context, opt metadata.ListFeatureOpt) (typ return sqlutil.ListFeature(ctx, tx, opt) } -func (tx *Tx) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (tx *Tx) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, tx, groupName, featureName) } diff --git a/internal/database/metadata/postgres/db.go b/internal/database/metadata/postgres/db.go index 35c3020fa..346984899 100644 --- a/internal/database/metadata/postgres/db.go +++ b/internal/database/metadata/postgres/db.go @@ -89,7 +89,7 @@ func (db *DB) ListFeature(ctx context.Context, opt metadata.ListFeatureOpt) (typ return sqlutil.ListFeature(ctx, db, opt) } -func (db *DB) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (db *DB) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, db, groupName, featureName) } diff --git a/internal/database/metadata/postgres/tx.go b/internal/database/metadata/postgres/tx.go index ccecbf500..acca7a2ea 100644 --- a/internal/database/metadata/postgres/tx.go +++ b/internal/database/metadata/postgres/tx.go @@ -68,7 +68,7 @@ func (tx *Tx) ListFeature(ctx context.Context, opt metadata.ListFeatureOpt) (typ return sqlutil.ListFeature(ctx, tx, opt) } -func (tx *Tx) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (tx *Tx) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, tx, groupName, featureName) } diff --git a/internal/database/metadata/sqlite/db.go b/internal/database/metadata/sqlite/db.go index 80b815602..447dca0d2 100644 --- a/internal/database/metadata/sqlite/db.go +++ b/internal/database/metadata/sqlite/db.go @@ -66,7 +66,7 @@ func (db *DB) GetFeature(ctx context.Context, id int) (*types.Feature, error) { return sqlutil.GetFeature(ctx, db, id) } -func (db *DB) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (db *DB) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, db, groupName, featureName) } diff --git a/internal/database/metadata/sqlite/tx.go b/internal/database/metadata/sqlite/tx.go index 3ad61d7c2..c9b3d59ba 100644 --- a/internal/database/metadata/sqlite/tx.go +++ b/internal/database/metadata/sqlite/tx.go @@ -68,7 +68,7 @@ func (tx *Tx) ListFeature(ctx context.Context, opt metadata.ListFeatureOpt) (typ return sqlutil.ListFeature(ctx, tx, opt) } -func (tx *Tx) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (tx *Tx) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return sqlutil.GetFeatureByName(ctx, tx, groupName, featureName) } diff --git a/internal/database/metadata/sqlutil/feature.go b/internal/database/metadata/sqlutil/feature.go index b5fd2f4bb..2c1512a70 100644 --- a/internal/database/metadata/sqlutil/feature.go +++ b/internal/database/metadata/sqlutil/feature.go @@ -55,7 +55,7 @@ func GetFeature(ctx context.Context, sqlxCtx metadata.SqlxContext, id int) (*typ return &feature, nil } -func GetFeatureByName(ctx context.Context, sqlxCtx metadata.SqlxContext, groupName string, featureName string) (*types.Feature, error) { +func GetFeatureByName(ctx context.Context, sqlxCtx metadata.SqlxContext, groupName, featureName string) (*types.Feature, error) { var ( feature types.Feature group *types.Group diff --git a/internal/database/metadata/test_impl/feature.go b/internal/database/metadata/test_impl/feature.go index 44b50d649..aa3f0ff22 100644 --- a/internal/database/metadata/test_impl/feature.go +++ b/internal/database/metadata/test_impl/feature.go @@ -12,7 +12,7 @@ import ( "github.com/oom-ai/oomstore/pkg/oomstore/types" ) -func prepareEntityAndGroup(t *testing.T, ctx context.Context, store metadata.Store) (entityID int, groupID int) { +func prepareEntityAndGroup(t *testing.T, ctx context.Context, store metadata.Store) (entityID, groupID int) { entityID, err := store.CreateEntity(ctx, metadata.CreateEntityOpt{ CreateEntityOpt: types.CreateEntityOpt{ EntityName: "device", diff --git a/internal/database/metadata/test_impl/revision.go b/internal/database/metadata/test_impl/revision.go index 39eb831fc..caa9e4323 100644 --- a/internal/database/metadata/test_impl/revision.go +++ b/internal/database/metadata/test_impl/revision.go @@ -374,7 +374,7 @@ func ignoreCreateAndModifyTime(revision *types.Revision) { revision.ModifyTime = time.Time{} } -func prepareRevisions(t *testing.T, ctx context.Context, store metadata.Store) (entityID int, groupID int, revisionIDs []int, revisions types.RevisionList) { +func prepareRevisions(t *testing.T, ctx context.Context, store metadata.Store) (entityID, groupID int, revisionIDs []int, revisions types.RevisionList) { entityID, err := store.CreateEntity(ctx, metadata.CreateEntityOpt{ CreateEntityOpt: types.CreateEntityOpt{ EntityName: "device", diff --git a/internal/database/offline/test_impl/export.go b/internal/database/offline/test_impl/export.go index 1ad95865c..6dd1efc07 100644 --- a/internal/database/offline/test_impl/export.go +++ b/internal/database/offline/test_impl/export.go @@ -102,7 +102,7 @@ func TestExport(t *testing.T, prepareStore PrepareStoreFn, destroyStore DestroyS } } -func prepareFeaturesForExport() (batchFeatures types.FeatureList, streamFeatures types.FeatureList) { +func prepareFeaturesForExport() (batchFeatures, streamFeatures types.FeatureList) { batchGroup := &types.Group{ ID: 1, Name: "device", diff --git a/internal/database/offline/test_impl/join.go b/internal/database/offline/test_impl/join.go index f0fe2e7b1..b2d0cbfaa 100644 --- a/internal/database/offline/test_impl/join.go +++ b/internal/database/offline/test_impl/join.go @@ -196,7 +196,7 @@ func prepareFeatures(oneGroup bool) (features types.FeatureList, featureMap map[ } } -func prepareRevisionRanges(oneGroup bool, stream bool) map[string][]*offline.RevisionRange { +func prepareRevisionRanges(oneGroup, stream bool) map[string][]*offline.RevisionRange { basic := []*offline.RevisionRange{ { MinRevision: 5, @@ -232,7 +232,7 @@ func prepareRevisionRanges(oneGroup bool, stream bool) map[string][]*offline.Rev } } -func prepareEntityRows(isEmpty bool, withValue bool) <-chan types.EntityRow { +func prepareEntityRows(isEmpty, withValue bool) <-chan types.EntityRow { entityRows := make(chan types.EntityRow) rows := []types.EntityRow{ { @@ -276,7 +276,7 @@ func prepareEntityRows(isEmpty bool, withValue bool) <-chan types.EntityRow { return entityRows } -func prepareResult(oneGroup bool, withValue bool, values []map[string]interface{}) *types.JoinResult { +func prepareResult(oneGroup, withValue bool, values []map[string]interface{}) *types.JoinResult { header := []string{"entity_key", "unix_milli", "device_basic.model", "device_basic.price", "device_advanced.is_active"} if withValue { header = []string{"entity_key", "unix_milli", "value_1", "value_2", "device_basic.model", "device_basic.price", "device_advanced.is_active"} diff --git a/internal/database/online/tikv/util.go b/internal/database/online/tikv/util.go index c04eb3a01..ba38af706 100644 --- a/internal/database/online/tikv/util.go +++ b/internal/database/online/tikv/util.go @@ -8,14 +8,14 @@ import ( const keyDelimiter = ':' -func getKeyOfBatchFeature(revisionID string, entityKey string, featureID string) []byte { +func getKeyOfBatchFeature(revisionID, entityKey, featureID string) []byte { return []byte(kvutil.KeyPrefixForBatchFeature + getKey(revisionID, entityKey, featureID)) } -func getKeyOfStreamFeature(groupID string, entityKey string, featureID string) []byte { +func getKeyOfStreamFeature(groupID, entityKey, featureID string) []byte { return []byte(kvutil.KeyPrefixForStreamFeature + getKey(groupID, entityKey, featureID)) } -func getKey(revisionID string, entityKey string, featureID string) string { +func getKey(revisionID, entityKey, featureID string) string { return strings.Join([]string{revisionID, entityKey, featureID}, string(keyDelimiter)) } diff --git a/pkg/oomstore/feature.go b/pkg/oomstore/feature.go index b89402dde..2685ab685 100644 --- a/pkg/oomstore/feature.go +++ b/pkg/oomstore/feature.go @@ -27,7 +27,7 @@ func (s *OomStore) GetFeatureByFullName(ctx context.Context, fullName string) (* } // GetFeatureByName gets metadata of a feature by group name and feature name. -func (s *OomStore) GetFeatureByName(ctx context.Context, groupName string, featureName string) (*types.Feature, error) { +func (s *OomStore) GetFeatureByName(ctx context.Context, groupName, featureName string) (*types.Feature, error) { return s.metadata.GetFeatureByName(ctx, groupName, featureName) } diff --git a/pkg/oomstore/push_processor.go b/pkg/oomstore/push_processor.go index 062e97bbd..b01b640b5 100644 --- a/pkg/oomstore/push_processor.go +++ b/pkg/oomstore/push_processor.go @@ -149,6 +149,6 @@ func (p *PushProcessor) pushToOffline(ctx context.Context, s *OomStore, groupID } // lastRevisionForStream returns the latest revision before unixMill. -func lastRevisionForStream(snapshotInterval int64, unixMill int64) int64 { +func lastRevisionForStream(snapshotInterval, unixMill int64) int64 { return (unixMill / snapshotInterval) * snapshotInterval } diff --git a/pkg/oomstore/util/feature.go b/pkg/oomstore/util/feature.go index c9c77cb53..221bae1ea 100644 --- a/pkg/oomstore/util/feature.go +++ b/pkg/oomstore/util/feature.go @@ -8,11 +8,11 @@ import ( const SepFullFeatureName = "." -func ComposeFullFeatureName(groupName string, featureName string) string { +func ComposeFullFeatureName(groupName, featureName string) string { return groupName + SepFullFeatureName + featureName } -func SplitFullFeatureName(fullName string) (groupName string, featureName string, err error) { +func SplitFullFeatureName(fullName string) (groupName, featureName string, err error) { parts := strings.SplitN(fullName, SepFullFeatureName, 2) if len(parts) != 2 { return "", "", errdefs.Errorf("invalid full feature name: '%s'", fullName) From f98f053ea2b3625b44ea0848f5a55e81e31a9dbd Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Wed, 23 Feb 2022 10:57:28 +0800 Subject: [PATCH 09/17] chore(oomstore): put a space between // and comment text --- pkg/oomstore/import_batch.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/oomstore/import_batch.go b/pkg/oomstore/import_batch.go index 07679e6b9..c8e576e63 100644 --- a/pkg/oomstore/import_batch.go +++ b/pkg/oomstore/import_batch.go @@ -5,16 +5,17 @@ import ( "context" "time" + "github.com/spf13/cast" + "github.com/oom-ai/oomstore/internal/database/dbutil" "github.com/oom-ai/oomstore/internal/database/metadata" "github.com/oom-ai/oomstore/internal/database/offline" "github.com/oom-ai/oomstore/pkg/oomstore/types" - "github.com/spf13/cast" ) // csvReaderImportBatch imports batch feature from external data source to offline store through csv reader. func (s *OomStore) csvReaderImportBatch(ctx context.Context, opt *importOpt, dataSource *types.CsvReaderDataSource) (int, error) { - //make sure csv data source has all defined columns + // make sure csv data source has all defined columns reader := bufio.NewReader(dataSource.Reader) source := &offline.CSVSource{ Reader: reader, From e4b3b4ca951b9bf289ea07d7cd45f03b7798cfd9 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Wed, 23 Feb 2022 11:13:43 +0800 Subject: [PATCH 10/17] refactor: remove unnecessary conversion --- internal/database/dbutil/serialize.go | 6 +++--- oomcli/cmd/export_helper.go | 4 ++-- pkg/oomstore/export.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/database/dbutil/serialize.go b/internal/database/dbutil/serialize.go index 43725eb9d..19db4c6a4 100644 --- a/internal/database/dbutil/serialize.go +++ b/internal/database/dbutil/serialize.go @@ -45,7 +45,7 @@ func kvSerializerByValueType(i interface{}, valueType types.ValueType) (s interf case types.String: return i.(string), nil case types.Int64: - return strconv.FormatInt(int64(i.(int64)), serializeIntBase), nil + return strconv.FormatInt(i.(int64), serializeIntBase), nil case types.Float64: return strconv.FormatFloat(i.(float64), 'f', -1, 64), nil case types.Bool: @@ -108,7 +108,7 @@ func kvSerializerByValue(i interface{}) (string, error) { case int: return strconv.FormatInt(int64(s), serializeIntBase), nil case int64: - return strconv.FormatInt(int64(s), serializeIntBase), nil + return strconv.FormatInt(s, serializeIntBase), nil case int32: return strconv.FormatInt(int64(s), serializeIntBase), nil case int16: @@ -124,7 +124,7 @@ func kvSerializerByValue(i interface{}) (string, error) { case uint: return strconv.FormatUint(uint64(s), serializeIntBase), nil case uint64: - return strconv.FormatUint(uint64(s), serializeIntBase), nil + return strconv.FormatUint(s, serializeIntBase), nil case uint32: return strconv.FormatUint(uint64(s), serializeIntBase), nil case uint16: diff --git a/oomcli/cmd/export_helper.go b/oomcli/cmd/export_helper.go index 2d164de22..77e936877 100644 --- a/oomcli/cmd/export_helper.go +++ b/oomcli/cmd/export_helper.go @@ -52,7 +52,7 @@ func printExportResultInCSV(exportResult *types.ExportResult) error { return row.Error } - if err := w.Write(cast.ToStringSlice([]interface{}(row.Record))); err != nil { + if err := w.Write(cast.ToStringSlice(row.Record)); err != nil { return err } } @@ -69,7 +69,7 @@ func printExportResultInASCIITable(exportResult *types.ExportResult) error { return row.Error } - table.Append(cast.ToStringSlice([]interface{}(row.Record))) + table.Append(cast.ToStringSlice(row.Record)) } table.Render() diff --git a/pkg/oomstore/export.go b/pkg/oomstore/export.go index 9998527f6..a615d92b2 100644 --- a/pkg/oomstore/export.go +++ b/pkg/oomstore/export.go @@ -120,7 +120,7 @@ func (s *OomStore) Export(ctx context.Context, opt types.ExportOpt) error { return row.Error } - if err := w.Write(cast.ToStringSlice([]interface{}(row.Record))); err != nil { + if err := w.Write(cast.ToStringSlice(row.Record)); err != nil { return err } } From 2d04d4f4ba1e2d6138b942168230464fb1ecd164 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Wed, 23 Feb 2022 11:23:05 +0800 Subject: [PATCH 11/17] refactor: refactor naked return --- internal/database/dbutil/schema.go | 2 +- oomcli/cmd/types.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/database/dbutil/schema.go b/internal/database/dbutil/schema.go index c2fba54df..9bc20de8c 100644 --- a/internal/database/dbutil/schema.go +++ b/internal/database/dbutil/schema.go @@ -114,5 +114,5 @@ func parseColumns(entityName string, hasUnixMilli bool, features types.FeatureLi rs = append(rs, Column{Name: f.Name, DbType: dbType, ValueType: f.ValueType}) } } - return + return rs } diff --git a/oomcli/cmd/types.go b/oomcli/cmd/types.go index 30f08d2fa..d028faeb3 100644 --- a/oomcli/cmd/types.go +++ b/oomcli/cmd/types.go @@ -71,7 +71,7 @@ func parseTokenLists(i interface{}) (headerTokens TokenList, dataTokens []TokenL ModifyTime: e.ModifyTime, }) if err != nil { - return + return nil, nil, err } dataTokens = append(dataTokens, tokens) } @@ -91,7 +91,7 @@ func parseTokenLists(i interface{}) (headerTokens TokenList, dataTokens []TokenL ModifyTime: e.ModifyTime, }) if err != nil { - return + return nil, nil, err } dataTokens = append(dataTokens, tokens) } @@ -110,7 +110,7 @@ func parseTokenLists(i interface{}) (headerTokens TokenList, dataTokens []TokenL ModifyTime: e.ModifyTime, }) if err != nil { - return + return nil, nil, err } dataTokens = append(dataTokens, tokens) } @@ -138,7 +138,7 @@ func parseTokenLists(i interface{}) (headerTokens TokenList, dataTokens []TokenL } headerTokens, err = parseTokens(element) if err != nil { - return + return nil, nil, err } - return + return headerTokens, dataTokens, nil } From 55857884e1431c53e37e273d08331fa54535590c Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Thu, 24 Feb 2022 09:55:41 +0800 Subject: [PATCH 12/17] refactor: modify 'if err =' to 'if err :=' --- internal/database/offline/sqlutil/export_helper.go | 4 ++-- internal/database/offline/sqlutil/join.go | 4 ++-- internal/database/offline/sqlutil/join_helper.go | 10 +++++----- internal/database/offline/sqlutil/snapshot.go | 2 +- internal/database/online/dynamodb/get.go | 2 +- internal/database/online/dynamodb/import.go | 2 +- internal/database/online/sqlutil/import.go | 6 +++--- oomcli/cmd/edit_entity.go | 2 +- oomcli/cmd/edit_feature.go | 2 +- oomcli/cmd/edit_group.go | 2 +- pkg/oomstore/apply.go | 4 ++-- pkg/oomstore/export.go | 2 +- pkg/oomstore/feature.go | 2 +- pkg/oomstore/import_batch.go | 4 ++-- pkg/oomstore/import_stream.go | 8 ++++---- pkg/oomstore/join.go | 2 +- pkg/oomstore/push_processor.go | 2 +- pkg/oomstore/revision.go | 4 ++-- pkg/oomstore/snapshot.go | 4 ++-- pkg/oomstore/sync.go | 12 ++++++------ 20 files changed, 40 insertions(+), 40 deletions(-) diff --git a/internal/database/offline/sqlutil/export_helper.go b/internal/database/offline/sqlutil/export_helper.go index bc2a4838f..a62e0e361 100644 --- a/internal/database/offline/sqlutil/export_helper.go +++ b/internal/database/offline/sqlutil/export_helper.go @@ -156,7 +156,7 @@ func prepareEntityTable(ctx context.Context, dbOpt dbutil.DBOpt, opt offline.Exp %s ); `, qtTableName, strings.Join(columnDefs, ",\n")) - if err = dbOpt.ExecContext(ctx, schema); err != nil { + if err := dbOpt.ExecContext(ctx, schema); err != nil { return "", err } @@ -173,7 +173,7 @@ func prepareEntityTable(ctx context.Context, dbOpt dbutil.DBOpt, opt offline.Exp if err != nil { return "", err } - if err = dbOpt.ExecContext(ctx, query, args...); err != nil { + if err := dbOpt.ExecContext(ctx, query, args...); err != nil { return "", err } diff --git a/internal/database/offline/sqlutil/join.go b/internal/database/offline/sqlutil/join.go index 4116434ba..b5c32b1fd 100644 --- a/internal/database/offline/sqlutil/join.go +++ b/internal/database/offline/sqlutil/join.go @@ -163,7 +163,7 @@ func joinOneGroup(ctx context.Context, dbOpt dbutil.DBOpt, opt joinOneGroupOpt) if err != nil { return nil, err } - if err = dbOpt.ExecContext(ctx, query, r.MinRevision, r.MaxRevision); err != nil { + if err := dbOpt.ExecContext(ctx, query, r.MinRevision, r.MaxRevision); err != nil { return nil, err } } @@ -192,7 +192,7 @@ func joinOneGroup(ctx context.Context, dbOpt dbutil.DBOpt, opt joinOneGroupOpt) if err != nil { return nil, err } - if err = dbOpt.ExecContext(ctx, query, r.MinRevision, r.MaxRevision); err != nil { + if err := dbOpt.ExecContext(ctx, query, r.MinRevision, r.MaxRevision); err != nil { return nil, err } } diff --git a/internal/database/offline/sqlutil/join_helper.go b/internal/database/offline/sqlutil/join_helper.go index f17382d67..0c8d83a2c 100644 --- a/internal/database/offline/sqlutil/join_helper.go +++ b/internal/database/offline/sqlutil/join_helper.go @@ -104,7 +104,7 @@ func prepareJoinedTable( ); ` schema = fmt.Sprintf(schema, qtTableName, strings.Join(columnDefs, ",\n")) - if err = dbOpt.ExecContext(ctx, schema); err != nil { + if err := dbOpt.ExecContext(ctx, schema); err != nil { return "", err } @@ -112,7 +112,7 @@ func prepareJoinedTable( // Step 2: create index on table joined_ if supportIndex(dbOpt.Backend) { index := fmt.Sprintf(`CREATE INDEX %s ON %s (unix_milli, entity_key)`, qt("idx_"+tableName), qtTableName) - if err = dbOpt.ExecContext(ctx, index); err != nil { + if err := dbOpt.ExecContext(ctx, index); err != nil { return "", err } } @@ -141,19 +141,19 @@ func prepareEntityRowsTable(ctx context.Context, ); `, qtTableName, strings.Join(columnDefs, ",\n")) - if err = dbOpt.ExecContext(ctx, schema); err != nil { + if err := dbOpt.ExecContext(ctx, schema); err != nil { return "", err } // Step 2: populate dataset to the table - if err = insertEntityRows(ctx, dbOpt, tableName, entityRows, valueNames); err != nil { + if err := insertEntityRows(ctx, dbOpt, tableName, entityRows, valueNames); err != nil { return "", err } // Step 3: create index on table entity_rows if supportIndex(dbOpt.Backend) { index := fmt.Sprintf(`CREATE INDEX idx_%s ON %s (unix_milli, entity_key)`, tableName, tableName) - if err = dbOpt.ExecContext(ctx, index); err != nil { + if err := dbOpt.ExecContext(ctx, index); err != nil { return "", err } } diff --git a/internal/database/offline/sqlutil/snapshot.go b/internal/database/offline/sqlutil/snapshot.go index c6d8e4e72..965ec725f 100644 --- a/internal/database/offline/sqlutil/snapshot.go +++ b/internal/database/offline/sqlutil/snapshot.go @@ -91,7 +91,7 @@ func Snapshot(ctx context.Context, dbOpt dbutil.DBOpt, opt offline.SnapshotOpt) if err != nil { return err } - if err = dbOpt.ExecContext(ctx, query); err != nil { + if err := dbOpt.ExecContext(ctx, query); err != nil { return err } diff --git a/internal/database/online/dynamodb/get.go b/internal/database/online/dynamodb/get.go index cdb1c8f83..a9447ba97 100644 --- a/internal/database/online/dynamodb/get.go +++ b/internal/database/online/dynamodb/get.go @@ -75,7 +75,7 @@ func (db *DB) MultiGet(ctx context.Context, opt online.MultiGetOpt) (map[string] entityName: entityKeyValue, }) if len(keys) == BatchGetItemCapacity { - if err = batchGetItem(ctx, db, keys, tableName, entityName, opt.Features, res); err != nil { + if err := batchGetItem(ctx, db, keys, tableName, entityName, opt.Features, res); err != nil { return nil, err } keys = make([]map[string]types.AttributeValue, 0, BatchGetItemCapacity) diff --git a/internal/database/online/dynamodb/import.go b/internal/database/online/dynamodb/import.go index 0cc1d70b0..7adb78dac 100644 --- a/internal/database/online/dynamodb/import.go +++ b/internal/database/online/dynamodb/import.go @@ -60,7 +60,7 @@ func (db *DB) Import(ctx context.Context, opt online.ImportOpt) error { }, }) if len(items) == BatchWriteItemCapacity { - if err = batchWrite(ctx, db, tableName, items); err != nil { + if err := batchWrite(ctx, db, tableName, items); err != nil { return err } items = make([]types.WriteRequest, 0, BatchWriteItemCapacity) diff --git a/internal/database/online/sqlutil/import.go b/internal/database/online/sqlutil/import.go index c29245e39..da148ee06 100644 --- a/internal/database/online/sqlutil/import.go +++ b/internal/database/online/sqlutil/import.go @@ -52,19 +52,19 @@ func Import(ctx context.Context, db *sqlx.DB, opt online.ImportOpt, backend type records = append(records, record.Record) if len(records) == importBatchSize { - if err = dbutil.InsertRecordsToTableTx(tx, ctx, tableName, records, columns, backend); err != nil { + if err := dbutil.InsertRecordsToTableTx(tx, ctx, tableName, records, columns, backend); err != nil { return err } records = make([]interface{}, 0, importBatchSize) } } - if err = dbutil.InsertRecordsToTableTx(tx, ctx, tableName, records, columns, backend); err != nil { + if err := dbutil.InsertRecordsToTableTx(tx, ctx, tableName, records, columns, backend); err != nil { return err } if opt.Group.Category == types.CategoryStream { streamTableName := dbutil.OnlineStreamTableName(opt.Group.ID) - if err = PurgeTx(ctx, tx, streamTableName, backend); err != nil { + if err := PurgeTx(ctx, tx, streamTableName, backend); err != nil { return err } query := fmt.Sprintf(`ALTER TABLE %s RENAME TO %s;`, tableName, streamTableName) diff --git a/oomcli/cmd/edit_entity.go b/oomcli/cmd/edit_entity.go index 133628f24..d6a8ff44f 100644 --- a/oomcli/cmd/edit_entity.go +++ b/oomcli/cmd/edit_entity.go @@ -65,7 +65,7 @@ func writeEntitiesToTempFile(ctx context.Context, oomStore *oomstore.OomStore, e _ = tempFile.Close() }() - if err = outputEntity(ctx, entities, outputParams{ + if err := outputEntity(ctx, entities, outputParams{ writer: tempFile, oomStore: oomStore, outputOpt: YAML, diff --git a/oomcli/cmd/edit_feature.go b/oomcli/cmd/edit_feature.go index 93331f7e5..6dbfdf714 100644 --- a/oomcli/cmd/edit_feature.go +++ b/oomcli/cmd/edit_feature.go @@ -71,7 +71,7 @@ func writeFeaturesToTempFile(features types.FeatureList) (string, error) { _ = tempFile.Close() }() - if err = outputFeature(features, outputParams{ + if err := outputFeature(features, outputParams{ writer: tempFile, outputOpt: YAML, }); err != nil { diff --git a/oomcli/cmd/edit_group.go b/oomcli/cmd/edit_group.go index f08b1e9c7..fc15e7808 100644 --- a/oomcli/cmd/edit_group.go +++ b/oomcli/cmd/edit_group.go @@ -75,7 +75,7 @@ func writeGroupsToTempFile(ctx context.Context, oomStore *oomstore.OomStore, gro _ = tempFile.Close() }() - if err = outputGroup(ctx, groups, outputParams{ + if err := outputGroup(ctx, groups, outputParams{ writer: tempFile, oomStore: oomStore, outputOpt: YAML, diff --git a/pkg/oomstore/apply.go b/pkg/oomstore/apply.go index 47a0b9035..e587be6ec 100644 --- a/pkg/oomstore/apply.go +++ b/pkg/oomstore/apply.go @@ -24,7 +24,7 @@ func (s *OomStore) Apply(ctx context.Context, opt apply.ApplyOpt) error { } onlineJobs := make([]func() error, 0) - if err = s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { + if err := s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { // apply entity for _, entity := range stage.NewEntities { if err2 := s.applyEntity(c, tx, entity); err2 != nil { @@ -59,7 +59,7 @@ func (s *OomStore) Apply(ctx context.Context, opt apply.ApplyOpt) error { } for _, job := range onlineJobs { - if err = job(); err != nil { + if err := job(); err != nil { return err } } diff --git a/pkg/oomstore/export.go b/pkg/oomstore/export.go index a615d92b2..dadfe5b9a 100644 --- a/pkg/oomstore/export.go +++ b/pkg/oomstore/export.go @@ -62,7 +62,7 @@ func (s *OomStore) ChannelExport(ctx context.Context, opt types.ChannelExportOpt return nil, errdefs.Errorf("group %s no feature values up to %d, use a later timestamp", group.Name, opt.UnixMilli) } else { if revision.SnapshotTable == "" { - if err = s.Snapshot(ctx, group.Name); err != nil { + if err := s.Snapshot(ctx, group.Name); err != nil { return nil, err } } diff --git a/pkg/oomstore/feature.go b/pkg/oomstore/feature.go index 2685ab685..57d16c6a7 100644 --- a/pkg/oomstore/feature.go +++ b/pkg/oomstore/feature.go @@ -111,7 +111,7 @@ func (s *OomStore) CreateFeature(ctx context.Context, opt types.CreateFeatureOpt if err != nil { return 0, err } - if err = s.online.CreateTable(ctx, online.CreateTableOpt{ + if err := s.online.CreateTable(ctx, online.CreateTableOpt{ EntityName: group.Entity.Name, TableName: dbutil.OnlineStreamTableName(group.ID), Features: features, diff --git a/pkg/oomstore/import_batch.go b/pkg/oomstore/import_batch.go index c8e576e63..d4f59024b 100644 --- a/pkg/oomstore/import_batch.go +++ b/pkg/oomstore/import_batch.go @@ -55,7 +55,7 @@ func (s *OomStore) csvReaderImportBatch(ctx context.Context, opt *importOpt, dat if opt.Revision != nil { revision = *opt.Revision } - if err = s.metadata.UpdateRevision(ctx, metadata.UpdateRevisionOpt{ + if err := s.metadata.UpdateRevision(ctx, metadata.UpdateRevisionOpt{ RevisionID: newRevisionID, NewRevision: &revision, NewSnapshotTable: &snapshotTableName, @@ -77,7 +77,7 @@ func (s *OomStore) tableLinkImportBatch(ctx context.Context, opt *importOpt, dat if err != nil { return 0, err } - if err = validateTableSchema(tableSchema, opt.features); err != nil { + if err := validateTableSchema(tableSchema, opt.features); err != nil { return 0, err } var revision int64 diff --git a/pkg/oomstore/import_stream.go b/pkg/oomstore/import_stream.go index 29393b5df..9fa9c98a8 100644 --- a/pkg/oomstore/import_stream.go +++ b/pkg/oomstore/import_stream.go @@ -75,10 +75,10 @@ func (s *OomStore) tableLinkImportStream(ctx context.Context, opt *importOpt, da return err } // validation - if err = validateTableSchema(tableSchema, opt.features); err != nil { + if err := validateTableSchema(tableSchema, opt.features); err != nil { return err } - if err = s.validateRevisions(ctx, opt.group.ID, tableSchema); err != nil { + if err := s.validateRevisions(ctx, opt.group.ID, tableSchema); err != nil { return err } @@ -119,11 +119,11 @@ func (s *OomStore) pushStreamingRecords(ctx context.Context, records []types.Str return err } - if err = s.createRevisionAndCdcTable(ctx, group.ID, revision); err != nil { + if err := s.createRevisionAndCdcTable(ctx, group.ID, revision); err != nil { return err } // push data to new offline stream cdc table - if err = s.offline.Push(ctx, pushOpt); err != nil { + if err := s.offline.Push(ctx, pushOpt); err != nil { return err } } diff --git a/pkg/oomstore/join.go b/pkg/oomstore/join.go index 3b52354be..c01d34d39 100644 --- a/pkg/oomstore/join.go +++ b/pkg/oomstore/join.go @@ -114,7 +114,7 @@ func (s *OomStore) buildRevisionRanges(ctx context.Context, group *types.Group) }) for _, revision := range revisions { if revision.SnapshotTable == "" { - if err = s.Snapshot(ctx, group.Name); err != nil { + if err := s.Snapshot(ctx, group.Name); err != nil { return nil, err } } diff --git a/pkg/oomstore/push_processor.go b/pkg/oomstore/push_processor.go index b01b640b5..9c8e7891f 100644 --- a/pkg/oomstore/push_processor.go +++ b/pkg/oomstore/push_processor.go @@ -138,7 +138,7 @@ func (p *PushProcessor) pushToOffline(ctx context.Context, s *OomStore, groupID if err != nil { return err } - if err = s.pushStreamingRecords(ctx, b.records, entity.Name, group, features); err != nil { + if err := s.pushStreamingRecords(ctx, b.records, entity.Name, group, features); err != nil { return err } diff --git a/pkg/oomstore/revision.go b/pkg/oomstore/revision.go index 49a9e9954..cbbef7d1e 100644 --- a/pkg/oomstore/revision.go +++ b/pkg/oomstore/revision.go @@ -105,7 +105,7 @@ func (s *OomStore) createSnapshotAndCdcTable(ctx context.Context, revisionID int return err } - if err = s.offline.CreateTable(ctx, offline.CreateTableOpt{ + if err := s.offline.CreateTable(ctx, offline.CreateTableOpt{ TableName: snapshotTableName, EntityName: revision.Group.Entity.Name, Features: features, @@ -117,7 +117,7 @@ func (s *OomStore) createSnapshotAndCdcTable(ctx context.Context, revisionID int var cdcTable *string if revision.Group.Category == types.CategoryStream { tableName := dbutil.OfflineStreamCdcTableName(revision.GroupID, revision.Revision) - if err = s.offline.CreateTable(ctx, offline.CreateTableOpt{ + if err := s.offline.CreateTable(ctx, offline.CreateTableOpt{ TableName: tableName, EntityName: revision.Group.Entity.Name, Features: features, diff --git a/pkg/oomstore/snapshot.go b/pkg/oomstore/snapshot.go index 07215a70f..8c1e7f0a5 100644 --- a/pkg/oomstore/snapshot.go +++ b/pkg/oomstore/snapshot.go @@ -41,7 +41,7 @@ func (s *OomStore) Snapshot(ctx context.Context, groupName string) error { continue } tableName := dbutil.OfflineStreamSnapshotTableName(group.ID, revision.Revision) - if err = s.offline.Snapshot(ctx, offline.SnapshotOpt{ + if err := s.offline.Snapshot(ctx, offline.SnapshotOpt{ Group: *group, Features: features, Revision: revisions[i].Revision, @@ -49,7 +49,7 @@ func (s *OomStore) Snapshot(ctx context.Context, groupName string) error { }); err != nil { return err } - if err = s.metadata.UpdateRevision(ctx, metadata.UpdateRevisionOpt{ + if err := s.metadata.UpdateRevision(ctx, metadata.UpdateRevisionOpt{ RevisionID: revision.ID, NewSnapshotTable: &tableName, }); err != nil { diff --git a/pkg/oomstore/sync.go b/pkg/oomstore/sync.go index 973b5cd4c..f0b042854 100644 --- a/pkg/oomstore/sync.go +++ b/pkg/oomstore/sync.go @@ -55,7 +55,7 @@ func (s *OomStore) syncBatch(ctx context.Context, opt types.SyncOpt, group *type return err } - if err = s.online.Import(ctx, online.ImportOpt{ + if err := s.online.Import(ctx, online.ImportOpt{ Group: *group, Features: features, ExportStream: exportResult.Data, @@ -64,19 +64,19 @@ func (s *OomStore) syncBatch(ctx context.Context, opt types.SyncOpt, group *type return err } - if err = s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { + if err := s.metadata.WithTransaction(ctx, func(c context.Context, tx metadata.DBStore) error { // Update the online revision id of the feature group upon sync success - if err2 := tx.UpdateGroup(c, metadata.UpdateGroupOpt{ + if err := tx.UpdateGroup(c, metadata.UpdateGroupOpt{ GroupID: group.ID, NewOnlineRevisionID: &revision.ID, - }); err2 != nil { - return err2 + }); err != nil { + return err } if !revision.Anchored { newRevision := time.Now().UnixMilli() newChored := true // Update revision timestamp using current timestamp - if err = tx.UpdateRevision(c, metadata.UpdateRevisionOpt{ + if err := tx.UpdateRevision(c, metadata.UpdateRevisionOpt{ RevisionID: revision.ID, NewRevision: &newRevision, NewAnchored: &newChored, From 84425ffa0003cea419573d839099ed9b2d3dc840 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Thu, 24 Feb 2022 10:23:55 +0800 Subject: [PATCH 13/17] refactor(dbutil/deserialize): modify cassandraDeserializer singleCaseSwitch --- internal/database/dbutil/deserialize.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/database/dbutil/deserialize.go b/internal/database/dbutil/deserialize.go index 9b49c31dd..b819fe262 100644 --- a/internal/database/dbutil/deserialize.go +++ b/internal/database/dbutil/deserialize.go @@ -58,9 +58,8 @@ func rdbDeserializer(i interface{}, valueType types.ValueType) (interface{}, err } func cassandraDeserializer(i interface{}, valueType types.ValueType) (interface{}, error) { - switch i.(type) { - case string: - if i == "" { + if v, ok := i.(string); ok { + if v == "" { return nil, nil } } From ac471b8724101b751f12256b1bdb2251c51f4253 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Thu, 24 Feb 2022 10:52:13 +0800 Subject: [PATCH 14/17] refactor(oomcli|oomagent): use utf8.DecodeRunInString --- oomagent/server.go | 3 ++- oomcli/cmd/import.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/oomagent/server.go b/oomagent/server.go index 1f2f7dd42..aee4d2179 100644 --- a/oomagent/server.go +++ b/oomagent/server.go @@ -6,6 +6,7 @@ import ( "io" "log" "time" + "unicode/utf8" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -179,7 +180,7 @@ func (s *server) Import(ctx context.Context, req *codegen.ImportRequest) (*codeg description = *req.Description } if req.Delimiter != nil { - delimiter = []rune(*req.Delimiter)[0] + delimiter, _ = utf8.DecodeRuneInString(*req.Delimiter) } else { delimiter = ',' } diff --git a/oomcli/cmd/import.go b/oomcli/cmd/import.go index bd9d19c1d..57216cb27 100644 --- a/oomcli/cmd/import.go +++ b/oomcli/cmd/import.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "os" + "unicode/utf8" - "github.com/oom-ai/oomstore/pkg/oomstore/types" "github.com/spf13/cobra" + + "github.com/oom-ai/oomstore/pkg/oomstore/types" ) var importOpt types.ImportOpt @@ -18,7 +20,7 @@ var importCmd = &cobra.Command{ Use: "import", Short: "Import feature data from a data source", PreRunE: func(cmd *cobra.Command, args []string) error { - importCSVFileDataSource.Delimiter = []rune(delimitre)[0] + importCSVFileDataSource.Delimiter, _ = utf8.DecodeRuneInString(delimitre) if !cmd.Flags().Changed("revision") { importOpt.Revision = nil From e923e012101e365a6bd51e25ee2ace3bfed47618 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Thu, 24 Feb 2022 10:54:48 +0800 Subject: [PATCH 15/17] refactor: use %q instead of "%s" for quoted string --- internal/database/offline/bigquery/table_schema.go | 2 +- internal/database/online/cassandra/import.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/offline/bigquery/table_schema.go b/internal/database/offline/bigquery/table_schema.go index ab521c4c6..83570a9f9 100644 --- a/internal/database/offline/bigquery/table_schema.go +++ b/internal/database/offline/bigquery/table_schema.go @@ -14,7 +14,7 @@ import ( ) func (db *DB) TableSchema(ctx context.Context, opt offline.TableSchemaOpt) (*types.DataTableSchema, error) { - q := fmt.Sprintf(`SELECT column_name, data_type FROM %s.INFORMATION_SCHEMA.COLUMNS WHERE table_name = "%s"`, db.datasetID, opt.TableName) + q := fmt.Sprintf(`SELECT column_name, data_type FROM %s.INFORMATION_SCHEMA.COLUMNS WHERE table_name = %q`, db.datasetID, opt.TableName) rows, err := db.Query(q).Read(ctx) if err != nil { return nil, errdefs.WithStack(err) diff --git a/internal/database/online/cassandra/import.go b/internal/database/online/cassandra/import.go index b8470d465..f9ee2213a 100644 --- a/internal/database/online/cassandra/import.go +++ b/internal/database/online/cassandra/import.go @@ -64,7 +64,7 @@ func buildInsertStatement(tableName string, columns []string) string { valueFlags = append(valueFlags, "?") } - return fmt.Sprintf(`INSERT INTO "%s" (%s) VALUES (%s)`, + return fmt.Sprintf(`INSERT INTO %q (%s) VALUES (%s)`, tableName, dbutil.QuoteFn(Backend)(columns...), strings.Join(valueFlags, ",")) From 881f5ee6c8ecb9e312dd10503ac6687fca4bfcf3 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Thu, 24 Feb 2022 11:02:55 +0800 Subject: [PATCH 16/17] refactor(oomcli|oomagent): replace ioutil.ReadFile with os.ReadFile --- oomagent/root.go | 3 +-- oomcli/cmd/root.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/oomagent/root.go b/oomagent/root.go index 5e6f73835..59ca50c46 100644 --- a/oomagent/root.go +++ b/oomagent/root.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "io/ioutil" "log" "net" "os" @@ -102,7 +101,7 @@ func initConfig() { if envCfgFile := os.Getenv("OOMAGENT_CONFIG"); envCfgFile != "" { cfgFile = envCfgFile } - cfgContent, err := ioutil.ReadFile(filepath.Clean(cfgFile)) + cfgContent, err := os.ReadFile(filepath.Clean(cfgFile)) if err != nil { fmt.Fprintf(os.Stderr, "failed reading config file: %v\n", err) os.Exit(1) diff --git a/oomcli/cmd/root.go b/oomcli/cmd/root.go index dc308d08b..5e298fb73 100644 --- a/oomcli/cmd/root.go +++ b/oomcli/cmd/root.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "io/ioutil" "os" "path/filepath" @@ -52,7 +51,7 @@ func initConfig() { if envCfgFile := os.Getenv("OOMCLI_CONFIG"); envCfgFile != "" { cfgFile = envCfgFile } - cfgContent, err := ioutil.ReadFile(filepath.Clean(cfgFile)) + cfgContent, err := os.ReadFile(filepath.Clean(cfgFile)) if err != nil { exitf("failed reading config file: %v\n", err) } From 39e6751efb4e8e7480afcd54facf06136a2e2570 Mon Sep 17 00:00:00 2001 From: lianxmfor <1034552569@qq.com> Date: Thu, 24 Feb 2022 15:13:14 +0800 Subject: [PATCH 17/17] fix: fix unused func argument --- oomcli/cmd/docgen.go | 4 ++-- oomcli/cmd/serialize.go | 3 +-- pkg/oomstore/online_query.go | 10 ++++------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/oomcli/cmd/docgen.go b/oomcli/cmd/docgen.go index 249511da3..906015b3c 100644 --- a/oomcli/cmd/docgen.go +++ b/oomcli/cmd/docgen.go @@ -76,13 +76,13 @@ func genMarkdown(cmd *cobra.Command, w io.Writer) error { buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.Example)) } - printOptions(buf, cmd, name) + printOptions(buf, cmd) _, err := buf.WriteTo(w) return err } -func printOptions(buf *bytes.Buffer, cmd *cobra.Command, name string) { +func printOptions(buf *bytes.Buffer, cmd *cobra.Command) { flags := cmd.NonInheritedFlags() flags.SetOutput(buf) if flags.HasAvailableFlags() { diff --git a/oomcli/cmd/serialize.go b/oomcli/cmd/serialize.go index ebda472c8..7771edbcc 100644 --- a/oomcli/cmd/serialize.go +++ b/oomcli/cmd/serialize.go @@ -3,7 +3,6 @@ package cmd import ( "encoding/csv" "io" - "os" "reflect" "time" @@ -126,7 +125,7 @@ func serializeInCSV(w io.Writer, header []string, records [][]string) error { } func serializeInASCIITable(w io.Writer, header []string, records [][]string, border bool) error { - table := tablewriter.NewWriter(os.Stdout) + table := tablewriter.NewWriter(w) table.SetAutoWrapText(false) table.SetHeader(header) table.SetAutoFormatHeaders(false) diff --git a/pkg/oomstore/online_query.go b/pkg/oomstore/online_query.go index 896c0fe31..963359f19 100644 --- a/pkg/oomstore/online_query.go +++ b/pkg/oomstore/online_query.go @@ -3,12 +3,10 @@ package oomstore import ( "context" - "github.com/oom-ai/oomstore/internal/database/metadata" - - "github.com/oom-ai/oomstore/pkg/errdefs" - "github.com/oom-ai/oomstore/internal/database/dbutil" + "github.com/oom-ai/oomstore/internal/database/metadata" "github.com/oom-ai/oomstore/internal/database/online" + "github.com/oom-ai/oomstore/pkg/errdefs" "github.com/oom-ai/oomstore/pkg/oomstore/types" "github.com/oom-ai/oomstore/pkg/oomstore/util" ) @@ -87,7 +85,7 @@ func (s *OomStore) OnlineMultiGet(ctx context.Context, opt types.OnlineMultiGetO featureMap := features.GroupByGroupID() // entity_key -> feature_name -> feature_value - featureValueMap, err := s.getFeatureValueMap(ctx, opt.EntityKeys, featureMap, entity) + featureValueMap, err := s.getFeatureValueMap(ctx, opt.EntityKeys, featureMap) if err != nil { return nil, err } @@ -106,7 +104,7 @@ func (s *OomStore) OnlineMultiGet(ctx context.Context, opt types.OnlineMultiGetO return result, nil } -func (s *OomStore) getFeatureValueMap(ctx context.Context, entityKeys []string, featureMap map[int]types.FeatureList, entity *types.Entity) (map[string]dbutil.RowMap, error) { +func (s *OomStore) getFeatureValueMap(ctx context.Context, entityKeys []string, featureMap map[int]types.FeatureList) (map[string]dbutil.RowMap, error) { // entity_key -> types.RecordMap featureValueMap := make(map[string]dbutil.RowMap)