diff --git a/.gitignore b/.gitignore
index fe7f69d..60a2d0b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,7 +15,7 @@ result.db.*
dump.data
runtime.trace
-bin/redis-full-check
+bin/redis-full-check*
.DS_Store
diff --git a/README.md b/README.md
index 8daf72e..8c065ca 100644
--- a/README.md
+++ b/README.md
@@ -45,9 +45,21 @@ Application Options:
-s, --source=SOURCE Set host:port of source redis.
-p, --sourcepassword=Password Set source redis password (format: password or username:password)
--sourceauthtype=AUTH-TYPE useless for opensource redis, valid value:auth/adminauth (default: auth)
+ --sourcetls Enable TLS for all source Redis connections
+ --sourcetlscacert=FILE CA certificate file for source Redis TLS verification
+ --sourcetlscert=FILE Client certificate file for source Redis mutual TLS
+ --sourcetlskey=FILE Client private key file for source Redis mutual TLS
+ --sourcetlsservername=NAME Override the source Redis TLS certificate server name
+ --sourcetlsskipverify Skip source Redis TLS certificate verification (insecure)
-t, --target=TARGET Set host:port of target redis.
-a, --targetpassword=Password Set target redis password (format: password or username:password)
--targetauthtype=AUTH-TYPE useless for opensource redis, valid value:auth/adminauth (default: auth)
+ --targettls Enable TLS for all target Redis connections
+ --targettlscacert=FILE CA certificate file for target Redis TLS verification
+ --targettlscert=FILE Client certificate file for target Redis mutual TLS
+ --targettlskey=FILE Client private key file for target Redis mutual TLS
+ --targettlsservername=NAME Override the target Redis TLS certificate server name
+ --targettlsskipverify Skip target Redis TLS certificate verification (insecure)
-d, --db=Sqlite3-DB-FILE sqlite3 db file for store result. If exist, it will be removed and a new file is created. (default: result.db)
--comparetimes=COUNT Total compare count, at least 1. In the first round, all keys will be compared. The subsequent rounds of the comparison
will be done on the previous results. (default: 3)
@@ -82,6 +94,26 @@ Or you can build redis-full-check yourself according to the following steps:
* ./build.sh
* ./bin/redis-full-check -s $(source_redis_ip_port) -p $(source_password) -t $(target_redis_ip_port) -a $(target_password) # these parameters should be given by users
+## TLS
+
+TLS is configured independently for the source and target. If the CA is already trusted by the operating system, the `tlscacert` option can be omitted. Client certificate and key options must be specified together when mutual TLS is required.
+
+```sh
+./bin/redis-full-check \
+ -s source.example.com:6379 -p source_password --sourcetls --sourcetlscacert /path/to/ca.crt \
+ -t target.example.com:6379 -a target_password --targettls --targettlscacert /path/to/ca.crt
+```
+
+In Redis Cluster mode, TLS applies to the startup node, every primary returned by `CLUSTER SLOTS`, redirects, and the per-node full scan connections. Use the role prefix to discover all primary nodes from one startup node:
+
+```sh
+./bin/redis-full-check \
+ -s 'master@redis-cluster.example.com:6379' --sourcedbtype 1 --sourcetls --sourcetlscacert /path/to/ca.crt \
+ -t 'master@target-cluster.example.com:6379' --targetdbtype 1 --targettls --targettlscacert /path/to/ca.crt
+```
+
+By default, each cluster node certificate is verified against the hostname or IP advertised by Redis. Set `sourcetlsservername` or `targettlsservername` only when every node certificate uses the same explicit server name. The `tlsskipverify` options disable certificate verification and should only be used for temporary diagnostics.
+
Here comes the sqlite3 example to display the conflict result:
```
$ sqlite3 result.db.3 # result.db.x shows the x-round comparison conflict result. len == -1 means inconsistent key type.
@@ -114,4 +146,4 @@ We also provide some tools for synchronization in Shake series.
# License
- On `20230427` and later, we distribute this library under the new [Apache2.0](https://www.apache.org/licenses/LICENSE-2.0) protocol, `1.4.10` is the first version to support the Apache2.0 protocol.
-- Prior to 20230427, it was distributed under the [GPLV3.0](https://www.gnu.org/licenses/gpl-3.0.html) protocol.
\ No newline at end of file
+- Prior to 20230427, it was distributed under the [GPLV3.0](https://www.gnu.org/licenses/gpl-3.0.html) protocol.
diff --git a/src/full_check/client/address.go b/src/full_check/client/address.go
index 2dee8ec..640c938 100644
--- a/src/full_check/client/address.go
+++ b/src/full_check/client/address.go
@@ -1,8 +1,9 @@
package client
import (
- "strings"
+ "crypto/tls"
"fmt"
+ "strings"
"full_check/common"
)
@@ -15,7 +16,7 @@ const (
RoleSlave = "slave"
)
-func HandleAddress(address, password, authType string) ([]string, error) {
+func HandleAddress(address, password, authType string, tlsConfig *tls.Config) ([]string, error) {
if strings.Contains(address, AddressSplitter) {
arr := strings.Split(address, AddressSplitter)
if len(arr) != 2 {
@@ -33,7 +34,7 @@ func HandleAddress(address, password, authType string) ([]string, error) {
role = RoleMaster
}
- return fetchNodeList(clusterList[0], password, authType, role)
+ return fetchNodeList(clusterList[0], password, authType, role, tlsConfig)
} else {
clusterList := strings.Split(address, AddressClusterSplitter)
if len(clusterList) <= 1 {
@@ -41,7 +42,7 @@ func HandleAddress(address, password, authType string) ([]string, error) {
}
// fetch master
- masterList, err := fetchNodeList(clusterList[0], password, authType, common.TypeMaster)
+ masterList, err := fetchNodeList(clusterList[0], password, authType, common.TypeMaster, tlsConfig)
if err != nil {
return nil, err
}
@@ -50,7 +51,7 @@ func HandleAddress(address, password, authType string) ([]string, error) {
return clusterList, nil
}
- slaveList, err := fetchNodeList(clusterList[0], password, authType, common.TypeSlave)
+ slaveList, err := fetchNodeList(clusterList[0], password, authType, common.TypeSlave, tlsConfig)
if err != nil {
return nil, err
}
@@ -65,12 +66,13 @@ func HandleAddress(address, password, authType string) ([]string, error) {
}
}
-func fetchNodeList(oneNode, password, authType, role string) ([]string, error) {
+func fetchNodeList(oneNode, password, authType, role string, tlsConfig *tls.Config) ([]string, error) {
// create client to fetch
client, err := NewRedisClient(RedisHost{
- Addr: []string{oneNode},
- Password: password,
- Authtype: authType,
+ Addr: []string{oneNode},
+ Password: password,
+ Authtype: authType,
+ TLSConfig: tlsConfig,
}, 0)
if err != nil {
return nil, fmt.Errorf("fetch cluster info failed[%v]", err)
diff --git a/src/full_check/client/client.go b/src/full_check/client/client.go
index 84444a1..9f0a4de 100644
--- a/src/full_check/client/client.go
+++ b/src/full_check/client/client.go
@@ -1,13 +1,14 @@
package client
import (
+ "crypto/tls"
+ "errors"
"fmt"
"io"
"net"
"strconv"
"strings"
"time"
- "errors"
"full_check/common"
@@ -28,6 +29,7 @@ type RedisHost struct {
Authtype string // "auth" or "adminauth"
DBType int
DBFilterList map[int]struct{} // whitelist
+ TLSConfig *tls.Config
}
func (p RedisHost) String() string {
@@ -91,12 +93,15 @@ func (p *RedisClient) Connect() error {
var err error
if p.redisHost.IsCluster() == false {
// single db or proxy
- if p.redisHost.TimeoutMs == 0 {
- p.conn, err = redis.Dial("tcp", p.redisHost.Addr[0])
- } else {
- p.conn, err = redis.DialTimeout("tcp", p.redisHost.Addr[0], time.Millisecond*time.Duration(p.redisHost.TimeoutMs),
- time.Millisecond*time.Duration(p.redisHost.TimeoutMs), time.Millisecond*time.Duration(p.redisHost.TimeoutMs))
- }
+ p.conn, err = p.dial(p.redisHost.Addr[0])
+ } else if p.redisHost.TLSConfig != nil {
+ p.conn, err = NewTLSClusterConn(TLSClusterOptions{
+ StartNodes: p.redisHost.Addr,
+ Password: p.redisHost.Password,
+ AuthType: p.redisHost.Authtype,
+ Timeout: time.Duration(p.redisHost.TimeoutMs) * time.Millisecond,
+ TLSConfig: p.redisHost.TLSConfig,
+ })
} else {
// cluster
cluster, err := redigoCluster.NewCluster(
@@ -140,6 +145,18 @@ func (p *RedisClient) Connect() error {
return nil
}
+func (p *RedisClient) dial(address string) (redis.Conn, error) {
+ options := make([]redis.DialOption, 0, 5)
+ if p.redisHost.TimeoutMs != 0 {
+ timeout := time.Duration(p.redisHost.TimeoutMs) * time.Millisecond
+ options = append(options, redis.DialConnectTimeout(timeout), redis.DialReadTimeout(timeout), redis.DialWriteTimeout(timeout))
+ }
+ if p.redisHost.TLSConfig != nil {
+ options = append(options, redis.DialUseTLS(true), redis.DialTLSConfig(p.redisHost.TLSConfig))
+ }
+ return redis.Dial("tcp", address, options...)
+}
+
func (p *RedisClient) Do(commandName string, args ...interface{}) (interface{}, error) {
var err error
var result interface{}
@@ -179,7 +196,7 @@ type combine struct {
}
func (c combine) String() string {
- all := make([]string, 0, len(c.params) + 1)
+ all := make([]string, 0, len(c.params)+1)
all = append(all, c.command)
for _, ele := range c.params {
all = append(all, string(ele.([]byte)))
diff --git a/src/full_check/client/tls.go b/src/full_check/client/tls.go
new file mode 100644
index 0000000..84fd94a
--- /dev/null
+++ b/src/full_check/client/tls.go
@@ -0,0 +1,53 @@
+package client
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "fmt"
+ "os"
+)
+
+func LoadTLSConfig(enabled bool, caFile, certFile, keyFile, serverName string, skipVerify bool) (*tls.Config, error) {
+ if !enabled {
+ if caFile != "" || certFile != "" || keyFile != "" || serverName != "" || skipVerify {
+ return nil, fmt.Errorf("TLS options require TLS to be enabled")
+ }
+ return nil, nil
+ }
+
+ if (certFile == "") != (keyFile == "") {
+ return nil, fmt.Errorf("TLS client certificate and key must be specified together")
+ }
+
+ config := &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ ServerName: serverName,
+ InsecureSkipVerify: skipVerify,
+ }
+
+ if caFile != "" {
+ caPEM, err := os.ReadFile(caFile)
+ if err != nil {
+ return nil, fmt.Errorf("read TLS CA certificate %q: %w", caFile, err)
+ }
+
+ rootCAs, err := x509.SystemCertPool()
+ if err != nil || rootCAs == nil {
+ rootCAs = x509.NewCertPool()
+ }
+ if !rootCAs.AppendCertsFromPEM(caPEM) {
+ return nil, fmt.Errorf("TLS CA certificate %q does not contain a valid PEM certificate", caFile)
+ }
+ config.RootCAs = rootCAs
+ }
+
+ if certFile != "" {
+ certificate, err := tls.LoadX509KeyPair(certFile, keyFile)
+ if err != nil {
+ return nil, fmt.Errorf("load TLS client certificate %q and key %q: %w", certFile, keyFile, err)
+ }
+ config.Certificates = []tls.Certificate{certificate}
+ }
+
+ return config, nil
+}
diff --git a/src/full_check/client/tls_cluster.go b/src/full_check/client/tls_cluster.go
new file mode 100644
index 0000000..11e7662
--- /dev/null
+++ b/src/full_check/client/tls_cluster.go
@@ -0,0 +1,413 @@
+package client
+
+import (
+ "crypto/tls"
+ "fmt"
+ "net"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/gomodule/redigo/redis"
+ redigoCluster "github.com/najoast/redis-go-cluster"
+)
+
+const redisClusterSlotCount = 16384
+
+type TLSClusterOptions struct {
+ StartNodes []string
+ Password string
+ AuthType string
+ Timeout time.Duration
+ TLSConfig *tls.Config
+}
+
+type tlsClusterCommand struct {
+ name string
+ args []interface{}
+}
+
+type tlsClusterReply struct {
+ value interface{}
+ err error
+}
+
+// TLSClusterConn implements redigo.Conn while routing every cluster node
+// connection through TLS. It is intentionally used only for TLS clusters;
+// the existing cluster client remains unchanged for plaintext connections.
+type TLSClusterConn struct {
+ options TLSClusterOptions
+
+ topologyMu sync.RWMutex
+ slots [redisClusterSlotCount]string
+ pools map[string]*redis.Pool
+ nodes []string
+
+ pipelineMu sync.Mutex
+ pending []tlsClusterCommand
+ replies []tlsClusterReply
+
+ closeMu sync.RWMutex
+ closed bool
+}
+
+func NewTLSClusterConn(options TLSClusterOptions) (redis.Conn, error) {
+ if len(options.StartNodes) == 0 {
+ return nil, fmt.Errorf("TLS cluster requires at least one start node")
+ }
+ if options.TLSConfig == nil {
+ return nil, fmt.Errorf("TLS cluster requires a TLS configuration")
+ }
+ if options.AuthType == "" {
+ options.AuthType = "auth"
+ }
+
+ cluster := &TLSClusterConn{
+ options: options,
+ pools: make(map[string]*redis.Pool),
+ }
+
+ errors := make([]string, 0, len(options.StartNodes))
+ for _, address := range options.StartNodes {
+ if err := cluster.refresh(address); err == nil {
+ return cluster, nil
+ } else {
+ errors = append(errors, fmt.Sprintf("%s: %v", address, err))
+ }
+ }
+ cluster.Close()
+ return nil, fmt.Errorf("initialize TLS cluster from %v failed: %s", options.StartNodes, strings.Join(errors, "; "))
+}
+
+func (c *TLSClusterConn) dial(address string) (redis.Conn, error) {
+ options := []redis.DialOption{
+ redis.DialUseTLS(true),
+ redis.DialTLSConfig(c.options.TLSConfig),
+ }
+ if c.options.Timeout > 0 {
+ options = append(options,
+ redis.DialConnectTimeout(c.options.Timeout),
+ redis.DialReadTimeout(c.options.Timeout),
+ redis.DialWriteTimeout(c.options.Timeout),
+ )
+ }
+
+ conn, err := redis.Dial("tcp", address, options...)
+ if err != nil {
+ return nil, err
+ }
+ if c.options.Password == "" {
+ return conn, nil
+ }
+
+ authArgs := make([]interface{}, 0, 2)
+ for _, arg := range strings.SplitN(c.options.Password, ":", 2) {
+ authArgs = append(authArgs, arg)
+ }
+ if _, err := conn.Do(c.options.AuthType, authArgs...); err != nil {
+ conn.Close()
+ return nil, err
+ }
+ return conn, nil
+}
+
+func (c *TLSClusterConn) pool(address string) (*redis.Pool, error) {
+ c.closeMu.RLock()
+ closed := c.closed
+ c.closeMu.RUnlock()
+ if closed {
+ return nil, fmt.Errorf("TLS cluster connection is closed")
+ }
+
+ c.topologyMu.Lock()
+ defer c.topologyMu.Unlock()
+ if pool, ok := c.pools[address]; ok {
+ return pool, nil
+ }
+
+ pool := &redis.Pool{
+ MaxIdle: 16,
+ IdleTimeout: 60 * time.Second,
+ Dial: func() (redis.Conn, error) {
+ return c.dial(address)
+ },
+ }
+ c.pools[address] = pool
+ return pool, nil
+}
+
+func (c *TLSClusterConn) refresh(seedAddress string) error {
+ pool, err := c.pool(seedAddress)
+ if err != nil {
+ return err
+ }
+ conn := pool.Get()
+ defer conn.Close()
+ if err := conn.Err(); err != nil {
+ return err
+ }
+
+ reply, err := redis.Values(conn.Do("CLUSTER", "SLOTS"))
+ if err != nil {
+ return fmt.Errorf("CLUSTER SLOTS: %w", err)
+ }
+
+ var slots [redisClusterSlotCount]string
+ covered := 0
+ nodeSet := make(map[string]struct{})
+ seedHost, _, _ := net.SplitHostPort(seedAddress)
+ for _, rawSlot := range reply {
+ values, err := redis.Values(rawSlot, nil)
+ if err != nil || len(values) < 3 {
+ return fmt.Errorf("invalid CLUSTER SLOTS entry %v", rawSlot)
+ }
+ start, err := redis.Int(values[0], nil)
+ if err != nil {
+ return fmt.Errorf("invalid cluster slot start: %w", err)
+ }
+ end, err := redis.Int(values[1], nil)
+ if err != nil || start < 0 || end < start || end >= redisClusterSlotCount {
+ return fmt.Errorf("invalid cluster slot range %d-%d", start, end)
+ }
+
+ primary, err := redis.Values(values[2], nil)
+ if err != nil || len(primary) < 2 {
+ return fmt.Errorf("invalid cluster primary entry %v", values[2])
+ }
+ host, err := redis.String(primary[0], nil)
+ if err != nil {
+ return fmt.Errorf("invalid cluster primary host: %w", err)
+ }
+ if host == "" {
+ host = seedHost
+ }
+ port, err := redis.Int(primary[1], nil)
+ if err != nil {
+ return fmt.Errorf("invalid cluster primary port: %w", err)
+ }
+ address := net.JoinHostPort(strings.Trim(host, "[]"), strconv.Itoa(port))
+
+ for slot := start; slot <= end; slot++ {
+ if slots[slot] == "" {
+ covered++
+ }
+ slots[slot] = address
+ }
+ nodeSet[address] = struct{}{}
+ }
+ if covered != redisClusterSlotCount {
+ return fmt.Errorf("CLUSTER SLOTS covers %d of %d slots", covered, redisClusterSlotCount)
+ }
+
+ nodes := make([]string, 0, len(nodeSet))
+ for address := range nodeSet {
+ nodes = append(nodes, address)
+ if _, err := c.pool(address); err != nil {
+ return err
+ }
+ }
+ sort.Strings(nodes)
+
+ c.topologyMu.Lock()
+ c.slots = slots
+ c.nodes = nodes
+ c.topologyMu.Unlock()
+ return nil
+}
+
+func (c *TLSClusterConn) route(commandName string, args []interface{}) (string, error) {
+ command := strings.ToUpper(commandName)
+ if command == "PING" || command == "CLUSTER" || command == "INFO" {
+ c.topologyMu.RLock()
+ defer c.topologyMu.RUnlock()
+ if len(c.nodes) == 0 {
+ return "", fmt.Errorf("TLS cluster has no available nodes")
+ }
+ return c.nodes[0], nil
+ }
+ if len(args) == 0 {
+ return "", fmt.Errorf("cluster command %s requires a key", commandName)
+ }
+
+ slot, err := redigoCluster.GetSlot(args[0])
+ if err != nil {
+ return "", err
+ }
+ c.topologyMu.RLock()
+ address := c.slots[slot]
+ c.topologyMu.RUnlock()
+ if address == "" {
+ return "", fmt.Errorf("no cluster node for slot %d", slot)
+ }
+ return address, nil
+}
+
+func (c *TLSClusterConn) executeAt(address, commandName string, args ...interface{}) (interface{}, error) {
+ pool, err := c.pool(address)
+ if err != nil {
+ return nil, err
+ }
+ conn := pool.Get()
+ defer conn.Close()
+ if err := conn.Err(); err != nil {
+ return nil, err
+ }
+ return conn.Do(commandName, args...)
+}
+
+func parseClusterRedirect(err error) (kind, address string, ok bool) {
+ if err == nil {
+ return "", "", false
+ }
+ fields := strings.Fields(err.Error())
+ if len(fields) != 3 || (fields[0] != "MOVED" && fields[0] != "ASK") {
+ return "", "", false
+ }
+ return fields[0], fields[2], true
+}
+
+func (c *TLSClusterConn) execute(commandName string, args ...interface{}) (interface{}, error) {
+ command := strings.ToUpper(commandName)
+ if command == "AUTH" || command == "ADMINAUTH" || command == "SELECT" {
+ return "OK", nil
+ }
+
+ address, err := c.route(commandName, args)
+ if err != nil {
+ return nil, err
+ }
+ for redirects := 0; redirects < 3; redirects++ {
+ reply, err := c.executeAt(address, commandName, args...)
+ kind, redirectAddress, redirected := parseClusterRedirect(err)
+ if !redirected {
+ return reply, err
+ }
+
+ address = redirectAddress
+ if kind == "ASK" {
+ pool, poolErr := c.pool(address)
+ if poolErr != nil {
+ return nil, poolErr
+ }
+ conn := pool.Get()
+ if conn.Err() != nil {
+ err = conn.Err()
+ conn.Close()
+ return nil, err
+ }
+ if _, err = conn.Do("ASKING"); err == nil {
+ reply, err = conn.Do(commandName, args...)
+ }
+ conn.Close()
+ return reply, err
+ }
+ _ = c.refresh(address)
+ }
+ return nil, fmt.Errorf("too many cluster redirects for %s", commandName)
+}
+
+func (c *TLSClusterConn) Do(commandName string, args ...interface{}) (interface{}, error) {
+ return c.execute(commandName, args...)
+}
+
+func (c *TLSClusterConn) Send(commandName string, args ...interface{}) error {
+ c.pipelineMu.Lock()
+ defer c.pipelineMu.Unlock()
+ c.pending = append(c.pending, tlsClusterCommand{name: commandName, args: args})
+ return nil
+}
+
+func (c *TLSClusterConn) Flush() error {
+ c.pipelineMu.Lock()
+ commands := c.pending
+ c.pending = nil
+ c.replies = nil
+ c.pipelineMu.Unlock()
+
+ replies := make([]tlsClusterReply, len(commands))
+ groups := make(map[string][]int)
+ for index, command := range commands {
+ address, err := c.route(command.name, command.args)
+ if err != nil {
+ replies[index].err = err
+ continue
+ }
+ groups[address] = append(groups[address], index)
+ }
+
+ for address, indexes := range groups {
+ pool, err := c.pool(address)
+ if err != nil {
+ return err
+ }
+ conn := pool.Get()
+ if err := conn.Err(); err != nil {
+ conn.Close()
+ return err
+ }
+ for _, index := range indexes {
+ command := commands[index]
+ if err := conn.Send(command.name, command.args...); err != nil {
+ conn.Close()
+ return err
+ }
+ }
+ if err := conn.Flush(); err != nil {
+ conn.Close()
+ return err
+ }
+ for _, index := range indexes {
+ reply, err := conn.Receive()
+ if _, _, redirected := parseClusterRedirect(err); redirected {
+ command := commands[index]
+ reply, err = c.execute(command.name, command.args...)
+ }
+ replies[index] = tlsClusterReply{value: reply, err: err}
+ }
+ conn.Close()
+ }
+
+ c.pipelineMu.Lock()
+ c.replies = replies
+ c.pipelineMu.Unlock()
+ return nil
+}
+
+func (c *TLSClusterConn) Receive() (interface{}, error) {
+ c.pipelineMu.Lock()
+ defer c.pipelineMu.Unlock()
+ if len(c.replies) == 0 {
+ return nil, fmt.Errorf("no pending TLS cluster replies")
+ }
+ reply := c.replies[0]
+ c.replies = c.replies[1:]
+ return reply.value, reply.err
+}
+
+func (c *TLSClusterConn) Err() error {
+ c.closeMu.RLock()
+ defer c.closeMu.RUnlock()
+ if c.closed {
+ return fmt.Errorf("TLS cluster connection is closed")
+ }
+ return nil
+}
+
+func (c *TLSClusterConn) Close() error {
+ c.closeMu.Lock()
+ if c.closed {
+ c.closeMu.Unlock()
+ return nil
+ }
+ c.closed = true
+ c.closeMu.Unlock()
+
+ c.topologyMu.Lock()
+ defer c.topologyMu.Unlock()
+ for _, pool := range c.pools {
+ pool.Close()
+ }
+ return nil
+}
diff --git a/src/full_check/client/tls_cluster_test.go b/src/full_check/client/tls_cluster_test.go
new file mode 100644
index 0000000..7ddf510
--- /dev/null
+++ b/src/full_check/client/tls_cluster_test.go
@@ -0,0 +1,326 @@
+package client
+
+import (
+ "bufio"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/pem"
+ "fmt"
+ "io"
+ "math/big"
+ "net"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ redigoCluster "github.com/najoast/redis-go-cluster"
+)
+
+type testTLSCluster struct {
+ servers []*testTLSServer
+ config *tls.Config
+}
+
+type testTLSServer struct {
+ id string
+ listener net.Listener
+ cluster *testTLSCluster
+
+ mu sync.Mutex
+ commands []string
+ wg sync.WaitGroup
+}
+
+func newTestTLSCertificate(t *testing.T) (tls.Certificate, []byte, *x509.CertPool) {
+ t.Helper()
+ key, err := rsa.GenerateKey(rand.Reader, 2048)
+ if err != nil {
+ t.Fatal(err)
+ }
+ template := &x509.Certificate{
+ SerialNumber: big.NewInt(1),
+ Subject: pkix.Name{CommonName: "localhost"},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(time.Hour),
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
+ DNSNames: []string{"localhost"},
+ IsCA: true,
+ BasicConstraintsValid: true,
+ }
+ der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+ keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+ certificate, err := tls.X509KeyPair(certificatePEM, keyPEM)
+ if err != nil {
+ t.Fatal(err)
+ }
+ roots := x509.NewCertPool()
+ roots.AppendCertsFromPEM(certificatePEM)
+ return certificate, certificatePEM, roots
+}
+
+func newTestTLSCluster(t *testing.T) *testTLSCluster {
+ t.Helper()
+ certificate, _, roots := newTestTLSCertificate(t)
+ cluster := &testTLSCluster{
+ config: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots},
+ }
+ for _, id := range []string{"node-a", "node-b"} {
+ listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ Certificates: []tls.Certificate{certificate},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ server := &testTLSServer{id: id, listener: listener, cluster: cluster}
+ cluster.servers = append(cluster.servers, server)
+ }
+ for _, server := range cluster.servers {
+ server.wg.Add(1)
+ go server.serve()
+ }
+ return cluster
+}
+
+func (c *testTLSCluster) Close() {
+ for _, server := range c.servers {
+ server.listener.Close()
+ }
+ for _, server := range c.servers {
+ server.wg.Wait()
+ }
+}
+
+func (s *testTLSServer) serve() {
+ defer s.wg.Done()
+ for {
+ conn, err := s.listener.Accept()
+ if err != nil {
+ return
+ }
+ s.wg.Add(1)
+ go func() {
+ defer s.wg.Done()
+ defer conn.Close()
+ s.serveConn(conn)
+ }()
+ }
+}
+
+func (s *testTLSServer) serveConn(conn net.Conn) {
+ reader := bufio.NewReader(conn)
+ writer := bufio.NewWriter(conn)
+ for {
+ command, err := readRESPCommand(reader)
+ if err != nil {
+ return
+ }
+ s.mu.Lock()
+ s.commands = append(s.commands, strings.Join(command, " "))
+ s.mu.Unlock()
+
+ switch strings.ToUpper(command[0]) {
+ case "PING":
+ writeSimpleString(writer, "PONG")
+ case "AUTH", "ADMINAUTH", "SELECT", "ASKING":
+ writeSimpleString(writer, "OK")
+ case "CLUSTER":
+ if len(command) > 1 && strings.EqualFold(command[1], "SLOTS") {
+ s.writeClusterSlots(writer)
+ } else if len(command) > 1 && strings.EqualFold(command[1], "NODES") {
+ s.writeClusterNodes(writer)
+ } else {
+ writeError(writer, "ERR unsupported CLUSTER command")
+ }
+ case "GET":
+ writeBulkString(writer, s.id+":"+command[1])
+ case "TYPE":
+ writeSimpleString(writer, "string")
+ default:
+ writeError(writer, "ERR unsupported command")
+ }
+ if err := writer.Flush(); err != nil {
+ return
+ }
+ }
+}
+
+func (s *testTLSServer) writeClusterSlots(writer *bufio.Writer) {
+ fmt.Fprint(writer, "*2\r\n")
+ for index, bounds := range [][2]int{{0, 8191}, {8192, 16383}} {
+ host, portText, _ := net.SplitHostPort(s.cluster.servers[index].listener.Addr().String())
+ port, _ := strconv.Atoi(portText)
+ fmt.Fprintf(writer, "*3\r\n:%d\r\n:%d\r\n*3\r\n", bounds[0], bounds[1])
+ writeBulkString(writer, host)
+ fmt.Fprintf(writer, ":%d\r\n", port)
+ writeBulkString(writer, s.cluster.servers[index].id)
+ }
+}
+
+func (s *testTLSServer) writeClusterNodes(writer *bufio.Writer) {
+ var lines strings.Builder
+ for index, server := range s.cluster.servers {
+ start, end := 0, 8191
+ if index == 1 {
+ start, end = 8192, 16383
+ }
+ fmt.Fprintf(&lines, "%s %s@1 master - 0 0 %d connected %d-%d\n",
+ server.id, server.listener.Addr().String(), index+1, start, end)
+ }
+ writeBulkString(writer, lines.String())
+}
+
+func (s *testTLSServer) sawCommand(command string) bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for _, actual := range s.commands {
+ if actual == command {
+ return true
+ }
+ }
+ return false
+}
+
+func readRESPCommand(reader *bufio.Reader) ([]string, error) {
+ line, err := reader.ReadString('\n')
+ if err != nil {
+ return nil, err
+ }
+ if len(line) < 4 || line[0] != '*' {
+ return nil, fmt.Errorf("invalid RESP array %q", line)
+ }
+ count, err := strconv.Atoi(strings.TrimSpace(line[1:]))
+ if err != nil {
+ return nil, err
+ }
+ command := make([]string, count)
+ for index := range command {
+ line, err = reader.ReadString('\n')
+ if err != nil || len(line) < 4 || line[0] != '$' {
+ return nil, fmt.Errorf("invalid RESP bulk length %q: %v", line, err)
+ }
+ length, err := strconv.Atoi(strings.TrimSpace(line[1:]))
+ if err != nil {
+ return nil, err
+ }
+ value := make([]byte, length+2)
+ if _, err = io.ReadFull(reader, value); err != nil {
+ return nil, err
+ }
+ command[index] = string(value[:length])
+ }
+ return command, nil
+}
+
+func writeSimpleString(writer *bufio.Writer, value string) {
+ fmt.Fprintf(writer, "+%s\r\n", value)
+}
+
+func writeBulkString(writer *bufio.Writer, value string) {
+ fmt.Fprintf(writer, "$%d\r\n%s\r\n", len(value), value)
+}
+
+func writeError(writer *bufio.Writer, value string) {
+ fmt.Fprintf(writer, "-%s\r\n", value)
+}
+
+func keyInSlotRange(t *testing.T, start, end int) []byte {
+ t.Helper()
+ for index := 0; index < 100000; index++ {
+ key := []byte(fmt.Sprintf("key-%d", index))
+ slot, err := redigoCluster.GetSlot(key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if int(slot) >= start && int(slot) <= end {
+ return key
+ }
+ }
+ t.Fatal("could not find key in slot range")
+ return nil
+}
+
+func TestRedisClientUsesTLS(t *testing.T) {
+ cluster := newTestTLSCluster(t)
+ defer cluster.Close()
+
+ client, err := NewRedisClient(RedisHost{
+ Addr: []string{cluster.servers[0].listener.Addr().String()},
+ Authtype: "auth",
+ TLSConfig: cluster.config,
+ }, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ client.Close()
+ if !cluster.servers[0].sawCommand("ping") {
+ t.Fatal("TLS server did not receive PING")
+ }
+}
+
+func TestTLSClusterRoutesAndPipelines(t *testing.T) {
+ cluster := newTestTLSCluster(t)
+ defer cluster.Close()
+
+ conn, err := NewTLSClusterConn(TLSClusterOptions{
+ StartNodes: []string{cluster.servers[0].listener.Addr().String()},
+ AuthType: "auth",
+ Timeout: 5 * time.Second,
+ TLSConfig: cluster.config,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+
+ firstKey := keyInSlotRange(t, 0, 8191)
+ secondKey := keyInSlotRange(t, 8192, 16383)
+ firstReply, err := conn.Do("GET", firstKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ secondReply, err := conn.Do("GET", secondKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(firstReply.([]byte)) != "node-a:"+string(firstKey) {
+ t.Fatalf("unexpected first node reply %q", firstReply)
+ }
+ if string(secondReply.([]byte)) != "node-b:"+string(secondKey) {
+ t.Fatalf("unexpected second node reply %q", secondReply)
+ }
+
+ if err := conn.Send("TYPE", firstKey); err != nil {
+ t.Fatal(err)
+ }
+ if err := conn.Send("TYPE", secondKey); err != nil {
+ t.Fatal(err)
+ }
+ if err := conn.Flush(); err != nil {
+ t.Fatal(err)
+ }
+ for index := 0; index < 2; index++ {
+ reply, err := conn.Receive()
+ if err != nil || reply != "string" {
+ t.Fatalf("pipeline reply %d = %v, %v", index, reply, err)
+ }
+ }
+
+ if !cluster.servers[0].sawCommand("GET " + string(firstKey)) {
+ t.Fatalf("first key %q was not routed to node-a", firstKey)
+ }
+ if !cluster.servers[1].sawCommand("GET " + string(secondKey)) {
+ t.Fatalf("second key %q was not routed to node-b", secondKey)
+ }
+}
diff --git a/src/full_check/client/tls_test.go b/src/full_check/client/tls_test.go
new file mode 100644
index 0000000..f945e5f
--- /dev/null
+++ b/src/full_check/client/tls_test.go
@@ -0,0 +1,38 @@
+package client
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestLoadTLSConfigValidation(t *testing.T) {
+ if config, err := LoadTLSConfig(false, "", "", "", "", false); err != nil || config != nil {
+ t.Fatalf("disabled TLS returned config=%v err=%v", config, err)
+ }
+ if _, err := LoadTLSConfig(false, "ca.pem", "", "", "", false); err == nil {
+ t.Fatal("expected TLS options without TLS enabled to fail")
+ }
+ if _, err := LoadTLSConfig(true, "", "client.pem", "", "", false); err == nil {
+ t.Fatal("expected an incomplete client certificate pair to fail")
+ }
+}
+
+func TestLoadTLSConfigCA(t *testing.T) {
+ _, certificatePEM, _ := newTestTLSCertificate(t)
+ caFile := filepath.Join(t.TempDir(), "ca.pem")
+ if err := os.WriteFile(caFile, certificatePEM, 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ config, err := LoadTLSConfig(true, caFile, "", "", "redis.internal", false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if config.RootCAs == nil {
+ t.Fatal("expected custom root CA pool")
+ }
+ if config.ServerName != "redis.internal" {
+ t.Fatalf("unexpected server name %q", config.ServerName)
+ }
+}
diff --git a/src/full_check/configure/conf.go b/src/full_check/configure/conf.go
index dca7612..de0da46 100644
--- a/src/full_check/configure/conf.go
+++ b/src/full_check/configure/conf.go
@@ -1,32 +1,44 @@
package conf
var Opts struct {
- SourceAddr string `short:"s" long:"source" value-name:"SOURCE" description:"Set host:port of source redis. If db type is cluster, split by semicolon(;'), e.g., 10.1.1.1:1000;10.2.2.2:2000;10.3.3.3:3000. We also support auto-detection, so \"master@10.1.1.1:1000\" or \"slave@10.1.1.1:1000\" means choose master or slave. Only need to give a role in the master or slave."`
- SourcePassword string `short:"p" long:"sourcepassword" value-name:"Password" description:"Set source redis password (format: password or username:password)"`
- SourceAuthType string `long:"sourceauthtype" value-name:"AUTH-TYPE" default:"auth" description:"useless for opensource redis, valid value:auth/adminauth" `
- SourceDBType int `long:"sourcedbtype" default:"0" description:"0: db, 1: cluster 2: aliyun proxy, 3: tencent proxy"`
- SourceDBFilterList string `long:"sourcedbfilterlist" default:"-1" description:"db white list that need to be compared, -1 means fetch all, \"0;5;15\" means fetch db 0, 5, and 15"`
- TargetAddr string `short:"t" long:"target" value-name:"TARGET" description:"Set host:port of target redis. If db type is cluster, split by semicolon(;'), e.g., 10.1.1.1:1000;10.2.2.2:2000;10.3.3.3:3000. We also support auto-detection, so \"master@10.1.1.1:1000\" or \"slave@10.1.1.1:1000\" means choose master or slave. Only need to give a role in the master or slave."`
- TargetPassword string `short:"a" long:"targetpassword" value-name:"Password" description:"Set target redis password (format: password or username:password)"`
- TargetAuthType string `long:"targetauthtype" value-name:"AUTH-TYPE" default:"auth" description:"useless for opensource redis, valid value:auth/adminauth" `
- TargetDBType int `long:"targetdbtype" default:"0" description:"0: db, 1: cluster 2: aliyun proxy 3: tencent proxy"`
- TargetDBFilterList string `long:"targetdbfilterlist" default:"-1" description:"db white list that need to be compared, -1 means fetch all, \"0;5;15\" means fetch db 0, 5, and 15"`
- ResultDBFile string `short:"d" long:"db" value-name:"Sqlite3-DB-FILE" default:"result.db" description:"sqlite3 db file for store result. If exist, it will be removed and a new file is created."`
- ResultFile string `long:"result" value-name:"FILE" description:"store all diff result into the file, format is 'db\tdiff-type\tkey\tfield'"`
- CompareTimes string `long:"comparetimes" value-name:"COUNT" default:"3" description:"Total compare count, at least 1. In the first round, all keys will be compared. The subsequent rounds of the comparison will be done on the previous results."`
- CompareMode int `short:"m" long:"comparemode" default:"2" description:"compare mode, 1: compare full value, 2: only compare value length, 3: only compare keys outline, 4: compare full value, but only compare value length when meets big key"`
- Id string `long:"id" default:"unknown" description:"used in metric, run id, useless for open source"`
- JobId string `long:"jobid" default:"unknown" description:"used in metric, job id, useless for open source"`
- TaskId string `long:"taskid" default:"unknown" description:"used in metric, task id, useless for open source"`
- Qps int `short:"q" long:"qps" default:"15000" description:"max batch qps limit: e.g., if qps is 10, full-check fetches 10 * $batch keys every second"`
- Interval int `long:"interval" value-name:"Second" default:"5" description:"The time interval for each round of comparison(Second)"`
- BatchCount string `long:"batchcount" value-name:"COUNT" default:"256" description:"the count of key/field per batch compare, valid value [1, 10000]"`
- Parallel int `long:"parallel" value-name:"COUNT" default:"5" description:"concurrent goroutine number for comparison, valid value [1, 100]"`
- LogFile string `long:"log" value-name:"FILE" description:"log file, if not specified, log is put to console"`
- LogLevel string `long:"loglevel" value-name:"LEVEL" description:"log level: 'debug', 'info', 'warn', 'error', default is 'info'"`
- MetricPrint bool `long:"metric" value-name:"BOOL" description:"print metric in log"`
- BigKeyThreshold int64 `long:"bigkeythreshold" value-name:"COUNT" default:"16384"`
- FilterList string `short:"f" long:"filterlist" value-name:"FILTER" default:"" description:"if the filter list isn't empty, all elements in list will be synced. The input should be split by '|'. The end of the string is followed by a * to indicate a prefix match, otherwise it is a full match. e.g.: 'abc*|efg|m*' matches 'abc', 'abc1', 'efg', 'm', 'mxyz', but 'efgh', 'p' aren't'"`
- SystemProfile uint `long:"systemprofile" value-name:"SYSTEM-PROFILE" default:"20445" description:"port that used to print golang inner head and stack message"`
- Version bool `short:"v" long:"version"`
+ SourceAddr string `short:"s" long:"source" value-name:"SOURCE" description:"Set host:port of source redis. If db type is cluster, split by semicolon(;'), e.g., 10.1.1.1:1000;10.2.2.2:2000;10.3.3.3:3000. We also support auto-detection, so \"master@10.1.1.1:1000\" or \"slave@10.1.1.1:1000\" means choose master or slave. Only need to give a role in the master or slave."`
+ SourcePassword string `short:"p" long:"sourcepassword" value-name:"Password" description:"Set source redis password (format: password or username:password)"`
+ SourceAuthType string `long:"sourceauthtype" value-name:"AUTH-TYPE" default:"auth" description:"useless for opensource redis, valid value:auth/adminauth" `
+ SourceDBType int `long:"sourcedbtype" default:"0" description:"0: db, 1: cluster 2: aliyun proxy, 3: tencent proxy"`
+ SourceDBFilterList string `long:"sourcedbfilterlist" default:"-1" description:"db white list that need to be compared, -1 means fetch all, \"0;5;15\" means fetch db 0, 5, and 15"`
+ SourceTLS bool `long:"sourcetls" description:"Enable TLS for all source Redis connections"`
+ SourceTLSCAFile string `long:"sourcetlscacert" value-name:"FILE" description:"CA certificate file for source Redis TLS verification"`
+ SourceTLSCertFile string `long:"sourcetlscert" value-name:"FILE" description:"Client certificate file for source Redis mutual TLS"`
+ SourceTLSKeyFile string `long:"sourcetlskey" value-name:"FILE" description:"Client private key file for source Redis mutual TLS"`
+ SourceTLSServerName string `long:"sourcetlsservername" value-name:"NAME" description:"Override the source Redis TLS certificate server name"`
+ SourceTLSSkipVerify bool `long:"sourcetlsskipverify" description:"Skip source Redis TLS certificate verification (insecure)"`
+ TargetAddr string `short:"t" long:"target" value-name:"TARGET" description:"Set host:port of target redis. If db type is cluster, split by semicolon(;'), e.g., 10.1.1.1:1000;10.2.2.2:2000;10.3.3.3:3000. We also support auto-detection, so \"master@10.1.1.1:1000\" or \"slave@10.1.1.1:1000\" means choose master or slave. Only need to give a role in the master or slave."`
+ TargetPassword string `short:"a" long:"targetpassword" value-name:"Password" description:"Set target redis password (format: password or username:password)"`
+ TargetAuthType string `long:"targetauthtype" value-name:"AUTH-TYPE" default:"auth" description:"useless for opensource redis, valid value:auth/adminauth" `
+ TargetDBType int `long:"targetdbtype" default:"0" description:"0: db, 1: cluster 2: aliyun proxy 3: tencent proxy"`
+ TargetDBFilterList string `long:"targetdbfilterlist" default:"-1" description:"db white list that need to be compared, -1 means fetch all, \"0;5;15\" means fetch db 0, 5, and 15"`
+ TargetTLS bool `long:"targettls" description:"Enable TLS for all target Redis connections"`
+ TargetTLSCAFile string `long:"targettlscacert" value-name:"FILE" description:"CA certificate file for target Redis TLS verification"`
+ TargetTLSCertFile string `long:"targettlscert" value-name:"FILE" description:"Client certificate file for target Redis mutual TLS"`
+ TargetTLSKeyFile string `long:"targettlskey" value-name:"FILE" description:"Client private key file for target Redis mutual TLS"`
+ TargetTLSServerName string `long:"targettlsservername" value-name:"NAME" description:"Override the target Redis TLS certificate server name"`
+ TargetTLSSkipVerify bool `long:"targettlsskipverify" description:"Skip target Redis TLS certificate verification (insecure)"`
+ ResultDBFile string `short:"d" long:"db" value-name:"Sqlite3-DB-FILE" default:"result.db" description:"sqlite3 db file for store result. If exist, it will be removed and a new file is created."`
+ ResultFile string `long:"result" value-name:"FILE" description:"store all diff result into the file, format is 'db\tdiff-type\tkey\tfield'"`
+ CompareTimes string `long:"comparetimes" value-name:"COUNT" default:"3" description:"Total compare count, at least 1. In the first round, all keys will be compared. The subsequent rounds of the comparison will be done on the previous results."`
+ CompareMode int `short:"m" long:"comparemode" default:"2" description:"compare mode, 1: compare full value, 2: only compare value length, 3: only compare keys outline, 4: compare full value, but only compare value length when meets big key"`
+ Id string `long:"id" default:"unknown" description:"used in metric, run id, useless for open source"`
+ JobId string `long:"jobid" default:"unknown" description:"used in metric, job id, useless for open source"`
+ TaskId string `long:"taskid" default:"unknown" description:"used in metric, task id, useless for open source"`
+ Qps int `short:"q" long:"qps" default:"15000" description:"max batch qps limit: e.g., if qps is 10, full-check fetches 10 * $batch keys every second"`
+ Interval int `long:"interval" value-name:"Second" default:"5" description:"The time interval for each round of comparison(Second)"`
+ BatchCount string `long:"batchcount" value-name:"COUNT" default:"256" description:"the count of key/field per batch compare, valid value [1, 10000]"`
+ Parallel int `long:"parallel" value-name:"COUNT" default:"5" description:"concurrent goroutine number for comparison, valid value [1, 100]"`
+ LogFile string `long:"log" value-name:"FILE" description:"log file, if not specified, log is put to console"`
+ LogLevel string `long:"loglevel" value-name:"LEVEL" description:"log level: 'debug', 'info', 'warn', 'error', default is 'info'"`
+ MetricPrint bool `long:"metric" value-name:"BOOL" description:"print metric in log"`
+ BigKeyThreshold int64 `long:"bigkeythreshold" value-name:"COUNT" default:"16384"`
+ FilterList string `short:"f" long:"filterlist" value-name:"FILTER" default:"" description:"if the filter list isn't empty, all elements in list will be synced. The input should be split by '|'. The end of the string is followed by a * to indicate a prefix match, otherwise it is a full match. e.g.: 'abc*|efg|m*' matches 'abc', 'abc1', 'efg', 'm', 'mxyz', but 'efgh', 'p' aren't'"`
+ SystemProfile uint `long:"systemprofile" value-name:"SYSTEM-PROFILE" default:"20445" description:"port that used to print golang inner head and stack message"`
+ Version bool `short:"v" long:"version"`
}
diff --git a/src/full_check/go.mod b/src/full_check/go.mod
index d3cc12a..fff56ba 100644
--- a/src/full_check/go.mod
+++ b/src/full_check/go.mod
@@ -11,7 +11,6 @@ require (
github.com/mattn/go-sqlite3 v1.14.16
github.com/najoast/redis-go-cluster v1.0.0
github.com/stretchr/testify v1.8.1
- github.com/vinllen/redis-go-cluster v1.0.0
)
require (
diff --git a/src/full_check/go.sum b/src/full_check/go.sum
index 23529b9..5c755af 100644
--- a/src/full_check/go.sum
+++ b/src/full_check/go.sum
@@ -1,6 +1,7 @@
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs=
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
@@ -14,6 +15,7 @@ github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwp
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/najoast/redis-go-cluster v1.0.0 h1:GJhtiwitgaQ0Kc9ZcRE9FJCcu1GLCIIW7u7vpRrgE6k=
github.com/najoast/redis-go-cluster v1.0.0/go.mod h1:lGMMsVLZW+0gAuA+oo1YrFTZjjaIhkmhR6cA77/etiw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -21,10 +23,12 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/vinllen/redis-go-cluster v1.0.0/go.mod h1:xig5hQAOZX1K+KNUVDqAbhTRzMTPcb257nJl7OCHrI4=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/src/full_check/main.go b/src/full_check/main.go
index 1b9ec3a..05e4876 100644
--- a/src/full_check/main.go
+++ b/src/full_check/main.go
@@ -6,14 +6,14 @@ import (
"strconv"
"strings"
- "full_check/configure"
- "full_check/full_check"
"full_check/checker"
"full_check/client"
"full_check/common"
+ "full_check/configure"
+ "full_check/full_check"
- "github.com/jessevdk/go-flags"
"github.com/gugemichael/nimo4go"
+ "github.com/jessevdk/go-flags"
)
var VERSION = "$"
@@ -89,6 +89,18 @@ func main() {
if conf.Opts.TargetAuthType != "auth" && conf.Opts.TargetAuthType != "adminauth" {
panic(common.Logger.Errorf("invalid targetauthtype %s, expect auth/adminauth", conf.Opts.TargetAuthType))
}
+ sourceTLSConfig, err := client.LoadTLSConfig(conf.Opts.SourceTLS, conf.Opts.SourceTLSCAFile,
+ conf.Opts.SourceTLSCertFile, conf.Opts.SourceTLSKeyFile, conf.Opts.SourceTLSServerName,
+ conf.Opts.SourceTLSSkipVerify)
+ if err != nil {
+ panic(common.Logger.Errorf("invalid source TLS configuration: %v", err))
+ }
+ targetTLSConfig, err := client.LoadTLSConfig(conf.Opts.TargetTLS, conf.Opts.TargetTLSCAFile,
+ conf.Opts.TargetTLSCertFile, conf.Opts.TargetTLSKeyFile, conf.Opts.TargetTLSServerName,
+ conf.Opts.TargetTLSSkipVerify)
+ if err != nil {
+ panic(common.Logger.Errorf("invalid target TLS configuration: %v", err))
+ }
if conf.Opts.CompareMode < full_check.FullValue || conf.Opts.CompareMode > full_check.FullValueWithOutline {
panic(common.Logger.Errorf("invalid compare mode %d", conf.Opts.CompareMode))
}
@@ -100,7 +112,8 @@ func main() {
common.BigKeyThreshold = conf.Opts.BigKeyThreshold
}
- sourceAddressList, err := client.HandleAddress(conf.Opts.SourceAddr, conf.Opts.SourcePassword, conf.Opts.SourceAuthType)
+ sourceAddressList, err := client.HandleAddress(conf.Opts.SourceAddr, conf.Opts.SourcePassword,
+ conf.Opts.SourceAuthType, sourceTLSConfig)
if err != nil {
panic(common.Logger.Errorf("source address[%v] illegal[%v]", conf.Opts.SourceAddr, err))
} else if len(sourceAddressList) > 1 && conf.Opts.SourceDBType != 1 {
@@ -109,7 +122,8 @@ func main() {
panic(common.Logger.Errorf("input source address is empty"))
}
- targetAddressList, err := client.HandleAddress(conf.Opts.TargetAddr, conf.Opts.TargetPassword, conf.Opts.TargetAuthType)
+ targetAddressList, err := client.HandleAddress(conf.Opts.TargetAddr, conf.Opts.TargetPassword,
+ conf.Opts.TargetAuthType, targetTLSConfig)
if err != nil {
panic(common.Logger.Errorf("target address[%v] illegal[%v]", conf.Opts.TargetAddr, err))
} else if len(targetAddressList) > 1 && conf.Opts.TargetDBType != 1 {
@@ -146,6 +160,7 @@ func main() {
Authtype: conf.Opts.SourceAuthType,
DBType: conf.Opts.SourceDBType,
DBFilterList: common.FilterDBList(conf.Opts.SourceDBFilterList),
+ TLSConfig: sourceTLSConfig,
},
TargetHost: client.RedisHost{
Addr: targetAddressList,
@@ -155,6 +170,7 @@ func main() {
Authtype: conf.Opts.TargetAuthType,
DBType: conf.Opts.TargetDBType,
DBFilterList: common.FilterDBList(conf.Opts.TargetDBFilterList),
+ TLSConfig: targetTLSConfig,
},
ResultDBFile: conf.Opts.ResultDBFile,
CompareCount: compareCount,
@@ -164,7 +180,14 @@ func main() {
FilterTree: filterTree,
}
- common.Logger.Info("configuration: ", conf.Opts)
+ loggedOpts := conf.Opts
+ if loggedOpts.SourcePassword != "" {
+ loggedOpts.SourcePassword = "[REDACTED]"
+ }
+ if loggedOpts.TargetPassword != "" {
+ loggedOpts.TargetPassword = "[REDACTED]"
+ }
+ common.Logger.Info("configuration: ", loggedOpts)
common.Logger.Info("---------")
fullCheck := full_check.NewFullCheck(fullCheckParameter, full_check.CheckType(conf.Opts.CompareMode))