diff --git a/go-redis/go-redis.go b/go-redis/go-redis.go index 22f1aeb..b401727 100644 --- a/go-redis/go-redis.go +++ b/go-redis/go-redis.go @@ -121,7 +121,7 @@ func (provider *Redis) ListKeys() []string { keys := []string{} - iter := provider.inClient.Scan(provider.ctx, 0, provider.hashtags+core.MappingKeyPrefix+"*", 0).Iterator() + iter := provider.inClient.Scan(provider.ctx, 0, provider.hashtags+core.MappingKeyPrefix+"*", 100).Iterator() for iter.Next(provider.ctx) { value := provider.Get(iter.Val()) @@ -157,7 +157,7 @@ func (provider *Redis) MapKeys(prefix string) map[string]string { mapKeys := map[string]string{} keys := []string{} - iter := provider.inClient.Scan(provider.ctx, 0, prefix+"*", 0).Iterator() + iter := provider.inClient.Scan(provider.ctx, 0, prefix+"*", 100).Iterator() for iter.Next(provider.ctx) { keys = append(keys, iter.Val()) } @@ -319,12 +319,17 @@ func (provider *Redis) DeleteMany(key string) { } keys := []string{} - iter := provider.inClient.Scan(provider.ctx, 0, "*", 0).Iterator() + iter := provider.inClient.Scan(provider.ctx, 0, "*", 100).Iterator() for iter.Next(provider.ctx) { if rgKey.MatchString(iter.Val()) { keys = append(keys, iter.Val()) } + + if len(keys) >= 100 { + provider.inClient.Unlink(provider.ctx, keys...) + keys = keys[:0] + } } if iter.Err() != nil && !provider.reconnecting { @@ -333,7 +338,10 @@ func (provider *Redis) DeleteMany(key string) { return } - provider.inClient.Del(provider.ctx, keys...) + // unlink the rest + if len(keys) > 0 { + provider.inClient.Unlink(provider.ctx, keys...) + } } // Init method will. diff --git a/go-redis/go-redis_test.go b/go-redis/go-redis_test.go index 3e3f47e..c701dfc 100644 --- a/go-redis/go-redis_test.go +++ b/go-redis/go-redis_test.go @@ -152,18 +152,18 @@ func TestRedis_DeleteMany(t *testing.T) { client, _ := getRedisInstance() if len(client.MapKeys("")) != 12 { - t.Error("The map should contain 12 elements") + t.Errorf("The map should contain 12 elements, %d given", len(client.MapKeys(""))) } client.DeleteMany("MAP_KEYS_PREFIX_*") if len(client.MapKeys("")) != 2 { - t.Error("The map should contain 2 element") + t.Errorf("The map should contain 2 element, %d given", len(client.MapKeys(""))) } client.DeleteMany(".*") if len(client.MapKeys("")) != 0 { - t.Error("The map should be empty") + t.Errorf("The map should be empty, %d given", len(client.MapKeys(""))) } } diff --git a/go-redis/go.mod b/go-redis/go.mod index fa900d1..c4dc198 100644 --- a/go-redis/go.mod +++ b/go-redis/go.mod @@ -7,13 +7,14 @@ replace github.com/darkweak/storages/core => ../core require ( github.com/darkweak/storages/core v0.0.19 github.com/pierrec/lz4/v4 v4.1.23 - github.com/redis/go-redis/v9 v9.17.2 + github.com/redis/go-redis/v9 v9.18.0 go.uber.org/zap v1.27.0 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.10.0 // indirect google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/go-redis/go.sum b/go-redis/go.sum index 63f8b7c..c2e9b1a 100644 --- a/go-redis/go.sum +++ b/go-redis/go.sum @@ -10,14 +10,20 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= -github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= diff --git a/redis/go.mod b/redis/go.mod index 43e9ae8..6d9185a 100644 --- a/redis/go.mod +++ b/redis/go.mod @@ -7,12 +7,12 @@ replace github.com/darkweak/storages/core => ../core require ( github.com/darkweak/storages/core v0.0.19 github.com/pierrec/lz4/v4 v4.1.23 - github.com/redis/rueidis v1.0.54 + github.com/redis/rueidis v1.0.73 go.uber.org/zap v1.27.0 ) require ( go.uber.org/multierr v1.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/sys v0.39.0 // indirect google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/redis/go.sum b/redis/go.sum index 70e169c..481fcc4 100644 --- a/redis/go.sum +++ b/redis/go.sum @@ -1,15 +1,15 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= -github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= +github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/rueidis v1.0.54 h1:QGlQXic7ILkcogpbMTV2bjIyKsUsZk5mPMveoe9eMPs= -github.com/redis/rueidis v1.0.54/go.mod h1:HqQFoIupoJzcRnOlI6URmujTCXfNgbm6kYpczVrs4Pw= +github.com/redis/rueidis v1.0.73 h1:0Enrg0VuMdaYyNDDj0lLIheWY0uybCeQOh+jTp2GG3M= +github.com/redis/rueidis v1.0.73/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -18,12 +18,14 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/redis/redis.go b/redis/redis.go index 98bb84c..348232b 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/http" + "regexp" "strings" "time" @@ -109,7 +110,7 @@ func (provider *Redis) ListKeys() []string { provider.logger.Debugf("Call the ListKeys function in redis") for more := true; more; more = scan.Cursor != 0 { - if scan, err = provider.inClient.Do(context.Background(), provider.inClient.B().Scan().Cursor(scan.Cursor).Match(provider.hashtags+core.MappingKeyPrefix+"*").Build()).AsScanEntry(); err != nil { + if scan, err = provider.inClient.Do(provider.ctx, provider.inClient.B().Scan().Cursor(scan.Cursor).Match(provider.hashtags+core.MappingKeyPrefix+"*").Count(100).Build()).AsScanEntry(); err != nil { provider.logger.Errorf("Cannot scan: %v", err) } @@ -146,7 +147,7 @@ func (provider *Redis) MapKeys(prefix string) map[string]string { provider.logger.Debugf("Call the MapKeys in redis with the prefix %s", prefix) for more := true; more; more = scan.Cursor != 0 { - if scan, err = provider.inClient.Do(context.Background(), provider.inClient.B().Scan().Cursor(scan.Cursor).Match(prefix+"*").Build()).AsScanEntry(); err != nil { + if scan, err = provider.inClient.Do(provider.ctx, provider.inClient.B().Scan().Cursor(scan.Cursor).Match(prefix+"*").Count(100).Build()).AsScanEntry(); err != nil { provider.logger.Errorf("Cannot scan: %v", err) } @@ -229,7 +230,7 @@ func (provider *Redis) Get(key string) []byte { return r } -// Set method will store the response in Etcd provider. +// Set method will store the response in Redis provider. func (provider *Redis) Set(key string, value []byte, duration time.Duration) error { var cmd redis.Completed if duration == -1 { @@ -246,30 +247,44 @@ func (provider *Redis) Set(key string, value []byte, duration time.Duration) err return err } -// Delete method will delete the response in Etcd provider if exists corresponding to key param. +// Delete method will delete the response in Redis provider if exists corresponding to key param. func (provider *Redis) Delete(key string) { _ = provider.inClient.Do(provider.ctx, provider.inClient.B().Del().Key(key).Build()) } // DeleteMany method will delete the responses in Redis provider if exists corresponding to the regex key param. func (provider *Redis) DeleteMany(key string) { - var scan redis.ScanEntry + provider.logger.Debugf("Call the DeleteMany function in redis") var err error - elements := []string{} + var scan redis.ScanEntry - provider.logger.Debugf("Call the DeleteMany function in redis") + rgKey, err := regexp.Compile(key) + if err != nil { + return + } for more := true; more; more = scan.Cursor != 0 { - if scan, err = provider.inClient.Do(context.Background(), provider.inClient.B().Scan().Cursor(scan.Cursor).Match(key).Build()).AsScanEntry(); err != nil { + if scan, err = provider.inClient.Do(provider.ctx, provider.inClient.B().Scan().Cursor(scan.Cursor).Match("*").Count(100).Build()).AsScanEntry(); err != nil { provider.logger.Errorf("Cannot scan: %v", err) } - elements = append(elements, scan.Elements...) - } + elements := []string{} + + for _, element := range scan.Elements { + if rgKey.MatchString(element) { + elements = append(elements, element) + } + } - _ = provider.inClient.Do(provider.ctx, provider.inClient.B().Del().Key(elements...).Build()) + // only unlink item if elements are found in the current iteration + if len(elements) > 0 { + if err = provider.inClient.Do(provider.ctx, provider.inClient.B().Unlink().Key(elements...).Build()).Error(); err != nil { + provider.logger.Errorf("Cannot unlink: %v", err) + } + } + } } // Init method will. diff --git a/redis/redis_test.go b/redis/redis_test.go index 5f5d116..275db43 100644 --- a/redis/redis_test.go +++ b/redis/redis_test.go @@ -135,18 +135,18 @@ func TestRedis_DeleteMany(t *testing.T) { client, _ := getRedisInstance() if len(client.MapKeys("")) != 12 { - t.Error("The map should contain 12 elements") + t.Errorf("The map should contain 12 elements, %d given", len(client.MapKeys(""))) } - client.DeleteMany("MAP_KEYS_PREFIX_*") + client.DeleteMany("MAP_KEYS_PREFIX_") if len(client.MapKeys("")) != 2 { - t.Error("The map should contain 2 element") + t.Errorf("The map should contain 2 elements, %d given", len(client.MapKeys(""))) } - client.DeleteMany("*") + client.DeleteMany(".+") if len(client.MapKeys("")) != 0 { - t.Error("The map should be empty") + t.Errorf("The map should be empty, %d given", len(client.MapKeys(""))) } }