fix(redis): Resolve DeleteMany failure by supporting regex matching and optimizing deletion

go-redis/go-redis.go

@@ -121,7 +121,7 @@ func (provider *Redis) ListKeys() []string {
 
 	keys := []string{}
 
-	iter := provider.inClient.Scan(provider.ctx, 0, provider.hashtags+core.MappingKeyPrefix+"*", 0).Iterator()
+	iter := provider.inClient.Scan(provider.ctx, 0, provider.hashtags+core.MappingKeyPrefix+"*", 100).Iterator()
 	for iter.Next(provider.ctx) {
 		value := provider.Get(iter.Val())
 
@@ -157,7 +157,7 @@ func (provider *Redis) MapKeys(prefix string) map[string]string {
 	mapKeys := map[string]string{}
 	keys := []string{}
 
-	iter := provider.inClient.Scan(provider.ctx, 0, prefix+"*", 0).Iterator()
+	iter := provider.inClient.Scan(provider.ctx, 0, prefix+"*", 100).Iterator()
 	for iter.Next(provider.ctx) {
 		keys = append(keys, iter.Val())
 	}
@@ -319,12 +319,17 @@ func (provider *Redis) DeleteMany(key string) {
 	}
 
 	keys := []string{}
-	iter := provider.inClient.Scan(provider.ctx, 0, "*", 0).Iterator()
+	iter := provider.inClient.Scan(provider.ctx, 0, "*", 100).Iterator()
 
 	for iter.Next(provider.ctx) {
 		if rgKey.MatchString(iter.Val()) {
 			keys = append(keys, iter.Val())
 		}
+
+		if len(keys) >= 100 {
+			provider.inClient.Unlink(provider.ctx, keys...)
+			keys = keys[:0]
+		}
 	}
 
 	if iter.Err() != nil && !provider.reconnecting {
@@ -333,7 +338,10 @@ func (provider *Redis) DeleteMany(key string) {
 		return
 	}
 
-	provider.inClient.Del(provider.ctx, keys...)
+	// unlink the rest
+	if len(keys) > 0 {
+		provider.inClient.Unlink(provider.ctx, keys...)
+	}
 }
 
 // Init method will.

go-redis/go-redis_test.go

@@ -152,18 +152,18 @@ func TestRedis_DeleteMany(t *testing.T) {
 	client, _ := getRedisInstance()
 
 	if len(client.MapKeys("")) != 12 {
-		t.Error("The map should contain 12 elements")
+		t.Errorf("The map should contain 12 elements, %d given", len(client.MapKeys("")))
 	}
 
 	client.DeleteMany("MAP_KEYS_PREFIX_*")
 
 	if len(client.MapKeys("")) != 2 {
-		t.Error("The map should contain 2 element")
+		t.Errorf("The map should contain 2 element, %d given", len(client.MapKeys("")))
 	}
 
 	client.DeleteMany(".*")
 
 	if len(client.MapKeys("")) != 0 {
-		t.Error("The map should be empty")
+		t.Errorf("The map should be empty, %d given", len(client.MapKeys("")))
 	}
 }

go-redis/go.mod

@@ -7,13 +7,14 @@ replace github.com/darkweak/storages/core => ../core
 require (
 	github.com/darkweak/storages/core v0.0.19
 	github.com/pierrec/lz4/v4 v4.1.23
-	github.com/redis/go-redis/v9 v9.17.2
+	github.com/redis/go-redis/v9 v9.18.0
 	go.uber.org/zap v1.27.0
 )
 
 require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
 )

go-redis/go.sum

@@ -10,14 +10,20 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU=
 github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
-github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
+github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=

redis/go.mod

@@ -7,12 +7,12 @@ replace github.com/darkweak/storages/core => ../core
 require (
 	github.com/darkweak/storages/core v0.0.19
 	github.com/pierrec/lz4/v4 v4.1.23
-	github.com/redis/rueidis v1.0.54
+	github.com/redis/rueidis v1.0.73
 	go.uber.org/zap v1.27.0
 )
 
 require (
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
 )

redis/go.sum

@@ -1,15 +1,15 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
-github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM=
+github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4=
 github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU=
 github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/rueidis v1.0.54 h1:QGlQXic7ILkcogpbMTV2bjIyKsUsZk5mPMveoe9eMPs=
-github.com/redis/rueidis v1.0.54/go.mod h1:HqQFoIupoJzcRnOlI6URmujTCXfNgbm6kYpczVrs4Pw=
+github.com/redis/rueidis v1.0.73 h1:0Enrg0VuMdaYyNDDj0lLIheWY0uybCeQOh+jTp2GG3M=
+github.com/redis/rueidis v1.0.73/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -18,12 +18,14 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
-golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
 google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

redis/redis.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"regexp"
 	"strings"
 	"time"
 
@@ -109,7 +110,7 @@ func (provider *Redis) ListKeys() []string {
 	provider.logger.Debugf("Call the ListKeys function in redis")
 
 	for more := true; more; more = scan.Cursor != 0 {
-		if scan, err = provider.inClient.Do(context.Background(), provider.inClient.B().Scan().Cursor(scan.Cursor).Match(provider.hashtags+core.MappingKeyPrefix+"*").Build()).AsScanEntry(); err != nil {
+		if scan, err = provider.inClient.Do(provider.ctx, provider.inClient.B().Scan().Cursor(scan.Cursor).Match(provider.hashtags+core.MappingKeyPrefix+"*").Count(100).Build()).AsScanEntry(); err != nil {
 			provider.logger.Errorf("Cannot scan: %v", err)
 		}
 
@@ -146,7 +147,7 @@ func (provider *Redis) MapKeys(prefix string) map[string]string {
 	provider.logger.Debugf("Call the MapKeys in redis with the prefix %s", prefix)
 
 	for more := true; more; more = scan.Cursor != 0 {
-		if scan, err = provider.inClient.Do(context.Background(), provider.inClient.B().Scan().Cursor(scan.Cursor).Match(prefix+"*").Build()).AsScanEntry(); err != nil {
+		if scan, err = provider.inClient.Do(provider.ctx, provider.inClient.B().Scan().Cursor(scan.Cursor).Match(prefix+"*").Count(100).Build()).AsScanEntry(); err != nil {
 			provider.logger.Errorf("Cannot scan: %v", err)
 		}
 
@@ -229,7 +230,7 @@ func (provider *Redis) Get(key string) []byte {
 	return r
 }
 
-// Set method will store the response in Etcd provider.
+// Set method will store the response in Redis provider.
 func (provider *Redis) Set(key string, value []byte, duration time.Duration) error {
 	var cmd redis.Completed
 	if duration == -1 {
@@ -246,30 +247,44 @@ func (provider *Redis) Set(key string, value []byte, duration time.Duration) err
 	return err
 }
 
-// Delete method will delete the response in Etcd provider if exists corresponding to key param.
+// Delete method will delete the response in Redis provider if exists corresponding to key param.
 func (provider *Redis) Delete(key string) {
 	_ = provider.inClient.Do(provider.ctx, provider.inClient.B().Del().Key(key).Build())
 }
 
 // DeleteMany method will delete the responses in Redis provider if exists corresponding to the regex key param.
 func (provider *Redis) DeleteMany(key string) {
-	var scan redis.ScanEntry
+	provider.logger.Debugf("Call the DeleteMany function in redis")
 
 	var err error
 
-	elements := []string{}
+	var scan redis.ScanEntry
 
-	provider.logger.Debugf("Call the DeleteMany function in redis")
+	rgKey, err := regexp.Compile(key)
+	if err != nil {
+		return
+	}
 
 	for more := true; more; more = scan.Cursor != 0 {
-		if scan, err = provider.inClient.Do(context.Background(), provider.inClient.B().Scan().Cursor(scan.Cursor).Match(key).Build()).AsScanEntry(); err != nil {
+		if scan, err = provider.inClient.Do(provider.ctx, provider.inClient.B().Scan().Cursor(scan.Cursor).Match("*").Count(100).Build()).AsScanEntry(); err != nil {
 			provider.logger.Errorf("Cannot scan: %v", err)
 		}
 
-		elements = append(elements, scan.Elements...)
-	}
+		elements := []string{}
+
+		for _, element := range scan.Elements {
+			if rgKey.MatchString(element) {
+				elements = append(elements, element)
+			}
+		}
 
-	_ = provider.inClient.Do(provider.ctx, provider.inClient.B().Del().Key(elements...).Build())
+		// only unlink item if elements are found in the current iteration
+		if len(elements) > 0 {
+			if err = provider.inClient.Do(provider.ctx, provider.inClient.B().Unlink().Key(elements...).Build()).Error(); err != nil {
+				provider.logger.Errorf("Cannot unlink: %v", err)
+			}
+		}
+	}
 }
 
 // Init method will.

redis/redis_test.go

@@ -135,18 +135,18 @@ func TestRedis_DeleteMany(t *testing.T) {
 	client, _ := getRedisInstance()
 
 	if len(client.MapKeys("")) != 12 {
-		t.Error("The map should contain 12 elements")
+		t.Errorf("The map should contain 12 elements, %d given", len(client.MapKeys("")))
 	}
 
-	client.DeleteMany("MAP_KEYS_PREFIX_*")
+	client.DeleteMany("MAP_KEYS_PREFIX_")
 
 	if len(client.MapKeys("")) != 2 {
-		t.Error("The map should contain 2 element")
+		t.Errorf("The map should contain 2 elements, %d given", len(client.MapKeys("")))
 	}
 
-	client.DeleteMany("*")
+	client.DeleteMany(".+")
 
 	if len(client.MapKeys("")) != 0 {
-		t.Error("The map should be empty")
+		t.Errorf("The map should be empty, %d given", len(client.MapKeys("")))
 	}
 }