Skip to content

TTL Heap implementation #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
osteensco wants to merge 11 commits into
base: main
Choose a base branch
from
Draft

TTL Heap implementation #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c411d22
TTL heap implemented
osteensco Nov 25, 2024
8960808
adding additional tests
osteensco Nov 25, 2024
71fe095
resolved conflicts
osteensco Nov 26, 2024
4224a80
working implementation, debug race conditions
osteensco Nov 29, 2024
f8173de
race conditions addressed
osteensco Dec 3, 2024
585e351
Evict Expired TTL test WIP
osteensco Dec 4, 2024
968559b
Test_Cluster failing
osteensco Dec 17, 2024
ebd428d
still debugging
osteensco Dec 17, 2024
cbe0eed
minor touchups, more debugging
osteensco Dec 18, 2024
f9d7ba6
merge conf resolved
osteensco Feb 9, 2025
9c9fa27
additional debugging
osteensco Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,22 @@ build-modules-test:
test:
env RACE=false OUT=internal/modules/admin/testdata make build-modules-test && \
env RACE=false OUT=sugardb/testdata make build-modules-test && \
CGO_ENABLED=1 go test ./... -coverprofile coverage/coverage.out && \
CGO_ENABLED=1 go test ./... -timeout 1m -coverprofile coverage/coverage.out && \
rm -rf ./internal/modules/admin/testdata && \
rm -rf ./sugardb/testdata && \
rm -rf ./sugardb/aof

test-race:
env RACE=true OUT=internal/modules/admin/testdata make build-modules-test && \
env RACE=true OUT=sugardb/testdata make build-modules-test && \
CGO_ENABLED=1 go test ./... --race && \
CGO_ENABLED=1 go test ./... -timeout 1m --race && \
rm -rf ./internal/modules/admin/testdata && \
rm -rf ./sugardb/testdata && \
rm -rf ./sugardb/aof

testenv-run:
docker-compose -f test_env/run/docker-compose.yaml build
docker-compose -f test_env/run/docker-compose.yaml run projenv
docker-compose -f test_env/run/docker-compose.yaml run --rm projenv

testenv-test:
docker-compose -f test_env/test/docker-compose.yaml up --build
Expand Down
10,207 changes: 5,227 additions & 4,980 deletions coverage/coverage.out
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 0 additions & 3 deletions internal/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ type Config struct {
AOFSyncStrategy string `json:"AOFSyncStrategy" yaml:"AOFSyncStrategy"`
MaxMemory uint64 `json:"MaxMemory" yaml:"MaxMemory"`
EvictionPolicy string `json:"EvictionPolicy" yaml:"EvictionPolicy"`
EvictionSample uint `json:"EvictionSample" yaml:"EvictionSample"`
EvictionInterval time.Duration `json:"EvictionInterval" yaml:"EvictionInterval"`
ElectionTimeout time.Duration `json:"ElectionTimeout" yaml:"ElectionTimeout"`
HeartbeatTimeout time.Duration `json:"HeartbeatTimeout" yaml:"HeartbeatTimeout"`
Expand Down Expand Up @@ -162,7 +161,6 @@ There is no limit by default.`, func(memory string) error {
snapshotInterval := flag.Duration("snapshot-interval", 5*time.Minute, "The time interval between snapshots (in seconds). Default is 5 minutes.")
restoreSnapshot := flag.Bool("restore-snapshot", false, "This flag prompts the echovault to restore state from snapshot when set to true. Only works in standalone mode. Higher priority than restoreAOF.")
restoreAOF := flag.Bool("restore-aof", false, "This flag prompts the echovault to restore state from append-only logs. Only works in standalone mode. Lower priority than restoreSnapshot.")
evictionSample := flag.Uint("eviction-sample", 20, "An integer specifying the number of keys to sample when checking for expired keys.")
evictionInterval := flag.Duration("eviction-interval", 100*time.Millisecond, "The interval between each sampling of keys to evict.")
electionTimeout := flag.Duration("election-timeout", 1000*time.Millisecond, "The maximum duration the leader will wait for followers to reach consensus on an election before starting a new election")
heartbeatTimeout := flag.Duration("heartbeat-timeout", 1000*time.Millisecond, "The interval between heartbeats sent by the leader to followers. In other words, the time in candidate state without leader contact.")
Expand Down Expand Up @@ -222,7 +220,6 @@ It is a plain text value by default but you can provide a SHA256 hash by adding
AOFSyncStrategy: aofSyncStrategy,
MaxMemory: maxMemory,
EvictionPolicy: evictionPolicy,
EvictionSample: *evictionSample,
EvictionInterval: *evictionInterval,
ElectionTimeout: *electionTimeout,
HeartbeatTimeout: *heartbeatTimeout,
Expand Down
2 changes: 0 additions & 2 deletions internal/config/default.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
func DefaultConfig() Config {
raftBindAddr, _ := internal.GetIPAddress()
raftBindPort, _ := internal.GetFreePort()

return Config{
TLS: false,
MTLS: false,
Expand All @@ -36,7 +35,6 @@ func DefaultConfig() Config {
AOFSyncStrategy: "everysec",
MaxMemory: 0,
EvictionPolicy: constants.NoEviction,
EvictionSample: 20,
EvictionInterval: 100 * time.Millisecond,
ElectionTimeout: 1000 * time.Millisecond,
HeartbeatTimeout: 1000 * time.Millisecond,
Expand Down
6 changes: 6 additions & 0 deletions internal/memberlist/memberlist.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ func NewMemberList(opts Opts) *MemberList {
}
}

// !!!!!!!!!!!!
// delete me probably
func (m *MemberList) ShowNumNodes() int {
return m.noOfNodes
}

func (m *MemberList) MemberListInit(ctx context.Context) {
cfg := memberlist.DefaultWANConfig()
cfg.RequireNodeNames = true
Expand Down
18 changes: 7 additions & 11 deletions internal/modules/acl/commands_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"encoding/hex"
"fmt"
"github.com/echovault/sugardb/internal"
"github.com/echovault/sugardb/internal/config"
"github.com/echovault/sugardb/internal/constants"
"github.com/echovault/sugardb/sugardb"
"github.com/tidwall/resp"
Expand All @@ -32,18 +31,15 @@ import (
)

func setUpServer(port int, requirePass bool, aclConfig string) (*sugardb.SugarDB, error) {
conf := config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.NoEviction,
RequirePass: requirePass,
Password: "password1",
AclConfig: aclConfig,
}

mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(conf),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.NoEviction),
sugardb.WithRequirePass(requirePass),
sugardb.WithPassword("password1"),
sugardb.WithAclConfig(aclConfig),
)
if err != nil {
return nil, err
Expand Down
18 changes: 8 additions & 10 deletions internal/modules/connection/commands_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,20 @@ import (
"testing"

"github.com/echovault/sugardb/internal"
"github.com/echovault/sugardb/internal/config"
"github.com/echovault/sugardb/internal/constants"
"github.com/echovault/sugardb/sugardb"
"github.com/tidwall/resp"
)

func setUpServer(port int, requirePass bool, aclConfig string) (*sugardb.SugarDB, error) {
conf := config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.NoEviction,
RequirePass: requirePass,
Password: "password1",
AclConfig: aclConfig,
}
conf := sugardb.DefaultConfig()
conf.BindAddr = "localhost"
conf.Port = uint16(port)
conf.DataDir = ""
conf.EvictionPolicy = constants.NoEviction
conf.RequirePass = requirePass
conf.Password = "password1"
conf.AclConfig = aclConfig

mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(conf),
Expand Down
4 changes: 3 additions & 1 deletion internal/modules/generic/commands.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ func handleGet(params internal.HandlerFuncParams) ([]byte, error) {
return nil, err
}
key := keys.ReadKeys[0]
keyExists := params.KeysExist(params.Context, []string{key})[key]

keyExists := params.KeysExist(params.Context, []string{key})[key]
if !keyExists {
return []byte("$-1\r\n"), nil
}
Expand Down Expand Up @@ -753,6 +753,7 @@ func handleGetdel(params internal.HandlerFuncParams) ([]byte, error) {

value := params.GetValues(params.Context, []string{key})[key]
delkey := keys.WriteKeys[0]

err = params.DeleteKey(params.Context, delkey)
if err != nil {
return nil, err
Expand Down Expand Up @@ -983,6 +984,7 @@ func handleMove(params internal.HandlerFuncParams) ([]byte, error) {
}

// remove key from source db

err = params.DeleteKey(params.Context, key)
if err != nil {
return nil, err
Expand Down
99 changes: 60 additions & 39 deletions internal/modules/generic/commands_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

"github.com/echovault/sugardb/internal"
"github.com/echovault/sugardb/internal/clock"
"github.com/echovault/sugardb/internal/config"
"github.com/echovault/sugardb/internal/constants"
"github.com/echovault/sugardb/internal/modules/set"
"github.com/echovault/sugardb/internal/modules/sorted_set"
Expand All @@ -47,12 +46,10 @@ func Test_Generic(t *testing.T) {
}

mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.NoEviction,
}),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.NoEviction),
)
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -1091,9 +1088,9 @@ func Test_Generic(t *testing.T) {
name: "2. Return expire time in milliseconds",
command: []string{"PEXPIRETIME", "ExpireTimeKey2"},
presetValues: map[string]KeyData{
"ExpireTimeKey2": {Value: "value2", ExpireAt: mockClock.Now().Add(4096 * time.Millisecond)},
"ExpireTimeKey2": {Value: "value2", ExpireAt: mockClock.Now().Add(409600 * time.Millisecond)},
},
expectedResponse: int(mockClock.Now().Add(4096 * time.Millisecond).UnixMilli()),
expectedResponse: int(mockClock.Now().Add(409600 * time.Millisecond).UnixMilli()),
expectedError: nil,
},
{
Expand Down Expand Up @@ -1320,6 +1317,7 @@ func Test_Generic(t *testing.T) {
expectedResponse int
expectedValues map[string]KeyData
expectedError error
checkExpired bool
}{
{
name: "1. Set new expire by seconds",
Expand All @@ -1335,13 +1333,13 @@ func Test_Generic(t *testing.T) {
},
{
name: "2. Set new expire by milliseconds",
command: []string{"PEXPIRE", "ExpireKey2", "1000"},
command: []string{"PEXPIRE", "ExpireKey2", "1000000"},
presetValues: map[string]KeyData{
"ExpireKey2": {Value: "value2", ExpireAt: time.Time{}},
},
expectedResponse: 1,
expectedValues: map[string]KeyData{
"ExpireKey2": {Value: "value2", ExpireAt: mockClock.Now().Add(1000 * time.Millisecond)},
"ExpireKey2": {Value: "value2", ExpireAt: mockClock.Now().Add(1000000 * time.Millisecond)},
},
expectedError: nil,
},
Expand Down Expand Up @@ -1499,6 +1497,20 @@ func Test_Generic(t *testing.T) {
expectedValues: nil,
expectedError: errors.New(constants.WrongArgsResponse),
},
{
name: "17. Ensure Keys expire as expected",
command: []string{"EXPIRE", "ExpireKey16", "1"},
presetValues: map[string]KeyData{
"ExpireKey16": {Value: "valueShouldBeExpired", ExpireAt: time.Time{}},
},
expectedResponse: 1,
expectedValues: nil,
// map[string]KeyData{
// "ExpireKey16": {Value: "value1", ExpireAt: mockClock.Now().Add(1*time.Second)},
// },
expectedError: nil,
checkExpired: true,
},
}

for _, test := range tests {
Expand Down Expand Up @@ -1551,6 +1563,23 @@ func Test_Generic(t *testing.T) {
}

if test.expectedValues == nil {

if test.checkExpired {
time.Sleep(5 * time.Second)
for key, _ := range test.presetValues {

if err = client.WriteArray([]resp.Value{resp.StringValue("GET"), resp.StringValue(key)}); err != nil {
t.Error(err)
}
res, _, err = client.ReadValue()
if err != nil {
t.Error(err)
}
if res.String() != "" {
t.Errorf("Key should be expired, expected value '', got %q", res.String())
}
}
}
return
}

Expand Down Expand Up @@ -2855,12 +2884,10 @@ func Test_Generic(t *testing.T) {
return
}
mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.NoEviction,
}),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.NoEviction),
)
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -3763,12 +3790,10 @@ func Test_Generic(t *testing.T) {
return
}
mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.NoEviction,
}),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.NoEviction),
)
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -3907,14 +3932,12 @@ func Test_LFU_Generic(t *testing.T) {
duration := time.Duration(30) * time.Second

mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.AllKeysLFU,
EvictionInterval: duration,
MaxMemory: 550,
}),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.AllKeysLFU),
sugardb.WithEvictionInterval(duration),
sugardb.WithMaxMemory(550),
)
if err != nil {
t.Error(err)
Expand Down Expand Up @@ -4088,14 +4111,12 @@ func Test_LRU_Generic(t *testing.T) {
duration := time.Duration(30) * time.Second

mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.AllKeysLRU,
EvictionInterval: duration,
MaxMemory: 550,
}),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.AllKeysLRU),
sugardb.WithEvictionInterval(duration),
sugardb.WithMaxMemory(550),
)
if err != nil {
t.Error(err)
Expand Down
1 change: 0 additions & 1 deletion internal/modules/hash/commands.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,7 +723,6 @@ func handleHEXPIRE(params internal.HandlerFuncParams) ([]byte, error) {
continue
}
currentExpireAt := hash[f].ExpireAt
//TODO
if currentExpireAt == (time.Time{}) || expireAt.Before(currentExpireAt) {
resp = resp + ":0\r\n"
continue
Expand Down
11 changes: 4 additions & 7 deletions internal/modules/hash/commands_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (

"github.com/echovault/sugardb/internal"
"github.com/echovault/sugardb/internal/clock"
"github.com/echovault/sugardb/internal/config"
"github.com/echovault/sugardb/internal/constants"
"github.com/echovault/sugardb/internal/modules/hash"
"github.com/echovault/sugardb/sugardb"
Expand All @@ -41,12 +40,10 @@ func Test_Hash(t *testing.T) {
}

mockServer, err := sugardb.NewSugarDB(
sugardb.WithConfig(config.Config{
BindAddr: "localhost",
Port: uint16(port),
DataDir: "",
EvictionPolicy: constants.NoEviction,
}),
sugardb.WithBindAddr("localhost"),
sugardb.WithPort(uint16(port)),
sugardb.WithDataDir(""),
sugardb.WithEvictionPolicy(constants.NoEviction),
)
if err != nil {
t.Error(err)
Expand Down
Loading