From 1b9a84c136e98983ad517c61ce3b35a5d79653bc Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 27 May 2025 16:10:32 -0700 Subject: [PATCH 01/30] start index --- grids/index.go | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/grids/index.go b/grids/index.go index bb593854..9f83f3f3 100644 --- a/grids/index.go +++ b/grids/index.go @@ -2,7 +2,6 @@ package grids import ( "context" - "fmt" "strings" "github.com/bmeg/grip/gripql" @@ -16,18 +15,15 @@ func normalizePath(path string) string { } // AddVertexIndex add index to vertices -func (ggraph *Graph) AddVertexIndex(label string, field string) error { +func (ggraph *Graph) AddVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") - field = normalizePath(field) - //TODO kick off background process to reindex existing data - return ggraph.bsonkv.AddField(fmt.Sprintf("%s.v.%s.%s", ggraph.graphID, label, field)) + return ggraph.bsonkv.AddField(VTABLE_PREFIX+label, field) } // DeleteVertexIndex delete index from vertices -func (ggraph *Graph) DeleteVertexIndex(label string, field string) error { +func (ggraph *Graph) DeleteVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") - field = normalizePath(field) - return ggraph.bsonkv.RemoveField(fmt.Sprintf("%s.v.%s.%s", ggraph.graphID, label, field)) + return ggraph.bsonkv.RemoveField(VTABLE_PREFIX+label, field) } // GetVertexIndexList lists out all the vertex indices for a graph @@ -36,29 +32,42 @@ func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { out := make(chan *gripql.IndexID) go func() { defer close(out) - fields := ggraph.bsonkv.ListFields() - for _, f := range fields { - t := strings.Split(f, ".") - if len(t) > 3 { - out <- &gripql.IndexID{Graph: ggraph.graphID, Label: t[2], Field: t[3]} - } + for _, f := range ggraph.bsonkv.ListFields() { + out <- &gripql.IndexID{Graph: ggraph.graphID, Label: f.Label, Field: f.Field} } }() return out } +// Vertex Filter Scan produces a channel of all vertex ids in a graph that match the field - value filter +func (ggraph *Graph) VertexFilterScan(ctx context.Context, label string, field string, value string) (chan string, error) { + log.WithFields(log.Fields{"field": field, "value": value}).Info("Running VertexFilterScan") + if label[:2] != VTABLE_PREFIX { + label = VTABLE_PREFIX + label + } + return ggraph.bsonkv.RowIdsByFieldValue(field, value) +} + +// Vertex Filter Scan produces a channel of all vertex ids in a graph that match the field - value filter +func (ggraph *Graph) VertexFilterLabelScan(ctx context.Context, label string, field string, value string) (chan string, error) { + log.WithFields(log.Fields{"label": label, "field": field, "value": value}).Info("Running VertexFilterLabelScan") + if label[:2] != VTABLE_PREFIX { + label = VTABLE_PREFIX + label + } + return ggraph.bsonkv.RowIdsByLabelFieldValue(label, field, value) +} + // VertexLabelScan produces a channel of all vertex ids in a graph // that match a given label func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan string { - log.WithFields(log.Fields{"label": label}).Debug("Running VertexLabelScan") + log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") //TODO: Make this work better out := make(chan string, 100) - if label[:2] != "v_" { - label = "v_" + label + if label[:2] != VTABLE_PREFIX { + label = VTABLE_PREFIX + label } go func() { defer close(out) - log.Infof("Searching %s %s", fmt.Sprintf("%s.label", ggraph.graphID), label) for i := range ggraph.bsonkv.GetIDsForLabel(label) { out <- i } From 6a4672cfacab6995fdcff0f0586f4fded90aa8b1 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 30 May 2025 09:16:44 -0700 Subject: [PATCH 02/30] start indexing --- conformance/tests/ot_index.py | 8 ++++++- engine/core/optimize.go | 32 +++++++++++++++++++++---- engine/core/processors.go | 40 +++++++++++++++++++++++++++++-- engine/core/statement_compiler.go | 7 ++++-- existing-sql/graph.go | 4 ++++ gdbi/interface.go | 1 + grids/graph.go | 9 +++++++ grids/index.go | 15 ++---------- gripper/graph.go | 4 ++++ gripql/custom_statements.go | 12 ++++++++-- gripql/inspect/inspect.go | 7 +++--- kvgraph/graph.go | 4 ++++ mongo/graph.go | 4 ++++ psql/graph.go | 4 ++++ sqlite/graph.go | 4 ++++ 15 files changed, 127 insertions(+), 28 deletions(-) diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index 991d42db..c0d0eb71 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -1,4 +1,4 @@ - +import gripql def test_index(man): errors = [] @@ -25,9 +25,15 @@ def test_index(man): resp = G.listIndices() found = False for i in resp: + print("I: ", i) if i["field"] == "name" and i["label"] == "Person": found = True if not found: errors.append("Expected index not found") + + resp2 = G.query().V().has(gripql.eq("name","marko")) + for i in resp2: + print("I: ", i) + return errors diff --git a/engine/core/optimize.go b/engine/core/optimize.go index d40d8af7..83625192 100644 --- a/engine/core/optimize.go +++ b/engine/core/optimize.go @@ -1,6 +1,8 @@ package core import ( + "github.com/bmeg/grip/log" + "github.com/bmeg/grip/gdbi/tpath" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/util/protoutil" @@ -13,8 +15,7 @@ func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement optimized := []*gripql.GraphStatement{} //var lookupV *gripql.GraphStatement_V - hasIDIdx := []int{} - hasLabelIdx := []int{} + hasIDIdx, hasLabelIdx, hasCondIdx := []int{}, []int{}, []int{} isDone := false for i, step := range pipe { if isDone { @@ -48,13 +49,14 @@ func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement } if cond := s.Has.GetCondition(); cond != nil { path := tpath.NormalizePath(cond.Key) + log.Infof("KEY: %s PATH: %s", cond.Key, path) switch path { case "$_current._id": hasIDIdx = append(hasIDIdx, i) case "$_current._label": hasLabelIdx = append(hasLabelIdx, i) default: - // do nothing + hasCondIdx = append(hasCondIdx, i) } } default: @@ -91,13 +93,28 @@ func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement } if len(labels) > 0 { labelOpt = true - hIdx := &gripql.GraphStatement_LookupVertsIndex{Labels: labels} + hIdx := &gripql.GraphStatement_LookupVertsLabelIndex{Labels: labels} optimized = append(optimized, &gripql.GraphStatement{Statement: hIdx}) } } + hasCondOpt := false + if len(hasCondIdx) > 0 { + idx := hasCondIdx[0] + if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { + cond := has.Has.GetCondition() + optimized = append(optimized, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertexHasCondIndex{ + Key: cond.Key, Value: cond.GetValue().String(), + }}, + ) + hasCondOpt = true + log.Infoln("OPTIMiZED: ", optimized) + } + } + for i, step := range pipe { - if idOpt || labelOpt { + if idOpt || labelOpt || hasCondOpt { if i == 0 { continue } @@ -114,6 +131,11 @@ func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement optimized = append(optimized, step) } } + if hasCondOpt { + if i != hasCondIdx[0] { + optimized = append(optimized, step) + } + } } return optimized diff --git a/engine/core/processors.go b/engine/core/processors.go index 7c7108ed..886f19be 100644 --- a/engine/core/processors.go +++ b/engine/core/processors.go @@ -7,6 +7,7 @@ import ( "github.com/bmeg/grip/engine/logic" "github.com/bmeg/grip/gdbi" + "github.com/bmeg/grip/log" "github.com/bmeg/grip/util/copy" "github.com/spf13/cast" ) @@ -56,17 +57,52 @@ func (l *LookupVerts) Process(ctx context.Context, man gdbi.Manager, in gdbi.InP return ctx } +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertsCondIndex look up vertices by indexed +type LookupVertsCondIndex struct { + db gdbi.GraphInterface + key string + value string + loadData bool +} + +func (l *LookupVertsCondIndex) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + queryChan := make(chan gdbi.ElementLookup, 100) + go func() { + defer close(queryChan) + for t := range in { + log.Infof("HELLO: %s %s", l.key, l.value) + for id := range l.db.VertexHasConditionScan(ctx, l.key, l.value) { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } + } + + } + }() + + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + return ctx +} + //////////////////////////////////////////////////////////////////////////////// // LookupVertsIndex look up vertices by indexed based feature -type LookupVertsIndex struct { +type LookupVertsLabelIndex struct { db gdbi.GraphInterface labels []string loadData bool } // Process LookupVertsIndex -func (l *LookupVertsIndex) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { +func (l *LookupVertsLabelIndex) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { queryChan := make(chan gdbi.ElementLookup, 100) go func() { defer close(queryChan) diff --git a/engine/core/statement_compiler.go b/engine/core/statement_compiler.go index 7e2b1822..36a123e0 100644 --- a/engine/core/statement_compiler.go +++ b/engine/core/statement_compiler.go @@ -242,9 +242,12 @@ func (sc *DefaultStmtCompiler) Custom(gs *gripql.GraphStatement, ps *gdbi.State) switch stmt := gs.GetStatement().(type) { //Custom graph statements - case *gripql.GraphStatement_LookupVertsIndex: + case *gripql.GraphStatement_LookupVertsLabelIndex: ps.LastType = gdbi.VertexData - return &LookupVertsIndex{db: sc.db, labels: stmt.Labels, loadData: ps.StepLoadData()}, nil + return &LookupVertsLabelIndex{db: sc.db, labels: stmt.Labels, loadData: ps.StepLoadData()}, nil + case *gripql.GraphStatement_LookupVertexHasCondIndex: + ps.LastType = gdbi.VertexData + return &LookupVertsCondIndex{db: sc.db, key: stmt.Key, value: stmt.Value, loadData: ps.StepLoadData()}, nil case *gripql.GraphStatement_EngineCustom: proc := stmt.Custom.(gdbi.CustomProcGen) diff --git a/existing-sql/graph.go b/existing-sql/graph.go index 1b75c2dd..f4e05627 100644 --- a/existing-sql/graph.go +++ b/existing-sql/graph.go @@ -814,3 +814,7 @@ func (g *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } + +func (g *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { + panic("not implemented") +} diff --git a/gdbi/interface.go b/gdbi/interface.go index 4de3e1b6..c82c56d9 100644 --- a/gdbi/interface.go +++ b/gdbi/interface.go @@ -168,6 +168,7 @@ type GraphInterface interface { DelVertex(key string) error DelEdge(key string) error + VertexHasConditionScan(ctx context.Context, key, value string) chan string VertexLabelScan(ctx context.Context, label string) chan string // EdgeLabelScan(ctx context.Context, label string) chan string ListVertexLabels() ([]string, error) diff --git a/grids/graph.go b/grids/graph.go index f2c08b99..9c1d4c48 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -112,6 +112,15 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge) error { if err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}); err != nil { return fmt.Errorf("indexEdge: table.AddRow: %s", err) } + + _, fieldsExist := ggraph.bsonkv.Fields[edgeLabel] + if fieldsExist { + for field := range ggraph.bsonkv.Fields[edgeLabel] { + if val, ok := edge.Data[field]; ok { + table.Pb.Db.Set(benchtop.FieldKey(edgeLabel, field, val, []byte(edge.ID)), []byte{}, nil) + } + } + } return nil } diff --git a/grids/index.go b/grids/index.go index 9f83f3f3..5f04f415 100644 --- a/grids/index.go +++ b/grids/index.go @@ -40,11 +40,8 @@ func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { } // Vertex Filter Scan produces a channel of all vertex ids in a graph that match the field - value filter -func (ggraph *Graph) VertexFilterScan(ctx context.Context, label string, field string, value string) (chan string, error) { +func (ggraph *Graph) VertexHasConditionScan(ctx context.Context, field string, value string) chan string { log.WithFields(log.Fields{"field": field, "value": value}).Info("Running VertexFilterScan") - if label[:2] != VTABLE_PREFIX { - label = VTABLE_PREFIX + label - } return ggraph.bsonkv.RowIdsByFieldValue(field, value) } @@ -61,16 +58,8 @@ func (ggraph *Graph) VertexFilterLabelScan(ctx context.Context, label string, fi // that match a given label func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan string { log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") - //TODO: Make this work better - out := make(chan string, 100) if label[:2] != VTABLE_PREFIX { label = VTABLE_PREFIX + label } - go func() { - defer close(out) - for i := range ggraph.bsonkv.GetIDsForLabel(label) { - out <- i - } - }() - return out + return ggraph.bsonkv.GetIDsForLabel(label) } diff --git a/gripper/graph.go b/gripper/graph.go index 801993c9..65c0d6d0 100644 --- a/gripper/graph.go +++ b/gripper/graph.go @@ -766,3 +766,7 @@ func (t *TabularGraph) GetInEdgeChannel(ctx context.Context, req chan gdbi.Eleme }() return out } + +func (T *TabularGraph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { + panic("not implemented") +} diff --git a/gripql/custom_statements.go b/gripql/custom_statements.go index c65bcb8f..995a514e 100644 --- a/gripql/custom_statements.go +++ b/gripql/custom_statements.go @@ -7,11 +7,19 @@ package gripql //in the traversal that the optimizer may add in, but can't be coded by a //serialized user request -type GraphStatement_LookupVertsIndex struct { +type GraphStatement_LookupVertsLabelIndex struct { Labels []string `protobuf:"bytes,1,rep,name=labels" json:"labels,omitempty"` } -func (*GraphStatement_LookupVertsIndex) isGraphStatement_Statement() {} +func (*GraphStatement_LookupVertsLabelIndex) isGraphStatement_Statement() {} + +type GraphStatement_LookupVertexHasCondIndex struct { + Key string `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` + Value string `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"` +} + +func (*GraphStatement_LookupVertexHasCondIndex) isGraphStatement_Statement() {} + type GraphStatement_EngineCustom struct { Desc string `protobuf:"bytes,1,opt,name=desc" json:"desc,omitempty"` diff --git a/gripql/inspect/inspect.go b/gripql/inspect/inspect.go index da4911d9..6a061a94 100644 --- a/gripql/inspect/inspect.go +++ b/gripql/inspect/inspect.go @@ -45,7 +45,8 @@ func PipelineSteps(stmts []*gripql.GraphStatement) []string { *gripql.GraphStatement_Set, *gripql.GraphStatement_Increment, *gripql.GraphStatement_Mark, *gripql.GraphStatement_Jump, *gripql.GraphStatement_Sort, *gripql.GraphStatement_Pivot, *gripql.GraphStatement_Group, *gripql.GraphStatement_Totype: - case *gripql.GraphStatement_LookupVertsIndex, *gripql.GraphStatement_EngineCustom: + case *gripql.GraphStatement_LookupVertexHasCondIndex, *gripql.GraphStatement_LookupVertsLabelIndex, + *gripql.GraphStatement_EngineCustom: default: log.Errorf("Unknown Graph Statement: %T", gs.GetStatement()) } @@ -145,7 +146,7 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st out[steps[i]] = []string{"*"} } onLast = false - case *gripql.GraphStatement_LookupVertsIndex: + case *gripql.GraphStatement_LookupVertsLabelIndex: if onLast { out[steps[i]] = []string{"*"} } @@ -157,7 +158,7 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st } else { out[steps[i]] = []string{"_label"} } - case *gripql.GraphStatement_Has: + case *gripql.GraphStatement_Has, *gripql.GraphStatement_LookupVertexHasCondIndex: out[steps[i]] = []string{"*"} } } diff --git a/kvgraph/graph.go b/kvgraph/graph.go index e8b11e10..e8056e2f 100644 --- a/kvgraph/graph.go +++ b/kvgraph/graph.go @@ -699,3 +699,7 @@ func (kgdb *KVInterfaceGDB) ListEdgeLabels() ([]string, error) { } return labels, nil } + +func (kgdb *KVInterfaceGDB) VertexHasConditionScan(ctx context.Context, key, value string) chan string { + panic("not implemented") +} diff --git a/mongo/graph.go b/mongo/graph.go index 7ba33ec2..2af0bf6f 100644 --- a/mongo/graph.go +++ b/mongo/graph.go @@ -693,3 +693,7 @@ func (mg *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } + +func (mg *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { + panic("not implemented") +} diff --git a/psql/graph.go b/psql/graph.go index a6947c24..492e15db 100644 --- a/psql/graph.go +++ b/psql/graph.go @@ -929,3 +929,7 @@ func (g *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } + +func (g *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { + panic("not implemented") +} diff --git a/sqlite/graph.go b/sqlite/graph.go index 0fa117a7..446c4183 100644 --- a/sqlite/graph.go +++ b/sqlite/graph.go @@ -921,3 +921,7 @@ func (g *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } + +func (g *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { + panic("not implemented") +} From 20b78bdb1b1f14efa8d026471fb0c68973208c94 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 30 May 2025 15:14:05 -0700 Subject: [PATCH 03/30] Add custom processor for grids indexing --- engine/core/optimize.go | 247 +++++++++++++----------------- engine/core/processors.go | 36 ----- engine/core/statement_compiler.go | 4 - existing-sql/graph.go | 4 - gdbi/interface.go | 1 - grids/graph.go | 2 +- grids/index.go | 7 - grids/optimize.go | 167 ++++++++++++++++++++ gripper/graph.go | 4 - gripql/custom_statements.go | 8 - gripql/inspect/inspect.go | 4 +- kvgraph/graph.go | 4 - mongo/graph.go | 4 - psql/graph.go | 4 - sqlite/graph.go | 4 - 15 files changed, 276 insertions(+), 224 deletions(-) create mode 100644 grids/optimize.go diff --git a/engine/core/optimize.go b/engine/core/optimize.go index 83625192..577dcd7b 100644 --- a/engine/core/optimize.go +++ b/engine/core/optimize.go @@ -1,164 +1,129 @@ package core import ( - "github.com/bmeg/grip/log" - "github.com/bmeg/grip/gdbi/tpath" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/util/protoutil" ) -// IndexStartOptimize looks at processor pipeline for queries like -// V().Has(Eq("$._label", "Person")) and V().Has(Eq("$._id", "1")), -// streamline into a single index lookup -func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - optimized := []*gripql.GraphStatement{} +// OptimizationRule defines a structure for matching and replacing query pipeline patterns. +type OptimizationRule struct { + Match func(pipe []*gripql.GraphStatement) bool + Replace func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement +} - //var lookupV *gripql.GraphStatement_V - hasIDIdx, hasLabelIdx, hasCondIdx := []int{}, []int{}, []int{} - isDone := false - for i, step := range pipe { - if isDone { - break - } - if i == 0 { - if v, ok := step.GetStatement().(*gripql.GraphStatement_V); ok { - if v.V != nil && len(v.V.Values) > 0 { - break - } - } else { - break +// startOptimizations is a list of rules to optimize the query pipeline. +var startOptimizations = []OptimizationRule{ + { + // Matches V().HasId(...) + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false } - continue - } - switch s := step.GetStatement().(type) { - case *gripql.GraphStatement_HasId: - hasIDIdx = append(hasIDIdx, i) - case *gripql.GraphStatement_HasLabel: - hasLabelIdx = append(hasLabelIdx, i) - case *gripql.GraphStatement_Has: - if and := s.Has.GetAnd(); and != nil { - stmts := and.GetExpressions() - newPipe := []*gripql.GraphStatement{} - newPipe = append(newPipe, pipe[:i]...) - for _, stmt := range stmts { - newPipe = append(newPipe, &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{Has: stmt}}) - } - newPipe = append(newPipe, pipe[i+1:]...) - return IndexStartOptimize(newPipe) + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false } - if cond := s.Has.GetCondition(); cond != nil { - path := tpath.NormalizePath(cond.Key) - log.Infof("KEY: %s PATH: %s", cond.Key, path) - switch path { - case "$_current._id": - hasIDIdx = append(hasIDIdx, i) - case "$_current._label": - hasLabelIdx = append(hasLabelIdx, i) - default: - hasCondIdx = append(hasCondIdx, i) - } + if hasId, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasId); ok { + return len(hasId.HasId.Values) > 0 } - default: - isDone = true - } - } - - idOpt := false - if len(hasIDIdx) > 0 { - ids := []string{} - idx := hasIDIdx[0] - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { - ids = append(ids, extractHasVals(has)...) - } - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_HasId); ok { - ids = append(ids, protoutil.AsStringList(has.HasId)...) - } - if len(ids) > 0 { - idOpt = true - hIdx := &gripql.GraphStatement_V{V: protoutil.NewListFromStrings(ids)} - optimized = append(optimized, &gripql.GraphStatement{Statement: hIdx}) - } - } - - labelOpt := false - if len(hasLabelIdx) > 0 && !idOpt { - labels := []string{} - idx := hasLabelIdx[0] - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { - labels = append(labels, extractHasVals(has)...) - } - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_HasLabel); ok { - labels = append(labels, protoutil.AsStringList(has.HasLabel)...) - } - if len(labels) > 0 { - labelOpt = true - hIdx := &gripql.GraphStatement_LookupVertsLabelIndex{Labels: labels} - optimized = append(optimized, &gripql.GraphStatement{Statement: hIdx}) - } - } - - hasCondOpt := false - if len(hasCondIdx) > 0 { - idx := hasCondIdx[0] - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { - cond := has.Has.GetCondition() - optimized = append(optimized, - &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertexHasCondIndex{ - Key: cond.Key, Value: cond.GetValue().String(), - }}, - ) - hasCondOpt = true - log.Infoln("OPTIMiZED: ", optimized) - } - } - - for i, step := range pipe { - if idOpt || labelOpt || hasCondOpt { - if i == 0 { - continue + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + ids := protoutil.AsStringList(pipe[1].GetHasId()) + optimized := []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_V{V: protoutil.NewListFromStrings(ids)}}, } - } else { - optimized = append(optimized, step) - } - if idOpt { - if i != hasIDIdx[0] { - optimized = append(optimized, step) + return append(optimized, pipe[2:]...) + }, + }, + { + // Matches V().HasLabel(...) + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false } - } - if labelOpt { - if i != hasLabelIdx[0] { - optimized = append(optimized, step) + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false } - } - if hasCondOpt { - if i != hasCondIdx[0] { - optimized = append(optimized, step) + if hasLabel, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); ok { + return len(hasLabel.HasLabel.GetValues()) > 0 } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + optimized := []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: labels}}, + } + return append(optimized, pipe[2:]...) + }, + }, + { + // Matches V().Has(Eq(key, value)) for _id, _label, or other conditions + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if has, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { + cond := has.Has.GetCondition() + return cond != nil && cond.Condition == gripql.Condition_EQ + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[1].GetHas() + cond := has.GetCondition() + path := tpath.NormalizePath(cond.Key) + value := cond.Value.String() + var optimized []*gripql.GraphStatement + switch path { + case "$_current._id": + optimized = []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_V{V: protoutil.NewListFromStrings([]string{value})}}, + } + case "$_current._label": + optimized = []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{value}}}, + } + default: + } + return append(optimized, pipe[2:]...) + }, + }, +} + +// expandHasAnd preprocesses the pipeline to split Has statements with And expressions. +func expandHasAnd(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + expanded := []*gripql.GraphStatement{} + for _, step := range pipe { + if has, ok := step.GetStatement().(*gripql.GraphStatement_Has); ok { + if and := has.Has.GetAnd(); and != nil { + for _, expr := range and.Expressions { + expanded = append(expanded, &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{Has: expr}}) + } + } else { + expanded = append(expanded, step) + } + } else { + expanded = append(expanded, step) } } - - return optimized + return expanded } -func extractHasVals(h *gripql.GraphStatement_Has) []string { - vals := []string{} - if cond := h.Has.GetCondition(); cond != nil { - // path := jsonpath.GetJSONPath(cond.Key) - val := cond.Value.AsInterface() - switch cond.Condition { - case gripql.Condition_EQ: - if l, ok := val.(string); ok { - vals = []string{l} - } - case gripql.Condition_WITHIN: - v := val.([]interface{}) - for _, x := range v { - vals = append(vals, x.(string)) - } - default: - // do nothing +// IndexStartOptimize applies optimization rules to the query pipeline. +func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + // Preprocess to handle Has with And expressions + pipe = expandHasAnd(pipe) + // Apply the first matching optimization rule + for _, rule := range startOptimizations { + if rule.Match(pipe) { + return rule.Replace(pipe) } } - return vals + // Return the original pipeline if no optimizations apply + return pipe } diff --git a/engine/core/processors.go b/engine/core/processors.go index 886f19be..059d9e99 100644 --- a/engine/core/processors.go +++ b/engine/core/processors.go @@ -7,7 +7,6 @@ import ( "github.com/bmeg/grip/engine/logic" "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/log" "github.com/bmeg/grip/util/copy" "github.com/spf13/cast" ) @@ -57,41 +56,6 @@ func (l *LookupVerts) Process(ctx context.Context, man gdbi.Manager, in gdbi.InP return ctx } -// ////////////////////////////////////////////////////////////////////////////// -// LookupVertsCondIndex look up vertices by indexed -type LookupVertsCondIndex struct { - db gdbi.GraphInterface - key string - value string - loadData bool -} - -func (l *LookupVertsCondIndex) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - queryChan := make(chan gdbi.ElementLookup, 100) - go func() { - defer close(queryChan) - for t := range in { - log.Infof("HELLO: %s %s", l.key, l.value) - for id := range l.db.VertexHasConditionScan(ctx, l.key, l.value) { - queryChan <- gdbi.ElementLookup{ - ID: id, - Ref: t, - } - } - - } - }() - - go func() { - defer close(out) - for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { - i := v.Ref - out <- i.AddCurrent(v.Vertex.Copy()) - } - }() - return ctx -} - //////////////////////////////////////////////////////////////////////////////// // LookupVertsIndex look up vertices by indexed based feature diff --git a/engine/core/statement_compiler.go b/engine/core/statement_compiler.go index 36a123e0..2709938a 100644 --- a/engine/core/statement_compiler.go +++ b/engine/core/statement_compiler.go @@ -245,10 +245,6 @@ func (sc *DefaultStmtCompiler) Custom(gs *gripql.GraphStatement, ps *gdbi.State) case *gripql.GraphStatement_LookupVertsLabelIndex: ps.LastType = gdbi.VertexData return &LookupVertsLabelIndex{db: sc.db, labels: stmt.Labels, loadData: ps.StepLoadData()}, nil - case *gripql.GraphStatement_LookupVertexHasCondIndex: - ps.LastType = gdbi.VertexData - return &LookupVertsCondIndex{db: sc.db, key: stmt.Key, value: stmt.Value, loadData: ps.StepLoadData()}, nil - case *gripql.GraphStatement_EngineCustom: proc := stmt.Custom.(gdbi.CustomProcGen) ps.LastType = proc.GetType() diff --git a/existing-sql/graph.go b/existing-sql/graph.go index f4e05627..1b75c2dd 100644 --- a/existing-sql/graph.go +++ b/existing-sql/graph.go @@ -814,7 +814,3 @@ func (g *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } - -func (g *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { - panic("not implemented") -} diff --git a/gdbi/interface.go b/gdbi/interface.go index c82c56d9..4de3e1b6 100644 --- a/gdbi/interface.go +++ b/gdbi/interface.go @@ -168,7 +168,6 @@ type GraphInterface interface { DelVertex(key string) error DelEdge(key string) error - VertexHasConditionScan(ctx context.Context, key, value string) chan string VertexLabelScan(ctx context.Context, label string) chan string // EdgeLabelScan(ctx context.Context, label string) chan string ListVertexLabels() ([]string, error) diff --git a/grids/graph.go b/grids/graph.go index 9c1d4c48..39360c07 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -125,7 +125,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge) error { } func (ggraph *Graph) Compiler() gdbi.Compiler { - return core.NewCompiler(ggraph, core.IndexStartOptimize) + return core.NewCompiler(ggraph, GripOptimizer, core.IndexStartOptimize) } // AddVertex adds an edge to the graph, if it already exists diff --git a/grids/index.go b/grids/index.go index 5f04f415..fff8d70f 100644 --- a/grids/index.go +++ b/grids/index.go @@ -2,18 +2,11 @@ package grids import ( "context" - "strings" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" ) -func normalizePath(path string) string { - path = strings.TrimPrefix(path, "$.") - path = strings.TrimPrefix(path, "data.") - return path -} - // AddVertexIndex add index to vertices func (ggraph *Graph) AddVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") diff --git a/grids/optimize.go b/grids/optimize.go new file mode 100644 index 00000000..58aa3bde --- /dev/null +++ b/grids/optimize.go @@ -0,0 +1,167 @@ +package grids + +import ( + "context" + "fmt" + + "github.com/bmeg/grip/gdbi" + "github.com/bmeg/grip/gdbi/tpath" + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/util/protoutil" +) + +type OptimizationRule struct { + Match func(pipe []*gripql.GraphStatement) bool + Replace func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement +} + +func expandHasAnd(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + expanded := []*gripql.GraphStatement{} + for _, step := range pipe { + if has, ok := step.GetStatement().(*gripql.GraphStatement_Has); ok { + if and := has.Has.GetAnd(); and != nil { + for _, expr := range and.Expressions { + expanded = append(expanded, &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{Has: expr}}) + } + } else { + expanded = append(expanded, step) + } + } else { + expanded = append(expanded, step) + } + } + return expanded +} + +func GripOptimizer(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + // Preprocess to handle Has with And expressions + pipe = expandHasAnd(pipe) + // Apply the first matching optimization rule + for _, rule := range startOptimizations { + if rule.Match(pipe) { + return rule.Replace(pipe) + } + } + // Return the original pipeline if no optimizations apply + return pipe +} + +var startOptimizations = []OptimizationRule{ + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if has, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { + cond := has.Has.GetCondition() + return cond != nil && cond.Condition == gripql.Condition_EQ + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[1].GetHas() + cond := has.GetCondition() + path := tpath.NormalizePath(cond.Key) + value := cond.Value.GetStringValue() + var optimized []*gripql.GraphStatement + switch path { + case "$_current._id": + optimized = []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_V{V: protoutil.NewListFromStrings([]string{value})}}, + } + case "$_current._label": + optimized = []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{value}}}, + } + default: + optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids Has Level Indexing", + Custom: lookupVertsCondIndexStep{key: cond.Key, value: value}, + }, + }, + } + } + return append(optimized, pipe[2:]...) + }, + }, +} + +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertsCondIndex look up vertices by indexed +type lookupVertsCondIndexStep struct { + key string + value string +} + +func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { + graph := db.(*Graph) + // If Field is indexed, use the special processor + for _, fields := range graph.bsonkv.Fields { + for field := range fields { + if field == t.key { + return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: false}, nil + } + } + } + return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: true}, nil +} + +func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { + return gdbi.VertexData +} + +type lookupVertsCondIndexProc struct { + db *Graph + key string + value string + loadData bool + fallback bool +} + +func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + queryChan := make(chan gdbi.ElementLookup, 100) + if l.fallback { + go func() { + defer close(queryChan) + for t := range in { + for v := range l.db.GetVertexList(ctx, true) { + val, keyExists := v.Data[l.key] + // In cases where eq comparisons to 'None' values are made + if l.value == "" && (!keyExists || val == nil) { + queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} + } else if keyExists && (val == l.value || fmt.Sprintf("%v", val) == l.value) { + queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} + } + } + } + }() + } else { + go func() { + defer close(queryChan) + for t := range in { + for id := range l.db.VertexHasConditionScan(ctx, l.key, l.value) { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } + } + + } + }() + } + + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + return ctx + +} diff --git a/gripper/graph.go b/gripper/graph.go index 65c0d6d0..801993c9 100644 --- a/gripper/graph.go +++ b/gripper/graph.go @@ -766,7 +766,3 @@ func (t *TabularGraph) GetInEdgeChannel(ctx context.Context, req chan gdbi.Eleme }() return out } - -func (T *TabularGraph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { - panic("not implemented") -} diff --git a/gripql/custom_statements.go b/gripql/custom_statements.go index 995a514e..e9cbc792 100644 --- a/gripql/custom_statements.go +++ b/gripql/custom_statements.go @@ -13,14 +13,6 @@ type GraphStatement_LookupVertsLabelIndex struct { func (*GraphStatement_LookupVertsLabelIndex) isGraphStatement_Statement() {} -type GraphStatement_LookupVertexHasCondIndex struct { - Key string `protobuf:"bytes,1,opt,name=key" json:"key,omitempty"` - Value string `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"` -} - -func (*GraphStatement_LookupVertexHasCondIndex) isGraphStatement_Statement() {} - - type GraphStatement_EngineCustom struct { Desc string `protobuf:"bytes,1,opt,name=desc" json:"desc,omitempty"` Custom interface{} `protobuf:"bytes,2,opt,name=custom" json:"custom,omitempty"` diff --git a/gripql/inspect/inspect.go b/gripql/inspect/inspect.go index 6a061a94..40e8f9d6 100644 --- a/gripql/inspect/inspect.go +++ b/gripql/inspect/inspect.go @@ -45,7 +45,7 @@ func PipelineSteps(stmts []*gripql.GraphStatement) []string { *gripql.GraphStatement_Set, *gripql.GraphStatement_Increment, *gripql.GraphStatement_Mark, *gripql.GraphStatement_Jump, *gripql.GraphStatement_Sort, *gripql.GraphStatement_Pivot, *gripql.GraphStatement_Group, *gripql.GraphStatement_Totype: - case *gripql.GraphStatement_LookupVertexHasCondIndex, *gripql.GraphStatement_LookupVertsLabelIndex, + case *gripql.GraphStatement_LookupVertsLabelIndex, *gripql.GraphStatement_EngineCustom: default: log.Errorf("Unknown Graph Statement: %T", gs.GetStatement()) @@ -158,7 +158,7 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st } else { out[steps[i]] = []string{"_label"} } - case *gripql.GraphStatement_Has, *gripql.GraphStatement_LookupVertexHasCondIndex: + case *gripql.GraphStatement_Has: out[steps[i]] = []string{"*"} } } diff --git a/kvgraph/graph.go b/kvgraph/graph.go index e8056e2f..e8b11e10 100644 --- a/kvgraph/graph.go +++ b/kvgraph/graph.go @@ -699,7 +699,3 @@ func (kgdb *KVInterfaceGDB) ListEdgeLabels() ([]string, error) { } return labels, nil } - -func (kgdb *KVInterfaceGDB) VertexHasConditionScan(ctx context.Context, key, value string) chan string { - panic("not implemented") -} diff --git a/mongo/graph.go b/mongo/graph.go index 2af0bf6f..7ba33ec2 100644 --- a/mongo/graph.go +++ b/mongo/graph.go @@ -693,7 +693,3 @@ func (mg *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } - -func (mg *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { - panic("not implemented") -} diff --git a/psql/graph.go b/psql/graph.go index 492e15db..a6947c24 100644 --- a/psql/graph.go +++ b/psql/graph.go @@ -929,7 +929,3 @@ func (g *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } - -func (g *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { - panic("not implemented") -} diff --git a/sqlite/graph.go b/sqlite/graph.go index 446c4183..0fa117a7 100644 --- a/sqlite/graph.go +++ b/sqlite/graph.go @@ -921,7 +921,3 @@ func (g *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } - -func (g *Graph) VertexHasConditionScan(ctx context.Context, key, value string) chan string { - panic("not implemented") -} From de2d99665124aa0e49c58174355138c869759b52 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 2 Jun 2025 15:41:18 -0700 Subject: [PATCH 04/30] update test --- conformance/tests/ot_index.py | 56 +++++++++++-- grids/graph.go | 25 ++++-- grids/optimize.go | 152 +++++++++++++++++++++++++++++++++- 3 files changed, 219 insertions(+), 14 deletions(-) diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index c0d0eb71..076a3743 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -1,15 +1,15 @@ import gripql + def test_index(man): errors = [] G = man.writeTest() - G.addIndex("Person", "name") G.addVertex("1", "Person", {"name": "marko", "age": "29"}) G.addVertex("2", "Person", {"name": "vadas", "age": "27"}) - G.addVertex("3", "Software", {"name": "lop", "lang": "java"}) + G.addVertex("3", "Software", {"name": "lop", "lang": "java"}) G.addVertex("4", "Person", {"name": "josh", "age": "32"}) G.addVertex("5", "Software", {"name": "ripple", "lang": "java"}) G.addVertex("6", "Person", {"name": "peter", "age": "35"}) @@ -31,9 +31,55 @@ def test_index(man): if not found: errors.append("Expected index not found") + count = 0 + for i in G.query().V().has(gripql.eq("name","marko")): + count += 1 + if "name" not in i: + errors.append("'name' field not found in vertex") + if i["name"] != "marko": + errors.append("Filtering on field name, value marko but got '%s' instead" % i["name"]) + print("J: ", i) + if count != 2: + errors.append("Expecting 2 vertices returned but got %d instead" % (count)) + return errors - resp2 = G.query().V().has(gripql.eq("name","marko")) - for i in resp2: - print("I: ", i) + +def test_bulk_index(man): + errors = [] + + G = man.writeTest() + G.addIndex("Person", "age") + + bulk = G.bulkAdd() + + bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) + bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) + bulk.addVertex("4", "Person", {"name": "josh", "age": "32"}) + bulk.addVertex("6", "Person", {"name": "peter", "age": "35"}) + bulk.addVertex("7", "Person", {"name": "alice", "age": "31"}) + bulk.addVertex("8", "Person", {"name": "bob", "age": "32"}) + bulk.addVertex("9", "Person", {"name": "charlie", "age": "28"}) + bulk.addVertex("10", "Person", {"name": "diana", "age": "32"}) + bulk.addVertex("11", "Person", {"name": "eve", "age": "30"}) + bulk.addVertex("12", "Person", {"name": "frank", "age": "33"}) + bulk.addVertex("13", "Person", {"name": "grace", "age": "26"}) + bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"}) + bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"}) + bulk.addVertex("16", "Person", {"name": "judy", "age": "34"}) + + + res = bulk.execute() + print("RES: ", res) + + count = 0 + resp3 = G.query().V().has(gripql.eq("age","32")) + for i in resp3: + count += 1 + if "age" not in i: + errors.append("field 'age' not found in vertex") + if "age" in i and i["age"] != "32": + errors.append("filtering on field age value '32' but got %s instead" %s (i["age"])) + if count != 4: + errors.append("expected count 4 but got %d instead" % (count)) return errors diff --git a/grids/graph.go b/grids/graph.go index 39360c07..43681559 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -39,7 +39,7 @@ func insertVertex(tx *pebblebulk.PebbleBulk, keyMap *KeyMap, vertex *gdbi.Vertex return nil } -func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex) error { +func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) error { vertexLabel := VTABLE_PREFIX + vertex.Label ggraph.bsonkv.Lock.Lock() table, ok := ggraph.bsonkv.Tables[vertexLabel] @@ -55,9 +55,20 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex) error { ggraph.bsonkv.Tables[vertexLabel] = table ggraph.bsonkv.Lock.Unlock() } - if err := table.AddRow(benchtop.Row{Id: []byte(vertex.ID), TableName: vertexLabel, Data: vertex.Data}); err != nil { + if err := table.AddRow(benchtop.Row{Id: []byte(vertex.ID), TableName: vertexLabel, Data: vertex.Data}, tx); err != nil { return fmt.Errorf("AddVertex Error %s", err) } + + _, fieldsExist := ggraph.bsonkv.Fields[vertexLabel] + if fieldsExist { + for field := range ggraph.bsonkv.Fields[vertexLabel] { + if val, ok := vertex.Data[field]; ok { + log.Debugln("Field: ", field, "Value: ", val, "Id: ", vertex.ID) + tx.Set(benchtop.FieldKey(vertexLabel, field, val, []byte(vertex.ID)), []byte{}, nil) + } + } + } + return nil } @@ -92,7 +103,7 @@ func insertEdge(tx *pebblebulk.PebbleBulk, keyMap *KeyMap, edge *gdbi.Edge) erro return nil } -func (ggraph *Graph) indexEdge(edge *gdbi.Edge) error { +func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error { edgeLabel := ETABLE_PREFIX + edge.Label ggraph.bsonkv.Lock.Lock() table, ok := ggraph.bsonkv.Tables[edgeLabel] @@ -109,7 +120,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge) error { ggraph.bsonkv.Tables[edgeLabel] = table ggraph.bsonkv.Lock.Unlock() } - if err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}); err != nil { + if err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}, tx); err != nil { return fmt.Errorf("indexEdge: table.AddRow: %s", err) } @@ -117,7 +128,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge) error { if fieldsExist { for field := range ggraph.bsonkv.Fields[edgeLabel] { if val, ok := edge.Data[field]; ok { - table.Pb.Db.Set(benchtop.FieldKey(edgeLabel, field, val, []byte(edge.ID)), []byte{}, nil) + tx.Set(benchtop.FieldKey(edgeLabel, field, val, []byte(edge.ID)), []byte{}, nil) } } } @@ -146,7 +157,7 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { - if err := ggraph.indexVertex(vert); err != nil { + if err := ggraph.indexVertex(vert, tx); err != nil { bulkErr = multierror.Append(bulkErr, err) log.Errorf("IndexVertex Error %s", err) } @@ -176,7 +187,7 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, edge := range edges { - if err := ggraph.indexEdge(edge); err != nil { + if err := ggraph.indexEdge(edge, tx); err != nil { bulkErr = multierror.Append(bulkErr, err) } } diff --git a/grids/optimize.go b/grids/optimize.go index 58aa3bde..18e1edb6 100644 --- a/grids/optimize.go +++ b/grids/optimize.go @@ -4,9 +4,11 @@ import ( "context" "fmt" + "github.com/bmeg/benchtop" "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gdbi/tpath" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" "github.com/bmeg/grip/util/protoutil" ) @@ -89,6 +91,151 @@ var startOptimizations = []OptimizationRule{ return append(optimized, pipe[2:]...) }, }, + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 3 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if hasLabel, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); ok { + return len(hasLabel.HasLabel.GetValues()) > 0 + } + if has, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { + cond := has.Has.GetCondition() + return cond != nil && cond.Condition == gripql.Condition_EQ + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[2].GetHas() + cond := has.GetCondition() + path := tpath.NormalizePath(cond.Key) + value := cond.Value.GetStringValue() + var optimized []*gripql.GraphStatement + switch path { + default: + optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids Has Level Indexing", + Custom: lookupVertsHasLabelCondIndexStep{key: cond.Key, value: value}, + }, + }, + } + } + return append(optimized, pipe[3:]...) + }, + }, +} + +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertexHasLabelCondIndex look up vertices has label +type lookupVertsHasLabelCondIndexStep struct { + key string + label string + value string + op benchtop.OperatorType +} + +func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { + graph := db.(*Graph) + // If Field is indexed, use the special processor + for _, fields := range graph.bsonkv.Fields { + for field := range fields { + if field == t.key { + return &lookupVertsHasLabelCondIndexProc{ + db: graph, + key: t.key, + value: t.value, + label: t.label, + op: t.op, + fallback: false, + loadData: true}, nil + } + } + } + return &lookupVertsHasLabelCondIndexProc{ + db: graph, + key: t.key, + value: t.value, + label: t.label, + op: t.op, + fallback: true, + loadData: true}, nil +} + +func (t lookupVertsHasLabelCondIndexStep) GetType() gdbi.DataType { + return gdbi.VertexData +} + +type lookupVertsHasLabelCondIndexProc struct { + db *Graph + key string + value string + label string + op benchtop.OperatorType + loadData bool + fallback bool +} + +func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + queryChan := make(chan gdbi.ElementLookup, 100) + if l.fallback { + log.Debugf("No index found for %s falling back to GetVertexList", l.key) + go func() { + defer close(queryChan) + for t := range in { + rowChan, err := l.db.bsonkv.Tables[l.label].Scan( + true, + []benchtop.FieldFilter{ + {Field: l.key, Value: l.value, Operator: benchtop.OpEqual}, + }, + ) + if err != nil { + log.Errorln("Scan Process Err: ", err) + } + for v := range rowChan { + val, keyExists := v[l.key] + id, idExists := v["_id"].(string) + // In cases where eq comparisons to 'None' values are made + if l.value == "" && (!keyExists && idExists || val == nil) { + queryChan <- gdbi.ElementLookup{ID: id, Ref: t} + } else if (keyExists && idExists) && (val == l.value || fmt.Sprintf("%v", val) == l.value) { + queryChan <- gdbi.ElementLookup{ID: id, Ref: t} + } + } + } + }() + } else { + go func() { + defer close(queryChan) + for t := range in { + rowChan, err := l.db.VertexFilterLabelScan(ctx, l.label, l.key, l.value) + if err != nil { + log.Errorln("VertexFilterLabelScan Process Err: ", err) + } + for id := range rowChan { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } + } + + } + }() + } + + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + return ctx + } // ////////////////////////////////////////////////////////////////////////////// @@ -104,11 +251,11 @@ func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.P for _, fields := range graph.bsonkv.Fields { for field := range fields { if field == t.key { - return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: false}, nil + return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: false, loadData: true}, nil } } } - return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: true}, nil + return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: true, loadData: true}, nil } func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { @@ -126,6 +273,7 @@ type lookupVertsCondIndexProc struct { func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { queryChan := make(chan gdbi.ElementLookup, 100) if l.fallback { + log.Debugf("No index found for %s falling back to GetVertexList", l.key) go func() { defer close(queryChan) for t := range in { From 64d4cdc77f1c9d5adc082ff4380509aa049f6d75 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 3 Jun 2025 16:09:51 -0700 Subject: [PATCH 05/30] start make indexing work with data in grids --- conformance/tests/ot_index.py | 45 ++++++++++++++++- engine/logic/match.go | 6 +-- grids/filters.go | 39 ++++++++++++++ grids/index.go | 2 +- grids/optimize.go | 95 +++++++++++++++++++++-------------- 5 files changed, 144 insertions(+), 43 deletions(-) create mode 100644 grids/filters.go diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index 076a3743..d379ee10 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -43,7 +43,7 @@ def test_index(man): errors.append("Expecting 2 vertices returned but got %d instead" % (count)) return errors - +""" def test_bulk_index(man): errors = [] @@ -83,3 +83,46 @@ def test_bulk_index(man): errors.append("expected count 4 but got %d instead" % (count)) return errors + + + + +def test_index_after_write(man): + errors = [] + + G = man.writeTest() + + bulk = G.bulkAdd() + bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) + bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) + bulk.addVertex("4", "Person", {"name": "josh", "age": "32"}) + bulk.addVertex("6", "Person", {"name": "peter", "age": "35"}) + bulk.addVertex("7", "Person", {"name": "alice", "age": "31"}) + bulk.addVertex("8", "Person", {"name": "bob", "age": "32"}) + bulk.addVertex("9", "Person", {"name": "charlie", "age": "28"}) + bulk.addVertex("10", "Person", {"name": "diana", "age": "32"}) + bulk.addVertex("11", "Person", {"name": "eve", "age": "30"}) + bulk.addVertex("12", "Person", {"name": "frank", "age": "33"}) + bulk.addVertex("13", "Person", {"name": "grace", "age": "26"}) + bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"}) + bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"}) + bulk.addVertex("16", "Person", {"name": "judy", "age": "34"}) + res = bulk.execute() + print("RES: ", res) + + + G.addIndex("Person", "age") + + count = 0 + resp3 = G.query().V().has(gripql.eq("age","32")) + for i in resp3: + count += 1 + if "age" not in i: + errors.append("field 'age' not found in vertex") + if "age" in i and i["age"] != "32": + errors.append("filtering on field age value '32' but got %s instead" %s (i["age"])) + if count != 4: + errors.append("expected count 4 but got %d instead" % (count)) + + return errors +""" diff --git a/engine/logic/match.go b/engine/logic/match.go index 4001a010..2c3bbe9c 100644 --- a/engine/logic/match.go +++ b/engine/logic/match.go @@ -12,8 +12,8 @@ import ( ) func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { - var val interface{} - var condVal interface{} + var val any + var condVal any val = gdbi.TravelerPathLookup(trav, cond.Key) condVal = cond.Value.AsInterface() @@ -194,7 +194,7 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { case gripql.Condition_WITHOUT: found := false switch condVal := condVal.(type) { - case []interface{}: + case []any: for _, v := range condVal { if reflect.DeepEqual(val, v) { found = true diff --git a/grids/filters.go b/grids/filters.go new file mode 100644 index 00000000..b55ae706 --- /dev/null +++ b/grids/filters.go @@ -0,0 +1,39 @@ +package grids + +import ( + "github.com/bmeg/benchtop" + "github.com/bmeg/grip/gripql" +) + +func MapConditionToOperator(condition gripql.Condition) benchtop.OperatorType { + switch condition { + case gripql.Condition_EQ: + return benchtop.OP_EQ + case gripql.Condition_NEQ: + return benchtop.OP_NEQ + case gripql.Condition_GT: + return benchtop.OP_GT + case gripql.Condition_GTE: + return benchtop.OP_GTE + case gripql.Condition_LT: + return benchtop.OP_LT + case gripql.Condition_LTE: + return benchtop.OP_LTE + case gripql.Condition_INSIDE: + return benchtop.OP_INSIDE + case gripql.Condition_OUTSIDE: + return benchtop.OP_OUTSIDE + case gripql.Condition_BETWEEN: + return benchtop.OP_BETWEEN + case gripql.Condition_WITHIN: + return benchtop.OP_WITHIN + case gripql.Condition_WITHOUT: + return benchtop.OP_WITHOUT + case gripql.Condition_CONTAINS: + return benchtop.OP_CONTAINS + default: + // For Condition_UNKNOWN_CONDITION or any other unmapped value, + // return an empty string or a specific "UNKNOWN" operator type if preferred. + return "" + } +} diff --git a/grids/index.go b/grids/index.go index fff8d70f..faab34c3 100644 --- a/grids/index.go +++ b/grids/index.go @@ -16,7 +16,7 @@ func (ggraph *Graph) AddVertexIndex(label, field string) error { // DeleteVertexIndex delete index from vertices func (ggraph *Graph) DeleteVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") - return ggraph.bsonkv.RemoveField(VTABLE_PREFIX+label, field) + return ggraph.bsonkv.RemoveField(VTABLE_PREFIX+label, field, nil, nil) } // GetVertexIndexList lists out all the vertex indices for a graph diff --git a/grids/optimize.go b/grids/optimize.go index 18e1edb6..de90af42 100644 --- a/grids/optimize.go +++ b/grids/optimize.go @@ -99,8 +99,8 @@ var startOptimizations = []OptimizationRule{ if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { return false } - if hasLabel, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); ok { - return len(hasLabel.HasLabel.GetValues()) > 0 + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); !ok { + return false } if has, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { cond := has.Has.GetCondition() @@ -110,7 +110,16 @@ var startOptimizations = []OptimizationRule{ }, Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { has := pipe[2].GetHas() + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + for i, label := range labels { + if label[:2] != VTABLE_PREFIX { + labels[i] = VTABLE_PREFIX + label + } + } + fmt.Println("PIPE 2: ", pipe[2].GetStatement()) + cond := has.GetCondition() + fmt.Println("COND: ", cond) path := tpath.NormalizePath(cond.Key) value := cond.Value.GetStringValue() var optimized []*gripql.GraphStatement @@ -119,8 +128,13 @@ var startOptimizations = []OptimizationRule{ optimized = []*gripql.GraphStatement{ { Statement: &gripql.GraphStatement_EngineCustom{ - Desc: "Grids Has Level Indexing", - Custom: lookupVertsHasLabelCondIndexStep{key: cond.Key, value: value}, + Desc: "Grids Has Level Indexing", + Custom: lookupVertsHasLabelCondIndexStep{ + key: cond.Key, + value: value, + labels: labels, + op: MapConditionToOperator(cond.GetCondition()), + }, }, }, } @@ -133,10 +147,10 @@ var startOptimizations = []OptimizationRule{ // ////////////////////////////////////////////////////////////////////////////// // LookupVertexHasLabelCondIndex look up vertices has label type lookupVertsHasLabelCondIndexStep struct { - key string - label string - value string - op benchtop.OperatorType + key string + labels []string + value string + op benchtop.OperatorType } func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { @@ -149,7 +163,7 @@ func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, p db: graph, key: t.key, value: t.value, - label: t.label, + labels: t.labels, op: t.op, fallback: false, loadData: true}, nil @@ -160,7 +174,7 @@ func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, p db: graph, key: t.key, value: t.value, - label: t.label, + labels: t.labels, op: t.op, fallback: true, loadData: true}, nil @@ -174,7 +188,7 @@ type lookupVertsHasLabelCondIndexProc struct { db *Graph key string value string - label string + labels []string op benchtop.OperatorType loadData bool fallback bool @@ -183,27 +197,30 @@ type lookupVertsHasLabelCondIndexProc struct { func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { queryChan := make(chan gdbi.ElementLookup, 100) if l.fallback { - log.Debugf("No index found for %s falling back to GetVertexList", l.key) + log.Debugf("lookupVertsHasLabelCondIndexProc: No index found for %s falling back to GetVertexList", l.key) go func() { defer close(queryChan) for t := range in { - rowChan, err := l.db.bsonkv.Tables[l.label].Scan( - true, - []benchtop.FieldFilter{ - {Field: l.key, Value: l.value, Operator: benchtop.OpEqual}, - }, - ) - if err != nil { - log.Errorln("Scan Process Err: ", err) - } - for v := range rowChan { - val, keyExists := v[l.key] - id, idExists := v["_id"].(string) - // In cases where eq comparisons to 'None' values are made - if l.value == "" && (!keyExists && idExists || val == nil) { - queryChan <- gdbi.ElementLookup{ID: id, Ref: t} - } else if (keyExists && idExists) && (val == l.value || fmt.Sprintf("%v", val) == l.value) { - queryChan <- gdbi.ElementLookup{ID: id, Ref: t} + for _, label := range l.labels { + tableFound, ok := l.db.bsonkv.Tables[label] + if !ok { + log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) + continue + } + rowChan, err := tableFound.Scan( + true, + []benchtop.FieldFilter{ + {Field: l.key, Value: l.value, Operator: l.op}, + }, + ) + if err != nil { + log.Errorln("Scan Process Err: ", err) + } + for v := range rowChan { + id, idExists := v["_key"].(string) + if idExists { + queryChan <- gdbi.ElementLookup{ID: id, Ref: t} + } } } } @@ -212,14 +229,16 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi go func() { defer close(queryChan) for t := range in { - rowChan, err := l.db.VertexFilterLabelScan(ctx, l.label, l.key, l.value) - if err != nil { - log.Errorln("VertexFilterLabelScan Process Err: ", err) - } - for id := range rowChan { - queryChan <- gdbi.ElementLookup{ - ID: id, - Ref: t, + for _, label := range l.labels { + rowChan, err := l.db.VertexFilterLabelScan(ctx, label, l.key, l.value) + if err != nil { + log.Errorln("VertexFilterLabelScan Process Err: ", err) + } + for id := range rowChan { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } } } @@ -273,7 +292,7 @@ type lookupVertsCondIndexProc struct { func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { queryChan := make(chan gdbi.ElementLookup, 100) if l.fallback { - log.Debugf("No index found for %s falling back to GetVertexList", l.key) + log.Debugf("lookupVertsCondIndexProc: No index found for %s falling back to GetVertexList", l.key) go func() { defer close(queryChan) for t := range in { From 5770297969107d5a32188493abcbb4b78e2d0b50 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 9 Jun 2025 10:56:13 -0700 Subject: [PATCH 06/30] most tests passing --- conformance/tests/ot_has.py | 3 +- conformance/tests/ot_index.py | 35 +++++----- engine/core/optimize.go | 36 ---------- engine/logic/match.go | 3 +- grids/index.go | 19 +---- grids/optimize.go | 126 ++++++++++++++++++---------------- 6 files changed, 90 insertions(+), 132 deletions(-) diff --git a/conformance/tests/ot_has.py b/conformance/tests/ot_has.py index 84469671..4b1f6b1a 100644 --- a/conformance/tests/ot_has.py +++ b/conformance/tests/ot_has.py @@ -137,7 +137,6 @@ def test_has_prev(man): q = q.has(gripql.neq("$1._id", "$._id")) count = 0 for i in q.render(["$1._id", "$._id"]): - print(i) if i[0] == i[1]: errors.append("History based filter failed: %s" % (i[0]) ) count += 1 @@ -171,6 +170,7 @@ def test_has_neq(man): "Fail: G.query().V().has(gripql.not_(gripql.eq(\"_label\", \"Character\"))) %s != %s" % (count, 21)) + count = 0 for i in G.query().V().hasLabel("Character").has(gripql.neq("eye_color", "brown")): count += 1 @@ -181,6 +181,7 @@ def test_has_neq(man): "Fail: G.query().V().has(gripql.not_(gripql.eq(\"eye_color\", \"brown\"))) %s != %s" % (count, 14)) + return errors diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index d379ee10..de2f9db1 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -1,6 +1,6 @@ import gripql - +""" def test_index(man): errors = [] @@ -25,7 +25,6 @@ def test_index(man): resp = G.listIndices() found = False for i in resp: - print("I: ", i) if i["field"] == "name" and i["label"] == "Person": found = True if not found: @@ -38,12 +37,11 @@ def test_index(man): errors.append("'name' field not found in vertex") if i["name"] != "marko": errors.append("Filtering on field name, value marko but got '%s' instead" % i["name"]) - print("J: ", i) if count != 2: errors.append("Expecting 2 vertices returned but got %d instead" % (count)) return errors -""" + def test_bulk_index(man): errors = [] @@ -74,6 +72,7 @@ def test_bulk_index(man): count = 0 resp3 = G.query().V().has(gripql.eq("age","32")) for i in resp3: + print("I: ", i) count += 1 if "age" not in i: errors.append("field 'age' not found in vertex") @@ -85,14 +84,13 @@ def test_bulk_index(man): return errors - +""" def test_index_after_write(man): errors = [] G = man.writeTest() - - bulk = G.bulkAdd() + """bulk = G.bulkAdd() bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) bulk.addVertex("4", "Person", {"name": "josh", "age": "32"}) @@ -108,21 +106,22 @@ def test_index_after_write(man): bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"}) bulk.addVertex("16", "Person", {"name": "judy", "age": "34"}) res = bulk.execute() - print("RES: ", res) - + if res["errorCount"] > 0: + errors.append("errorCount on bulk add > 0") G.addIndex("Person", "age") count = 0 - resp3 = G.query().V().has(gripql.eq("age","32")) - for i in resp3: + restwo = G.query().V().has(gripql.within("name", ["marko", "vadas", "eve", "ivan", "charlie", "nothere"])) + for i in restwo: count += 1 - if "age" not in i: - errors.append("field 'age' not found in vertex") - if "age" in i and i["age"] != "32": - errors.append("filtering on field age value '32' but got %s instead" %s (i["age"])) - if count != 4: - errors.append("expected count 4 but got %d instead" % (count)) + if count != 5: + errors.append("Expected 5 names from filter but got %d instead" % (count)) + """ + G.addIndex("Starship", "cost_in_credits") + + respthree = G.query().V().hasLabel("Starship").has(gripql.lt("cost_in_credits", 150000000)) + for i in respthree: + print("HELLO ", i) return errors -""" diff --git a/engine/core/optimize.go b/engine/core/optimize.go index 577dcd7b..1840317a 100644 --- a/engine/core/optimize.go +++ b/engine/core/optimize.go @@ -1,7 +1,6 @@ package core import ( - "github.com/bmeg/grip/gdbi/tpath" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/util/protoutil" ) @@ -58,41 +57,6 @@ var startOptimizations = []OptimizationRule{ return append(optimized, pipe[2:]...) }, }, - { - // Matches V().Has(Eq(key, value)) for _id, _label, or other conditions - Match: func(pipe []*gripql.GraphStatement) bool { - if len(pipe) < 2 { - return false - } - if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { - return false - } - if has, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { - cond := has.Has.GetCondition() - return cond != nil && cond.Condition == gripql.Condition_EQ - } - return false - }, - Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - has := pipe[1].GetHas() - cond := has.GetCondition() - path := tpath.NormalizePath(cond.Key) - value := cond.Value.String() - var optimized []*gripql.GraphStatement - switch path { - case "$_current._id": - optimized = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_V{V: protoutil.NewListFromStrings([]string{value})}}, - } - case "$_current._label": - optimized = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{value}}}, - } - default: - } - return append(optimized, pipe[2:]...) - }, - }, } // expandHasAnd preprocesses the pipeline to split Has statements with And expressions. diff --git a/engine/logic/match.go b/engine/logic/match.go index 2c3bbe9c..41678a1d 100644 --- a/engine/logic/match.go +++ b/engine/logic/match.go @@ -26,7 +26,6 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { //TODO: Add escape for $ user string } //If filtering on nil or no match was found on float64 casting operators return false - log.Debug("val: ", val, "condVal: ", condVal) if (val == nil || condVal == nil) && cond.Condition != gripql.Condition_EQ && cond.Condition != gripql.Condition_NEQ && @@ -239,7 +238,7 @@ func MatchesHasExpression(trav gdbi.Traveler, stmt *gripql.HasExpression) bool { switch stmt.Expression.(type) { case *gripql.HasExpression_Condition: cond := stmt.GetCondition() - log.Debug("COND: ", cond) + log.Debug("COND IN ENGINE: ", cond) return MatchesCondition(trav, cond) case *gripql.HasExpression_And: diff --git a/grids/index.go b/grids/index.go index faab34c3..0e89f7e7 100644 --- a/grids/index.go +++ b/grids/index.go @@ -2,6 +2,7 @@ package grids import ( "context" + "fmt" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" @@ -32,27 +33,13 @@ func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { return out } -// Vertex Filter Scan produces a channel of all vertex ids in a graph that match the field - value filter -func (ggraph *Graph) VertexHasConditionScan(ctx context.Context, field string, value string) chan string { - log.WithFields(log.Fields{"field": field, "value": value}).Info("Running VertexFilterScan") - return ggraph.bsonkv.RowIdsByFieldValue(field, value) -} - -// Vertex Filter Scan produces a channel of all vertex ids in a graph that match the field - value filter -func (ggraph *Graph) VertexFilterLabelScan(ctx context.Context, label string, field string, value string) (chan string, error) { - log.WithFields(log.Fields{"label": label, "field": field, "value": value}).Info("Running VertexFilterLabelScan") - if label[:2] != VTABLE_PREFIX { - label = VTABLE_PREFIX + label - } - return ggraph.bsonkv.RowIdsByLabelFieldValue(label, field, value) -} - // VertexLabelScan produces a channel of all vertex ids in a graph // that match a given label func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan string { - log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") if label[:2] != VTABLE_PREFIX { label = VTABLE_PREFIX + label } + fmt.Println("HELLO LABEL:", label) + log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") return ggraph.bsonkv.GetIDsForLabel(label) } diff --git a/grids/optimize.go b/grids/optimize.go index de90af42..29495f91 100644 --- a/grids/optimize.go +++ b/grids/optimize.go @@ -2,9 +2,9 @@ package grids import ( "context" - "fmt" "github.com/bmeg/benchtop" + "github.com/bmeg/benchtop/bsontable" "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gdbi/tpath" "github.com/bmeg/grip/gripql" @@ -58,36 +58,24 @@ var startOptimizations = []OptimizationRule{ return false } if has, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { - cond := has.Has.GetCondition() - return cond != nil && cond.Condition == gripql.Condition_EQ + return has.Has.GetCondition() != nil } return false }, Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - has := pipe[1].GetHas() - cond := has.GetCondition() - path := tpath.NormalizePath(cond.Key) - value := cond.Value.GetStringValue() - var optimized []*gripql.GraphStatement - switch path { - case "$_current._id": - optimized = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_V{V: protoutil.NewListFromStrings([]string{value})}}, - } - case "$_current._label": - optimized = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{value}}}, - } - default: - optimized = []*gripql.GraphStatement{ - { - Statement: &gripql.GraphStatement_EngineCustom{ - Desc: "Grids Has Level Indexing", - Custom: lookupVertsCondIndexStep{key: cond.Key, value: value}, - }, + cond := pipe[1].GetHas().GetCondition() + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids Has Level Indexing", + Custom: lookupVertsCondIndexStep{ + key: cond.Key, + value: cond.Value.AsInterface(), + op: MapConditionToOperator(cond.GetCondition())}, }, - } + }, } + return append(optimized, pipe[2:]...) }, }, @@ -103,8 +91,7 @@ var startOptimizations = []OptimizationRule{ return false } if has, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { - cond := has.Has.GetCondition() - return cond != nil && cond.Condition == gripql.Condition_EQ + return has.Has.GetCondition() != nil } return false }, @@ -116,29 +103,21 @@ var startOptimizations = []OptimizationRule{ labels[i] = VTABLE_PREFIX + label } } - fmt.Println("PIPE 2: ", pipe[2].GetStatement()) - cond := has.GetCondition() - fmt.Println("COND: ", cond) - path := tpath.NormalizePath(cond.Key) - value := cond.Value.GetStringValue() - var optimized []*gripql.GraphStatement - switch path { - default: - optimized = []*gripql.GraphStatement{ - { - Statement: &gripql.GraphStatement_EngineCustom{ - Desc: "Grids Has Level Indexing", - Custom: lookupVertsHasLabelCondIndexStep{ - key: cond.Key, - value: value, - labels: labels, - op: MapConditionToOperator(cond.GetCondition()), - }, + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids Has Level Indexing", + Custom: lookupVertsHasLabelCondIndexStep{ + key: cond.Key, + value: cond.Value.AsInterface(), + labels: labels, + op: MapConditionToOperator(cond.GetCondition()), }, }, - } + }, } + return append(optimized, pipe[3:]...) }, }, @@ -146,10 +125,11 @@ var startOptimizations = []OptimizationRule{ // ////////////////////////////////////////////////////////////////////////////// // LookupVertexHasLabelCondIndex look up vertices has label + type lookupVertsHasLabelCondIndexStep struct { key string labels []string - value string + value any op benchtop.OperatorType } @@ -187,7 +167,7 @@ func (t lookupVertsHasLabelCondIndexStep) GetType() gdbi.DataType { type lookupVertsHasLabelCondIndexProc struct { db *Graph key string - value string + value any labels []string op benchtop.OperatorType loadData bool @@ -195,6 +175,7 @@ type lookupVertsHasLabelCondIndexProc struct { } func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) if l.fallback { log.Debugf("lookupVertsHasLabelCondIndexProc: No index found for %s falling back to GetVertexList", l.key) @@ -207,6 +188,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } + log.Debugln("OP: ", l.op, "KEY: ", l.key, "VAL: ", l.value) rowChan, err := tableFound.Scan( true, []benchtop.FieldFilter{ @@ -230,7 +212,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi defer close(queryChan) for t := range in { for _, label := range l.labels { - rowChan, err := l.db.VertexFilterLabelScan(ctx, label, l.key, l.value) + rowChan, err := l.db.bsonkv.RowIdsByLabelFieldValue(label, l.key, l.value, l.op) if err != nil { log.Errorln("VertexFilterLabelScan Process Err: ", err) } @@ -261,7 +243,8 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi // LookupVertsCondIndex look up vertices by indexed type lookupVertsCondIndexStep struct { key string - value string + value any + op benchtop.OperatorType } func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { @@ -270,11 +253,23 @@ func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.P for _, fields := range graph.bsonkv.Fields { for field := range fields { if field == t.key { - return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: false, loadData: true}, nil + return &lookupVertsCondIndexProc{ + db: graph, + key: t.key, + value: t.value, + op: t.op, + fallback: false, + loadData: true}, nil } } } - return &lookupVertsCondIndexProc{db: graph, key: t.key, value: t.value, fallback: true, loadData: true}, nil + return &lookupVertsCondIndexProc{ + db: graph, + key: t.key, + value: t.value, + op: t.op, + fallback: true, + loadData: true}, nil } func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { @@ -284,12 +279,24 @@ func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { type lookupVertsCondIndexProc struct { db *Graph key string - value string + value any + op benchtop.OperatorType loadData bool fallback bool } +func AddSpecialFields(v *gdbi.Vertex, path string) any { + switch tpath.NormalizePath(path) { + case "$_current._label": + v.Data["_label"] = v.Label + case "$_current._id": + v.Data["_id"] = v.ID + } + return v.Data +} + func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + log.Debugln("Entering lookupVertsCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) if l.fallback { log.Debugf("lookupVertsCondIndexProc: No index found for %s falling back to GetVertexList", l.key) @@ -297,13 +304,14 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager defer close(queryChan) for t := range in { for v := range l.db.GetVertexList(ctx, true) { - val, keyExists := v.Data[l.key] - // In cases where eq comparisons to 'None' values are made - if l.value == "" && (!keyExists || val == nil) { - queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} - } else if keyExists && (val == l.value || fmt.Sprintf("%v", val) == l.value) { + if bsontable.PassesFilters( + AddSpecialFields(v, l.key), + []benchtop.FieldFilter{ + {Field: l.key, Value: l.value, Operator: l.op}, + }) { queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} } + } } }() @@ -311,7 +319,7 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager go func() { defer close(queryChan) for t := range in { - for id := range l.db.VertexHasConditionScan(ctx, l.key, l.value) { + for id := range l.db.bsonkv.RowIdsByHas(l.key, l.value, l.op) { queryChan <- gdbi.ElementLookup{ ID: id, Ref: t, From 2c1890c0b6fc918c9c3f12273c7ebefc8a543eee Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 10 Jun 2025 09:23:24 -0700 Subject: [PATCH 07/30] delete index working --- accounts/interface.go | 1 + conformance/tests/ot_index.py | 83 ++++++++++++++++++++++++++++------- grids/graph.go | 10 ++--- grids/index.go | 2 +- grids/optimize.go | 7 +-- gripql/python/gripql/graph.py | 8 ++++ 6 files changed, 83 insertions(+), 28 deletions(-) diff --git a/accounts/interface.go b/accounts/interface.go index 7f86438c..e75fbc3f 100644 --- a/accounts/interface.go +++ b/accounts/interface.go @@ -43,6 +43,7 @@ var MethodMap = map[string]Operation{ "/gripql.Edit/DeleteVertex": Write, "/gripql.Edit/DeleteEdge": Write, "/gripql.Edit/AddIndex": Write, + "/gripql.Edit/DeleteIndex": Write, "/gripql.Edit/AddSchema": Write, "/gripql.Edit/AddJsonSchema": Write, "/gripql.Edit/AddMapping": Write, diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index de2f9db1..d5beecb6 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -1,6 +1,6 @@ import gripql -""" + def test_index(man): errors = [] @@ -22,14 +22,6 @@ def test_index(man): G.addEdge("6", "3", "created", {"weight": 0.2}) G.addEdge("4", "5", "created", {"weight": 1.0}) - resp = G.listIndices() - found = False - for i in resp: - if i["field"] == "name" and i["label"] == "Person": - found = True - if not found: - errors.append("Expected index not found") - count = 0 for i in G.query().V().has(gripql.eq("name","marko")): count += 1 @@ -67,12 +59,10 @@ def test_bulk_index(man): res = bulk.execute() - print("RES: ", res) count = 0 resp3 = G.query().V().has(gripql.eq("age","32")) for i in resp3: - print("I: ", i) count += 1 if "age" not in i: errors.append("field 'age' not found in vertex") @@ -84,13 +74,12 @@ def test_bulk_index(man): return errors -""" - def test_index_after_write(man): errors = [] G = man.writeTest() - """bulk = G.bulkAdd() + + bulk = G.bulkAdd() bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) bulk.addVertex("4", "Person", {"name": "josh", "age": "32"}) @@ -117,11 +106,73 @@ def test_index_after_write(man): count += 1 if count != 5: errors.append("Expected 5 names from filter but got %d instead" % (count)) - """ + + return errors + + +def test_index_filter(man): + errors = [] + G = man.setGraph("swapi") + G.addIndex("Starship", "cost_in_credits") + G.addIndex("Starship", "cargo_capacity") + respthree = G.query().V().hasLabel("Starship").has(gripql.lt("cost_in_credits", 150000000)) + count = 0 for i in respthree: - print("HELLO ", i) + count += 1 + if i['cost_in_credits'] > 149999999: + errors.append("filtering on ships that cost less than 150000000, but %d > 149999999" % (i['cost_in_credits'])) + + if count != 5: + errors.append("Expected 5 results got %d instead" % (count)) + + indices = G.listIndices() + found = False + count = 0 + for i in indices: + count +=1 + if i["field"] == "cost_in_credits" and i["label"] == "Starship": + found = True + if not found: + errors.append("Expected index not found") + if count != 2: + errors.append("Expected to find 2 indices but found %d instead" % (count)) + + G.deleteIndex("Starship", "cost_in_credits") + count = 0 + indices_two = G.listIndices() + found = False + for i in indices_two: + count += 1 + if i["field"] == "cost_in_credits" and i["label"] == "Starship": + found = True + if found: + errors.append("Expected index not found, but it was found") + if count != 1: + errors.append("Expected to find 1 index but found %d instead" % (count)) + + return errors + + +def test_consistent_results(man): + errors = [] + G = man.setGraph("swapi") + + resp = G.query().V().has(gripql.contains("eye_colors", "yellow")) + count = 0 + for i in resp: + count += 1 + if count != 2: + errors.append("Expected 2 results but got %d instead" % (count)) + + G.addIndex("Species", "eye_colors") + resp = G.query().V().has(gripql.contains("eye_colors", "yellow")) + count = 0 + for i in resp: + count += 1 + if count != 2: + errors.append("Expected 2 results but got %d instead" % (count)) return errors diff --git a/grids/graph.go b/grids/graph.go index 43681559..b3e31f34 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -45,7 +45,7 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) table, ok := ggraph.bsonkv.Tables[vertexLabel] ggraph.bsonkv.Lock.Unlock() if !ok { - log.Debugf("Creating new table for: %s on graph %s", vertex.Label, ggraph.graphID) + log.Debugf("Creating new table %s for label %s on graph %s", vertexLabel, vertex.Label, ggraph.graphID) newTable, err := ggraph.bsonkv.New(vertexLabel, nil) if err != nil { return fmt.Errorf("indexVertex: %s", err) @@ -63,8 +63,7 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) if fieldsExist { for field := range ggraph.bsonkv.Fields[vertexLabel] { if val, ok := vertex.Data[field]; ok { - log.Debugln("Field: ", field, "Value: ", val, "Id: ", vertex.ID) - tx.Set(benchtop.FieldKey(vertexLabel, field, val, []byte(vertex.ID)), []byte{}, nil) + tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) } } } @@ -110,7 +109,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error ggraph.bsonkv.Lock.Unlock() if !ok { - log.Debugf("Creating new table for: %s on graph %s", edge.Label, ggraph.graphID) + log.Debugf("Creating new table %s for label %s on graph %s", edgeLabel, edge.Label, ggraph.graphID) newTable, err := ggraph.bsonkv.New(edgeLabel, nil) if err != nil { return fmt.Errorf("indexEdge: bsonkv.New: %s", err) @@ -128,7 +127,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error if fieldsExist { for field := range ggraph.bsonkv.Fields[edgeLabel] { if val, ok := edge.Data[field]; ok { - tx.Set(benchtop.FieldKey(edgeLabel, field, val, []byte(edge.ID)), []byte{}, nil) + tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) } } } @@ -506,6 +505,7 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element if id.IsSignal() { data <- elementData{req: id} } else { + log.Debugln("ID: ", id.ID) key, _ := ggraph.keyMap.GetVertexKey(id.ID, ggraph.bsonkv.Pb.Db) ed := elementData{key: key, req: id} if load { diff --git a/grids/index.go b/grids/index.go index 0e89f7e7..4dd4fb97 100644 --- a/grids/index.go +++ b/grids/index.go @@ -17,7 +17,7 @@ func (ggraph *Graph) AddVertexIndex(label, field string) error { // DeleteVertexIndex delete index from vertices func (ggraph *Graph) DeleteVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") - return ggraph.bsonkv.RemoveField(VTABLE_PREFIX+label, field, nil, nil) + return ggraph.bsonkv.RemoveField(VTABLE_PREFIX+label, field) } // GetVertexIndexList lists out all the vertex indices for a graph diff --git a/grids/optimize.go b/grids/optimize.go index 29495f91..7f115a76 100644 --- a/grids/optimize.go +++ b/grids/optimize.go @@ -188,7 +188,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } - log.Debugln("OP: ", l.op, "KEY: ", l.key, "VAL: ", l.value) rowChan, err := tableFound.Scan( true, []benchtop.FieldFilter{ @@ -212,11 +211,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi defer close(queryChan) for t := range in { for _, label := range l.labels { - rowChan, err := l.db.bsonkv.RowIdsByLabelFieldValue(label, l.key, l.value, l.op) - if err != nil { - log.Errorln("VertexFilterLabelScan Process Err: ", err) - } - for id := range rowChan { + for id := range l.db.bsonkv.RowIdsByLabelFieldValue(label, l.key, l.value, l.op) { queryChan <- gdbi.ElementLookup{ ID: id, Ref: t, diff --git a/gripql/python/gripql/graph.py b/gripql/python/gripql/graph.py index 6ba38e30..a70aa3d3 100644 --- a/gripql/python/gripql/graph.py +++ b/gripql/python/gripql/graph.py @@ -174,6 +174,14 @@ def addIndex(self, label, field): raise_for_status(response) return response.json() + def deleteIndex(self, label, field): + url = self.url + f"/index/{label}/{field}" + response = self.session.delete( + url, + ) + raise_for_status(response) + return response.json() + def listIndices(self): url = self.url + "/index" response = self.session.get( From 8261f693543fa655e218b0d819c51cf04f4972a9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 10 Jun 2025 15:11:17 -0700 Subject: [PATCH 08/30] adds indexing --- .github/workflows/tests.yml | 4 +- conformance/tests/ot_index.py | 35 ++++++++++--- go.mod | 2 +- go.sum | 8 +-- psql/index.go | 96 +++++++++++++++++++++++++++++++---- sqlite/index.go | 93 +++++++++++++++++++++++++++++---- 6 files changed, 205 insertions(+), 33 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fd151e6..7e8d0231 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -171,7 +171,7 @@ jobs: sleep 15 ./grip server --rpc-port 18202 --http-port 18201 --config ./test/psql.yml & sleep 5 - python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations + python conformance/run_conformance.py http://localhost:18201 --exclude aggregations sqliteTest: @@ -192,7 +192,7 @@ jobs: chmod +x grip ./grip server --rpc-port 18202 --http-port 18201 --config ./test/sqlite.yml & sleep 5 - python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations + python conformance/run_conformance.py http://localhost:18201 --exclude aggregations gripperTest: diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index d5beecb6..75fae9c9 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -1,6 +1,31 @@ import gripql +def test_index_create_and_delete(man): + errors = [] + G = man.writeTest() + G.addIndex("Person", "name") + found = False + indices = G.listIndices() + for i in indices: + if i["field"] == "name" and i["label"] == "Person": + found = True + if not found: + errors.append("Expected index to be found") + + G.deleteIndex("Person", "name") + indices = G.listIndices() + + found = False + for i in indices: + if i["field"] == "name" and i["label"] == "Person": + found = True + if found: + errors.append("Expected index not found") + + return errors + + def test_index(man): errors = [] @@ -31,6 +56,7 @@ def test_index(man): errors.append("Filtering on field name, value marko but got '%s' instead" % i["name"]) if count != 2: errors.append("Expecting 2 vertices returned but got %d instead" % (count)) + return errors @@ -56,9 +82,9 @@ def test_bulk_index(man): bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"}) bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"}) bulk.addVertex("16", "Person", {"name": "judy", "age": "34"}) - - res = bulk.execute() + if res["errorCount"] > 0: + errors.append("errorCount on bulk add > 0") count = 0 resp3 = G.query().V().has(gripql.eq("age","32")) @@ -76,9 +102,7 @@ def test_bulk_index(man): def test_index_after_write(man): errors = [] - G = man.writeTest() - bulk = G.bulkAdd() bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) @@ -113,11 +137,9 @@ def test_index_after_write(man): def test_index_filter(man): errors = [] G = man.setGraph("swapi") - G.addIndex("Starship", "cost_in_credits") G.addIndex("Starship", "cargo_capacity") - respthree = G.query().V().hasLabel("Starship").has(gripql.lt("cost_in_credits", 150000000)) count = 0 for i in respthree: @@ -128,6 +150,7 @@ def test_index_filter(man): if count != 5: errors.append("Expected 5 results got %d instead" % (count)) + indices = G.listIndices() found = False count = 0 diff --git a/go.mod b/go.mod index f1f9f6e1..0ce407d3 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250418205623-cbcef7cb3ad1 + github.com/bmeg/benchtop v0.0.0-20250610220432-e643fa628598 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index fd7a1d58..31e90364 100644 --- a/go.sum +++ b/go.sum @@ -33,12 +33,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250407225705-b34b3f7eeae3 h1:R2kBnE790WVMJ7/cyOk5t/JdDbAhWfRm39R9scq8JKA= -github.com/bmeg/benchtop v0.0.0-20250407225705-b34b3f7eeae3/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= -github.com/bmeg/benchtop v0.0.0-20250417164945-eae1ff034cce h1:ffXdKMWrziCNnYiVrLwApRWr9iRR7GTVHYEEGYZfU/w= -github.com/bmeg/benchtop v0.0.0-20250417164945-eae1ff034cce/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= -github.com/bmeg/benchtop v0.0.0-20250418205623-cbcef7cb3ad1 h1:xKFGnYOIKul7kVHly2ov6mI5GzJWWOPlW45zURcUzC8= -github.com/bmeg/benchtop v0.0.0-20250418205623-cbcef7cb3ad1/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= +github.com/bmeg/benchtop v0.0.0-20250610220432-e643fa628598 h1:3ZFSnoln2g4ogaiV6ExX9HztV08WmbiU3mKxBMtNbmI= +github.com/bmeg/benchtop v0.0.0-20250610220432-e643fa628598/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/psql/index.go b/psql/index.go index 23cf5fec..1d6eff01 100644 --- a/psql/index.go +++ b/psql/index.go @@ -1,24 +1,102 @@ package psql import ( - "errors" + "fmt" + "strings" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" + "github.com/lib/pq" ) -// AddVertexIndex add index to vertices -func (g *Graph) AddVertexIndex(label string, field string) error { - return errors.New("not implemented") +func (g *Graph) AddVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf( + "CREATE INDEX IF NOT EXISTS %s ON %s ((data->>'%s')) WHERE label = '%s'", + pq.QuoteIdentifier(indexName), + pq.QuoteIdentifier(g.v), + field, + vertexLabel, + ) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error adding index %s: %v", indexName, err) + return fmt.Errorf("failed to add index %s: %w", indexName, err) + } + log.Infof("Successfully added index %s for vertex label %s field %s", indexName, vertexLabel, field) + return nil } -// DeleteVertexIndex delete index from vertices -func (g *Graph) DeleteVertexIndex(label string, field string) error { - return errors.New("not implemented") +// DeleteVertexIndex drops the specified index. +func (g *Graph) DeleteVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf("DROP INDEX IF EXISTS %s", pq.QuoteIdentifier(indexName)) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error deleting index %s: %v", indexName, err) + return fmt.Errorf("failed to delete index %s: %w", indexName, err) + } + log.Infof("Successfully deleted index %s", indexName) + return nil } // GetVertexIndexList lists indices func (g *Graph) GetVertexIndexList() <-chan *gripql.IndexID { - o := make(chan *gripql.IndexID) - defer close(o) + o := make(chan *gripql.IndexID, 100) + go func() { + defer close(o) + // Query indices only on the vertex table (g.v) + query := ` + SELECT i.relname AS index_name + FROM pg_class t, + pg_class i, + pg_index ix + WHERE t.oid = ix.indrelid + AND i.oid = ix.indexrelid + AND t.relkind = 'r' + AND i.relkind = 'i' + AND NOT ix.indisprimary + AND NOT ix.indisunique + AND t.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = current_schema()) + AND t.relname = $1 + ` + rows, err := g.db.Queryx(query, g.v) + if err != nil { + log.Errorf("Error getting index list: %v", err) + return + } + defer rows.Close() + + prefix := fmt.Sprintf("%s_", g.graph) + for rows.Next() { + var indexName string + if err := rows.Scan(&indexName); err != nil { + log.Errorf("Error scanning index row: %v", err) + continue + } + if !strings.HasPrefix(indexName, prefix) { + log.Infof("Skipping index '%s': does not match graph '%s'", indexName, g.graph) + continue + } + rest := strings.TrimPrefix(indexName, prefix) + if strings.HasPrefix(rest, "vertices_") { + continue + } else { + parts := strings.SplitN(rest, "_", 2) + if len(parts) == 2 { + o <- &gripql.IndexID{ + Graph: g.graph, + Label: parts[0], // Vertex label, e.g., "Person" + Field: parts[1], // Field, e.g., "age" + } + } else { + log.Infof("Skipping index '%s': unrecognized format", indexName) + } + } + } + if err := rows.Err(); err != nil { + log.Errorf("Error during index row iteration: %v", err) + } + }() return o } diff --git a/sqlite/index.go b/sqlite/index.go index 2862fd58..ca2f0530 100644 --- a/sqlite/index.go +++ b/sqlite/index.go @@ -1,24 +1,99 @@ package sqlite import ( - "errors" + "fmt" + "strings" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" + "github.com/lib/pq" ) -// AddVertexIndex add index to vertices -func (g *Graph) AddVertexIndex(label string, field string) error { - return errors.New("not implemented") +func (g *Graph) AddVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf( + "CREATE INDEX IF NOT EXISTS %s ON %s ((data->>'%s')) WHERE label = '%s'", + pq.QuoteIdentifier(indexName), + pq.QuoteIdentifier(g.v), + field, + vertexLabel, + ) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error adding index %s: %v", indexName, err) + return fmt.Errorf("failed to add index %s: %w", indexName, err) + } + log.Infof("Successfully added index %s for vertex label %s field %s", indexName, vertexLabel, field) + return nil } -// DeleteVertexIndex delete index from vertices -func (g *Graph) DeleteVertexIndex(label string, field string) error { - return errors.New("not implemented") +// DeleteVertexIndex drops the specified index. +func (g *Graph) DeleteVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf("DROP INDEX IF EXISTS %s", pq.QuoteIdentifier(indexName)) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error deleting index %s: %v", indexName, err) + return fmt.Errorf("failed to delete index %s: %w", indexName, err) + } + log.Infof("Successfully deleted index %s", indexName) + return nil } // GetVertexIndexList lists indices func (g *Graph) GetVertexIndexList() <-chan *gripql.IndexID { - o := make(chan *gripql.IndexID) - defer close(o) + o := make(chan *gripql.IndexID, 100) + go func() { + defer close(o) + // Query indices only on the vertex table (g.v) + query := ` + SELECT name + FROM sqlite_master + WHERE type = 'index' AND tbl_name = ? + AND name NOT LIKE 'sqlite_autoindex_%' -- Exclude auto-generated primary key indices + AND name IN ( + SELECT name + FROM pragma_index_list(?) + WHERE "unique" = 0 -- Exclude unique indices + ) + ` + rows, err := g.db.Queryx(query, g.v, g.v) + if err != nil { + log.Errorf("Error getting index list: %v", err) + return + } + defer rows.Close() + + prefix := fmt.Sprintf("%s_", g.graph) + for rows.Next() { + var indexName string + if err := rows.Scan(&indexName); err != nil { + log.Errorf("Error scanning index row: %v", err) + continue + } + if !strings.HasPrefix(indexName, prefix) { + log.Infof("Skipping index '%s': does not match graph '%s'", indexName, g.graph) + continue + } + rest := strings.TrimPrefix(indexName, prefix) + if strings.HasPrefix(rest, "vertices_") { + continue + } else { + parts := strings.SplitN(rest, "_", 2) + if len(parts) == 2 { + o <- &gripql.IndexID{ + Graph: g.graph, + Label: parts[0], // Vertex label, e.g., "Person" + Field: parts[1], // Field, e.g., "age" + } + } else { + log.Infof("Skipping index '%s': unrecognized format", indexName) + } + } + } + if err := rows.Err(); err != nil { + log.Errorf("Error during index row iteration: %v", err) + } + }() return o } From 96c12b1009e9bd61e0843d124d0b4c26a403f27a Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 11 Jun 2025 12:04:46 -0700 Subject: [PATCH 09/30] fix bug in grids driver --- .github/workflows/tests.yml | 10 +--------- accounts/basic.go | 6 ++++-- conformance/tests/auth_basic.py | 10 ++++------ conformance/tests/ot_index.py | 16 +++++++++++++++- conformance/tests/ot_job.py | 1 - engine/core/optimizer_test.go | 10 +++++----- engine/logic/sorter_kv.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- grids/graph.go | 6 ++---- grids/index.go | 2 -- grids/optimize.go | 16 +++++----------- gripql/python/gripql/util.py | 4 ++++ mongo/compile.go | 2 +- mongo/index.go | 2 +- server/marshaler.go | 4 ++-- server/server.go | 2 +- 17 files changed, 50 insertions(+), 51 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e8d0231..20637332 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,7 @@ jobs: name: gripBin path: grip + unitTests: needs: build name: Unit tests @@ -65,7 +66,6 @@ jobs: sleep 5 python conformance/run_kafka.py - badgerTest: needs: build name: Badger Conformance @@ -86,7 +86,6 @@ jobs: sleep 5 make test-conformance - pebbleTest: needs: build name: Pebble Conformance @@ -107,7 +106,6 @@ jobs: sleep 5 make test-conformance - mongoTest: needs: build name: Mongo Test @@ -129,7 +127,6 @@ jobs: sleep 5 make test-conformance - mongoCoreTest: needs: build name: Mongo Core Test @@ -173,7 +170,6 @@ jobs: sleep 5 python conformance/run_conformance.py http://localhost:18201 --exclude aggregations - sqliteTest: needs: build name: Sqlite Test @@ -194,7 +190,6 @@ jobs: sleep 5 python conformance/run_conformance.py http://localhost:18201 --exclude aggregations - gripperTest: needs: build name: Gripper Test @@ -220,7 +215,6 @@ jobs: sleep 5 python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude pivot - authTest: needs: build name: Auth Conformance @@ -251,8 +245,6 @@ jobs: # run specialized role based tests make test-authorization ARGS="--grip_config_file_path test/pebble-auth.yml" - - gridsTest: needs: build name: GRIDs Conformance diff --git a/accounts/basic.go b/accounts/basic.go index ffbf6b11..3e8f1723 100644 --- a/accounts/basic.go +++ b/accounts/basic.go @@ -4,6 +4,8 @@ import ( "encoding/base64" "fmt" "strings" + + "github.com/bmeg/grip/log" ) // BasicCredential describes a username and password for use with Funnel's basic auth. @@ -18,7 +20,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { var auth []string var ok bool - fmt.Printf("Running BasicAuth: %#v\n", md) + log.Infof("Running BasicAuth: %#v\n", md) if auth, ok = md["Authorization"]; !ok { if auth, ok = md["authorization"]; !ok { @@ -28,7 +30,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { if len(auth) > 0 { user, password, ok := parseBasicAuth(auth[0]) - fmt.Printf("User: %s Password: %s OK: %s\n", user, password, ok) + log.Infof("User: %s Password: %s OK: %t\n", user, password, ok) for _, c := range ba { if c.User == user && c.Password == password { return user, nil diff --git a/conformance/tests/auth_basic.py b/conformance/tests/auth_basic.py index b22a1ee0..1f9a579f 100644 --- a/conformance/tests/auth_basic.py +++ b/conformance/tests/auth_basic.py @@ -1,7 +1,5 @@ from __future__ import absolute_import -import requests - def test_current_user_has_policy(manager): """Ensure current user has a policy defined.""" @@ -18,7 +16,7 @@ def test_current_user_can_query(manager): account = manager.current_user_account() assert account, f"Could not find account for {manager.user}" policies = account.policies - assert len(policies) > 0, f"Should have at least one policy" + assert len(policies) > 0, "Should have at least one policy" errors = [] if not account.is_admin: @@ -56,7 +54,7 @@ def test_current_user_can_read(manager): account = manager.current_user_account() assert account, f"Could not find account for {manager.user}" policies = account.policies - assert len(policies) > 0, f"Should have at least one policy" + assert len(policies) > 0, "Should have at least one policy" errors = [] # non admin user if not account.is_admin: @@ -92,7 +90,7 @@ def test_current_user_can_write(manager): account = manager.current_user_account() assert account, f"Could not find account for {manager.user}" policies = account.policies - assert len(policies) > 0, f"Should have at least one policy" + assert len(policies) > 0, "Should have at least one policy" errors = [] # non admin user if not account.is_admin: @@ -109,7 +107,7 @@ def test_current_user_can_write(manager): try: manager.test_write('dummy') errors.append(f"{manager.user} should not be able to write dummy graph") - except AssertionError as e: + except AssertionError: pass else: diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index 75fae9c9..391f6664 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -93,7 +93,7 @@ def test_bulk_index(man): if "age" not in i: errors.append("field 'age' not found in vertex") if "age" in i and i["age"] != "32": - errors.append("filtering on field age value '32' but got %s instead" %s (i["age"])) + errors.append("filtering on field age value '32' but got %s instead" % (i["age"])) if count != 4: errors.append("expected count 4 but got %d instead" % (count)) @@ -199,3 +199,17 @@ def test_consistent_results(man): errors.append("Expected 2 results but got %d instead" % (count)) return errors + + +def test_hasLabel_contains(man): + # If using the grids driver + no indexing this test uses the optimized scan function pipeline + errors = [] + G = man.setGraph("swapi") + resp = G.query().V().hasLabel("Species").has(gripql.contains("eye_colors", "yellow")) + count = 0 + for i in resp: + count += 1 + if count != 2: + errors.append("Expected 2 results but got %d instead" % (count)) + + return errors diff --git a/conformance/tests/ot_job.py b/conformance/tests/ot_job.py index 7190bdd9..fc93c24c 100644 --- a/conformance/tests/ot_job.py +++ b/conformance/tests/ot_job.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -import gripql import time def test_job(man): diff --git a/engine/core/optimizer_test.go b/engine/core/optimizer_test.go index 527649f6..44bbe8a3 100644 --- a/engine/core/optimizer_test.go +++ b/engine/core/optimizer_test.go @@ -211,7 +211,7 @@ func TestIndexStartOptimize(t *testing.T) { } expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Out{}}, } @@ -266,7 +266,7 @@ func TestIndexStartOptimize(t *testing.T) { } expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Out{}}, } @@ -294,7 +294,7 @@ func TestIndexStartOptimize(t *testing.T) { barValue, _ := structpb.NewValue("bar") expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Has{ Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ Condition: &gripql.HasCondition{ @@ -339,7 +339,7 @@ func TestIndexStartOptimize(t *testing.T) { bazValue, _ := structpb.NewValue("baz") expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Has{ Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ Condition: &gripql.HasCondition{ @@ -440,7 +440,7 @@ func TestIndexStartOptimize(t *testing.T) { // handle 'and' statements expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Has{ Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ Condition: &gripql.HasCondition{ diff --git a/engine/logic/sorter_kv.go b/engine/logic/sorter_kv.go index 8a4a97e3..31546e31 100644 --- a/engine/logic/sorter_kv.go +++ b/engine/logic/sorter_kv.go @@ -63,11 +63,11 @@ func (ks *KVSorter[T]) Sorted() chan T { func (ks *kvCompare[T]) compareEncoded(a, b []byte) int { aT, err := ks.conf.FromBytes(a) if err != nil { - log.Debug("error compareEncoded: %s\n", err) + log.Debugf("error compareEncoded: %s\n", err) } bT, err := ks.conf.FromBytes(b) if err != nil { - log.Debug("error compareEncoded: %s\n", err) + log.Debugf("error compareEncoded: %s\n", err) } return ks.conf.Compare(aT, bT) } diff --git a/go.mod b/go.mod index 0ce407d3..597df321 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250610220432-e643fa628598 + github.com/bmeg/benchtop v0.0.0-20250611185622-191ea4eaaed3 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 31e90364..20ba47e8 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250610220432-e643fa628598 h1:3ZFSnoln2g4ogaiV6ExX9HztV08WmbiU3mKxBMtNbmI= -github.com/bmeg/benchtop v0.0.0-20250610220432-e643fa628598/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= +github.com/bmeg/benchtop v0.0.0-20250611185622-191ea4eaaed3 h1:+Rjh3cpOgEuMVHJd2OHOE8iDSt4mt4javLFZjr6+FKQ= +github.com/bmeg/benchtop v0.0.0-20250611185622-191ea4eaaed3/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/graph.go b/grids/graph.go index b3e31f34..745a80cc 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -505,7 +505,6 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element if id.IsSignal() { data <- elementData{req: id} } else { - log.Debugln("ID: ", id.ID) key, _ := ggraph.keyMap.GetVertexKey(id.ID, ggraph.bsonkv.Pb.Db) ed := elementData{key: key, req: id} if load { @@ -531,6 +530,7 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element out := make(chan gdbi.ElementLookup, 100) go func() { defer close(out) + var err error for d := range data { if d.req.IsSignal() { out <- d.req @@ -538,7 +538,6 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element lKey := ggraph.keyMap.GetVertexLabel(d.key, ggraph.bsonkv.Pb.Db) lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) v := gdbi.Vertex{ID: d.req.ID, Label: lID} - var err error v.Data, err = protoutil.StructUnMarshal(d.data) v.Loaded = true if err != nil { @@ -568,7 +567,6 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen defer close(vertexChan) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { - if req.IsSignal() { vertexChan <- elementData{req: req} } else { @@ -605,6 +603,7 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen go func() { defer close(o) for req := range vertexChan { + var err error if req.req.IsSignal() { o <- req.req } else { @@ -622,7 +621,6 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen continue } v := &gdbi.Vertex{ID: id, Label: lid} - var err error v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lid].GetRow([]byte(id)) v.Loaded = true if err != nil { diff --git a/grids/index.go b/grids/index.go index 4dd4fb97..60135d3b 100644 --- a/grids/index.go +++ b/grids/index.go @@ -2,7 +2,6 @@ package grids import ( "context" - "fmt" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" @@ -39,7 +38,6 @@ func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan str if label[:2] != VTABLE_PREFIX { label = VTABLE_PREFIX + label } - fmt.Println("HELLO LABEL:", label) log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") return ggraph.bsonkv.GetIDsForLabel(label) } diff --git a/grids/optimize.go b/grids/optimize.go index 7f115a76..332dcbaf 100644 --- a/grids/optimize.go +++ b/grids/optimize.go @@ -178,7 +178,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) if l.fallback { - log.Debugf("lookupVertsHasLabelCondIndexProc: No index found for %s falling back to GetVertexList", l.key) + log.Debugf("lookupVertsHasLabelCondIndexProc: No index found for %s falling back to bsontable.Scan", l.key) go func() { defer close(queryChan) for t := range in { @@ -188,20 +188,14 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } - rowChan, err := tableFound.Scan( + for id := range tableFound.Scan( true, []benchtop.FieldFilter{ {Field: l.key, Value: l.value, Operator: l.op}, }, - ) - if err != nil { - log.Errorln("Scan Process Err: ", err) - } - for v := range rowChan { - id, idExists := v["_key"].(string) - if idExists { - queryChan <- gdbi.ElementLookup{ID: id, Ref: t} - } + ) { + queryChan <- gdbi.ElementLookup{ID: id.(string), Ref: t} + } } } diff --git a/gripql/python/gripql/util.py b/gripql/python/gripql/util.py index f1cb34de..6ab0cad4 100644 --- a/gripql/python/gripql/util.py +++ b/gripql/python/gripql/util.py @@ -64,6 +64,10 @@ def close(self): if self.i == 0: return + if self.start is None: + self.logger.error("Rate.close() called but Rate.init() was never called. self.start is None.") + return + now = datetime.now() dt = now - self.start rate = self.i / dt.total_seconds() diff --git a/mongo/compile.go b/mongo/compile.go index 26454f2c..6331cea9 100644 --- a/mongo/compile.go +++ b/mongo/compile.go @@ -1005,7 +1005,7 @@ func (comp *Compiler) Compile(stmts []*gripql.GraphStatement, opts *gdbi.Compile } } - bsonDoc, err := bson.Marshal(bson.D{{"doc", query}}) + bsonDoc, err := bson.Marshal(bson.D{{Key: "doc", Value: query}}) if err != nil { fmt.Printf("Error: %s\n", err) } diff --git a/mongo/index.go b/mongo/index.go index 3077d255..8b1d8393 100644 --- a/mongo/index.go +++ b/mongo/index.go @@ -25,7 +25,7 @@ func (mg *Graph) AddVertexIndex(label string, field string) error { _, err := idx.CreateOne( context.Background(), mongo.IndexModel{ - Keys: bson.D{{"label", 1}, {field, 1}}, + Keys: bson.D{{Key: "label", Value: 1}, {Key: field, Value: 1}}, Options: options.Index().SetUnique(false).SetSparse(true).SetBackground(true), }) if err != nil { diff --git a/server/marshaler.go b/server/marshaler.go index ee564c87..a25c89f9 100644 --- a/server/marshaler.go +++ b/server/marshaler.go @@ -21,8 +21,8 @@ type MarshalClean struct { func NewMarshaler() runtime.Marshaler { return &MarshalClean{ m: &runtime.JSONPb{ - protojson.MarshalOptions{EmitUnpopulated: true}, - protojson.UnmarshalOptions{}, + MarshalOptions: protojson.MarshalOptions{EmitUnpopulated: true}, + UnmarshalOptions: protojson.UnmarshalOptions{}, //EnumsAsInts: false, //EmitDefaults: true, //OrigName: true, diff --git a/server/server.go b/server/server.go index 7abc8462..fb891876 100644 --- a/server/server.go +++ b/server/server.go @@ -366,7 +366,7 @@ func (server *GripServer) Serve(pctx context.Context) error { } partition, offset, err := server.kafkaProducer.SendMessage(msg) if err != nil { - log.Errorf("Failed to send Kafka message to topic %s: %v", *&server.conf.Kafka.Topic, err) + log.Errorf("Failed to send Kafka message to topic %#v: %v", *&server.conf.Kafka.Topic, err) } else { log.Infof("Message sent to Kafka topic %s [partition %d, offset %d]", *server.conf.Kafka.Topic, partition, offset) } From ed8d464ee9b6d6072ccc34950fd96d17323db773 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 11 Jun 2025 13:58:45 -0700 Subject: [PATCH 10/30] cleanup --- .github/workflows/tests.yml | 2 +- mongo/graph.go | 33 +++++++-------- mongo/index.go | 82 ++++++++++++++++++++++--------------- 3 files changed, 65 insertions(+), 52 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20637332..2b521356 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -213,7 +213,7 @@ jobs: sleep 5 ./grip server --rpc-port 18202 --http-port 18201 --config ./gripper/test-graph/config.yaml --er tableServer=localhost:50051 & sleep 5 - python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude pivot + python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude index pivot authTest: needs: build diff --git a/mongo/graph.go b/mongo/graph.go index 7ba33ec2..e94b9cd6 100644 --- a/mongo/graph.go +++ b/mongo/graph.go @@ -4,9 +4,6 @@ import ( "context" "fmt" - //"io" - //"strings" - "time" "github.com/bmeg/grip/engine/core" @@ -45,13 +42,13 @@ func (mg *Graph) GetTimestamp() string { func (mg *Graph) GetVertex(id string, load bool) *gdbi.Vertex { opts := options.FindOne() if !load { - opts.SetProjection(map[string]interface{}{FIELD_ID: 1, FIELD_LABEL: 1}) + opts.SetProjection(map[string]any{FIELD_ID: 1, FIELD_LABEL: 1}) } result := mg.ar.VertexCollection(mg.graph).FindOne(context.Background(), bson.M{FIELD_ID: id}, opts) if result.Err() != nil { return nil } - d := map[string]interface{}{} + d := map[string]any{} if nil == result.Decode(d) { v := UnpackVertex(d) return v @@ -63,13 +60,13 @@ func (mg *Graph) GetVertex(id string, load bool) *gdbi.Vertex { func (mg *Graph) GetEdge(id string, load bool) *gdbi.Edge { opts := options.FindOne() if !load { - opts.SetProjection(map[string]interface{}{FIELD_ID: 1, FIELD_LABEL: 1, FIELD_FROM: 1, FIELD_TO: 1}) + opts.SetProjection(map[string]any{FIELD_ID: 1, FIELD_LABEL: 1, FIELD_FROM: 1, FIELD_TO: 1}) } result := mg.ar.EdgeCollection(mg.graph).FindOne(context.TODO(), bson.M{FIELD_ID: id}, opts) if result.Err() != nil { return nil } - d := map[string]interface{}{} + d := map[string]any{} if nil == result.Decode(d) { v := UnpackEdge(d) return v @@ -268,7 +265,7 @@ func (mg *Graph) GetVertexList(ctx context.Context, load bool) <-chan *gdbi.Vert return default: } - result := map[string]interface{}{} + result := map[string]any{} if err := query.Decode(&result); err == nil { v := UnpackVertex(result) o <- v @@ -303,7 +300,7 @@ func (mg *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdbi.Ed return default: } - result := map[string]interface{}{} + result := map[string]any{} if err := query.Decode(&result); err == nil { if _, ok := result[FIELD_TO]; ok { e := UnpackEdge(result) @@ -349,7 +346,7 @@ func (mg *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLook } chunk := map[string]*gdbi.Vertex{} for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { v := UnpackVertex(result) chunk[v.ID] = v @@ -412,9 +409,9 @@ func (mg *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLoo cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { - if dst, ok := result["dst"].(map[string]interface{}); ok { + if dst, ok := result["dst"].(map[string]any); ok { v := UnpackVertex(dst) fromID := result[FIELD_FROM].(string) r := batchMap[fromID] @@ -424,7 +421,7 @@ func (mg *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLoo o <- ri } } else { - log.WithFields(log.Fields{"result": result["dst"]}).Error("GetOutChannel: unable to cast result to map[string]interface{}") + log.WithFields(log.Fields{"result": result["dst"]}).Error("GetOutChannel: unable to cast result to map[string]any") } } else { log.WithFields(log.Fields{"result": result, "error": err}).Error("GetOutChannel: decode error") @@ -491,9 +488,9 @@ func (mg *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLook cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { - if src, ok := result["src"].(map[string]interface{}); ok { + if src, ok := result["src"].(map[string]any); ok { v := UnpackVertex(src) toID := result[FIELD_TO].(string) r := batchMap[toID] @@ -503,7 +500,7 @@ func (mg *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLook o <- ri } } else { - log.WithFields(log.Fields{"result": result["src"]}).Error("GetInChannel: unable to cast result to map[string]interface{}") + log.WithFields(log.Fields{"result": result["src"]}).Error("GetInChannel: unable to cast result to map[string]any") } } else { log.WithFields(log.Fields{"error": err}).Error("Decode") @@ -561,7 +558,7 @@ func (mg *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.Elemen cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { e := UnpackEdge(result) fromID := result[FIELD_FROM].(string) @@ -628,7 +625,7 @@ func (mg *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Element cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { e := UnpackEdge(result) toID := result[FIELD_TO].(string) diff --git a/mongo/index.go b/mongo/index.go index 8b1d8393..b81a952e 100644 --- a/mongo/index.go +++ b/mongo/index.go @@ -13,7 +13,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -// AddVertexIndex add index to vertices +// AddVertexIndex adds an index to vertices for a specific label and field func (mg *Graph) AddVertexIndex(label string, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") field = tpath.NormalizePath(field) @@ -22,37 +22,56 @@ func (mg *Graph) AddVertexIndex(label string, field string) error { idx := mg.ar.VertexCollection(mg.graph).Indexes() + // Create a compound index on _label and the specified field, filtered by the specific label _, err := idx.CreateOne( context.Background(), mongo.IndexModel{ - Keys: bson.D{{Key: "label", Value: 1}, {Key: field, Value: 1}}, - Options: options.Index().SetUnique(false).SetSparse(true).SetBackground(true), + Keys: bson.D{ + {Key: FIELD_LABEL, Value: 1}, + {Key: field, Value: 1}, + }, + Options: options.Index(). + SetUnique(false). + SetBackground(true). + SetPartialFilterExpression(bson.M{FIELD_LABEL: label}), }) if err != nil { - return fmt.Errorf("failed create index %s %s %s", label, field, err) + return fmt.Errorf("failed to create index for label %s on field %s: %s", label, field, err) } return nil } -// DeleteVertexIndex delete index from vertices +// DeleteVertexIndex deletes an index from vertices for a specific label and field func (mg *Graph) DeleteVertexIndex(label string, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") field = tpath.NormalizePath(field) field = tpath.ToLocalPath(field) - field = strings.TrimPrefix(field, "$.") //FIXME + field = strings.TrimPrefix(field, "$.") idx := mg.ar.VertexCollection(mg.graph).Indexes() cursor, err := idx.List(context.TODO()) + if err != nil { + return fmt.Errorf("failed to list indices: %s", err) + } + var results []bson.M if err = cursor.All(context.TODO(), &results); err != nil { - return err + return fmt.Errorf("failed to retrieve index list: %s", err) } + for _, rec := range results { - recKeys := rec["key"].(bson.M) - if _, ok := recKeys["label"]; ok { - if _, ok := recKeys[field]; ok { - if _, err := idx.DropOne(context.TODO(), rec["name"].(string)); err != nil { - return err + if recKeys, ok := rec["key"].(bson.M); ok { + if _, hasLabel := recKeys[FIELD_LABEL]; hasLabel { + if _, hasField := recKeys[field]; hasField { + if partialFilter, ok := rec["partialFilterExpression"].(bson.M); ok { + if partialLabel, ok := partialFilter[FIELD_LABEL].(string); ok && partialLabel == label { + log.Debugln("HELLO ", partialLabel) + if _, err := idx.DropOne(context.TODO(), rec["name"].(string)); err != nil { + return fmt.Errorf("failed to delete index for label %s on field %s: %s", label, field, err) + } + return nil + } + } } } } @@ -67,34 +86,31 @@ func (mg *Graph) GetVertexIndexList() <-chan *gripql.IndexID { go func() { defer close(out) - c := mg.ar.VertexCollection(mg.graph) - - labels, err := mg.ListVertexLabels() + idx := mg.ar.VertexCollection(mg.graph).Indexes() + cursor, err := idx.List(context.TODO()) if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: finding distinct labels") + log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: failed to list indices") + return } - // list indexed fields - idx := c.Indexes() - cursor, err := idx.List(context.TODO()) var idxList []bson.M if err = cursor.All(context.TODO(), &idxList); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: finding indexed fields") + log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: failed to retrieve index list") + return } + for _, rec := range idxList { - recKeys := rec["key"].(bson.M) - if len(recKeys) > 1 { - if _, ok := recKeys["label"]; ok { - key := "" - for k := range recKeys { - if k != "label" { - key = k - } - } - if len(key) > 0 { - f := strings.TrimPrefix(key, "data.") - for _, l := range labels { - out <- &gripql.IndexID{Graph: mg.graph, Label: l, Field: f} + if recKeys, ok := rec["key"].(bson.M); ok { + if _, hasLabelKey := recKeys[FIELD_LABEL]; hasLabelKey { + for key := range recKeys { + if key != FIELD_LABEL { + f := strings.TrimPrefix(key, "data.") + if partialFilter, ok := rec["partialFilterExpression"].(bson.M); ok { + log.Debugln("HELLO2 ", partialFilter) + if label, ok := partialFilter[FIELD_LABEL].(string); ok { + out <- &gripql.IndexID{Graph: mg.graph, Label: label, Field: f} + } + } } } } From dcc32711023c21af54be3fed83ec6184e8bdb18d Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 12 Jun 2025 11:24:12 -0700 Subject: [PATCH 11/30] fix small benchtop issues --- go.mod | 2 +- go.sum | 4 ++-- grids/graphdb.go | 2 ++ grids/keymap.go | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 597df321..872accfd 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250611185622-191ea4eaaed3 + github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 20ba47e8..67a70794 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250611185622-191ea4eaaed3 h1:+Rjh3cpOgEuMVHJd2OHOE8iDSt4mt4javLFZjr6+FKQ= -github.com/bmeg/benchtop v0.0.0-20250611185622-191ea4eaaed3/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= +github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc h1:VX7MB/9cxuSgaYrBR3eaZ1HxkQCFm9Sd47PeNgAwr5Q= +github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/graphdb.go b/grids/graphdb.go index 332e59ea..a35ded0c 100644 --- a/grids/graphdb.go +++ b/grids/graphdb.go @@ -46,7 +46,9 @@ func (kgraph *GDB) Graph(graph string) (gdbi.GraphInterface, error) { if err != nil { return nil, err } + mu.Lock() kgraph.drivers[graph] = g + mu.Unlock() return g, nil } return nil, fmt.Errorf("graph '%s' was not found", graph) diff --git a/grids/keymap.go b/grids/keymap.go index a6aeb706..30b914f7 100644 --- a/grids/keymap.go +++ b/grids/keymap.go @@ -210,8 +210,7 @@ func (km *KeyMap) GetLabelID(key uint64, db GetSet) (string, bool) { } func getIDKey(prefix []byte, id string, db GetSet) (uint64, bool) { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - v, closer, err := db.Get(k) + v, closer, err := db.Get(bytes.Join([][]byte{prefix, []byte(id)}, []byte{})) if v == nil || err != nil { return 0, false } From b6b69b49fb473e3d21c0cd761e0b2a56e682272e Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 12 Jun 2025 13:16:42 -0700 Subject: [PATCH 12/30] fix bugs --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 872accfd..7dd6cc99 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a @@ -63,6 +62,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bmeg/benchtop v0.0.0-20250612201304-3e907b3cd7b0 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 67a70794..6e21351e 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc h1:VX7MB/9cxuSgaYrBR3eaZ1HxkQCFm9Sd47PeNgAwr5Q= github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= +github.com/bmeg/benchtop v0.0.0-20250612201304-3e907b3cd7b0 h1:F+qZPWexaI9zwUZjrMNNsvuzRsHBS8ii4bjW2SI/gE8= +github.com/bmeg/benchtop v0.0.0-20250612201304-3e907b3cd7b0/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= From 575ac6c28bb2dc52cd69e364e0f8afc85bceaa02 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 16 Jun 2025 15:43:04 -0700 Subject: [PATCH 13/30] adds support for nested fields and compound filters --- conformance/tests/ot_index.py | 43 +++++ engine/logic/match.go | 4 +- gdbi/traveler_doc.go | 2 +- go.mod | 4 +- go.sum | 10 +- grids/filters.go | 97 ++++++++++ grids/graph.go | 6 +- grids/optimize.go | 331 ---------------------------------- grids/optimizer.go | 93 ++++++++++ grids/processor.go | 186 +++++++++++++++++++ 10 files changed, 430 insertions(+), 346 deletions(-) delete mode 100644 grids/optimize.go create mode 100644 grids/optimizer.go create mode 100644 grids/processor.go diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index 391f6664..ad60dbad 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -213,3 +213,46 @@ def test_hasLabel_contains(man): errors.append("Expected 2 results but got %d instead" % (count)) return errors + + + +def test_cond_nested_field(man): + # Tests nested filters on optimized grids query driver + + errors = [] + G = man.writeTest() + + G.addIndex("Observation", "code.coding.[0].code") + + G.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + G.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + + count = 0 + for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): + count +=1 + + if count != 1: + errors.append("Expected count == 1 but got %d instead" % (count)) + + return errors + + +def test_bulk_cond_nested_field(man): + errors = [] + G = man.writeTest() + G.addIndex("Observation", "code.coding.[0].code") + + bulk = G.bulkAdd() + bulk.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + bulk.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + + bulk.execute() + + count = 0 + for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): + count +=1 + + if count != 1: + errors.append("Expected count == 1 but got %d instead" % (count)) + + return errors diff --git a/engine/logic/match.go b/engine/logic/match.go index 41678a1d..e02156fc 100644 --- a/engine/logic/match.go +++ b/engine/logic/match.go @@ -35,7 +35,7 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { return false } - log.Debugf("match: %s %s %s", condVal, val, cond.Key) + //log.Debugf("match: %s %s %s", condVal, val, cond.Key) switch cond.Condition { case gripql.Condition_EQ: @@ -67,7 +67,6 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { return valN >= condN case gripql.Condition_LT: - //log.Debugf("match: %#v %#v %s", condVal, val, cond.Key) valN, err := cast.ToFloat64E(val) //log.Debugf("CAST: ", valN, "ERROR: ", err) if err != nil { @@ -238,7 +237,6 @@ func MatchesHasExpression(trav gdbi.Traveler, stmt *gripql.HasExpression) bool { switch stmt.Expression.(type) { case *gripql.HasExpression_Condition: cond := stmt.GetCondition() - log.Debug("COND IN ENGINE: ", cond) return MatchesCondition(trav, cond) case *gripql.HasExpression_And: diff --git a/gdbi/traveler_doc.go b/gdbi/traveler_doc.go index 131c250d..4aab7c89 100644 --- a/gdbi/traveler_doc.go +++ b/gdbi/traveler_doc.go @@ -79,7 +79,7 @@ func TravelerPathLookup(traveler Traveler, path string) interface{} { return doc } res, err := jsonpath.JsonPathLookup(doc, jpath) - log.Debug("field: ", field, " jpath: ", jpath, " namespace: ", namespace, " doc: ", doc, " res: ", res) + //log.Debug("field: ", field, " jpath: ", jpath, " namespace: ", namespace, " doc: ", doc, " res: ", res) if err != nil { return nil diff --git a/go.mod b/go.mod index 7dd6cc99..18e55b85 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 + github.com/bmeg/benchtop v0.0.0-20250616223533-d74bbae858e2 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a @@ -40,7 +41,7 @@ require ( github.com/robertkrimen/otto v0.4.0 github.com/segmentio/ksuid v1.0.4 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cast v1.6.0 + github.com/spf13/cast v1.9.2 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.0 @@ -62,7 +63,6 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bmeg/benchtop v0.0.0-20250612201304-3e907b3cd7b0 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 6e21351e..76086b54 100644 --- a/go.sum +++ b/go.sum @@ -33,10 +33,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc h1:VX7MB/9cxuSgaYrBR3eaZ1HxkQCFm9Sd47PeNgAwr5Q= -github.com/bmeg/benchtop v0.0.0-20250612181916-a4ff2e322bfc/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= -github.com/bmeg/benchtop v0.0.0-20250612201304-3e907b3cd7b0 h1:F+qZPWexaI9zwUZjrMNNsvuzRsHBS8ii4bjW2SI/gE8= -github.com/bmeg/benchtop v0.0.0-20250612201304-3e907b3cd7b0/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= +github.com/bmeg/benchtop v0.0.0-20250616223533-d74bbae858e2 h1:9aqw+Le7fm7KFWggeuwFTDk0m+qXObxB5HduCfvWsso= +github.com/bmeg/benchtop v0.0.0-20250616223533-d74bbae858e2/go.mod h1:4s+8vTKrryyznb6ebLHkC7sriL3Y3PCRVkVkfdbGw3k= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= @@ -346,8 +344,8 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE= +github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= diff --git a/grids/filters.go b/grids/filters.go index b55ae706..d63f256b 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -2,9 +2,106 @@ package grids import ( "github.com/bmeg/benchtop" + "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/bsontable/filters" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" ) +type GripQLFilter struct { + Expression *gripql.HasExpression +} + +// This method makes GripQLFilter satisfy the bsontable.RowFilter interface. +func (f *GripQLFilter) Matches(row map[string]any) bool { + return MatchesHasExpression(row, f.Expression) +} + +func (f *GripQLFilter) RequiredFields() []string { + return extractKeys(f.Expression) +} + +func extractKeys(expr *gripql.HasExpression) []string { + keys := map[string]struct{}{} + + var recurse func(*gripql.HasExpression) + recurse = func(e *gripql.HasExpression) { + if e == nil { + return + } + switch st := e.Expression.(type) { + case *gripql.HasExpression_Condition: + keys[st.Condition.GetKey()] = struct{}{} + case *gripql.HasExpression_And: + for _, subExpr := range st.And.GetExpressions() { + recurse(subExpr) + } + case *gripql.HasExpression_Or: + for _, subExpr := range st.Or.GetExpressions() { + recurse(subExpr) + } + case *gripql.HasExpression_Not: + recurse(st.Not.GetNot()) + } + } + + recurse(expr) + + out := make([]string, 0, len(keys)) + for k := range keys { + out = append(out, k) + } + return out +} + +func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { + switch stmt.Expression.(type) { + case *gripql.HasExpression_Condition: + cond := stmt.GetCondition() + return filters.ApplyFilterCondition( + bsontable.PathLookup(val.(map[string]any), cond.Key), + &benchtop.FieldFilter{ + Operator: MapConditionToOperator(cond.Condition), + Field: cond.Key, + Value: cond.Value.AsInterface(), + }, + ) + case *gripql.HasExpression_And: + and := stmt.GetAnd() + andRes := []bool{} + for _, e := range and.Expressions { + andRes = append(andRes, MatchesHasExpression(val, e)) + } + for _, r := range andRes { + if !r { + return false + } + } + return true + + case *gripql.HasExpression_Or: + or := stmt.GetOr() + orRes := []bool{} + for _, e := range or.Expressions { + orRes = append(orRes, MatchesHasExpression(val, e)) + } + for _, r := range orRes { + if r { + return true + } + } + return false + + case *gripql.HasExpression_Not: + e := stmt.GetNot() + return !MatchesHasExpression(val, e) + + default: + log.Errorf("unknown where expression type: %T", stmt.Expression) + return false + } +} + func MapConditionToOperator(condition gripql.Condition) benchtop.OperatorType { switch condition { case gripql.Condition_EQ: diff --git a/grids/graph.go b/grids/graph.go index 745a80cc..9de1dabf 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -62,7 +62,7 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) _, fieldsExist := ggraph.bsonkv.Fields[vertexLabel] if fieldsExist { for field := range ggraph.bsonkv.Fields[vertexLabel] { - if val, ok := vertex.Data[field]; ok { + if val := bsontable.PathLookup(vertex.Data, field); val != nil { tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) } } @@ -126,7 +126,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error _, fieldsExist := ggraph.bsonkv.Fields[edgeLabel] if fieldsExist { for field := range ggraph.bsonkv.Fields[edgeLabel] { - if val, ok := edge.Data[field]; ok { + if val := bsontable.PathLookup(edge.Data, field); val != nil { tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) } } @@ -135,7 +135,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error } func (ggraph *Graph) Compiler() gdbi.Compiler { - return core.NewCompiler(ggraph, GripOptimizer, core.IndexStartOptimize) + return core.NewCompiler(ggraph, GridsOptimizer, core.IndexStartOptimize) } // AddVertex adds an edge to the graph, if it already exists diff --git a/grids/optimize.go b/grids/optimize.go deleted file mode 100644 index 332dcbaf..00000000 --- a/grids/optimize.go +++ /dev/null @@ -1,331 +0,0 @@ -package grids - -import ( - "context" - - "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" - "github.com/bmeg/grip/gdbi" - "github.com/bmeg/grip/gdbi/tpath" - "github.com/bmeg/grip/gripql" - "github.com/bmeg/grip/log" - "github.com/bmeg/grip/util/protoutil" -) - -type OptimizationRule struct { - Match func(pipe []*gripql.GraphStatement) bool - Replace func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement -} - -func expandHasAnd(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - expanded := []*gripql.GraphStatement{} - for _, step := range pipe { - if has, ok := step.GetStatement().(*gripql.GraphStatement_Has); ok { - if and := has.Has.GetAnd(); and != nil { - for _, expr := range and.Expressions { - expanded = append(expanded, &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{Has: expr}}) - } - } else { - expanded = append(expanded, step) - } - } else { - expanded = append(expanded, step) - } - } - return expanded -} - -func GripOptimizer(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - // Preprocess to handle Has with And expressions - pipe = expandHasAnd(pipe) - // Apply the first matching optimization rule - for _, rule := range startOptimizations { - if rule.Match(pipe) { - return rule.Replace(pipe) - } - } - // Return the original pipeline if no optimizations apply - return pipe -} - -var startOptimizations = []OptimizationRule{ - { - Match: func(pipe []*gripql.GraphStatement) bool { - if len(pipe) < 2 { - return false - } - if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { - return false - } - if has, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { - return has.Has.GetCondition() != nil - } - return false - }, - Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - cond := pipe[1].GetHas().GetCondition() - var optimized = []*gripql.GraphStatement{ - { - Statement: &gripql.GraphStatement_EngineCustom{ - Desc: "Grids Has Level Indexing", - Custom: lookupVertsCondIndexStep{ - key: cond.Key, - value: cond.Value.AsInterface(), - op: MapConditionToOperator(cond.GetCondition())}, - }, - }, - } - - return append(optimized, pipe[2:]...) - }, - }, - { - Match: func(pipe []*gripql.GraphStatement) bool { - if len(pipe) < 3 { - return false - } - if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { - return false - } - if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); !ok { - return false - } - if has, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { - return has.Has.GetCondition() != nil - } - return false - }, - Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - has := pipe[2].GetHas() - labels := protoutil.AsStringList(pipe[1].GetHasLabel()) - for i, label := range labels { - if label[:2] != VTABLE_PREFIX { - labels[i] = VTABLE_PREFIX + label - } - } - cond := has.GetCondition() - var optimized = []*gripql.GraphStatement{ - { - Statement: &gripql.GraphStatement_EngineCustom{ - Desc: "Grids Has Level Indexing", - Custom: lookupVertsHasLabelCondIndexStep{ - key: cond.Key, - value: cond.Value.AsInterface(), - labels: labels, - op: MapConditionToOperator(cond.GetCondition()), - }, - }, - }, - } - - return append(optimized, pipe[3:]...) - }, - }, -} - -// ////////////////////////////////////////////////////////////////////////////// -// LookupVertexHasLabelCondIndex look up vertices has label - -type lookupVertsHasLabelCondIndexStep struct { - key string - labels []string - value any - op benchtop.OperatorType -} - -func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { - graph := db.(*Graph) - // If Field is indexed, use the special processor - for _, fields := range graph.bsonkv.Fields { - for field := range fields { - if field == t.key { - return &lookupVertsHasLabelCondIndexProc{ - db: graph, - key: t.key, - value: t.value, - labels: t.labels, - op: t.op, - fallback: false, - loadData: true}, nil - } - } - } - return &lookupVertsHasLabelCondIndexProc{ - db: graph, - key: t.key, - value: t.value, - labels: t.labels, - op: t.op, - fallback: true, - loadData: true}, nil -} - -func (t lookupVertsHasLabelCondIndexStep) GetType() gdbi.DataType { - return gdbi.VertexData -} - -type lookupVertsHasLabelCondIndexProc struct { - db *Graph - key string - value any - labels []string - op benchtop.OperatorType - loadData bool - fallback bool -} - -func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") - queryChan := make(chan gdbi.ElementLookup, 100) - if l.fallback { - log.Debugf("lookupVertsHasLabelCondIndexProc: No index found for %s falling back to bsontable.Scan", l.key) - go func() { - defer close(queryChan) - for t := range in { - for _, label := range l.labels { - tableFound, ok := l.db.bsonkv.Tables[label] - if !ok { - log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) - continue - } - for id := range tableFound.Scan( - true, - []benchtop.FieldFilter{ - {Field: l.key, Value: l.value, Operator: l.op}, - }, - ) { - queryChan <- gdbi.ElementLookup{ID: id.(string), Ref: t} - - } - } - } - }() - } else { - go func() { - defer close(queryChan) - for t := range in { - for _, label := range l.labels { - for id := range l.db.bsonkv.RowIdsByLabelFieldValue(label, l.key, l.value, l.op) { - queryChan <- gdbi.ElementLookup{ - ID: id, - Ref: t, - } - } - } - - } - }() - } - - go func() { - defer close(out) - for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { - i := v.Ref - out <- i.AddCurrent(v.Vertex.Copy()) - } - }() - return ctx - -} - -// ////////////////////////////////////////////////////////////////////////////// -// LookupVertsCondIndex look up vertices by indexed -type lookupVertsCondIndexStep struct { - key string - value any - op benchtop.OperatorType -} - -func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { - graph := db.(*Graph) - // If Field is indexed, use the special processor - for _, fields := range graph.bsonkv.Fields { - for field := range fields { - if field == t.key { - return &lookupVertsCondIndexProc{ - db: graph, - key: t.key, - value: t.value, - op: t.op, - fallback: false, - loadData: true}, nil - } - } - } - return &lookupVertsCondIndexProc{ - db: graph, - key: t.key, - value: t.value, - op: t.op, - fallback: true, - loadData: true}, nil -} - -func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { - return gdbi.VertexData -} - -type lookupVertsCondIndexProc struct { - db *Graph - key string - value any - op benchtop.OperatorType - loadData bool - fallback bool -} - -func AddSpecialFields(v *gdbi.Vertex, path string) any { - switch tpath.NormalizePath(path) { - case "$_current._label": - v.Data["_label"] = v.Label - case "$_current._id": - v.Data["_id"] = v.ID - } - return v.Data -} - -func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsCondIndexProc custom processor") - queryChan := make(chan gdbi.ElementLookup, 100) - if l.fallback { - log.Debugf("lookupVertsCondIndexProc: No index found for %s falling back to GetVertexList", l.key) - go func() { - defer close(queryChan) - for t := range in { - for v := range l.db.GetVertexList(ctx, true) { - if bsontable.PassesFilters( - AddSpecialFields(v, l.key), - []benchtop.FieldFilter{ - {Field: l.key, Value: l.value, Operator: l.op}, - }) { - queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} - } - - } - } - }() - } else { - go func() { - defer close(queryChan) - for t := range in { - for id := range l.db.bsonkv.RowIdsByHas(l.key, l.value, l.op) { - queryChan <- gdbi.ElementLookup{ - ID: id, - Ref: t, - } - } - - } - }() - } - - go func() { - defer close(out) - for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { - i := v.Ref - out <- i.AddCurrent(v.Vertex.Copy()) - } - }() - return ctx - -} diff --git a/grids/optimizer.go b/grids/optimizer.go new file mode 100644 index 00000000..300d4fd0 --- /dev/null +++ b/grids/optimizer.go @@ -0,0 +1,93 @@ +package grids + +import ( + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/util/protoutil" +) + +type OptimizationRule struct { + Match func(pipe []*gripql.GraphStatement) bool + Replace func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement +} + +func GridsOptimizer(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + // Preprocess to handle Has with And expressions + //pipe = expandHasAnd(pipe) + // Apply the first matching optimization rule + for _, rule := range startOptimizations { + if rule.Match(pipe) { + return rule.Replace(pipe) + } + } + // Return the original pipeline if no optimizations apply + return pipe +} + +var startOptimizations = []OptimizationRule{ + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { + return true + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[1].GetHas() + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids V.Has() indexing", + Custom: lookupVertsCondIndexStep{ + expr: has, + }, + }, + }, + } + return append(optimized, pipe[2:]...) + }, + }, + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 3 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); !ok { + return false + } + if _, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { + return true + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[2].GetHas() + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + for i, label := range labels { + if label[:2] != VTABLE_PREFIX { + labels[i] = VTABLE_PREFIX + label + } + } + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids V().HasLabel().Has() indexing", + Custom: lookupVertsHasLabelCondIndexStep{ + expr: has, + labels: labels, + }, + }, + }, + } + return append(optimized, pipe[3:]...) + }, + }, +} diff --git a/grids/processor.go b/grids/processor.go new file mode 100644 index 00000000..0db76502 --- /dev/null +++ b/grids/processor.go @@ -0,0 +1,186 @@ +package grids + +import ( + "context" + + "github.com/bmeg/grip/gdbi" + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" +) + +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertexHasLabelCondIndex look up vertices has label + +type lookupVertsHasLabelCondIndexStep struct { + labels []string + expr *gripql.HasExpression +} + +func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { + graph := db.(*Graph) + return &lookupVertsHasLabelCondIndexProc{ + db: graph, + expr: t.expr, + labels: t.labels, + loadData: true, + }, nil + +} + +func (t lookupVertsHasLabelCondIndexStep) GetType() gdbi.DataType { + return gdbi.VertexData +} + +type lookupVertsHasLabelCondIndexProc struct { + db *Graph + labels []string + expr *gripql.HasExpression + loadData bool +} + +func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") + queryChan := make(chan gdbi.ElementLookup, 100) + cond := l.expr.GetCondition() + var exists = false + if len(l.db.bsonkv.Fields) > 0 { + _, exists = l.db.bsonkv.Fields[cond.Key] + } + if cond == nil || !exists { + log.Debugf("lookupVertsHasLabelCondIndexProc: No index found") + go func() { + defer close(queryChan) + for t := range in { + for _, label := range l.labels { + tableFound, ok := l.db.bsonkv.Tables[label] + if !ok { + log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) + continue + } + for id := range tableFound.Scan(true, &GripQLFilter{Expression: l.expr}) { + queryChan <- gdbi.ElementLookup{ID: id.(string), Ref: t} + + } + } + } + }() + } else { + go func() { + defer close(queryChan) + for t := range in { + cond := l.expr.GetCondition() + for _, label := range l.labels { + for id := range l.db.bsonkv.RowIdsByLabelFieldValue( + label, + cond.Key, + cond.Value.AsInterface(), + MapConditionToOperator(cond.Condition), + ) { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } + } + } + + } + }() + } + + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + return ctx + +} + +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertsCondIndex look up vertices by indexed +type lookupVertsCondIndexStep struct { + expr *gripql.HasExpression +} + +func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { + graph := db.(*Graph) + return &lookupVertsCondIndexProc{ + db: graph, + expr: t.expr, + loadData: true}, nil +} + +func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { + return gdbi.VertexData +} + +type lookupVertsCondIndexProc struct { + db *Graph + expr *gripql.HasExpression + loadData bool + fallback bool +} + +func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + log.Debugln("Entering lookupVertsCondIndexProc custom processor", l.expr.Expression) + queryChan := make(chan gdbi.ElementLookup, 100) + cond := l.expr.GetCondition() + var exists = false + if len(l.db.bsonkv.Fields) > 0 { + _, exists = l.db.bsonkv.Fields[cond.Key] + } + /* Optimized indexing only works for Simple filters. / + / If compound filter or index doesn't exist use backup method */ + if cond == nil || !exists { + log.Debugf("lookupVertsCondIndexProc: falling back to GetVertexList since filter is not basic Condition filter") + go func() { + defer close(queryChan) + for t := range in { + for v := range l.db.GetVertexList(ctx, true) { + if MatchesHasExpression( + AddSpecialFields(v), + l.expr, + ) { + queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} + } + + } + } + }() + } else { + go func() { + defer close(queryChan) + for t := range in { + cond := l.expr.GetCondition() + for id := range l.db.bsonkv.RowIdsByHas( + cond.Key, + cond.Value.AsInterface(), + MapConditionToOperator(cond.Condition), + ) { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } + } + + } + }() + } + + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + return ctx +} + +func AddSpecialFields(v *gdbi.Vertex) any { + v.Data["_label"] = v.Label + v.Data["_id"] = v.ID + return v.Data +} From 581d70fee62dd1ea01fa770f78863d16834e5535 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 16 Jun 2025 15:45:02 -0700 Subject: [PATCH 14/30] censor password --- accounts/basic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/basic.go b/accounts/basic.go index 3e8f1723..780ae28a 100644 --- a/accounts/basic.go +++ b/accounts/basic.go @@ -30,7 +30,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { if len(auth) > 0 { user, password, ok := parseBasicAuth(auth[0]) - log.Infof("User: %s Password: %s OK: %t\n", user, password, ok) + log.Infof("Authenticating as User: %s OK: %t\n", user, ok) for _, c := range ba { if c.User == user && c.Password == password { return user, nil From 352daca4e6152c4dedd034ad0e286f20021478a3 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 16 Jun 2025 16:26:03 -0700 Subject: [PATCH 15/30] seperate nested index tests --- .github/workflows/tests.yml | 2 +- conformance/tests/ot_index.py | 43 ---------------------------- conformance/tests/ot_nested_index.py | 42 +++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 44 deletions(-) create mode 100644 conformance/tests/ot_nested_index.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2b521356..93ca2b22 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -125,7 +125,7 @@ jobs: make start-mongo ./grip server --rpc-port 18202 --http-port 18201 --config ./test/mongo.yml & sleep 5 - make test-conformance + python conformance/run_conformance.py http://localhost:18201 --exclude nested_index mongoCoreTest: needs: build diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index ad60dbad..391f6664 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -213,46 +213,3 @@ def test_hasLabel_contains(man): errors.append("Expected 2 results but got %d instead" % (count)) return errors - - - -def test_cond_nested_field(man): - # Tests nested filters on optimized grids query driver - - errors = [] - G = man.writeTest() - - G.addIndex("Observation", "code.coding.[0].code") - - G.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) - G.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) - - count = 0 - for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): - count +=1 - - if count != 1: - errors.append("Expected count == 1 but got %d instead" % (count)) - - return errors - - -def test_bulk_cond_nested_field(man): - errors = [] - G = man.writeTest() - G.addIndex("Observation", "code.coding.[0].code") - - bulk = G.bulkAdd() - bulk.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) - bulk.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) - - bulk.execute() - - count = 0 - for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): - count +=1 - - if count != 1: - errors.append("Expected count == 1 but got %d instead" % (count)) - - return errors diff --git a/conformance/tests/ot_nested_index.py b/conformance/tests/ot_nested_index.py new file mode 100644 index 00000000..7fb90150 --- /dev/null +++ b/conformance/tests/ot_nested_index.py @@ -0,0 +1,42 @@ +import gripql + +def test_cond_nested_field(man): + # Tests nested filters on optimized grids query driver + + errors = [] + G = man.writeTest() + + G.addIndex("Observation", "code.coding.[0].code") + + G.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + G.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + + count = 0 + for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): + count +=1 + + if count != 1: + errors.append("Expected count == 1 but got %d instead" % (count)) + + return errors + + +def test_bulk_cond_nested_field(man): + errors = [] + G = man.writeTest() + G.addIndex("Observation", "code.coding.[0].code") + + bulk = G.bulkAdd() + bulk.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + bulk.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + + bulk.execute() + + count = 0 + for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): + count +=1 + + if count != 1: + errors.append("Expected count == 1 but got %d instead" % (count)) + + return errors From bf471307cc928505ccdf4a451dd56469e48eb6ba Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 18 Jun 2025 11:37:12 -0700 Subject: [PATCH 16/30] bug fixes --- go.mod | 2 +- go.sum | 4 ++-- grids/graph.go | 2 +- grids/processor.go | 29 ++++++++++++----------------- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 18e55b85..0f391bec 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250616223533-d74bbae858e2 + github.com/bmeg/benchtop v0.0.0-20250618183006-bddc564a4bd6 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 76086b54..5b6f0da3 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250616223533-d74bbae858e2 h1:9aqw+Le7fm7KFWggeuwFTDk0m+qXObxB5HduCfvWsso= -github.com/bmeg/benchtop v0.0.0-20250616223533-d74bbae858e2/go.mod h1:4s+8vTKrryyznb6ebLHkC7sriL3Y3PCRVkVkfdbGw3k= +github.com/bmeg/benchtop v0.0.0-20250618183006-bddc564a4bd6 h1:qbFsvy7iPsG8f4gdWOlmxnndCBVfI478sBX73sNK3Yo= +github.com/bmeg/benchtop v0.0.0-20250618183006-bddc564a4bd6/go.mod h1:4s+8vTKrryyznb6ebLHkC7sriL3Y3PCRVkVkfdbGw3k= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/graph.go b/grids/graph.go index 9de1dabf..039120b9 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -511,7 +511,7 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element lKey := ggraph.keyMap.GetVertexLabel(key, ggraph.bsonkv.Pb.Db) lID, ok := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) if !ok || lID == "" { - log.Debugln("No LID for lkey: ", lKey) + log.Debugln("No LID for lkey: ", lKey, "and ID: ", id.ID) continue } vData, err := ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(id.ID)) diff --git a/grids/processor.go b/grids/processor.go index 0db76502..2cc7ee79 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -41,13 +41,19 @@ type lookupVertsHasLabelCondIndexProc struct { func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) - cond := l.expr.GetCondition() var exists = false if len(l.db.bsonkv.Fields) > 0 { - _, exists = l.db.bsonkv.Fields[cond.Key] + for _, label := range l.labels { + log.Debugln("Checking indexed fields %v", l.db.bsonkv.Fields, "LABEL: ", label) + _, exists = l.db.bsonkv.Fields[label] + if exists { + break + } + } } - if cond == nil || !exists { - log.Debugf("lookupVertsHasLabelCondIndexProc: No index found") + + if l.expr.GetCondition() == nil || !exists { + log.Debugf("cond == nil || !exists: ", l.expr.GetCondition(), exists) go func() { defer close(queryChan) for t := range in { @@ -59,7 +65,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } for id := range tableFound.Scan(true, &GripQLFilter{Expression: l.expr}) { queryChan <- gdbi.ElementLookup{ID: id.(string), Ref: t} - } } } @@ -70,19 +75,10 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi for t := range in { cond := l.expr.GetCondition() for _, label := range l.labels { - for id := range l.db.bsonkv.RowIdsByLabelFieldValue( - label, - cond.Key, - cond.Value.AsInterface(), - MapConditionToOperator(cond.Condition), - ) { - queryChan <- gdbi.ElementLookup{ - ID: id, - Ref: t, - } + for id := range l.db.bsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), MapConditionToOperator(cond.Condition)) { + queryChan <- gdbi.ElementLookup{ID: id, Ref: t} } } - } }() } @@ -95,7 +91,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } }() return ctx - } // ////////////////////////////////////////////////////////////////////////////// From e59cd79599a879ab0db3621e8abc50178118d9a7 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 25 Jun 2025 15:16:59 -0700 Subject: [PATCH 17/30] nuke 'grids' storage style. Optimize for performance --- gdbi/data_element.go | 2 +- grids/filters.go | 7 + grids/graph.go | 504 +++++++++++++--------------------- grids/graphdb.go | 1 + grids/keyindex.go | 234 ++++++++-------- grids/keymap.go | 318 --------------------- grids/new.go | 3 - grids/optimizer.go | 34 +++ grids/processor.go | 36 ++- gripql/python/gripql/graph.py | 5 +- server/server.go | 4 +- 11 files changed, 375 insertions(+), 773 deletions(-) delete mode 100644 grids/keymap.go diff --git a/gdbi/data_element.go b/gdbi/data_element.go index 31974d1a..1d643b3f 100644 --- a/gdbi/data_element.go +++ b/gdbi/data_element.go @@ -12,7 +12,7 @@ import ( func (elem *DataElement) ToVertex() *gripql.Vertex { sValue, err := structpb.NewStruct(elem.Data) if err != nil { - log.Errorf("Error: %s For elem.Data: '%#v'\n", err, elem.Data) + log.Errorf("ToVertex: %s For elem.Data: '%#v'\n", err, elem.Data) } return &gripql.Vertex{ Id: elem.ID, diff --git a/grids/filters.go b/grids/filters.go index d63f256b..5c5cd6a0 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -21,6 +21,11 @@ func (f *GripQLFilter) RequiredFields() []string { return extractKeys(f.Expression) } +func (f *GripQLFilter) IsNoOp() bool { + // A GripQLFilter is a no-op if its Expression is nil + return f.Expression == nil +} + func extractKeys(expr *gripql.HasExpression) []string { keys := map[string]struct{}{} @@ -55,7 +60,9 @@ func extractKeys(expr *gripql.HasExpression) []string { } func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { + switch stmt.Expression.(type) { + case *gripql.HasExpression_Condition: cond := stmt.GetCondition() return filters.ApplyFilterCondition( diff --git a/grids/graph.go b/grids/graph.go index 039120b9..b45edbbb 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -12,7 +12,6 @@ import ( "github.com/bmeg/grip/engine/core" "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/log" - "github.com/bmeg/grip/util/protoutil" "github.com/bmeg/grip/util/setcmp" multierror "github.com/hashicorp/go-multierror" ) @@ -27,13 +26,11 @@ func (ggraph *Graph) GetTimestamp() string { return ggraph.ts.Get(ggraph.graphID) } -func insertVertex(tx *pebblebulk.PebbleBulk, keyMap *KeyMap, vertex *gdbi.Vertex) error { +func insertVertex(tx *pebblebulk.PebbleBulk, vertex *gdbi.Vertex) error { if vertex.ID == "" { return fmt.Errorf("inserting null key vertex") } - vertexKey, _ := keyMap.GetsertVertexKeyLabel(vertex.ID, vertex.Label, tx) - key := VertexKey(vertexKey) - if err := tx.Set(key, nil, nil); err != nil { + if err := tx.Set(VertexKey(vertex.ID, vertex.Label), nil, nil); err != nil { return fmt.Errorf("AddVertex Error %s", err) } return nil @@ -71,31 +68,19 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) return nil } -func insertEdge(tx *pebblebulk.PebbleBulk, keyMap *KeyMap, edge *gdbi.Edge) error { - var err error - if edge.ID == "" { +func insertEdge(tx *pebblebulk.PebbleBulk, edge *gdbi.Edge) error { + if edge.ID == "" || edge.From == "" || edge.To == "" || edge.Label == "" { return fmt.Errorf("inserting null key edge") } - - eid, lid := keyMap.GetsertEdgeKey(edge.ID, edge.Label, tx) - /* providing a label doesn't matter if not going to use the label key anyway. - It can get set in the insertvertex func later */ - src := keyMap.GetsertVertexKey(edge.From, tx) - dst := keyMap.GetsertVertexKey(edge.To, tx) - - ekey := EdgeKey(eid, src, dst, lid) - skey := SrcEdgeKey(eid, src, dst, lid) - dkey := DstEdgeKey(eid, src, dst, lid) - - err = tx.Set(ekey, nil, nil) + err := tx.Set(EdgeKey(edge.ID, edge.From, edge.To, edge.Label), nil, nil) if err != nil { return err } - err = tx.Set(skey, []byte{}, nil) + err = tx.Set(SrcEdgeKey(edge.ID, edge.From, edge.To, edge.Label), []byte{}, nil) if err != nil { return err } - err = tx.Set(dkey, []byte{}, nil) + err = tx.Set(DstEdgeKey(edge.ID, edge.From, edge.To, edge.Label), []byte{}, nil) if err != nil { return err } @@ -144,7 +129,7 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { - if err := insertVertex(tx, ggraph.keyMap, vert); err != nil { + if err := insertVertex(tx, vert); err != nil { bulkErr = multierror.Append(bulkErr, err) log.Errorf("AddVertex Error %s", err) } @@ -172,7 +157,7 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { for _, edge := range edges { - err := insertEdge(tx, ggraph.keyMap, edge) + err := insertEdge(tx, edge) if err != nil { return err } @@ -229,12 +214,12 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { for elem := range insertStream { if elem.Vertex != nil { - if err := insertVertex(tx, ggraph.keyMap, elem.Vertex); err != nil { + if err := insertVertex(tx, elem.Vertex); err != nil { return fmt.Errorf("vertex insert error: %v", err) } } if elem.Edge != nil { - if err := insertEdge(tx, ggraph.keyMap, elem.Edge); err != nil { + if err := insertEdge(tx, elem.Edge); err != nil { return fmt.Errorf("edge insert error: %v", err) } } @@ -295,11 +280,8 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { } func (ggraph *Graph) DelEdge(eid string) error { - edgeKey, ok := ggraph.keyMap.GetEdgeKey(eid, ggraph.bsonkv.Pb.Db) - if !ok { - return fmt.Errorf("edge not found") - } - ekeyPrefix := EdgeKeyPrefix(edgeKey) + + ekeyPrefix := EdgeKeyPrefix(eid) var ekey []byte ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { @@ -311,10 +293,10 @@ func (ggraph *Graph) DelEdge(eid string) error { return fmt.Errorf("edge not found") } - eidParsed, sid, did, lbl := EdgeKeyParse(ekey) + eid, sid, did, lbl := EdgeKeyParse(ekey) - skey := SrcEdgeKey(eidParsed, sid, did, lbl) - dkey := DstEdgeKey(eidParsed, sid, did, lbl) + skey := SrcEdgeKey(eid, sid, did, lbl) + dkey := DstEdgeKey(eid, sid, did, lbl) var bulkErr *multierror.Error err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { @@ -333,10 +315,6 @@ func (ggraph *Graph) DelEdge(eid string) error { if err != nil { bulkErr = multierror.Append(bulkErr, err) } - - if err := ggraph.keyMap.DelEdgeKey(eid, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { bulkErr = multierror.Append(bulkErr, err) } @@ -346,13 +324,10 @@ func (ggraph *Graph) DelEdge(eid string) error { // DelVertex deletes vertex with id `key` func (ggraph *Graph) DelVertex(id string) error { - vertexKey, ok := ggraph.keyMap.GetVertexKey(id, ggraph.bsonkv.Pb.Db) - if !ok { - return fmt.Errorf("vertex %s not found", id) - } - vid := VertexKey(vertexKey) - skeyPrefix := SrcEdgePrefix(vertexKey) - dkeyPrefix := DstEdgePrefix(vertexKey) + + vid := VertexKeyPrefix(id) + skeyPrefix := SrcEdgePrefix(id) + dkeyPrefix := DstEdgePrefix(id) delKeys := make([][]byte, 0, 1000) @@ -368,13 +343,7 @@ func (ggraph *Graph) DelVertex(id string) error { dkey := DstEdgeKey(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) - edgeID, ok := ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - if ok { - if err := ggraph.keyMap.DelEdgeKey(edgeID, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - if err := ggraph.bsonkv.DeleteAnyRow([]byte(edgeID)); err != nil { + if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { bulkErr = multierror.Append(bulkErr, err) } } @@ -386,13 +355,7 @@ func (ggraph *Graph) DelVertex(id string) error { skey := SrcEdgeKey(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) - edgeID, ok := ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - if ok { - if err := ggraph.keyMap.DelEdgeKey(edgeID, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - if err := ggraph.bsonkv.DeleteAnyRow([]byte(edgeID)); err != nil { + if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { bulkErr = multierror.Append(bulkErr, err) } } @@ -402,12 +365,8 @@ func (ggraph *Graph) DelVertex(id string) error { bulkErr = multierror.Append(bulkErr, err) } - if err := ggraph.keyMap.DelVertexKey(id, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := tx.Delete(vid, nil); err != nil { + if err := tx.DeletePrefix(vid); err != nil { return err } for _, k := range delKeys { @@ -437,16 +396,11 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb return nil default: } - keyValue := it.Key() - ekey, skey, dkey, label := EdgeKeyParse(keyValue) - labelID, _ := ggraph.keyMap.GetLabelID(label, ggraph.bsonkv.Pb.Db) - sid, _ := ggraph.keyMap.GetVertexID(skey, ggraph.bsonkv.Pb.Db) - did, _ := ggraph.keyMap.GetVertexID(dkey, ggraph.bsonkv.Pb.Db) - eid, _ := ggraph.keyMap.GetEdgeID(ekey, ggraph.bsonkv.Pb.Db) - e := &gdbi.Edge{ID: eid, Label: labelID, From: sid, To: did} + eid, sid, did, label := EdgeKeyParse(it.Key()) + e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} if loadProp { var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+labelID].GetRow([]byte(eid)) + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(eid)) if err != nil { log.Errorf("GetEdgeList: GetRow error: %v", err) continue @@ -465,20 +419,21 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb // GetVertex loads a vertex given an id. It returns a nil if not found func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { - key, ok := ggraph.keyMap.GetVertexKey(id, ggraph.bsonkv.Pb.Db) - if !ok { + var v *gdbi.Vertex + rtasockey := benchtop.NewRowTableAsocKey([]byte(id)) + rtasocval, closer, err := ggraph.bsonkv.Pb.Db.Get(rtasockey) + if err != nil { return nil } - var v *gdbi.Vertex - lKey := ggraph.keyMap.GetVertexLabel(key, ggraph.bsonkv.Pb.Db) - lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) + defer closer.Close() + label := string(rtasocval) v = &gdbi.Vertex{ ID: id, - Label: lID, + Label: label[2:], } if loadProp { var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(id)) + v.Data, err = ggraph.bsonkv.Tables[label].GetRow([]byte(id)) if err != nil { return nil } @@ -486,166 +441,118 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { } else { v.Data = map[string]any{} } + return v } type elementData struct { - key uint64 - req gdbi.ElementLookup - data []byte + label string + req gdbi.ElementLookup + data []byte } -// GetVertexChannel is passed a channel of vertex ids and it produces a channel -// of vertices func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { - data := make(chan elementData, 100) - go func() { - defer close(data) - for id := range ids { - if id.IsSignal() { - data <- elementData{req: id} - } else { - key, _ := ggraph.keyMap.GetVertexKey(id.ID, ggraph.bsonkv.Pb.Db) - ed := elementData{key: key, req: id} - if load { - lKey := ggraph.keyMap.GetVertexLabel(key, ggraph.bsonkv.Pb.Db) - lID, ok := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - if !ok || lID == "" { - log.Debugln("No LID for lkey: ", lKey, "and ID: ", id.ID) - continue - } - vData, err := ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(id.ID)) - if err != nil { - log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) - continue - } - vdataM, _ := protoutil.StructMarshal(vData) - ed.data = vdataM - } - data <- ed - } - } - }() - out := make(chan gdbi.ElementLookup, 100) go func() { defer close(out) - var err error - for d := range data { - if d.req.IsSignal() { - out <- d.req - } else { - lKey := ggraph.keyMap.GetVertexLabel(d.key, ggraph.bsonkv.Pb.Db) - lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - v := gdbi.Vertex{ID: d.req.ID, Label: lID} - v.Data, err = protoutil.StructUnMarshal(d.data) - v.Loaded = true - if err != nil { - log.Errorf("GetVertexChannel: unmarshal error: %v", err) - continue - } + ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for id := range ids { + if id.IsSignal() { + out <- id + } else { + if load { + v := gdbi.Vertex{ID: id.ID} + prefix := benchtop.NewRowTableAsocKey([]byte(id.ID)) + for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { + fetchLabel, err := it.Value() + if err != nil { + log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) + continue + } + label := string(fetchLabel) + v.Label = label[2:] + v.Data, err = ggraph.bsonkv.Tables[label].GetRow([]byte(id.ID)) + if err != nil { + log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) + continue + } + v.Loaded = true + break + } + id.Vertex = &v + out <- id - d.req.Vertex = &v - out <- d.req + } else { + id.Vertex = &gdbi.Vertex{ID: id.ID} + out <- id + } + } } - } + return nil + }) }() return out } // GetOutChannel process requests of vertex ids and find the connected vertices on outgoing edges func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - vertexChan := make(chan elementData, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } + o := make(chan gdbi.ElementLookup, 100) go func() { - defer close(vertexChan) + defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { - vertexChan <- elementData{req: req} + o <- req } else { found := false - key, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) - if ok { - skeyPrefix := SrcEdgePrefix(key) - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - keyValue := it.Key() - _, _, dst, label := SrcEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - vkey := VertexKey(dst) - vertexChan <- elementData{ - data: vkey, - req: req, + skeyPrefix := SrcEdgePrefix(req.ID) + for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { + _, _, dst, label := SrcEdgeKeyParse(it.Key()) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + rtasockey := benchtop.NewRowTableAsocKey([]byte(dst)) + rtasocval, closer, err := ggraph.bsonkv.Pb.Db.Get(rtasockey) + if err != nil { + log.Errorf("GetInChannel: GetRow Db.Get() error: %v", err) + continue + } + defer closer.Close() + + vLabel := string(rtasocval) + log.Debugln("VLABEL: ", vLabel, "DST: ", dst) + + v := &gdbi.Vertex{ID: dst, Label: vLabel[2:]} + if load { + v.Data, err = ggraph.bsonkv.Tables[vLabel].GetRow([]byte(dst)) + if err != nil { + log.Errorf("GetInChannel: GetRow error: %v", err) + continue } - found = true + v.Loaded = true + } else { + v.Data = map[string]any{} } + req.Vertex = v + o <- req + found = true } } if !found && emitNull { - vertexChan <- elementData{ - data: nil, - req: req, - } + req.Vertex = nil + o <- req } + } } return nil }) }() - o := make(chan gdbi.ElementLookup, 100) - go func() { - defer close(o) - for req := range vertexChan { - var err error - if req.req.IsSignal() { - o <- req.req - } else { - if req.data == nil { - req.req.Vertex = nil - o <- req.req - continue - } - vkey := VertexKeyParse(req.data) - id, _ := ggraph.keyMap.GetVertexID(vkey, ggraph.bsonkv.Pb.Db) - lkey := ggraph.keyMap.GetVertexLabel(vkey, ggraph.bsonkv.Pb.Db) - lid, ok := ggraph.keyMap.GetLabelID(lkey, ggraph.bsonkv.Pb.Db) - if !ok || lid == "" { - log.Debugln("No LID for lkey: ", lkey) - continue - } - v := &gdbi.Vertex{ID: id, Label: lid} - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lid].GetRow([]byte(id)) - v.Loaded = true - if err != nil { - log.Errorf("GetOutChannel: GetRow error: %v", err) - continue - } - - req.req.Vertex = v - o <- req.req - } - } - }() return o } // GetInChannel process requests of vertex ids and find the connected vertices on incoming edges func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { @@ -654,34 +561,37 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element o <- req } else { found := false - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) - if ok { - dkeyPrefix := DstEdgePrefix(vkey) - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - _, src, _, label := DstEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - srcID, _ := ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - lKey := ggraph.keyMap.GetVertexLabel(src, ggraph.bsonkv.Pb.Db) - lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - v := &gdbi.Vertex{ID: srcID, Label: lID} - if load { - var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(srcID)) - if err != nil { - log.Errorf("GetInChannel: GetRow error: %v", err) - continue - } - v.Loaded = true - } else { - v.Data = map[string]any{} + dkeyPrefix := DstEdgePrefix(req.ID) + for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { + keyValue := it.Key() + _, sid, _, label := DstEdgeKeyParse(keyValue) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + + rtasockey := benchtop.NewRowTableAsocKey([]byte(sid)) + rtasocval, closer, _ := ggraph.bsonkv.Pb.Db.Get(rtasockey) + defer closer.Close() + vLabel := string(rtasocval) + + v := &gdbi.Vertex{ID: sid, Label: vLabel[2:]} + if load { + var err error + log.Debugln("GET ROW IN GET IN CHAN") + + v.Data, err = ggraph.bsonkv.Tables[vLabel].GetRow([]byte(sid)) + if err != nil { + log.Errorf("GetInChannel: GetRow error: %v", err) + continue } - req.Vertex = v - o <- req - found = true + v.Loaded = true + } else { + v.Data = map[string]any{} } + req.Vertex = v + o <- req + found = true } } + if !found && emitNull { req.Vertex = nil o <- req @@ -697,13 +607,6 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element // GetOutEdgeChannel process requests of vertex ids and find the connected outgoing edges func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { @@ -712,35 +615,34 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El o <- req } else { found := false - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) - if ok { - skeyPrefix := SrcEdgePrefix(vkey) - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := SrcEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - e := gdbi.Edge{} - e.ID, _ = ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - e.From, _ = ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - e.To, _ = ggraph.keyMap.GetVertexID(dst, ggraph.bsonkv.Pb.Db) - e.Label, _ = ggraph.keyMap.GetLabelID(label, ggraph.bsonkv.Pb.Db) - if load { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow([]byte(e.ID)) - if err != nil { - log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) - continue - } - e.Loaded = true - } else { - e.Data = map[string]any{} + skeyPrefix := SrcEdgePrefix(req.ID) + for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { + eid, src, dst, label := SrcEdgeKeyParse(it.Key()) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + e := gdbi.Edge{ + From: src, + To: dst, + Label: label, + ID: eid, + } + if load { + var err error + + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) + if err != nil { + log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) + continue } - req.Edge = &e - o <- req - found = true + e.Loaded = true + } else { + e.Data = map[string]any{} } + req.Edge = &e + o <- req + found = true } } + if !found && emitNull { req.Edge = nil o <- req @@ -757,13 +659,6 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El // GetInEdgeChannel process requests of vertex ids and find the connected incoming edges func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { @@ -771,36 +666,37 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele if req.IsSignal() { o <- req } else { - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) found := false - if ok { - dkeyPrefix := DstEdgePrefix(vkey) - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := DstEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - e := gdbi.Edge{} - e.ID, _ = ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - e.From, _ = ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - e.To, _ = ggraph.keyMap.GetVertexID(dst, ggraph.bsonkv.Pb.Db) - e.Label, _ = ggraph.keyMap.GetLabelID(label, ggraph.bsonkv.Pb.Db) - if load { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow([]byte(e.ID)) - if err != nil { - log.Errorf("GetInEdgeChannel: GetRow error: %v", err) - continue - } - e.Loaded = true - - } else { - e.Data = map[string]any{} + dkeyPrefix := DstEdgePrefix(req.ID) + for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { + keyValue := it.Key() + eid, src, dst, label := DstEdgeKeyParse(keyValue) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + e := gdbi.Edge{ + ID: eid, + From: src, + To: dst, + Label: label, + } + if load { + var err error + log.Debugln("GET ROW IN GET IN EDGE CHAN") + + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) + if err != nil { + log.Errorf("GetInEdgeChannel: GetRow error: %v", err) + continue } - req.Edge = &e - o <- req - found = true + e.Loaded = true + + } else { + e.Data = map[string]any{} } + req.Edge = &e + o <- req + found = true } + } if !found && emitNull { req.Edge = nil @@ -817,28 +713,21 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele // GetEdge loads an edge given an id. It returns nil if not found func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { - ekey, ok := ggraph.keyMap.GetEdgeKey(id, ggraph.bsonkv.Pb.Db) - if !ok { - return nil - } - ekeyPrefix := EdgeKeyPrefix(ekey) - + ekeyPrefix := EdgeKeyPrefix(id) var e *gdbi.Edge err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { - eid, src, dst, labelKey := EdgeKeyParse(it.Key()) - id, _ := ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - from, _ := ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - to, _ := ggraph.keyMap.GetVertexID(dst, ggraph.bsonkv.Pb.Db) - label, _ := ggraph.keyMap.GetLabelID(labelKey, ggraph.bsonkv.Pb.Db) + eid, src, dst, label := EdgeKeyParse(it.Key()) e = &gdbi.Edge{ - ID: id, - From: from, - To: to, + ID: eid, + From: src, + To: dst, Label: label, } + if loadProp { var err error + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(id)) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) @@ -870,15 +759,16 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g return nil default: } - v := &gdbi.Vertex{} - keyValue := it.Key() - vKey := VertexKeyParse(keyValue) - lKey := ggraph.keyMap.GetVertexLabel(vKey, ggraph.bsonkv.Pb.Db) - v.ID, _ = ggraph.keyMap.GetVertexID(vKey, ggraph.bsonkv.Pb.Db) - v.Label, _ = ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) + + id, label := VertexKeyParse(it.Key()) + v := &gdbi.Vertex{ + ID: id, + Label: label, + } if loadProp { var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow([]byte(v.ID)) + + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+label].GetRow([]byte(v.ID)) if err != nil { log.Errorf("GetVertexList: GetRow error: %v", err) continue diff --git a/grids/graphdb.go b/grids/graphdb.go index a35ded0c..2a6aa35f 100644 --- a/grids/graphdb.go +++ b/grids/graphdb.go @@ -49,6 +49,7 @@ func (kgraph *GDB) Graph(graph string) (gdbi.GraphInterface, error) { mu.Lock() kgraph.drivers[graph] = g mu.Unlock() + return g, nil } return nil, fmt.Errorf("graph '%s' was not found", graph) diff --git a/grids/keyindex.go b/grids/keyindex.go index 4f06fb92..5b1cd909 100644 --- a/grids/keyindex.go +++ b/grids/keyindex.go @@ -1,7 +1,7 @@ package grids import ( - "encoding/binary" + "bytes" ) var vertexPrefix = []byte(".") @@ -12,155 +12,139 @@ var dstEdgePrefix = []byte(">") var intSize = 10 // VertexKey generates the key given a vertexId -func VertexKey(id uint64) []byte { - out := make([]byte, intSize+1) - out[0] = vertexPrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out +func VertexKey(id, label string) []byte { + return bytes.Join([][]byte{ + vertexPrefix, + []byte(id), + []byte(label), + }, []byte{0}) } -// VertexKeyParse takes a byte array key and returns: -// `graphId`, `vertexId` -func VertexKeyParse(key []byte) uint64 { - id, _ := binary.Uvarint(key[1 : intSize+1]) - return id +func VertexKeyParse(key []byte) (id, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[1]), string(tmp[2]) } -// EdgeKey takes the required components of an edge key and returns the byte array -func EdgeKey(id, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = edgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], dst) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out +func VertexKeyPrefix(id string) []byte { + return bytes.Join([][]byte{ + vertexPrefix, + []byte(id), + {}, + }, []byte{0}) +} + +func VertexKeyPrefixParse(key []byte) (id string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[1]) } // EdgeKeyPrefix returns the byte array prefix for a particular edge id -func EdgeKeyPrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = edgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out +func EdgeKeyPrefix(id string) []byte { + return bytes.Join([][]byte{ + edgePrefix, + []byte(id), + {}, + }, []byte{0}) +} + +// SrcEdgePrefix returns a byte array prefix for all entries in the source +// edge index a particular vertex (the source vertex) +func SrcEdgePrefix(id string) []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + []byte(id), + {}, + }, []byte{0}) +} + +// DstEdgePrefix returns a byte array prefix for all entries in the dest +// edge index a particular vertex (the dest vertex) +func DstEdgePrefix(id string) []byte { + return bytes.Join([][]byte{ + dstEdgePrefix, + []byte(id), + {}, + }, []byte{0}) +} + +// EdgeKey takes the required components of an edge key and returns the byte array +func EdgeKey(id, src, dst, label string) []byte { + return bytes.Join([][]byte{ + edgePrefix, + []byte(id), + []byte(label), + []byte(src), + []byte(dst), + }, []byte{0}) +} + +func EdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[1]), string(tmp[3]), string(tmp[4]), string(tmp[2]) +} + +// SrcEdgeKey creates a src edge index key +func SrcEdgeKey(eid, src, dst, label string) []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + []byte(src), + []byte(dst), + []byte(eid), + []byte(label), + }, []byte{0}) } -// EdgeKeyParse takes a edge key and returns the elements encoded in it: -// `graph`, `edgeID`, `srcVertexId`, `dstVertexId`, `label` -func EdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - eid, _ := binary.Uvarint(key[1 : intSize+1]) - sid, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - did, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label +func SrcEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]) +} + +// DstEdgeKey creates a dest edge index key +func DstEdgeKey(eid, src, dst, label string) []byte { + return bytes.Join([][]byte{ + dstEdgePrefix, + []byte(dst), + []byte(src), + []byte(eid), + []byte(label), + }, []byte{0}) +} + +func DstEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[3]), string(tmp[2]), string(tmp[1]), string(tmp[4]) } // VertexListPrefix returns a byte array prefix for all vertices in a graph func VertexListPrefix() []byte { - out := make([]byte, 1) - out[0] = vertexPrefix[0] - return out + return bytes.Join([][]byte{ + vertexPrefix, + {}, + }, []byte{0}) } // EdgeListPrefix returns a byte array prefix for all edges in a graph func EdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = edgePrefix[0] - return out + return bytes.Join([][]byte{ + edgePrefix, + {}, + }, []byte{0}) } // SrcEdgeListPrefix returns a byte array prefix for all entries in the source // edge index for a graph func SrcEdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = srcEdgePrefix[0] - return out + return bytes.Join([][]byte{ + srcEdgePrefix, + {}, + }, []byte{0}) } // DstEdgeListPrefix returns a byte array prefix for all entries in the dest // edge index for a graph func DstEdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = dstEdgePrefix[0] - return out -} - -// SrcEdgeKey creates a src edge index key -func SrcEdgeKey(eid, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], src) - binary.PutUvarint(out[intSize+1:intSize*2+1], dst) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out -} - -// DstEdgeKey creates a dest edge index key -func DstEdgeKey(eid, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], dst) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out -} - -// SrcEdgeKeyParse takes a src index key entry and parses it into: -// `graph`, `edgeId`, `srcVertexId`, `dstVertexId`, `label`, `etype` -func SrcEdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - sid, _ := binary.Uvarint(key[1 : intSize+1]) - did, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - eid, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label -} - -// DstEdgeKeyParse takes a dest index key entry and parses it into: -// `graph`, `edgeId`, `dstVertexId`, `srcVertexId`, `label`, `etype` -func DstEdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - did, _ := binary.Uvarint(key[1 : intSize+1]) - sid, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - eid, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label -} - -// SrcEdgeKeyPrefix creates a byte array prefix for a src edge index entry -func SrcEdgeKeyPrefix(eid, src, dst uint64) []byte { - out := make([]byte, 1+intSize*3) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], src) - binary.PutUvarint(out[intSize+1:intSize*2+1], dst) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - return out -} - -// DstEdgeKeyPrefix creates a byte array prefix for a dest edge index entry -func DstEdgeKeyPrefix(eid, src, dst uint64) []byte { - out := make([]byte, 1+intSize*3) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], dst) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - return out -} - -// SrcEdgePrefix returns a byte array prefix for all entries in the source -// edge index a particular vertex (the source vertex) -func SrcEdgePrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out -} - -// DstEdgePrefix returns a byte array prefix for all entries in the dest -// edge index a particular vertex (the dest vertex) -func DstEdgePrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out + return bytes.Join([][]byte{ + dstEdgePrefix, + {}, + }, []byte{0}) } diff --git a/grids/keymap.go b/grids/keymap.go deleted file mode 100644 index 30b914f7..00000000 --- a/grids/keymap.go +++ /dev/null @@ -1,318 +0,0 @@ -package grids - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "sync" - - "github.com/cockroachdb/pebble" - - "github.com/bmeg/grip/log" - - ristretto "github.com/dgraph-io/ristretto/v2" -) - -type GetSet interface { - Get(key []byte) ([]byte, io.Closer, error) - Set(key, value []byte, _ *pebble.WriteOptions) error - Delete(key []byte, _ *pebble.WriteOptions) error -} - -type KeyMap struct { - cache ristretto.Cache[string, uint64] - - vIncCur uint64 - eIncCur uint64 - lIncCur uint64 - - vIncMut sync.Mutex - eIncMut sync.Mutex - lIncMut sync.Mutex -} - -var incMod uint64 = 1000 - -var vIDPrefix = []byte{'v'} -var eIDPrefix = []byte{'e'} -var lIDPrefix = []byte{'l'} - -var vKeyPrefix byte = 'V' -var eKeyPrefix byte = 'E' -var lKeyPrefix byte = 'L' - -var vLabelPrefix byte = 'x' -var eLabelPrefix byte = 'y' - -var vInc = []byte{'i', 'v'} -var eInc = []byte{'i', 'e'} -var lInc = []byte{'i', 'l'} - -func NewKeyMap() *KeyMap { - return &KeyMap{} -} - -func (km *KeyMap) Close() {} - -// GetsertVertexKey : Get or Insert Vertex Key -func (km *KeyMap) GetsertVertexKeyLabel(id, label string, db GetSet) (uint64, uint64) { - o, ok := getIDKey(vIDPrefix, id, db) - if !ok { - km.vIncMut.Lock() - var err error - o, err = dbInc(&km.vIncCur, vInc, db) - if err != nil { - log.Errorf("%s", err) - } - km.vIncMut.Unlock() - err = setKeyID(vKeyPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - err = setIDKey(vIDPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - } - lkey := km.GetsertLabelKey(label, db) - setIDLabel(vLabelPrefix, o, lkey, db) - return o, lkey -} - -func (km *KeyMap) GetsertVertexKey(id string, db GetSet) uint64 { - o, ok := getIDKey(vIDPrefix, id, db) - if !ok { - km.vIncMut.Lock() - var err error - o, err = dbInc(&km.vIncCur, vInc, db) - if err != nil { - log.Errorf("%s", err) - } - km.vIncMut.Unlock() - err = setKeyID(vKeyPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - err = setIDKey(vIDPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - } - return o -} - -func (km *KeyMap) GetVertexKey(id string, db GetSet) (uint64, bool) { - return getIDKey(vIDPrefix, id, db) -} - -// GetVertexID -func (km *KeyMap) GetVertexID(key uint64, db GetSet) (string, bool) { - return getKeyID(vKeyPrefix, key, db) -} - -func (km *KeyMap) GetVertexLabel(key uint64, db GetSet) uint64 { - k, _ := getIDLabel(vLabelPrefix, key, db) - return k -} - -// GetsertEdgeKey gets or inserts a new uint64 id for a given edge GID string -func (km *KeyMap) GetsertEdgeKey(id, label string, db GetSet) (uint64, uint64) { - o, ok := getIDKey(eIDPrefix, id, db) - if !ok { - km.eIncMut.Lock() - o, _ = dbInc(&km.eIncCur, eInc, db) - km.eIncMut.Unlock() - if err := setKeyID(eKeyPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - if err := setIDKey(eIDPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - } - lkey := km.GetsertLabelKey(label, db) - if err := setIDLabel(eLabelPrefix, o, lkey, db); err != nil { - log.Errorf("%s", err) - } - return o, lkey -} - -// GetEdgeKey gets the uint64 key for a given GID string -func (km *KeyMap) GetEdgeKey(id string, db GetSet) (uint64, bool) { - return getIDKey(eIDPrefix, id, db) -} - -// GetEdgeID gets the GID string for a given edge id uint64 -func (km *KeyMap) GetEdgeID(key uint64, db GetSet) (string, bool) { - return getKeyID(eKeyPrefix, key, db) -} - -func (km *KeyMap) GetEdgeLabel(key uint64, db GetSet) uint64 { - k, _ := getIDLabel(eLabelPrefix, key, db) - return k -} - -// DelVertexKey -func (km *KeyMap) DelVertexKey(id string, db GetSet) error { - key, ok := km.GetVertexKey(id, db) - if !ok { - return fmt.Errorf("%s vertexKey not found", id) - } - if err := delKeyID(vKeyPrefix, key, db); err != nil { - return err - } - if err := delIDKey(vIDPrefix, id, db); err != nil { - return err - } - return nil -} - -// DelEdgeKey -func (km *KeyMap) DelEdgeKey(id string, db GetSet) error { - key, ok := km.GetEdgeKey(id, db) - if !ok { - return fmt.Errorf("%s edgeKey not found", id) - } - if err := delKeyID(eKeyPrefix, key, db); err != nil { - return err - } - if err := delIDKey(eIDPrefix, id, db); err != nil { - return err - } - return nil -} - -// GetsertLabelKey gets-or-inserts a new label key uint64 for a given string -func (km *KeyMap) GetsertLabelKey(id string, db GetSet) uint64 { - u, ok := getIDKey(lIDPrefix, id, db) - if ok { - return u - } - km.lIncMut.Lock() - o, _ := dbInc(&km.lIncCur, lInc, db) - km.lIncMut.Unlock() - if err := setKeyID(lKeyPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - if err := setIDKey(lIDPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - return o -} - -func (km *KeyMap) GetLabelKey(id string, db GetSet) (uint64, bool) { - return getIDKey(lIDPrefix, id, db) -} - -// GetLabelID gets the GID for a given uint64 label key -func (km *KeyMap) GetLabelID(key uint64, db GetSet) (string, bool) { - return getKeyID(lKeyPrefix, key, db) -} - -func getIDKey(prefix []byte, id string, db GetSet) (uint64, bool) { - v, closer, err := db.Get(bytes.Join([][]byte{prefix, []byte(id)}, []byte{})) - if v == nil || err != nil { - return 0, false - } - key, _ := binary.Uvarint(v) - closer.Close() - return key, true -} - -func setIDKey(prefix []byte, id string, key uint64, db GetSet) error { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - b := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(b, key) - return db.Set(k, b, nil) -} - -func delIDKey(prefix []byte, id string, db GetSet) error { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - return db.Delete(k, nil) -} - -func getIDLabel(prefix byte, key uint64, db GetSet) (uint64, bool) { - k := make([]byte, 1+binary.MaxVarintLen64) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - v, closer, err := db.Get(k) - if v == nil || err != nil { - return 0, false - } - label, _ := binary.Uvarint(v) - closer.Close() - return label, true -} - -func setIDLabel(prefix byte, key uint64, label uint64, db GetSet) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - - b := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(b, label) - - err := db.Set(k, b, nil) - return err -} - -func setKeyID(prefix byte, id string, key uint64, db GetSet) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - return db.Set(k, []byte(id), nil) -} - -func getKeyID(prefix byte, key uint64, db GetSet) (string, bool) { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - b, closer, err := db.Get(k) - if b == nil || err != nil { - return "", false - } - out := string(b) - closer.Close() - return out, true -} - -func delKeyID(prefix byte, key uint64, db GetSet) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - return db.Delete(k, nil) -} - -func dbInc(inc *uint64, k []byte, db GetSet) (uint64, error) { - b := make([]byte, binary.MaxVarintLen64) - if *inc == 0 { - v, closer, _ := db.Get(k) - if v == nil { - binary.PutUvarint(b, incMod) - if err := db.Set(k, b, nil); err != nil { - return 0, err - } - (*inc) += 2 - return 1, nil - } - closer.Close() - newInc, _ := binary.Uvarint(v) - *inc = newInc - binary.PutUvarint(b, (*inc)+incMod) - if err := db.Set(k, b, nil); err != nil { - return 0, err - } - o := (*inc) - (*inc)++ - return o, nil - } - o := *inc - (*inc)++ - if *inc%incMod == 0 { - binary.PutUvarint(b, *inc+incMod) - if err := db.Set(k, b, nil); err != nil { - return 0, err - } - } - return o, nil -} diff --git a/grids/new.go b/grids/new.go index 42c52441..c861a2f9 100644 --- a/grids/new.go +++ b/grids/new.go @@ -16,7 +16,6 @@ import ( type Graph struct { graphID string - keyMap *KeyMap bsonkv *bsontable.BSONDriver ts *timestamp.Timestamp } @@ -68,7 +67,6 @@ func newGraph(baseDir, name string) (*Graph, error) { ts := timestamp.NewTimestamp() o := &Graph{ - keyMap: NewKeyMap(), bsonkv: bsonkv, ts: &ts, graphID: name, @@ -112,7 +110,6 @@ func getGraph(baseDir, name string) (*Graph, error) { ts := timestamp.NewTimestamp() o := &Graph{ - keyMap: NewKeyMap(), bsonkv: bsonkv, ts: &ts, graphID: name, diff --git a/grids/optimizer.go b/grids/optimizer.go index 300d4fd0..16912398 100644 --- a/grids/optimizer.go +++ b/grids/optimizer.go @@ -90,4 +90,38 @@ var startOptimizations = []OptimizationRule{ return append(optimized, pipe[3:]...) }, }, + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); ok { + return true + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + for i, label := range labels { + if label[:2] != VTABLE_PREFIX { + labels[i] = VTABLE_PREFIX + label + } + } + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids V().HasLabel()", + Custom: lookupVertsHasLabelCondIndexStep{ + expr: nil, + labels: labels, + }, + }, + }, + } + return append(optimized, pipe[2:]...) + }, + }, } diff --git a/grids/processor.go b/grids/processor.go index 2cc7ee79..7f3a2973 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -39,12 +39,11 @@ type lookupVertsHasLabelCondIndexProc struct { } func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") - queryChan := make(chan gdbi.ElementLookup, 100) + log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor", l.expr, l.labels, len(l.db.bsonkv.Fields)) var exists = false if len(l.db.bsonkv.Fields) > 0 { for _, label := range l.labels { - log.Debugln("Checking indexed fields %v", l.db.bsonkv.Fields, "LABEL: ", label) + log.Debugln("Checking indexed fields ", l.db.bsonkv.Fields, "LABEL: ", label) _, exists = l.db.bsonkv.Fields[label] if exists { break @@ -52,10 +51,9 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } } - if l.expr.GetCondition() == nil || !exists { - log.Debugf("cond == nil || !exists: ", l.expr.GetCondition(), exists) + if !exists || (l.expr == nil && l.expr.GetCondition() == nil) { go func() { - defer close(queryChan) + defer close(out) for t := range in { for _, label := range l.labels { tableFound, ok := l.db.bsonkv.Tables[label] @@ -63,13 +61,21 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } - for id := range tableFound.Scan(true, &GripQLFilter{Expression: l.expr}) { - queryChan <- gdbi.ElementLookup{ID: id.(string), Ref: t} + for roMaps := range tableFound.Scan(false, &GripQLFilter{Expression: l.expr}) { + v := gdbi.Vertex{ + ID: roMaps.(map[string]any)["_id"].(string), + Label: label[2:], + Data: roMaps.(map[string]any), + Loaded: true, + } + out <- t.AddCurrent(v.Copy()) + } } } }() } else { + queryChan := make(chan gdbi.ElementLookup, 100) go func() { defer close(queryChan) for t := range in { @@ -81,15 +87,15 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } } }() + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() } - go func() { - defer close(out) - for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { - i := v.Ref - out <- i.AddCurrent(v.Vertex.Copy()) - } - }() return ctx } diff --git a/gripql/python/gripql/graph.py b/gripql/python/gripql/graph.py index a70aa3d3..e2c8af4e 100644 --- a/gripql/python/gripql/graph.py +++ b/gripql/python/gripql/graph.py @@ -333,14 +333,13 @@ def __init__(self, url, graph, extraArgs=None, user=None, password=None, token=N super(BulkAddRaw, self).__init__(url, user, password, token, credential_file) self.url = self.base_url + "/v1/rawJson" self.graph = graph - self.extraArgs = {"auth_resource_path": "test-data"} self.elements = [] - def addJson(self, data={}): + def addJson(self, data={}, extra_args={}): payload = { "graph": self.graph, - "extra_args": self.extraArgs, + "extra_args": extra_args, "data": data } self.elements.append(json.dumps(payload)) diff --git a/server/server.go b/server/server.go index fb891876..4f1ab14c 100644 --- a/server/server.go +++ b/server/server.go @@ -495,7 +495,9 @@ func (server *GripServer) Serve(pctx context.Context) error { if isSchema(graph) { log.WithFields(log.Fields{"graph": graph}).Debug("Loading existing schema into cache") schema, err := server.getGraph(graph) - if err == nil { + if err != nil { + log.Errorln("Error in server.getGraph: ", err) + } else { server.schemas[strings.TrimSuffix(graph, schemaSuffix)] = schema } } else if isMapping(graph) { From 9962b0409f5270c41432898ec027efe1b5bc6604 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 26 Jun 2025 15:52:20 -0700 Subject: [PATCH 18/30] many optimizations on grids driver --- grids/graph.go | 168 ++++++++++++++++++++++++++++++---------------- grids/keyindex.go | 42 ++++++++++-- 2 files changed, 148 insertions(+), 62 deletions(-) diff --git a/grids/graph.go b/grids/graph.go index b45edbbb..cd4d2a10 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -68,25 +68,65 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) return nil } -func insertEdge(tx *pebblebulk.PebbleBulk, edge *gdbi.Edge) error { - if edge.ID == "" || edge.From == "" || edge.To == "" || edge.Label == "" { +func insertEdge(tx *pebblebulk.PebbleBulk, edge *completeEdge) error { + if edge.OEdge.ID == "" || edge.OEdge.From == "" || edge.OEdge.To == "" || edge.OEdge.Label == "" { return fmt.Errorf("inserting null key edge") } - err := tx.Set(EdgeKey(edge.ID, edge.From, edge.To, edge.Label), nil, nil) + err := tx.Set(EdgeKey(edge.OEdge.ID, edge.OEdge.From, edge.OEdge.To, edge.OEdge.Label), nil, nil) if err != nil { return err } - err = tx.Set(SrcEdgeKey(edge.ID, edge.From, edge.To, edge.Label), []byte{}, nil) + err = tx.Set(DstEdgeKey( + edge.OEdge.ID, + edge.OEdge.From, + edge.OEdge.To, + edge.OEdge.Label, + edge.FromLabel, + ), []byte{}, nil) if err != nil { return err } - err = tx.Set(DstEdgeKey(edge.ID, edge.From, edge.To, edge.Label), []byte{}, nil) + err = tx.Set(SrcEdgeKey( + edge.OEdge.ID, + edge.OEdge.From, + edge.OEdge.To, + edge.OEdge.Label, + edge.ToLabel, + ), []byte{}, nil) if err != nil { return err } return nil } +func (ggraph *Graph) bulkGet(edges []*completeEdge) <-chan *completeEdge { + resultsChan := make(chan *completeEdge, 100) + go func() { + defer close(resultsChan) + err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for i, edge := range edges { + err := it.Seek(VertexKeyPrefix(edge.OEdge.From)) + if err == nil { + tmp := bytes.Split(it.Key(), []byte{0}) + edges[i].FromLabel = tmp[2] + } + err = it.Seek(VertexKeyPrefix(edge.OEdge.To)) + if err == nil { + tmp := bytes.Split(it.Key(), []byte{0}) + edges[i].ToLabel = tmp[2] + } + resultsChan <- edges[i] + } + return nil + }) + if err != nil { + log.Errorf("Error in PebbleBulk BulkGet (ViewRange) %s", err) + } + }() + + return resultsChan +} + func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error { edgeLabel := ETABLE_PREFIX + edge.Label ggraph.bsonkv.Lock.Lock() @@ -155,15 +195,30 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { // AddEdge adds an edge to the graph, if the id is not "" and in already exists // in the graph, it is replaced func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - for _, edge := range edges { - err := insertEdge(tx, edge) - if err != nil { - return err + var err error = nil + err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, edge := range edges { + var fromLabel, toLabel []byte + err = it.Seek(VertexKeyPrefix(edge.From)) + if err == nil { + tmp := bytes.Split(it.Key(), []byte{0}) + fromLabel = tmp[2] + } + err = it.Seek(VertexKeyPrefix(edge.To)) + if err == nil { + tmp := bytes.Split(it.Key(), []byte{0}) + toLabel = tmp[2] + } + err = insertEdge(tx, &completeEdge{OEdge: edge, FromLabel: fromLabel, ToLabel: toLabel}) + if err != nil { + return err + } } - } + return err + }) ggraph.ts.Touch(ggraph.graphID) - return nil + return err }) if err != nil { return err @@ -199,16 +254,22 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { return bulkErr.ErrorOrNil() } +type completeEdge struct { + OEdge *gdbi.Edge // Pointer to the original edge + FromLabel []byte // Looked up label for 'From' + ToLabel []byte // Looked up label for 'To' +} + func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { var errs *multierror.Error insertStream := make(chan *gdbi.GraphElement, 100) indexStream := make(chan *benchtop.Row, 100) errChan := make(chan error, 2) + var edgesToWrite []*completeEdge var wg sync.WaitGroup wg.Add(2) - // Goroutine for inserting vertices and edges into graphkv go func() { defer wg.Done() err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { @@ -219,15 +280,30 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { } } if elem.Edge != nil { - if err := insertEdge(tx, elem.Edge); err != nil { - return fmt.Errorf("edge insert error: %v", err) - } + ce := &completeEdge{OEdge: elem.Edge} + edgesToWrite = append(edgesToWrite, ce) + } + } + return nil + }) + if err != nil { + log.Errorf("ERR in graph Bulk Add: %s", err) + return + } + + err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + for ce := range ggraph.bulkGet(edgesToWrite) { + if err := insertEdge(tx, ce); err != nil { + return fmt.Errorf("edge insert error: %v", err) } } ggraph.ts.Touch(ggraph.graphID) return nil }) - errChan <- err + if err != nil { + log.Errorf("ERR in BulkWrite: ", err) + return + } }() go func() { @@ -280,7 +356,6 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { } func (ggraph *Graph) DelEdge(eid string) error { - ekeyPrefix := EdgeKeyPrefix(eid) var ekey []byte ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { @@ -295,18 +370,18 @@ func (ggraph *Graph) DelEdge(eid string) error { eid, sid, did, lbl := EdgeKeyParse(ekey) - skey := SrcEdgeKey(eid, sid, did, lbl) - dkey := DstEdgeKey(eid, sid, did, lbl) + skey := SrcEdgeKeyPrefix(sid, did, eid, lbl) + dkey := DstEdgeKeyPrefix(sid, did, eid, lbl) var bulkErr *multierror.Error err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := tx.Delete(ekey, nil); err != nil { return err } - if err := tx.Delete(skey, nil); err != nil { + if err := tx.DeletePrefix(skey); err != nil { return err } - if err := tx.Delete(dkey, nil); err != nil { + if err := tx.DeletePrefix(dkey); err != nil { return err } ggraph.ts.Touch(ggraph.graphID) @@ -318,7 +393,6 @@ func (ggraph *Graph) DelEdge(eid string) error { if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { bulkErr = multierror.Append(bulkErr, err) } - return bulkErr.ErrorOrNil() } @@ -338,9 +412,9 @@ func (ggraph *Graph) DelVertex(id string) error { for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() // get edge ID from key - eid, sid, did, label := SrcEdgeKeyParse(skey) + eid, sid, did, label, _ := SrcEdgeKeyParse(skey) ekey := EdgeKey(eid, sid, did, label) - dkey := DstEdgeKey(eid, sid, did, label) + dkey := DstEdgeKeyPrefix(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { @@ -350,9 +424,9 @@ func (ggraph *Graph) DelVertex(id string) error { for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { dkey := it.Key() // get edge ID from key - eid, sid, did, label := DstEdgeKeyParse(dkey) + eid, sid, did, label := DstEdgeKeyPrefixParse(dkey) ekey := EdgeKey(eid, sid, did, label) - skey := SrcEdgeKey(eid, sid, did, label) + skey := SrcEdgeKeyPrefix(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { @@ -500,6 +574,7 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + var err error for req := range reqChan { if req.IsSignal() { o <- req @@ -507,22 +582,11 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen found := false skeyPrefix := SrcEdgePrefix(req.ID) for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - _, _, dst, label := SrcEdgeKeyParse(it.Key()) + _, _, dst, label, vLabel := SrcEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { - rtasockey := benchtop.NewRowTableAsocKey([]byte(dst)) - rtasocval, closer, err := ggraph.bsonkv.Pb.Db.Get(rtasockey) - if err != nil { - log.Errorf("GetInChannel: GetRow Db.Get() error: %v", err) - continue - } - defer closer.Close() - - vLabel := string(rtasocval) - log.Debugln("VLABEL: ", vLabel, "DST: ", dst) - - v := &gdbi.Vertex{ID: dst, Label: vLabel[2:]} + v := &gdbi.Vertex{ID: dst, Label: vLabel} if load { - v.Data, err = ggraph.bsonkv.Tables[vLabel].GetRow([]byte(dst)) + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(dst)) if err != nil { log.Errorf("GetInChannel: GetRow error: %v", err) continue @@ -564,20 +628,14 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element dkeyPrefix := DstEdgePrefix(req.ID) for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { keyValue := it.Key() - _, sid, _, label := DstEdgeKeyParse(keyValue) + _, sid, _, label, vLabel := DstEdgeKeyParse(keyValue) + log.Debugln("GET IN CHAN VLABEL: ", vLabel) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { - - rtasockey := benchtop.NewRowTableAsocKey([]byte(sid)) - rtasocval, closer, _ := ggraph.bsonkv.Pb.Db.Get(rtasockey) - defer closer.Close() - vLabel := string(rtasocval) - - v := &gdbi.Vertex{ID: sid, Label: vLabel[2:]} + v := &gdbi.Vertex{ID: sid, Label: vLabel} if load { var err error - log.Debugln("GET ROW IN GET IN CHAN") - v.Data, err = ggraph.bsonkv.Tables[vLabel].GetRow([]byte(sid)) + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(sid)) if err != nil { log.Errorf("GetInChannel: GetRow error: %v", err) continue @@ -617,7 +675,7 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El found := false skeyPrefix := SrcEdgePrefix(req.ID) for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - eid, src, dst, label := SrcEdgeKeyParse(it.Key()) + eid, src, dst, label, _ := SrcEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { e := gdbi.Edge{ From: src, @@ -669,8 +727,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele found := false dkeyPrefix := DstEdgePrefix(req.ID) for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := DstEdgeKeyParse(keyValue) + eid, src, dst, label, _ := DstEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { e := gdbi.Edge{ ID: eid, @@ -680,8 +737,6 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele } if load { var err error - log.Debugln("GET ROW IN GET IN EDGE CHAN") - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) if err != nil { log.Errorf("GetInEdgeChannel: GetRow error: %v", err) @@ -727,7 +782,6 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { if loadProp { var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(id)) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) diff --git a/grids/keyindex.go b/grids/keyindex.go index 5b1cd909..2992bfc1 100644 --- a/grids/keyindex.go +++ b/grids/keyindex.go @@ -84,37 +84,69 @@ func EdgeKeyParse(key []byte) (eid string, sid string, did string, label string) } // SrcEdgeKey creates a src edge index key -func SrcEdgeKey(eid, src, dst, label string) []byte { +func SrcEdgeKey(eid, src, dst, label string, fromLabel []byte) []byte { return bytes.Join([][]byte{ srcEdgePrefix, []byte(src), []byte(dst), []byte(eid), []byte(label), + fromLabel, }, []byte{0}) } -func SrcEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { +func SrcEdgeKeyParse(key []byte) (eid string, sid string, did string, label string, fromLabel string) { tmp := bytes.Split(key, []byte{0}) - return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]) + return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]), string(tmp[5]) } // DstEdgeKey creates a dest edge index key -func DstEdgeKey(eid, src, dst, label string) []byte { +func DstEdgeKey(eid, src, dst, label string, toLabel []byte) []byte { return bytes.Join([][]byte{ dstEdgePrefix, []byte(dst), []byte(src), []byte(eid), []byte(label), + toLabel, + }, []byte{0}) +} + +func DstEdgeKeyParse(key []byte) (eid string, sid string, did string, label string, toLabel string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[3]), string(tmp[2]), string(tmp[1]), string(tmp[4]), string(tmp[5]) +} + +func DstEdgeKeyPrefix(eid, sid, did, lbl string) []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + []byte(did), + []byte(sid), + []byte(eid), + []byte(lbl), }, []byte{0}) } -func DstEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { +func DstEdgeKeyPrefixParse(key []byte) (eid string, sid string, did string, label string) { tmp := bytes.Split(key, []byte{0}) return string(tmp[3]), string(tmp[2]), string(tmp[1]), string(tmp[4]) } +func SrcEdgeKeyPrefix(eid, sid, did, lbl string) []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + []byte(sid), + []byte(did), + []byte(eid), + []byte(lbl), + }, []byte{0}) +} + +func SrcEdgeKeyPrefixParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]) +} + // VertexListPrefix returns a byte array prefix for all vertices in a graph func VertexListPrefix() []byte { return bytes.Join([][]byte{ From 00a1ed63c7a8831dbbadd3830225c2a243f009e6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 1 Jul 2025 13:08:03 -0700 Subject: [PATCH 19/30] checkpoint. tests are passing --- grids/graph.go | 281 +++++++++++++++++++++++--------------- grids/index.go | 21 +++ grids/keyindex.go | 18 +-- grids/new.go | 12 +- grids/processor.go | 15 +- gripql/inspect/inspect.go | 4 +- test/processors_test.go | 12 +- 7 files changed, 223 insertions(+), 140 deletions(-) diff --git a/grids/graph.go b/grids/graph.go index cd4d2a10..99287b21 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -30,7 +30,7 @@ func insertVertex(tx *pebblebulk.PebbleBulk, vertex *gdbi.Vertex) error { if vertex.ID == "" { return fmt.Errorf("inserting null key vertex") } - if err := tx.Set(VertexKey(vertex.ID, vertex.Label), nil, nil); err != nil { + if err := tx.Set(VertexKey(vertex.ID), []byte(vertex.Label), nil); err != nil { return fmt.Errorf("AddVertex Error %s", err) } return nil @@ -105,15 +105,23 @@ func (ggraph *Graph) bulkGet(edges []*completeEdge) <-chan *completeEdge { defer close(resultsChan) err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for i, edge := range edges { - err := it.Seek(VertexKeyPrefix(edge.OEdge.From)) + err := it.Seek(VertexKey(edge.OEdge.From)) if err == nil { - tmp := bytes.Split(it.Key(), []byte{0}) - edges[i].FromLabel = tmp[2] + label, err := it.Value() + if err != nil { + log.Errorf("Err getting vertex label: %s", err) + continue + } + edges[i].FromLabel = label } - err = it.Seek(VertexKeyPrefix(edge.OEdge.To)) + err = it.Seek(VertexKey(edge.OEdge.To)) if err == nil { - tmp := bytes.Split(it.Key(), []byte{0}) - edges[i].ToLabel = tmp[2] + label, err := it.Value() + if err != nil { + log.Errorf("Err getting vertex label: %s", err) + continue + } + edges[i].ToLabel = label } resultsChan <- edges[i] } @@ -200,18 +208,28 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { err = ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for _, edge := range edges { var fromLabel, toLabel []byte - err = it.Seek(VertexKeyPrefix(edge.From)) + err = it.Seek(VertexKey(edge.From)) if err == nil { - tmp := bytes.Split(it.Key(), []byte{0}) - fromLabel = tmp[2] + label, err := it.Value() + if err != nil { + log.Errorf("Err getting vertex label: %s", err) + continue + } + fromLabel = label } - err = it.Seek(VertexKeyPrefix(edge.To)) + err = it.Seek(VertexKey(edge.To)) if err == nil { - tmp := bytes.Split(it.Key(), []byte{0}) - toLabel = tmp[2] + label, err := it.Value() + if err != nil { + log.Errorf("Err getting vertex label: %s", err) + continue + } + toLabel = label } + err = insertEdge(tx, &completeEdge{OEdge: edge, FromLabel: fromLabel, ToLabel: toLabel}) if err != nil { + log.Errorln("Err insertEdge: ", err) return err } } @@ -239,18 +257,29 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { var bulkErr *multierror.Error - for _, val := range data.Edges { - err := ggraph.DelEdge(val) + ggraph.tempDeletedEdges = make(map[string]struct{}) + ggraph.edgesMutex.Lock() + + for _, val := range data.Vertices { + err := ggraph.DelVertex(val) if err != nil { bulkErr = multierror.Append(bulkErr, err) } } - for _, val := range data.Vertices { - err := ggraph.DelVertex(val) + + for _, val := range data.Edges { + if _, ok := ggraph.tempDeletedEdges[val]; ok { + log.Debugf("Skipping edge %s: already deleted during vertex deletion", val) + continue + } + err := ggraph.DelEdge(val) if err != nil { bulkErr = multierror.Append(bulkErr, err) } } + + ggraph.tempDeletedEdges = nil // Clean up + defer ggraph.edgesMutex.Unlock() return bulkErr.ErrorOrNil() } @@ -301,7 +330,7 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { return nil }) if err != nil { - log.Errorf("ERR in BulkWrite: ", err) + log.Errorf("ERR in BulkWrite: %s", err) return } }() @@ -351,100 +380,81 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { } } - // Return any accumulated errors return errs.ErrorOrNil() } -func (ggraph *Graph) DelEdge(eid string) error { - ekeyPrefix := EdgeKeyPrefix(eid) - var ekey []byte - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { - ekey = it.Key() - } - return nil - }) - if ekey == nil { - return fmt.Errorf("edge not found") - } - - eid, sid, did, lbl := EdgeKeyParse(ekey) - - skey := SrcEdgeKeyPrefix(sid, did, eid, lbl) - dkey := DstEdgeKeyPrefix(sid, did, eid, lbl) - - var bulkErr *multierror.Error - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := tx.Delete(ekey, nil); err != nil { - return err - } - if err := tx.DeletePrefix(skey); err != nil { - return err - } - if err := tx.DeletePrefix(dkey); err != nil { - return err - } - ggraph.ts.Touch(ggraph.graphID) - return nil - }) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - return bulkErr.ErrorOrNil() -} - -// DelVertex deletes vertex with id `key` func (ggraph *Graph) DelVertex(id string) error { - - vid := VertexKeyPrefix(id) + vid := VertexKey(id) skeyPrefix := SrcEdgePrefix(id) dkeyPrefix := DstEdgePrefix(id) delKeys := make([][]byte, 0, 1000) + edgesToDelete := make(map[string]string) var bulkErr *multierror.Error err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var bulkErr *multierror.Error for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() - // get edge ID from key - eid, sid, did, label, _ := SrcEdgeKeyParse(skey) + eid, sid, did, label := SrcEdgeKeyPrefixParse(skey) + + if ggraph.tempDeletedEdges != nil { + if _, exists := ggraph.tempDeletedEdges[eid]; exists { + continue + } + } + if _, exists := edgesToDelete[eid]; exists { + continue + } + ekey := EdgeKey(eid, sid, did, label) dkey := DstEdgeKeyPrefix(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) - - if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } + edgesToDelete[eid] = label } + for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { dkey := it.Key() - // get edge ID from key eid, sid, did, label := DstEdgeKeyPrefixParse(dkey) + + if ggraph.tempDeletedEdges != nil { + if _, exists := ggraph.tempDeletedEdges[eid]; exists { + continue + } + } + if _, exists := edgesToDelete[eid]; exists { + continue + } + ekey := EdgeKey(eid, sid, did, label) skey := SrcEdgeKeyPrefix(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) - - if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } + edgesToDelete[eid] = label } - return bulkErr.ErrorOrNil() + return nil }) + if err != nil { - bulkErr = multierror.Append(bulkErr, err) + return err + } + + for eid, label := range edgesToDelete { + if err := ggraph.DeleteAnyRow(eid, label, true); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + + if ggraph.tempDeletedEdges != nil { + ggraph.tempDeletedEdges[eid] = struct{}{} + } } err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := tx.DeletePrefix(vid); err != nil { + if err := tx.Delete(vid, nil); err != nil { return err } for _, k := range delKeys { if err := tx.Delete(k, nil); err != nil { + log.Errorf("BulkWrite failed to delete key %s: %v", string(k), err) return err } } @@ -454,6 +464,55 @@ func (ggraph *Graph) DelVertex(id string) error { if err != nil { bulkErr = multierror.Append(bulkErr, err) } + + return bulkErr.ErrorOrNil() +} + +func (ggraph *Graph) DelEdge(eid string) error { + ekeyPrefix := EdgeKeyPrefix(eid) + var ekey []byte + err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { + ekey = it.Key() + } + return nil + }) + if err != nil { + return err + } + + if ekey == nil { + log.Debugf("Edge %s not found", eid) + return nil + } + + _, sid, did, lbl := EdgeKeyParse(ekey) + skey := SrcEdgeKeyPrefix(sid, did, eid, lbl) + dkey := DstEdgeKeyPrefix(sid, did, eid, lbl) + + var bulkErr *multierror.Error + err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + if err := tx.Delete(ekey, nil); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + if err := tx.Delete(skey, nil); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + if err := tx.Delete(dkey, nil); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + ggraph.ts.Touch(ggraph.graphID) + return nil + }) + + if err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + + if err := ggraph.DeleteAnyRow(eid, lbl, true); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + return bulkErr.ErrorOrNil() } @@ -463,6 +522,7 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + var err error = nil ePrefix := EdgeListPrefix() for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { select { @@ -473,7 +533,6 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb eid, sid, did, label := EdgeKeyParse(it.Key()) e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} if loadProp { - var err error e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(eid)) if err != nil { log.Errorf("GetEdgeList: GetRow error: %v", err) @@ -493,21 +552,26 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb // GetVertex loads a vertex given an id. It returns a nil if not found func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { - var v *gdbi.Vertex - rtasockey := benchtop.NewRowTableAsocKey([]byte(id)) - rtasocval, closer, err := ggraph.bsonkv.Pb.Db.Get(rtasockey) - if err != nil { + ekeyPrefix := VertexKey(id) + var byteLabel []byte = nil + var err error = nil + err = ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { + byteLabel, err = it.Value() + } + return nil + }) + if err != nil || byteLabel == nil { return nil } - defer closer.Close() - label := string(rtasocval) - v = &gdbi.Vertex{ + label := string(byteLabel) + + v := &gdbi.Vertex{ ID: id, - Label: label[2:], + Label: string(label), } if loadProp { - var err error - v.Data, err = ggraph.bsonkv.Tables[label].GetRow([]byte(id)) + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+label].GetRow([]byte(id)) if err != nil { return nil } @@ -536,22 +600,21 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element } else { if load { v := gdbi.Vertex{ID: id.ID} - prefix := benchtop.NewRowTableAsocKey([]byte(id.ID)) + prefix := VertexKey(id.ID) for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { - fetchLabel, err := it.Value() + label, err := it.Value() if err != nil { - log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) + log.Errorln("GetVertexChannel it.Value() err: ", err) continue } - label := string(fetchLabel) - v.Label = label[2:] - v.Data, err = ggraph.bsonkv.Tables[label].GetRow([]byte(id.ID)) + vLabel := string(label) + v.Label = vLabel + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(id.ID)) if err != nil { log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) continue } v.Loaded = true - break } id.Vertex = &v out <- id @@ -574,7 +637,7 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error + var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -600,11 +663,11 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen found = true } } + if !found && emitNull { req.Vertex = nil o <- req } - } } return nil @@ -620,6 +683,7 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -629,11 +693,9 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { keyValue := it.Key() _, sid, _, label, vLabel := DstEdgeKeyParse(keyValue) - log.Debugln("GET IN CHAN VLABEL: ", vLabel) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { v := &gdbi.Vertex{ID: sid, Label: vLabel} if load { - var err error v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(sid)) if err != nil { @@ -668,6 +730,7 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -684,8 +747,6 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El ID: eid, } if load { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) if err != nil { log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) @@ -720,6 +781,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -736,7 +798,6 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele Label: label, } if load { - var err error e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) if err != nil { log.Errorf("GetInEdgeChannel: GetRow error: %v", err) @@ -751,8 +812,8 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele o <- req found = true } - } + if !found && emitNull { req.Edge = nil o <- req @@ -771,6 +832,7 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { ekeyPrefix := EdgeKeyPrefix(id) var e *gdbi.Edge err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + var err error = nil for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { eid, src, dst, label := EdgeKeyParse(it.Key()) e = &gdbi.Edge{ @@ -781,7 +843,6 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { } if loadProp { - var err error e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(id)) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) @@ -813,15 +874,17 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g return nil default: } - - id, label := VertexKeyParse(it.Key()) + id := VertexKeyParse(it.Key()) + byteLabel, err := it.Value() + if err != nil { + log.Errorf("GetVertexList it.Value() error: %s", err) + } + label := string(byteLabel) v := &gdbi.Vertex{ ID: id, Label: label, } if loadProp { - var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+label].GetRow([]byte(v.ID)) if err != nil { log.Errorf("GetVertexList: GetRow error: %v", err) diff --git a/grids/index.go b/grids/index.go index 60135d3b..28302811 100644 --- a/grids/index.go +++ b/grids/index.go @@ -5,6 +5,7 @@ import ( "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" + "github.com/cockroachdb/pebble" ) // AddVertexIndex add index to vertices @@ -41,3 +42,23 @@ func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan str log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") return ggraph.bsonkv.GetIDsForLabel(label) } + +func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error { + ggraph.bsonkv.Lock.Lock() + defer ggraph.bsonkv.Lock.Unlock() + + var prefix string = "v_" + if edgeFlag { + prefix = "e_" + } + + err := ggraph.bsonkv.Tables[prefix+label].DeleteRow([]byte(id)) + if err != nil { + if err == pebble.ErrNotFound{ + log.Debugln("Pebble not Found: %s", err) + return nil + } + return err + } + return nil +} diff --git a/grids/keyindex.go b/grids/keyindex.go index 2992bfc1..3e58db10 100644 --- a/grids/keyindex.go +++ b/grids/keyindex.go @@ -12,28 +12,14 @@ var dstEdgePrefix = []byte(">") var intSize = 10 // VertexKey generates the key given a vertexId -func VertexKey(id, label string) []byte { +func VertexKey(id string) []byte { return bytes.Join([][]byte{ vertexPrefix, []byte(id), - []byte(label), - }, []byte{0}) -} - -func VertexKeyParse(key []byte) (id, label string) { - tmp := bytes.Split(key, []byte{0}) - return string(tmp[1]), string(tmp[2]) -} - -func VertexKeyPrefix(id string) []byte { - return bytes.Join([][]byte{ - vertexPrefix, - []byte(id), - {}, }, []byte{0}) } -func VertexKeyPrefixParse(key []byte) (id string) { +func VertexKeyParse(key []byte) (id string) { tmp := bytes.Split(key, []byte{0}) return string(tmp[1]) } diff --git a/grids/new.go b/grids/new.go index c861a2f9..189c11e7 100644 --- a/grids/new.go +++ b/grids/new.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/bmeg/benchtop/bsontable" "github.com/bmeg/grip/gripql" @@ -14,10 +15,12 @@ import ( // Graph implements the GDB interface using a genertic key/value storage driver type Graph struct { - graphID string + graphID string - bsonkv *bsontable.BSONDriver - ts *timestamp.Timestamp + bsonkv *bsontable.BSONDriver + ts *timestamp.Timestamp + tempDeletedEdges map[string]struct{} + edgesMutex sync.Mutex } // Close the connection @@ -39,6 +42,7 @@ func (kgraph *GDB) AddGraph(graph string) error { kgraph.drivers[graph] = g return nil } + func newGraph(baseDir, name string) (*Graph, error) { dbPath := filepath.Join(baseDir, name) fmt.Printf("Creating new GRIDS graph %s\n", name) @@ -70,6 +74,8 @@ func newGraph(baseDir, name string) (*Graph, error) { bsonkv: bsonkv, ts: &ts, graphID: name, + tempDeletedEdges: make(map[string]struct{}), + edgesMutex: sync.Mutex{}, } return o, nil } diff --git a/grids/processor.go b/grids/processor.go index 7f3a2973..fa2c782b 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -58,18 +58,21 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi for _, label := range l.labels { tableFound, ok := l.db.bsonkv.Tables[label] if !ok { - log.Errorf("BSONTable for label '%s' is nil. Cannot scan.", label) + log.Debugf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } for roMaps := range tableFound.Scan(false, &GripQLFilter{Expression: l.expr}) { + log.Debugln("RES RETURNED: ", roMaps.(map[string]any)) + + id := roMaps.(map[string]any)["_id"].(string) + delete(roMaps.(map[string]any), "_id") v := gdbi.Vertex{ - ID: roMaps.(map[string]any)["_id"].(string), + ID: id, Label: label[2:], Data: roMaps.(map[string]any), Loaded: true, } out <- t.AddCurrent(v.Copy()) - } } } @@ -130,12 +133,14 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager cond := l.expr.GetCondition() var exists = false if len(l.db.bsonkv.Fields) > 0 { - _, exists = l.db.bsonkv.Fields[cond.Key] + _, exists = l.db.bsonkv.Fields[VTABLE_PREFIX+cond.Key] } /* Optimized indexing only works for Simple filters. / / If compound filter or index doesn't exist use backup method */ + log.Debugln("COND: ", cond, cond == nil, l.db.bsonkv.Fields) + if cond == nil || !exists { - log.Debugf("lookupVertsCondIndexProc: falling back to GetVertexList since filter is not basic Condition filter") + log.Debugf("lookupVertsCondIndexProc: falling back to GetVertexList since filter is not basic Condition filter or not indexed") go func() { defer close(queryChan) for t := range in { diff --git a/gripql/inspect/inspect.go b/gripql/inspect/inspect.go index 40e8f9d6..be1a5831 100644 --- a/gripql/inspect/inspect.go +++ b/gripql/inspect/inspect.go @@ -115,7 +115,9 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st onLast = false case *gripql.GraphStatement_Pivot: - //TODO: figure out which fields are referenced + if onLast { + out[steps[i]] = []string{"*"} + } onLast = false case *gripql.GraphStatement_Group: diff --git a/test/processors_test.go b/test/processors_test.go index 584c9c44..b199f692 100644 --- a/test/processors_test.go +++ b/test/processors_test.go @@ -419,15 +419,15 @@ func compare(expect []*gripql.QueryResult) checker { sort.Strings(expectS) if !reflect.DeepEqual(actualS, expectS) { - for _, s := range actualS { - t.Log("actual", s) - } - for _, s := range expectS { - t.Log("expect", s) - } if len(expectS) != len(actualS) { t.Logf("expected # results: %d actual # results: %d", len(expectS), len(actualS)) + } else { + for i, s := range actualS { + t.Log("actual", s) + t.Log("expect", expectS[i]) + } } + t.Errorf("not equal") } } From 2a6b21c2bada9f0ac697b1aaa8e49d608c57bc20 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 7 Jul 2025 12:25:05 -0700 Subject: [PATCH 20/30] run good --- conformance/run_util.py | 7 +- engine/core/processors.go | 2 + go.mod | 14 +- go.sum | 25 ++- grids/graph.go | 282 +++++++++++++++++----------------- grids/index.go | 4 +- grids/keyindex.go | 41 +---- grids/processor.go | 6 +- gripql/python/gripql/graph.py | 2 +- test/main_test.go | 1 + 10 files changed, 181 insertions(+), 203 deletions(-) diff --git a/conformance/run_util.py b/conformance/run_util.py index dbee58f1..1a32dc00 100644 --- a/conformance/run_util.py +++ b/conformance/run_util.py @@ -180,19 +180,22 @@ def setGraph(self, name): self._conn.addGraph(self.curGraph) G = self._conn.graph(self.curGraph) + bulk = G.bulkAdd() with open(os.path.join(BASE, "graphs", "%s.vertices" % (name))) as handle: for line in handle: data = json.loads(line) - G.addVertex(data["_id"], data["_label"], self.collect_fields_dict(data)) + bulk.addVertex(data["_id"], data["_label"], self.collect_fields_dict(data)) with open(os.path.join(BASE, "graphs", "%s.edges" % (name))) as handle: for line in handle: data = json.loads(line) - G.addEdge(src=data["_from"], dst=data["_to"], + bulk.addEdge(src=data["_from"], dst=data["_to"], id=data.get("_id", None), label=data["_label"], data=self.collect_fields_dict(data)) + + bulk.execute() self.curName = name return G diff --git a/engine/core/processors.go b/engine/core/processors.go index 059d9e99..b47e7cea 100644 --- a/engine/core/processors.go +++ b/engine/core/processors.go @@ -7,6 +7,7 @@ import ( "github.com/bmeg/grip/engine/logic" "github.com/bmeg/grip/gdbi" + //"github.com/bmeg/grip/log" "github.com/bmeg/grip/util/copy" "github.com/spf13/cast" ) @@ -226,6 +227,7 @@ func (r *Unwind) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, continue } v := gdbi.TravelerPathLookup(t, r.Field) + //log.Debugln("UNWIND V RES: ", v) if a, ok := v.([]interface{}); ok { cur := t.GetCurrent() if len(a) > 0 { diff --git a/go.mod b/go.mod index 0f391bec..7a237c7c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/bmeg/grip -go 1.23.6 +go 1.24 + +toolchain go1.24.2 require ( github.com/IBM/sarama v1.45.1 @@ -8,7 +10,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250618183006-bddc564a4bd6 + github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a @@ -17,7 +19,6 @@ require ( github.com/cockroachdb/pebble v1.1.2 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.2007.4 - github.com/dgraph-io/ristretto/v2 v2.1.0 github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5 github.com/felixge/httpsnoop v1.0.4 github.com/go-sql-driver/mysql v1.8.1 @@ -71,13 +72,14 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.11.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/fatih/color v1.17.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/getsentry/sentry-go v0.28.1 // indirect @@ -86,7 +88,6 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect - github.com/golang/glog v1.2.3 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect @@ -108,6 +109,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/maypok86/otter/v2 v2.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/montanaflynn/stats v0.7.1 // indirect @@ -133,7 +135,7 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect - golang.org/x/sys v0.30.0 // indirect + golang.org/x/sys v0.33.0 // indirect golang.org/x/term v0.29.0 // indirect golang.org/x/text v0.22.0 // indirect gonum.org/v1/gonum v0.8.2 // indirect diff --git a/go.sum b/go.sum index 5b6f0da3..2430978d 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,10 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250618183006-bddc564a4bd6 h1:qbFsvy7iPsG8f4gdWOlmxnndCBVfI478sBX73sNK3Yo= -github.com/bmeg/benchtop v0.0.0-20250618183006-bddc564a4bd6/go.mod h1:4s+8vTKrryyznb6ebLHkC7sriL3Y3PCRVkVkfdbGw3k= +github.com/bmeg/benchtop v0.0.0-20250707180338-29cf9de6b32c h1:QpwgPbmtPO30Hn96dDe9NwAj5v2wTxQkcHn7jBpVQ/g= +github.com/bmeg/benchtop v0.0.0-20250707180338-29cf9de6b32c/go.mod h1:2JT0auT+55LAADY/elY8HMK7iLPdUVOIBxqfCpbM5eE= +github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634 h1:LHZ5E+VRqdONk6RCXmyre67fxLM8ZXJH3uBZDN6fuL8= +github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634/go.mod h1:2JT0auT+55LAADY/elY8HMK7iLPdUVOIBxqfCpbM5eE= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= @@ -52,7 +54,6 @@ github.com/casbin/govaluate v1.2.0 h1:wXCXFmqyY+1RwiKfYo3jMKyrtZmOL3kHwaqDyCPOYa github.com/casbin/govaluate v1.2.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= @@ -81,10 +82,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I= -github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -103,6 +102,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= +github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= @@ -138,9 +139,6 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= -github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -260,6 +258,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/maypok86/otter/v2 v2.1.0 h1:H+FO9NtLuSWYUlIUQ/kT6VNEpWSIF4w4GZJRDhxYb7k= +github.com/maypok86/otter/v2 v2.1.0/go.mod h1:jX2xEKz9PrNVbDqnk8JUuOt5kURK8h7jd1kDYI5QsZk= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo= @@ -456,12 +456,11 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/grids/graph.go b/grids/graph.go index 99287b21..e04c2f76 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -52,9 +52,25 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) ggraph.bsonkv.Tables[vertexLabel] = table ggraph.bsonkv.Lock.Unlock() } - if err := table.AddRow(benchtop.Row{Id: []byte(vertex.ID), TableName: vertexLabel, Data: vertex.Data}, tx); err != nil { + + rowLoc, err := table.AddRow( + benchtop.Row{ + Id: []byte(vertex.ID), + TableName: vertexLabel, + Data: vertex.Data, + }, + ) + if err != nil { return fmt.Errorf("AddVertex Error %s", err) } + table.AddTableEntryInfo(tx, []byte(vertex.ID), *rowLoc) + + _, ok = ggraph.bsonkv.PageCache.Set(vertex.ID, *rowLoc) + if !ok { + ggraph.bsonkv.PageCache.Invalidate(vertex.ID) + ggraph.bsonkv.PageCache.Set(vertex.ID, *rowLoc) + //log.Debugln("Replaced vals: ", vertex.ID, oldVal, newVal) + } _, fieldsExist := ggraph.bsonkv.Fields[vertexLabel] if fieldsExist { @@ -68,30 +84,32 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) return nil } -func insertEdge(tx *pebblebulk.PebbleBulk, edge *completeEdge) error { - if edge.OEdge.ID == "" || edge.OEdge.From == "" || edge.OEdge.To == "" || edge.OEdge.Label == "" { +func insertEdge(tx *pebblebulk.PebbleBulk, edge *gdbi.Edge) error { + if edge.ID == "" || + edge.From == "" || + edge.To == "" || + edge.Label == "" { + log.Errorln("insertEdge Err: ", edge) return fmt.Errorf("inserting null key edge") } - err := tx.Set(EdgeKey(edge.OEdge.ID, edge.OEdge.From, edge.OEdge.To, edge.OEdge.Label), nil, nil) + err := tx.Set(EdgeKey(edge.ID, edge.From, edge.To, edge.Label), nil, nil) if err != nil { return err } err = tx.Set(DstEdgeKey( - edge.OEdge.ID, - edge.OEdge.From, - edge.OEdge.To, - edge.OEdge.Label, - edge.FromLabel, + edge.ID, + edge.From, + edge.To, + edge.Label, ), []byte{}, nil) if err != nil { return err } err = tx.Set(SrcEdgeKey( - edge.OEdge.ID, - edge.OEdge.From, - edge.OEdge.To, - edge.OEdge.Label, - edge.ToLabel, + edge.ID, + edge.From, + edge.To, + edge.Label, ), []byte{}, nil) if err != nil { return err @@ -99,42 +117,6 @@ func insertEdge(tx *pebblebulk.PebbleBulk, edge *completeEdge) error { return nil } -func (ggraph *Graph) bulkGet(edges []*completeEdge) <-chan *completeEdge { - resultsChan := make(chan *completeEdge, 100) - go func() { - defer close(resultsChan) - err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - for i, edge := range edges { - err := it.Seek(VertexKey(edge.OEdge.From)) - if err == nil { - label, err := it.Value() - if err != nil { - log.Errorf("Err getting vertex label: %s", err) - continue - } - edges[i].FromLabel = label - } - err = it.Seek(VertexKey(edge.OEdge.To)) - if err == nil { - label, err := it.Value() - if err != nil { - log.Errorf("Err getting vertex label: %s", err) - continue - } - edges[i].ToLabel = label - } - resultsChan <- edges[i] - } - return nil - }) - if err != nil { - log.Errorf("Error in PebbleBulk BulkGet (ViewRange) %s", err) - } - }() - - return resultsChan -} - func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error { edgeLabel := ETABLE_PREFIX + edge.Label ggraph.bsonkv.Lock.Lock() @@ -152,9 +134,17 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error ggraph.bsonkv.Tables[edgeLabel] = table ggraph.bsonkv.Lock.Unlock() } - if err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}, tx); err != nil { + rowLoc, err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}) + if err != nil { return fmt.Errorf("indexEdge: table.AddRow: %s", err) } + table.AddTableEntryInfo(tx, []byte(edge.ID), *rowLoc) + + _, ok = ggraph.bsonkv.PageCache.Set(edge.ID, *rowLoc) + if !ok { + ggraph.bsonkv.PageCache.Invalidate(edge.ID) + ggraph.bsonkv.PageCache.Set(edge.ID, *rowLoc) + } _, fieldsExist := ggraph.bsonkv.Fields[edgeLabel] if fieldsExist { @@ -207,27 +197,7 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { err = ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for _, edge := range edges { - var fromLabel, toLabel []byte - err = it.Seek(VertexKey(edge.From)) - if err == nil { - label, err := it.Value() - if err != nil { - log.Errorf("Err getting vertex label: %s", err) - continue - } - fromLabel = label - } - err = it.Seek(VertexKey(edge.To)) - if err == nil { - label, err := it.Value() - if err != nil { - log.Errorf("Err getting vertex label: %s", err) - continue - } - toLabel = label - } - - err = insertEdge(tx, &completeEdge{OEdge: edge, FromLabel: fromLabel, ToLabel: toLabel}) + err = insertEdge(tx, edge) if err != nil { log.Errorln("Err insertEdge: ", err) return err @@ -283,19 +253,12 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { return bulkErr.ErrorOrNil() } -type completeEdge struct { - OEdge *gdbi.Edge // Pointer to the original edge - FromLabel []byte // Looked up label for 'From' - ToLabel []byte // Looked up label for 'To' -} - func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { var errs *multierror.Error insertStream := make(chan *gdbi.GraphElement, 100) indexStream := make(chan *benchtop.Row, 100) errChan := make(chan error, 2) - var edgesToWrite []*completeEdge var wg sync.WaitGroup wg.Add(2) @@ -309,8 +272,9 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { } } if elem.Edge != nil { - ce := &completeEdge{OEdge: elem.Edge} - edgesToWrite = append(edgesToWrite, ce) + if err := insertEdge(tx, elem.Edge); err != nil { + return fmt.Errorf("edge insert error: %v", err) + } } } return nil @@ -320,19 +284,6 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { return } - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - for ce := range ggraph.bulkGet(edgesToWrite) { - if err := insertEdge(tx, ce); err != nil { - return fmt.Errorf("edge insert error: %v", err) - } - } - ggraph.ts.Touch(ggraph.graphID) - return nil - }) - if err != nil { - log.Errorf("ERR in BulkWrite: %s", err) - return - } }() go func() { @@ -396,7 +347,7 @@ func (ggraph *Graph) DelVertex(id string) error { err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() - eid, sid, did, label := SrcEdgeKeyPrefixParse(skey) + eid, sid, did, label := SrcEdgeKeyParse(skey) if ggraph.tempDeletedEdges != nil { if _, exists := ggraph.tempDeletedEdges[eid]; exists { @@ -408,14 +359,14 @@ func (ggraph *Graph) DelVertex(id string) error { } ekey := EdgeKey(eid, sid, did, label) - dkey := DstEdgeKeyPrefix(eid, sid, did, label) + dkey := DstEdgeKey(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) edgesToDelete[eid] = label } for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { dkey := it.Key() - eid, sid, did, label := DstEdgeKeyPrefixParse(dkey) + eid, sid, did, label := DstEdgeKeyParse(dkey) if ggraph.tempDeletedEdges != nil { if _, exists := ggraph.tempDeletedEdges[eid]; exists { @@ -427,7 +378,7 @@ func (ggraph *Graph) DelVertex(id string) error { } ekey := EdgeKey(eid, sid, did, label) - skey := SrcEdgeKeyPrefix(eid, sid, did, label) + skey := SrcEdgeKey(eid, sid, did, label) delKeys = append(delKeys, ekey, skey, dkey) edgesToDelete[eid] = label } @@ -449,11 +400,11 @@ func (ggraph *Graph) DelVertex(id string) error { } err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := tx.Delete(vid, nil); err != nil { + if err := tx.DeletePrefix(vid); err != nil { return err } for _, k := range delKeys { - if err := tx.Delete(k, nil); err != nil { + if err := tx.DeletePrefix(k); err != nil { log.Errorf("BulkWrite failed to delete key %s: %v", string(k), err) return err } @@ -487,8 +438,8 @@ func (ggraph *Graph) DelEdge(eid string) error { } _, sid, did, lbl := EdgeKeyParse(ekey) - skey := SrcEdgeKeyPrefix(sid, did, eid, lbl) - dkey := DstEdgeKeyPrefix(sid, did, eid, lbl) + skey := SrcEdgeKey(sid, did, eid, lbl) + dkey := DstEdgeKey(sid, did, eid, lbl) var bulkErr *multierror.Error err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { @@ -522,7 +473,6 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error = nil ePrefix := EdgeListPrefix() for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { select { @@ -533,7 +483,12 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb eid, sid, did, label := EdgeKeyParse(it.Key()) e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} if loadProp { - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(eid)) + entry, err := ggraph.bsonkv.PageCache.Get(ctx, eid, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetEdgeList: PageCache.Get( error: %v", err) + continue + } + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow(entry) if err != nil { log.Errorf("GetEdgeList: GetRow error: %v", err) continue @@ -564,22 +519,26 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { if err != nil || byteLabel == nil { return nil } - label := string(byteLabel) v := &gdbi.Vertex{ ID: id, - Label: string(label), + Label: string(byteLabel), } if loadProp { - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+label].GetRow([]byte(id)) + entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), id, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetVertex: PageCache.Get( error: %v", err) + return nil + } + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { + log.Errorf("GetVertex: table.GetRow( error: %v", err) return nil } v.Loaded = true } else { v.Data = map[string]any{} } - return v } @@ -599,17 +558,22 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element out <- id } else { if load { - v := gdbi.Vertex{ID: id.ID} prefix := VertexKey(id.ID) + v := gdbi.Vertex{ID: id.ID} for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { label, err := it.Value() if err != nil { log.Errorln("GetVertexChannel it.Value() err: ", err) continue } - vLabel := string(label) - v.Label = vLabel - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(id.ID)) + v.Label = string(label) + + entry, err := ggraph.bsonkv.PageCache.Get(ctx, id.ID, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetVertexChannel: PageCache.Get( error: %v", err) + continue + } + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) continue @@ -637,7 +601,6 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -645,13 +608,27 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen found := false skeyPrefix := SrcEdgePrefix(req.ID) for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - _, _, dst, label, vLabel := SrcEdgeKeyParse(it.Key()) + _, _, dst, label := SrcEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + entry, err := ggraph.bsonkv.PageCache.Get(ctx, dst, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetOutChannel: PageCache.Get( error: %v", err) + continue + } + + vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] + if !ok { + log.Errorf("GetOutChannel: Label not a string %s", vLabel) + continue + } + + //log.Debugln("USING VLABEL: ", vLabel, "LOOKUP: ", dst, "LOAD: ", load) + v := &gdbi.Vertex{ID: dst, Label: vLabel} if load { - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(dst)) + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { - log.Errorf("GetInChannel: GetRow error: %v", err) + log.Errorf("GetOutChannel: GetRow on %s: %s error: %v", vLabel, dst, err) continue } v.Loaded = true @@ -683,7 +660,6 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -691,15 +667,25 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element found := false dkeyPrefix := DstEdgePrefix(req.ID) for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - _, sid, _, label, vLabel := DstEdgeKeyParse(keyValue) + _, sid, _, label := DstEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + entry, err := ggraph.bsonkv.PageCache.Get(ctx, sid, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetInChannel: PageCache.Get( error: %v", err) + continue + } + + vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] + if ! ok { + log.Errorf("GetInChannel Label lookup failed") + continue + } + v := &gdbi.Vertex{ID: sid, Label: vLabel} if load { - - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+vLabel].GetRow([]byte(sid)) + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { - log.Errorf("GetInChannel: GetRow error: %v", err) + log.Errorf("GetInChannel: GetRow on %s: %s error: %v", vLabel, sid, err) continue } v.Loaded = true @@ -730,7 +716,6 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -738,16 +723,21 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El found := false skeyPrefix := SrcEdgePrefix(req.ID) for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - eid, src, dst, label, _ := SrcEdgeKeyParse(it.Key()) + eid, src, dst, label := SrcEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { e := gdbi.Edge{ From: src, To: dst, Label: label, ID: eid, - } + } if load { - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) + entry, err := ggraph.bsonkv.PageCache.Get(ctx, e.ID, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetOutEdgeChannel: PageCache.Get( error: %v", err) + continue + } + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) continue @@ -770,7 +760,6 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El } return nil }) - }() return o } @@ -781,7 +770,6 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele go func() { defer close(o) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error = nil for req := range reqChan { if req.IsSignal() { o <- req @@ -789,7 +777,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele found := false dkeyPrefix := DstEdgePrefix(req.ID) for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - eid, src, dst, label, _ := DstEdgeKeyParse(it.Key()) + eid, src, dst, label := DstEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { e := gdbi.Edge{ ID: eid, @@ -798,13 +786,19 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele Label: label, } if load { - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(e.ID)) + entry, err := ggraph.bsonkv.PageCache.Get(ctx, e.ID, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetInEdgeChannel: PageCache.Get( error: %v", err) + continue + } + //log.Debugln("IN EDGE LABEL: ", e.Label, "ENTRY: ", entry, "ID: ", e.ID) + + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetInEdgeChannel: GetRow error: %v", err) continue } e.Loaded = true - } else { e.Data = map[string]any{} } @@ -832,7 +826,6 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { ekeyPrefix := EdgeKeyPrefix(id) var e *gdbi.Edge err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var err error = nil for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { eid, src, dst, label := EdgeKeyParse(it.Key()) e = &gdbi.Edge{ @@ -841,9 +834,14 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { To: dst, Label: label, } - if loadProp { - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(id)) + entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), e.ID, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetEdge: PageCache.Get( error: %v", err) + continue + } + + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) continue @@ -874,20 +872,24 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g return nil default: } - id := VertexKeyParse(it.Key()) byteLabel, err := it.Value() if err != nil { log.Errorf("GetVertexList it.Value() error: %s", err) } - label := string(byteLabel) v := &gdbi.Vertex{ - ID: id, - Label: label, + ID: VertexKeyParse(it.Key()), + Label: string(byteLabel), } if loadProp { - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+label].GetRow([]byte(v.ID)) + entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), v.ID, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetVertexList: PageCache.Get on %s error: %s",v.ID, err) + continue + } + + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { - log.Errorf("GetVertexList: GetRow error: %v", err) + log.Errorf("GetVertexList: table.GetRow error: %s", err) continue } v.Loaded = true diff --git a/grids/index.go b/grids/index.go index 28302811..8252dbb1 100644 --- a/grids/index.go +++ b/grids/index.go @@ -54,11 +54,13 @@ func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error err := ggraph.bsonkv.Tables[prefix+label].DeleteRow([]byte(id)) if err != nil { - if err == pebble.ErrNotFound{ + if err == pebble.ErrNotFound { log.Debugln("Pebble not Found: %s", err) return nil } return err } + ggraph.bsonkv.PageCache.Invalidate(id) + return nil } diff --git a/grids/keyindex.go b/grids/keyindex.go index 3e58db10..2eae94c0 100644 --- a/grids/keyindex.go +++ b/grids/keyindex.go @@ -70,68 +70,37 @@ func EdgeKeyParse(key []byte) (eid string, sid string, did string, label string) } // SrcEdgeKey creates a src edge index key -func SrcEdgeKey(eid, src, dst, label string, fromLabel []byte) []byte { +func SrcEdgeKey(eid, src, dst, label string) []byte { return bytes.Join([][]byte{ srcEdgePrefix, []byte(src), []byte(dst), []byte(eid), []byte(label), - fromLabel, }, []byte{0}) } -func SrcEdgeKeyParse(key []byte) (eid string, sid string, did string, label string, fromLabel string) { +func SrcEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { tmp := bytes.Split(key, []byte{0}) - return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]), string(tmp[5]) + return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]) } // DstEdgeKey creates a dest edge index key -func DstEdgeKey(eid, src, dst, label string, toLabel []byte) []byte { +func DstEdgeKey(eid, src, dst, label string) []byte { return bytes.Join([][]byte{ dstEdgePrefix, []byte(dst), []byte(src), []byte(eid), []byte(label), - toLabel, - }, []byte{0}) -} - -func DstEdgeKeyParse(key []byte) (eid string, sid string, did string, label string, toLabel string) { - tmp := bytes.Split(key, []byte{0}) - return string(tmp[3]), string(tmp[2]), string(tmp[1]), string(tmp[4]), string(tmp[5]) -} - -func DstEdgeKeyPrefix(eid, sid, did, lbl string) []byte { - return bytes.Join([][]byte{ - srcEdgePrefix, - []byte(did), - []byte(sid), - []byte(eid), - []byte(lbl), }, []byte{0}) } -func DstEdgeKeyPrefixParse(key []byte) (eid string, sid string, did string, label string) { +func DstEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { tmp := bytes.Split(key, []byte{0}) return string(tmp[3]), string(tmp[2]), string(tmp[1]), string(tmp[4]) } -func SrcEdgeKeyPrefix(eid, sid, did, lbl string) []byte { - return bytes.Join([][]byte{ - srcEdgePrefix, - []byte(sid), - []byte(did), - []byte(eid), - []byte(lbl), - }, []byte{0}) -} - -func SrcEdgeKeyPrefixParse(key []byte) (eid string, sid string, did string, label string) { - tmp := bytes.Split(key, []byte{0}) - return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]) -} // VertexListPrefix returns a byte array prefix for all vertices in a graph func VertexListPrefix() []byte { diff --git a/grids/processor.go b/grids/processor.go index fa2c782b..2ab3ab52 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -39,7 +39,7 @@ type lookupVertsHasLabelCondIndexProc struct { } func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor", l.expr, l.labels, len(l.db.bsonkv.Fields)) + log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") var exists = false if len(l.db.bsonkv.Fields) > 0 { for _, label := range l.labels { @@ -62,8 +62,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi continue } for roMaps := range tableFound.Scan(false, &GripQLFilter{Expression: l.expr}) { - log.Debugln("RES RETURNED: ", roMaps.(map[string]any)) - id := roMaps.(map[string]any)["_id"].(string) delete(roMaps.(map[string]any), "_id") v := gdbi.Vertex{ @@ -128,7 +126,7 @@ type lookupVertsCondIndexProc struct { } func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsCondIndexProc custom processor", l.expr.Expression) + log.Debugln("Entering lookupVertsCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) cond := l.expr.GetCondition() var exists = false diff --git a/gripql/python/gripql/graph.py b/gripql/python/gripql/graph.py index e2c8af4e..6f1c8e9d 100644 --- a/gripql/python/gripql/graph.py +++ b/gripql/python/gripql/graph.py @@ -315,7 +315,7 @@ def addEdge(self, src, dst, label, data={}, id=None): } } if id is not None: - payload["id"] = id + payload["edge"]["id"] = id self.elements.append(json.dumps(payload)) def execute(self): diff --git a/test/main_test.go b/test/main_test.go index 2a79d619..a725f5fb 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -78,6 +78,7 @@ func TestMain(m *testing.M) { panic(err) } for e := range edgeChan { + fmt.Printf("Adding edge: %s %#v\n", e.Id, e.Data.AsMap()) edges = append(edges, e) } From 26de51732030df053e9785414c2d2b9ae6597bc6 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 11 Jul 2025 09:39:25 -0700 Subject: [PATCH 21/30] fix grids processor so that indexed query optimizations only happen when appropriate --- conformance/tests/ot_index.py | 32 +++++++++++ go.mod | 7 ++- go.sum | 17 ++++++ grids/graph.go | 104 ++++++++++++++++++++-------------- grids/processor.go | 63 ++++++++++++-------- 5 files changed, 156 insertions(+), 67 deletions(-) diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index 391f6664..46e106f4 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -213,3 +213,35 @@ def test_hasLabel_contains(man): errors.append("Expected 2 results but got %d instead" % (count)) return errors + + +def test_multiple_labels_and_indices(man): + """ In this scenario both Starship:13 and Vehicle:6 and Vehicle:8 have the speed of 1200 + but since only one index has been declared, in the grids driver the general v.has() should be used""" + + errors = [] + G = man.setGraph("swapi") + G.addIndex("Vehicle", "max_atmosphering_speed") + + count = 0 + for i in G.query().V().has(gripql.eq("max_atmosphering_speed", 1200)): + count += 1 + if count != 3: + errors.append("Expected 3 results but got %d instead" % (count)) + + #Every vertex needs to be indexed for this "index" feature to work in grids driver + G.addIndex("Starship", "max_atmosphering_speed") + G.addIndex("Character", "max_atmosphering_speed") + G.addIndex("Planet", "max_atmosphering_speed") + G.addIndex("Species", "max_atmosphering_speed") + G.addIndex("Film", "max_atmosphering_speed") + + # Add the other index to verify that using the other execution pipline path won't change the end result + + count = 0 + for i in G.query().V().has(gripql.eq("max_atmosphering_speed", 1200)): + count += 1 + if count != 3: + errors.append("Expected 3 results but got %d instead" % (count)) + + return errors \ No newline at end of file diff --git a/go.mod b/go.mod index 7a237c7c..3ffc9733 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634 + github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a @@ -64,9 +64,12 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic v1.13.3 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect @@ -130,10 +133,12 @@ require ( github.com/rs/xid v1.5.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/term v0.29.0 // indirect diff --git a/go.sum b/go.sum index 2430978d..e8c2817c 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/bmeg/benchtop v0.0.0-20250707180338-29cf9de6b32c h1:QpwgPbmtPO30Hn96d github.com/bmeg/benchtop v0.0.0-20250707180338-29cf9de6b32c/go.mod h1:2JT0auT+55LAADY/elY8HMK7iLPdUVOIBxqfCpbM5eE= github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634 h1:LHZ5E+VRqdONk6RCXmyre67fxLM8ZXJH3uBZDN6fuL8= github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634/go.mod h1:2JT0auT+55LAADY/elY8HMK7iLPdUVOIBxqfCpbM5eE= +github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723 h1:dYm8LB1QiXnZSJmVSYBnLiopT7KdnC/598alPETxHAo= +github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723/go.mod h1:+BnQ17bYysNdHqYBI/5HZczidsvz3pvnRcsK8fJR61w= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= @@ -47,6 +49,11 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= +github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/casbin/casbin/v2 v2.97.0 h1:FFHIzY+6fLIcoAB/DKcG5xvscUo9XqRpBniRYhlPWkg= github.com/casbin/casbin/v2 v2.97.0/go.mod h1:jX8uoN4veP85O/n2674r2qtfSXI6myvxW85f6TH50fw= github.com/casbin/govaluate v1.1.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -56,6 +63,9 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= @@ -228,10 +238,12 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042 h1:Vzdm5hdlLdpJOKK+hKtkV5u7xGZmNW6aUBjGcTfwx84= github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042/go.mod h1:fYE0718xXI13XMYLc6iHtvXudfyCGMsZ9hxSM1Ommpg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -370,6 +382,8 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -395,6 +409,8 @@ go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiy go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -526,6 +542,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/grids/graph.go b/grids/graph.go index e04c2f76..b5c86ac5 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -595,55 +595,39 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element return out } +type lookup struct { + req gdbi.ElementLookup + key string +} + // GetOutChannel process requests of vertex ids and find the connected vertices on outgoing edges func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - o := make(chan gdbi.ElementLookup, 100) + // Todo: implement bulk cache get + bulk get row to try to make this faster + lookupChan := make(chan lookup, 1000) go func() { - defer close(o) + defer close(lookupChan) ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { - o <- req + lookupChan <- lookup{req: req} } else { found := false skeyPrefix := SrcEdgePrefix(req.ID) for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { _, _, dst, label := SrcEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { - entry, err := ggraph.bsonkv.PageCache.Get(ctx, dst, ggraph.bsonkv.PageLoader) - if err != nil { - log.Errorf("GetOutChannel: PageCache.Get( error: %v", err) - continue - } - - vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] - if !ok { - log.Errorf("GetOutChannel: Label not a string %s", vLabel) - continue - } - - //log.Debugln("USING VLABEL: ", vLabel, "LOOKUP: ", dst, "LOAD: ", load) - - v := &gdbi.Vertex{ID: dst, Label: vLabel} - if load { - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) - if err != nil { - log.Errorf("GetOutChannel: GetRow on %s: %s error: %v", vLabel, dst, err) - continue - } - v.Loaded = true - } else { - v.Data = map[string]any{} + lookupChan <- lookup{ + key: dst, + req: req, } - req.Vertex = v - o <- req found = true } } - if !found && emitNull { - req.Vertex = nil - o <- req + lookupChan <- lookup{ + req: req, + key: "", + } } } } @@ -651,6 +635,42 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen }) }() + o := make(chan gdbi.ElementLookup, 100) + go func() { + defer close(o) + for req := range lookupChan { + if req.req.IsSignal() { + o <- req.req + } else { + if req.key != "" { + entry, err := ggraph.bsonkv.PageCache.Get(ctx, req.key, ggraph.bsonkv.PageLoader) + if err != nil { + log.Errorf("GetOutChannel: PageCache.Get( error: %v", err) + continue + } + vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] + if !ok { + log.Errorf("GetOutChannel: Label not a string %s", vLabel) + continue + } + v := &gdbi.Vertex{ID: req.key, Label: vLabel} + if load { + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + if err != nil { + log.Errorf("GetOutChannel: GetRow on %s: %s error: %v", vLabel, req.key, err) + continue + } + v.Loaded = true + } + req.req.Vertex = v + o <- req.req + } else { + req.req.Vertex = nil + o <- req.req + } + } + } + }() return o } @@ -674,13 +694,13 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element log.Errorf("GetInChannel: PageCache.Get( error: %v", err) continue } - + vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] - if ! ok { + if !ok { log.Errorf("GetInChannel Label lookup failed") continue } - + v := &gdbi.Vertex{ID: sid, Label: vLabel} if load { v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) @@ -730,14 +750,14 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El To: dst, Label: label, ID: eid, - } + } if load { entry, err := ggraph.bsonkv.PageCache.Get(ctx, e.ID, ggraph.bsonkv.PageLoader) if err != nil { log.Errorf("GetOutEdgeChannel: PageCache.Get( error: %v", err) continue } - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) continue @@ -840,7 +860,7 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { log.Errorf("GetEdge: PageCache.Get( error: %v", err) continue } - + e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) @@ -883,10 +903,10 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g if loadProp { entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), v.ID, ggraph.bsonkv.PageLoader) if err != nil { - log.Errorf("GetVertexList: PageCache.Get on %s error: %s",v.ID, err) + log.Errorf("GetVertexList: PageCache.Get on %s error: %s", v.ID, err) continue } - + v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetVertexList: table.GetRow error: %s", err) @@ -907,7 +927,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g // ListVertexLabels returns a list of vertex types in the graph func (ggraph *Graph) ListVertexLabels() ([]string, error) { labels := []string{} - for i := range ggraph.bsonkv.GetLabels(false) { + for i := range ggraph.bsonkv.GetLabels(false, true) { labels = append(labels, i) } return labels, nil @@ -916,7 +936,7 @@ func (ggraph *Graph) ListVertexLabels() ([]string, error) { // ListEdgeLabels returns a list of edge types in the graph func (ggraph *Graph) ListEdgeLabels() ([]string, error) { labels := []string{} - for i := range ggraph.bsonkv.GetLabels(true) { + for i := range ggraph.bsonkv.GetLabels(true, true) { labels = append(labels, i) } return labels, nil diff --git a/grids/processor.go b/grids/processor.go index 2ab3ab52..33c5c464 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -40,15 +40,18 @@ type lookupVertsHasLabelCondIndexProc struct { func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") - var exists = false + var exists = true + // Here if one of l.labels doesn't exist then not going to be querying all the data so leave it like this. if len(l.db.bsonkv.Fields) > 0 { for _, label := range l.labels { log.Debugln("Checking indexed fields ", l.db.bsonkv.Fields, "LABEL: ", label) _, exists = l.db.bsonkv.Fields[label] - if exists { + if !exists { break } } + }else { + exists = false } if !exists || (l.expr == nil && l.expr.GetCondition() == nil) { @@ -129,35 +132,32 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager log.Debugln("Entering lookupVertsCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) cond := l.expr.GetCondition() - var exists = false + var allMatch = true + // Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter + // otherwise this lookup will not fetch everything that was asked for if len(l.db.bsonkv.Fields) > 0 { - _, exists = l.db.bsonkv.Fields[VTABLE_PREFIX+cond.Key] + for lbl := range l.db.bsonkv.GetLabels(false, false){ + if val, exists := l.db.bsonkv.Fields[lbl]; exists{ + if _, ok := val[cond.Key]; !ok{ + allMatch = false + break + } + }else { + allMatch = false + break + } + } + } else { + allMatch = false } + /* Optimized indexing only works for Simple filters. / / If compound filter or index doesn't exist use backup method */ - log.Debugln("COND: ", cond, cond == nil, l.db.bsonkv.Fields) - - if cond == nil || !exists { - log.Debugf("lookupVertsCondIndexProc: falling back to GetVertexList since filter is not basic Condition filter or not indexed") - go func() { - defer close(queryChan) - for t := range in { - for v := range l.db.GetVertexList(ctx, true) { - if MatchesHasExpression( - AddSpecialFields(v), - l.expr, - ) { - queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} - } - - } - } - }() - } else { + if cond != nil && allMatch { + log.Debugln("Chose index optimized V().Has() statement path") go func() { defer close(queryChan) for t := range in { - cond := l.expr.GetCondition() for id := range l.db.bsonkv.RowIdsByHas( cond.Key, cond.Value.AsInterface(), @@ -168,7 +168,22 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager Ref: t, } } + } + }() + } else { + log.Debugf("Base case GetVertexList is used. No indexing") + go func() { + defer close(queryChan) + for t := range in { + for v := range l.db.GetVertexList(ctx, true) { + if MatchesHasExpression( + AddSpecialFields(v), + l.expr, + ) { + queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} + } + } } }() } From 0e2524ceefb19d74e8c1cad51200fe5807420b49 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 11 Jul 2025 10:25:55 -0700 Subject: [PATCH 22/30] add unique mongo index name --- go.sum | 4 ---- mongo/index.go | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.sum b/go.sum index e8c2817c..8e1385fa 100644 --- a/go.sum +++ b/go.sum @@ -33,10 +33,6 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250707180338-29cf9de6b32c h1:QpwgPbmtPO30Hn96dDe9NwAj5v2wTxQkcHn7jBpVQ/g= -github.com/bmeg/benchtop v0.0.0-20250707180338-29cf9de6b32c/go.mod h1:2JT0auT+55LAADY/elY8HMK7iLPdUVOIBxqfCpbM5eE= -github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634 h1:LHZ5E+VRqdONk6RCXmyre67fxLM8ZXJH3uBZDN6fuL8= -github.com/bmeg/benchtop v0.0.0-20250707191927-a1cc7a765634/go.mod h1:2JT0auT+55LAADY/elY8HMK7iLPdUVOIBxqfCpbM5eE= github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723 h1:dYm8LB1QiXnZSJmVSYBnLiopT7KdnC/598alPETxHAo= github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723/go.mod h1:+BnQ17bYysNdHqYBI/5HZczidsvz3pvnRcsK8fJR61w= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= diff --git a/mongo/index.go b/mongo/index.go index b81a952e..d06b4e8e 100644 --- a/mongo/index.go +++ b/mongo/index.go @@ -21,7 +21,8 @@ func (mg *Graph) AddVertexIndex(label string, field string) error { field = strings.TrimPrefix(field, "$.") idx := mg.ar.VertexCollection(mg.graph).Indexes() - + indexName := fmt.Sprintf("label_%s_%s_idx", label, field) + // Create a compound index on _label and the specified field, filtered by the specific label _, err := idx.CreateOne( context.Background(), @@ -31,6 +32,7 @@ func (mg *Graph) AddVertexIndex(label string, field string) error { {Key: field, Value: 1}, }, Options: options.Index(). + SetName(indexName). SetUnique(false). SetBackground(true). SetPartialFilterExpression(bson.M{FIELD_LABEL: label}), From c1892d4f63b143baf1adbde1cfd9f218b909a9ac Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 14 Jul 2025 10:39:25 -0700 Subject: [PATCH 23/30] update grids driver with various bug fixes --- gdbi/pipeline.go | 1 + go.mod | 4 +-- go.sum | 4 +-- grids/filters.go | 46 +++++++++++++++++++----- grids/graph.go | 2 ++ grids/processor.go | 73 ++++++++++++++++++++++----------------- gripql/inspect/inspect.go | 8 +++++ 7 files changed, 94 insertions(+), 44 deletions(-) diff --git a/gdbi/pipeline.go b/gdbi/pipeline.go index ee406d2b..2a8ef056 100644 --- a/gdbi/pipeline.go +++ b/gdbi/pipeline.go @@ -9,6 +9,7 @@ import ( type PipelineState interface { GetLastType() DataType SetLastType(DataType) + StepLoadData() bool } type CustomProcGen interface { diff --git a/go.mod b/go.mod index 3ffc9733..89c5e64e 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,12 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723 + github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a github.com/boltdb/bolt v1.3.1 + github.com/bytedance/sonic v1.13.3 github.com/casbin/casbin/v2 v2.97.0 github.com/cockroachdb/pebble v1.1.2 github.com/davecgh/go-spew v1.1.1 @@ -64,7 +65,6 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.13.3 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect diff --git a/go.sum b/go.sum index 8e1385fa..3f7a6e16 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723 h1:dYm8LB1QiXnZSJmVSYBnLiopT7KdnC/598alPETxHAo= -github.com/bmeg/benchtop v0.0.0-20250711163045-6fa19d883723/go.mod h1:+BnQ17bYysNdHqYBI/5HZczidsvz3pvnRcsK8fJR61w= +github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3 h1:GmLf1/a53EdnKTuoFyVy31EULj19DWlBNldwbzuOr9E= +github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3/go.mod h1:+BnQ17bYysNdHqYBI/5HZczidsvz3pvnRcsK8fJR61w= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/filters.go b/grids/filters.go index 5c5cd6a0..d6b84d9b 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -6,16 +6,14 @@ import ( "github.com/bmeg/benchtop/bsontable/filters" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" + "github.com/bytedance/sonic" + "github.com/bytedance/sonic/ast" ) type GripQLFilter struct { Expression *gripql.HasExpression } -// This method makes GripQLFilter satisfy the bsontable.RowFilter interface. -func (f *GripQLFilter) Matches(row map[string]any) bool { - return MatchesHasExpression(row, f.Expression) -} func (f *GripQLFilter) RequiredFields() []string { return extractKeys(f.Expression) @@ -26,6 +24,11 @@ func (f *GripQLFilter) IsNoOp() bool { return f.Expression == nil } +func (f *GripQLFilter) Matches(row any) bool { + return MatchesHasExpression(row, f.Expression) +} + + func extractKeys(expr *gripql.HasExpression) []string { keys := map[string]struct{}{} @@ -58,21 +61,48 @@ func extractKeys(expr *gripql.HasExpression) []string { } return out } - func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { - switch stmt.Expression.(type) { - case *gripql.HasExpression_Condition: cond := stmt.GetCondition() + var lookupVal any + + // Handle lookup based on input type + switch v := val.(type) { + case map[string]any: + lookupVal = bsontable.PathLookup(v, cond.Key) + case []byte: + pathArr, err := bsontable.ConvertJSONPathToArray(cond.Key) + if err != nil { + log.Errorf("Error converting JSON path: %v", err) + return false + } + node, err := sonic.Get(v, pathArr...) + if err != nil { + if err != ast.ErrNotExist{ + log.Errorf("Sonic Fetch err for path: %s on doc %#v: %v", pathArr, string(v), err) + } + return false + } + lookupVal, err = node.Interface() + if err != nil { + log.Errorf("Error unmarshaling node: %v", err) + return false + } + default: + log.Errorf("Unsupported input type: %T", val) + return false + } + return filters.ApplyFilterCondition( - bsontable.PathLookup(val.(map[string]any), cond.Key), + lookupVal, &benchtop.FieldFilter{ Operator: MapConditionToOperator(cond.Condition), Field: cond.Key, Value: cond.Value.AsInterface(), }, ) + case *gripql.HasExpression_And: and := stmt.GetAnd() andRes := []bool{} diff --git a/grids/graph.go b/grids/graph.go index b5c86ac5..0918cd1e 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -661,6 +661,8 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen continue } v.Loaded = true + }else { + v.Data = map[string]any{} } req.req.Vertex = v o <- req.req diff --git a/grids/processor.go b/grids/processor.go index 33c5c464..4b8ae93b 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -12,8 +12,9 @@ import ( // LookupVertexHasLabelCondIndex look up vertices has label type lookupVertsHasLabelCondIndexStep struct { - labels []string - expr *gripql.HasExpression + labels []string + expr *gripql.HasExpression + loadData bool } func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { @@ -22,7 +23,7 @@ func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, p db: graph, expr: t.expr, labels: t.labels, - loadData: true, + loadData: ps.StepLoadData(), }, nil } @@ -39,22 +40,27 @@ type lookupVertsHasLabelCondIndexProc struct { } func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor") + log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor", l.loadData) var exists = true // Here if one of l.labels doesn't exist then not going to be querying all the data so leave it like this. - if len(l.db.bsonkv.Fields) > 0 { - for _, label := range l.labels { - log.Debugln("Checking indexed fields ", l.db.bsonkv.Fields, "LABEL: ", label) - _, exists = l.db.bsonkv.Fields[label] - if !exists { + cond := l.expr.GetCondition() + exists = len(l.db.bsonkv.Fields) > 0 && cond != nil + if exists { + for _, iterLabel := range l.labels { + label, ok := l.db.bsonkv.Fields[iterLabel] + if !ok { + exists = false + break + } + _, exists = label[cond.Key] + if !exists{ break } } - }else { - exists = false } - if !exists || (l.expr == nil && l.expr.GetCondition() == nil) { + count :=0 + if !exists || (l.expr == nil && cond == nil) { go func() { defer close(out) for t := range in { @@ -64,15 +70,20 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Debugf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } - for roMaps := range tableFound.Scan(false, &GripQLFilter{Expression: l.expr}) { - id := roMaps.(map[string]any)["_id"].(string) - delete(roMaps.(map[string]any), "_id") + for roMaps := range tableFound.Scan(l.loadData, &GripQLFilter{Expression: l.expr}) { v := gdbi.Vertex{ - ID: id, Label: label[2:], - Data: roMaps.(map[string]any), - Loaded: true, + Loaded: l.loadData, } + if l.loadData { + v.ID = roMaps.(map[string]any)["_id"].(string) + delete(roMaps.(map[string]any), "_id") + v.Data = roMaps.(map[string]any) + } else { + v.ID = roMaps.(string) + v.Data = map[string]any{} + } + count += 1 out <- t.AddCurrent(v.Copy()) } } @@ -114,7 +125,7 @@ func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.P return &lookupVertsCondIndexProc{ db: graph, expr: t.expr, - loadData: true}, nil + loadData: ps.StepLoadData()}, nil } func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { @@ -132,25 +143,23 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager log.Debugln("Entering lookupVertsCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) cond := l.expr.GetCondition() - var allMatch = true - // Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter - // otherwise this lookup will not fetch everything that was asked for - if len(l.db.bsonkv.Fields) > 0 { - for lbl := range l.db.bsonkv.GetLabels(false, false){ - if val, exists := l.db.bsonkv.Fields[lbl]; exists{ - if _, ok := val[cond.Key]; !ok{ + + /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter + otherwise this lookup will not fetch everything that was asked for */ + allMatch := len(l.db.bsonkv.Fields) > 0 && cond != nil + if allMatch { + for lbl := range l.db.bsonkv.GetLabels(false, false) { + if val, exists := l.db.bsonkv.Fields[lbl]; exists { + if _, ok := val[cond.Key]; !ok { allMatch = false break - } - }else { + } + } else { allMatch = false break } } - } else { - allMatch = false - } - + } /* Optimized indexing only works for Simple filters. / / If compound filter or index doesn't exist use backup method */ if cond != nil && allMatch { diff --git a/gripql/inspect/inspect.go b/gripql/inspect/inspect.go index be1a5831..5dcc1938 100644 --- a/gripql/inspect/inspect.go +++ b/gripql/inspect/inspect.go @@ -148,6 +148,14 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st out[steps[i]] = []string{"*"} } onLast = false + + case *gripql.GraphStatement_Sort, *gripql.GraphStatement_Totype, + *gripql.GraphStatement_Unwind, *gripql.GraphStatement_Aggregate: + if onLast { + out[steps[i]] = []string{"*"} + } + onLast = false + case *gripql.GraphStatement_LookupVertsLabelIndex: if onLast { out[steps[i]] = []string{"*"} From 900bd8c9c2989e7c93b5531051a38cbd7ea10f56 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 18 Aug 2025 09:11:49 -0700 Subject: [PATCH 24/30] add grids disclaimer message --- grids/graphdb.go | 2 ++ log/logger.go | 45 +++++++++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/grids/graphdb.go b/grids/graphdb.go index 2a6aa35f..14b7a637 100644 --- a/grids/graphdb.go +++ b/grids/graphdb.go @@ -8,6 +8,7 @@ import ( "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" ) // GridsGDB implements the GripInterface using a generic key/value storage driver @@ -18,6 +19,7 @@ type GDB struct { // NewKVGraphDB intitalize a new grids graph driver func NewGraphDB(baseDir string) (gdbi.GraphDB, error) { + log.Redf("Disclaimer: the Grids driver is an experimental database driver. Use with caution.") _, err := os.Stat(baseDir) if os.IsNotExist(err) { os.Mkdir(baseDir, 0700) diff --git a/log/logger.go b/log/logger.go index 9bb84edf..a316bf8d 100644 --- a/log/logger.go +++ b/log/logger.go @@ -11,13 +11,13 @@ import ( "strings" "time" + "golang.org/x/term" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "github.com/kr/pretty" "github.com/logrusorgru/aurora" "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh/terminal" ) var PanicLevel = logrus.PanicLevel @@ -110,7 +110,7 @@ type textFormatter struct { func checkIfTerminal(w io.Writer) bool { switch v := w.(type) { case *os.File: - return terminal.IsTerminal(int(v.Fd())) + return term.IsTerminal(int(v.Fd())) default: return false } @@ -141,7 +141,7 @@ func (f *textFormatter) Format(entry *logrus.Entry) ([]byte, error) { case logrus.DebugLevel: levelColor = aurora.MagentaFg case logrus.WarnLevel: - levelColor = aurora.BrownFg + levelColor = aurora.YellowFg case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel: levelColor = aurora.RedFg default: @@ -276,67 +276,67 @@ func ConfigureLogger(conf Logger) { } // Debug log message -func Debug(args ...interface{}) { +func Debug(args ...any) { logger.Debug(args...) } // Debugln log message -func Debugln(args ...interface{}) { +func Debugln(args ...any) { logger.Debugln(args...) } // Debugf log message -func Debugf(format string, args ...interface{}) { +func Debugf(format string, args ...any) { logger.Debugf(format, args...) } // Info log message -func Info(args ...interface{}) { +func Info(args ...any) { logger.Info(args...) } // Infoln log message -func Infoln(args ...interface{}) { +func Infoln(args ...any) { logger.Infoln(args...) } // Infof log message -func Infof(format string, args ...interface{}) { +func Infof(format string, args ...any) { logger.Infof(format, args...) } // Warning log message -func Warning(args ...interface{}) { +func Warning(args ...any) { logger.Warning(args...) } // Warningln log message -func Warningln(args ...interface{}) { +func Warningln(args ...any) { logger.Warningln(fmt.Sprint(args...)) } // Warningf log message -func Warningf(format string, args ...interface{}) { +func Warningf(format string, args ...any) { logger.Warningf(format, args...) } // Error log message -func Error(args ...interface{}) { +func Error(args ...any) { logger.Error(args...) } // Errorln log message -func Errorln(args ...interface{}) { +func Errorln(args ...any) { logger.Errorln(args...) } // Errorf log message -func Errorf(format string, args ...interface{}) { +func Errorf(format string, args ...any) { logger.Errorf(format, args...) } // WithField creates an entry from the standard logger and adds a field to it. -func WithField(key string, value interface{}) *Entry { +func WithField(key string, value any) *Entry { return logger.WithField(key, value) } @@ -358,3 +358,16 @@ func GetLogger() *logrus.Logger { func Sub(ns string) *Entry { return logger.WithFields(Fields{"namespace": ns}) } + +func Redf(format string, args ...any) { + if tf, ok := logger.Formatter.(*textFormatter); ok { + isColored := (tf.ForceColors || isColorTerminal(logger.Out)) && !tf.DisableColors + if isColored { + msg := fmt.Sprintf(format, args...) + redMsg := aurora.Red(msg).String() + fmt.Fprintln(logger.Out, redMsg) + return + } + } + logger.Errorf(format, args...) +} From 31b52c37452030ba0bf36e28d9d0cb03464f78a8 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 18 Aug 2025 12:44:27 -0700 Subject: [PATCH 25/30] filter logic updates --- go.mod | 24 +++++---- go.sum | 125 +++++++++++++++++++++++++++++++++------------ grids/filters.go | 53 ++++--------------- grids/processor.go | 16 +++--- 4 files changed, 124 insertions(+), 94 deletions(-) diff --git a/go.mod b/go.mod index 89c5e64e..a7baed6e 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,15 @@ go 1.24 toolchain go1.24.2 +replace github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3 => ../benchtop + require ( github.com/IBM/sarama v1.45.1 github.com/Shopify/sarama v1.38.1 github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3 + github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a @@ -23,7 +25,7 @@ require ( github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5 github.com/felixge/httpsnoop v1.0.4 github.com/go-sql-driver/mysql v1.8.1 - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.6.1 @@ -48,12 +50,12 @@ require ( github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.0 go.mongodb.org/mongo-driver v1.17.0 - golang.org/x/crypto v0.33.0 - golang.org/x/net v0.35.0 - golang.org/x/sync v0.11.0 - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb - google.golang.org/grpc v1.70.0 - google.golang.org/protobuf v1.36.5 + golang.org/x/net v0.37.0 + golang.org/x/sync v0.12.0 + golang.org/x/term v0.30.0 + google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a + google.golang.org/grpc v1.71.0 + google.golang.org/protobuf v1.36.7 sigs.k8s.io/yaml v1.4.0 ) @@ -139,12 +141,12 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/text v0.23.0 // indirect gonum.org/v1/gonum v0.8.2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 3f7a6e16..35671dd2 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= @@ -31,10 +32,11 @@ github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YT github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3 h1:GmLf1/a53EdnKTuoFyVy31EULj19DWlBNldwbzuOr9E= -github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3/go.mod h1:+BnQ17bYysNdHqYBI/5HZczidsvz3pvnRcsK8fJR61w= +github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36 h1:frNXEP0pgmYHu7WDgaXLrsf/pB4hv1SMT6SMK+bVk1g= +github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= @@ -55,13 +57,16 @@ github.com/casbin/casbin/v2 v2.97.0/go.mod h1:jX8uoN4veP85O/n2674r2qtfSXI6myvxW8 github.com/casbin/govaluate v1.1.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.2.0 h1:wXCXFmqyY+1RwiKfYo3jMKyrtZmOL3kHwaqDyCPOYak= github.com/casbin/govaluate v1.2.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= @@ -110,6 +115,10 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= @@ -130,6 +139,8 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -138,6 +149,7 @@ github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TC github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -145,10 +157,14 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -176,8 +192,8 @@ github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25d github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -240,6 +256,7 @@ github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZY github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042 h1:Vzdm5hdlLdpJOKK+hKtkV5u7xGZmNW6aUBjGcTfwx84= github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042/go.mod h1:fYE0718xXI13XMYLc6iHtvXudfyCGMsZ9hxSM1Ommpg= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -299,6 +316,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -317,6 +335,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= @@ -341,6 +360,7 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPO github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= @@ -363,9 +383,11 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -395,16 +417,22 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -413,18 +441,26 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -436,19 +472,23 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -462,6 +502,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -476,20 +517,25 @@ golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -506,21 +552,32 @@ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a h1:DMCgtIAIQGZqJXMVzJF4MV8BlWoJh2ZuFiRdAleyr58= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a/go.mod h1:y2yVLIE/CSMCPXaHnSKXxu1spLPnglFLegmgdY23uuE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -532,12 +589,16 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/grids/filters.go b/grids/filters.go index d6b84d9b..f26c9be7 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -1,9 +1,8 @@ package grids import ( - "github.com/bmeg/benchtop" "github.com/bmeg/benchtop/bsontable" - "github.com/bmeg/benchtop/bsontable/filters" + bFilters "github.com/bmeg/benchtop/filters" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/bytedance/sonic" @@ -14,11 +13,9 @@ type GripQLFilter struct { Expression *gripql.HasExpression } - -func (f *GripQLFilter) RequiredFields() []string { - return extractKeys(f.Expression) +func (f *GripQLFilter) GetFilter() any { + return f.Expression } - func (f *GripQLFilter) IsNoOp() bool { // A GripQLFilter is a no-op if its Expression is nil return f.Expression == nil @@ -28,6 +25,9 @@ func (f *GripQLFilter) Matches(row any) bool { return MatchesHasExpression(row, f.Expression) } +func (f *GripQLFilter) RequiredFields() []string { + return extractKeys(f.Expression) +} func extractKeys(expr *gripql.HasExpression) []string { keys := map[string]struct{}{} @@ -79,7 +79,7 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { } node, err := sonic.Get(v, pathArr...) if err != nil { - if err != ast.ErrNotExist{ + if err != ast.ErrNotExist { log.Errorf("Sonic Fetch err for path: %s on doc %#v: %v", pathArr, string(v), err) } return false @@ -94,10 +94,10 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { return false } - return filters.ApplyFilterCondition( + return bFilters.ApplyFilterCondition( lookupVal, - &benchtop.FieldFilter{ - Operator: MapConditionToOperator(cond.Condition), + &bFilters.FieldFilter{ + Operator: cond.Condition, Field: cond.Key, Value: cond.Value.AsInterface(), }, @@ -138,36 +138,3 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { return false } } - -func MapConditionToOperator(condition gripql.Condition) benchtop.OperatorType { - switch condition { - case gripql.Condition_EQ: - return benchtop.OP_EQ - case gripql.Condition_NEQ: - return benchtop.OP_NEQ - case gripql.Condition_GT: - return benchtop.OP_GT - case gripql.Condition_GTE: - return benchtop.OP_GTE - case gripql.Condition_LT: - return benchtop.OP_LT - case gripql.Condition_LTE: - return benchtop.OP_LTE - case gripql.Condition_INSIDE: - return benchtop.OP_INSIDE - case gripql.Condition_OUTSIDE: - return benchtop.OP_OUTSIDE - case gripql.Condition_BETWEEN: - return benchtop.OP_BETWEEN - case gripql.Condition_WITHIN: - return benchtop.OP_WITHIN - case gripql.Condition_WITHOUT: - return benchtop.OP_WITHOUT - case gripql.Condition_CONTAINS: - return benchtop.OP_CONTAINS - default: - // For Condition_UNKNOWN_CONDITION or any other unmapped value, - // return an empty string or a specific "UNKNOWN" operator type if preferred. - return "" - } -} diff --git a/grids/processor.go b/grids/processor.go index 4b8ae93b..bc8e5b73 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -43,7 +43,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor", l.loadData) var exists = true // Here if one of l.labels doesn't exist then not going to be querying all the data so leave it like this. - cond := l.expr.GetCondition() + cond := l.expr.GetCondition() exists = len(l.db.bsonkv.Fields) > 0 && cond != nil if exists { for _, iterLabel := range l.labels { @@ -53,13 +53,13 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi break } _, exists = label[cond.Key] - if !exists{ + if !exists { break } } } - count :=0 + count := 0 if !exists || (l.expr == nil && cond == nil) { go func() { defer close(out) @@ -96,7 +96,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi for t := range in { cond := l.expr.GetCondition() for _, label := range l.labels { - for id := range l.db.bsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), MapConditionToOperator(cond.Condition)) { + for id := range l.db.bsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), cond.Condition) { queryChan <- gdbi.ElementLookup{ID: id, Ref: t} } } @@ -143,9 +143,9 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager log.Debugln("Entering lookupVertsCondIndexProc custom processor") queryChan := make(chan gdbi.ElementLookup, 100) cond := l.expr.GetCondition() - + /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter - otherwise this lookup will not fetch everything that was asked for */ + otherwise this lookup will not fetch everything that was asked for */ allMatch := len(l.db.bsonkv.Fields) > 0 && cond != nil if allMatch { for lbl := range l.db.bsonkv.GetLabels(false, false) { @@ -159,7 +159,7 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager break } } - } + } /* Optimized indexing only works for Simple filters. / / If compound filter or index doesn't exist use backup method */ if cond != nil && allMatch { @@ -170,7 +170,7 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager for id := range l.db.bsonkv.RowIdsByHas( cond.Key, cond.Value.AsInterface(), - MapConditionToOperator(cond.Condition), + cond.Condition, ) { queryChan <- gdbi.ElementLookup{ ID: id, From 7835cf41771c5a61def1fa31e9712fa08e119936 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 18 Aug 2025 12:54:23 -0700 Subject: [PATCH 26/30] upgrade deps --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a7baed6e..bcb6b86b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36 + github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 35671dd2..88567178 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36 h1:frNXEP0pgmYHu7WDgaXLrsf/pB4hv1SMT6SMK+bVk1g= github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= +github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311 h1:oEc44MjeTPX8DArCOrg/DBQDtT7xrfhz6q8ScfYmJj8= +github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= From 95a786138218c2b6062dcab6d81bfc2094da19e7 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Thu, 21 Aug 2025 10:41:52 -0700 Subject: [PATCH 27/30] Dealing with module name change in benchtop --- go.mod | 2 +- go.sum | 2 + grids/filters.go | 6 +- grids/graph.go | 148 ++++++++++++++++++++++----------------------- grids/index.go | 16 ++--- grids/new.go | 30 ++++----- grids/processor.go | 16 ++--- grids/schema.go | 2 +- 8 files changed, 112 insertions(+), 110 deletions(-) diff --git a/go.mod b/go.mod index bcb6b86b..1857eab2 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311 + github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 88567178..3bb5dbac 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36 h1:frNXEP0pgmYHu7WDg github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311 h1:oEc44MjeTPX8DArCOrg/DBQDtT7xrfhz6q8ScfYmJj8= github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= +github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb h1:k3md0eKLb80Ve/fYuNh8komOw9qD00NVho+n/G9ZQus= +github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/filters.go b/grids/filters.go index f26c9be7..b0e620ad 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -1,8 +1,8 @@ package grids import ( - "github.com/bmeg/benchtop/bsontable" bFilters "github.com/bmeg/benchtop/filters" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/bytedance/sonic" @@ -70,9 +70,9 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { // Handle lookup based on input type switch v := val.(type) { case map[string]any: - lookupVal = bsontable.PathLookup(v, cond.Key) + lookupVal = jsontable.PathLookup(v, cond.Key) case []byte: - pathArr, err := bsontable.ConvertJSONPathToArray(cond.Key) + pathArr, err := jsontable.ConvertJSONPathToArray(cond.Key) if err != nil { log.Errorf("Error converting JSON path: %v", err) return false diff --git a/grids/graph.go b/grids/graph.go index 0918cd1e..44190f7c 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -7,7 +7,7 @@ import ( "sync" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/pebblebulk" "github.com/bmeg/grip/engine/core" "github.com/bmeg/grip/gdbi" @@ -38,19 +38,19 @@ func insertVertex(tx *pebblebulk.PebbleBulk, vertex *gdbi.Vertex) error { func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) error { vertexLabel := VTABLE_PREFIX + vertex.Label - ggraph.bsonkv.Lock.Lock() - table, ok := ggraph.bsonkv.Tables[vertexLabel] - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table, ok := ggraph.jsonkv.Tables[vertexLabel] + ggraph.jsonkv.Lock.Unlock() if !ok { log.Debugf("Creating new table %s for label %s on graph %s", vertexLabel, vertex.Label, ggraph.graphID) - newTable, err := ggraph.bsonkv.New(vertexLabel, nil) + newTable, err := ggraph.jsonkv.New(vertexLabel, nil) if err != nil { return fmt.Errorf("indexVertex: %s", err) } - ggraph.bsonkv.Lock.Lock() - table = newTable.(*bsontable.BSONTable) - ggraph.bsonkv.Tables[vertexLabel] = table - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table = newTable.(*jsontable.JSONTable) + ggraph.jsonkv.Tables[vertexLabel] = table + ggraph.jsonkv.Lock.Unlock() } rowLoc, err := table.AddRow( @@ -65,17 +65,17 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) } table.AddTableEntryInfo(tx, []byte(vertex.ID), *rowLoc) - _, ok = ggraph.bsonkv.PageCache.Set(vertex.ID, *rowLoc) + _, ok = ggraph.jsonkv.PageCache.Set(vertex.ID, *rowLoc) if !ok { - ggraph.bsonkv.PageCache.Invalidate(vertex.ID) - ggraph.bsonkv.PageCache.Set(vertex.ID, *rowLoc) + ggraph.jsonkv.PageCache.Invalidate(vertex.ID) + ggraph.jsonkv.PageCache.Set(vertex.ID, *rowLoc) //log.Debugln("Replaced vals: ", vertex.ID, oldVal, newVal) } - _, fieldsExist := ggraph.bsonkv.Fields[vertexLabel] + _, fieldsExist := ggraph.jsonkv.Fields[vertexLabel] if fieldsExist { - for field := range ggraph.bsonkv.Fields[vertexLabel] { - if val := bsontable.PathLookup(vertex.Data, field); val != nil { + for field := range ggraph.jsonkv.Fields[vertexLabel] { + if val := jsontable.PathLookup(vertex.Data, field); val != nil { tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) } } @@ -119,20 +119,20 @@ func insertEdge(tx *pebblebulk.PebbleBulk, edge *gdbi.Edge) error { func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error { edgeLabel := ETABLE_PREFIX + edge.Label - ggraph.bsonkv.Lock.Lock() - table, ok := ggraph.bsonkv.Tables[edgeLabel] - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table, ok := ggraph.jsonkv.Tables[edgeLabel] + ggraph.jsonkv.Lock.Unlock() if !ok { log.Debugf("Creating new table %s for label %s on graph %s", edgeLabel, edge.Label, ggraph.graphID) - newTable, err := ggraph.bsonkv.New(edgeLabel, nil) + newTable, err := ggraph.jsonkv.New(edgeLabel, nil) if err != nil { return fmt.Errorf("indexEdge: bsonkv.New: %s", err) } - ggraph.bsonkv.Lock.Lock() - table = newTable.(*bsontable.BSONTable) - ggraph.bsonkv.Tables[edgeLabel] = table - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table = newTable.(*jsontable.JSONTable) + ggraph.jsonkv.Tables[edgeLabel] = table + ggraph.jsonkv.Lock.Unlock() } rowLoc, err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}) if err != nil { @@ -140,16 +140,16 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error } table.AddTableEntryInfo(tx, []byte(edge.ID), *rowLoc) - _, ok = ggraph.bsonkv.PageCache.Set(edge.ID, *rowLoc) + _, ok = ggraph.jsonkv.PageCache.Set(edge.ID, *rowLoc) if !ok { - ggraph.bsonkv.PageCache.Invalidate(edge.ID) - ggraph.bsonkv.PageCache.Set(edge.ID, *rowLoc) + ggraph.jsonkv.PageCache.Invalidate(edge.ID) + ggraph.jsonkv.PageCache.Set(edge.ID, *rowLoc) } - _, fieldsExist := ggraph.bsonkv.Fields[edgeLabel] + _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] if fieldsExist { - for field := range ggraph.bsonkv.Fields[edgeLabel] { - if val := bsontable.PathLookup(edge.Data, field); val != nil { + for field := range ggraph.jsonkv.Fields[edgeLabel] { + if val := jsontable.PathLookup(edge.Data, field); val != nil { tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) } } @@ -164,7 +164,7 @@ func (ggraph *Graph) Compiler() gdbi.Compiler { // AddVertex adds an edge to the graph, if it already exists // in the graph, it is replaced func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { if err := insertVertex(tx, vert); err != nil { @@ -176,7 +176,7 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { return bulkErr.ErrorOrNil() }) - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { if err := ggraph.indexVertex(vert, tx); err != nil { @@ -194,8 +194,8 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { // in the graph, it is replaced func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { var err error = nil - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - err = ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for _, edge := range edges { err = insertEdge(tx, edge) if err != nil { @@ -211,7 +211,7 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { if err != nil { return err } - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, edge := range edges { if err := ggraph.indexEdge(edge, tx); err != nil { @@ -264,7 +264,7 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { go func() { defer wg.Done() - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { for elem := range insertStream { if elem.Vertex != nil { if err := insertVertex(tx, elem.Vertex); err != nil { @@ -288,8 +288,8 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { go func() { defer wg.Done() - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := ggraph.bsonkv.BulkLoad(indexStream, tx); err != nil { + err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + if err := ggraph.jsonkv.BulkLoad(indexStream, tx); err != nil { return fmt.Errorf("bsonkv bulk load error: %v", err) } ggraph.ts.Touch(ggraph.graphID) @@ -344,7 +344,7 @@ func (ggraph *Graph) DelVertex(id string) error { var bulkErr *multierror.Error - err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() eid, sid, did, label := SrcEdgeKeyParse(skey) @@ -399,7 +399,7 @@ func (ggraph *Graph) DelVertex(id string) error { } } - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := tx.DeletePrefix(vid); err != nil { return err } @@ -422,7 +422,7 @@ func (ggraph *Graph) DelVertex(id string) error { func (ggraph *Graph) DelEdge(eid string) error { ekeyPrefix := EdgeKeyPrefix(eid) var ekey []byte - err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { ekey = it.Key() } @@ -442,7 +442,7 @@ func (ggraph *Graph) DelEdge(eid string) error { dkey := DstEdgeKey(sid, did, eid, lbl) var bulkErr *multierror.Error - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := tx.Delete(ekey, nil); err != nil { bulkErr = multierror.Append(bulkErr, err) } @@ -472,7 +472,7 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb o := make(chan *gdbi.Edge, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { ePrefix := EdgeListPrefix() for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { select { @@ -483,12 +483,12 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb eid, sid, did, label := EdgeKeyParse(it.Key()) e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} if loadProp { - entry, err := ggraph.bsonkv.PageCache.Get(ctx, eid, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, eid, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetEdgeList: PageCache.Get( error: %v", err) continue } - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow(entry) + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+label].GetRow(entry) if err != nil { log.Errorf("GetEdgeList: GetRow error: %v", err) continue @@ -510,7 +510,7 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { ekeyPrefix := VertexKey(id) var byteLabel []byte = nil var err error = nil - err = ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err = ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { byteLabel, err = it.Value() } @@ -525,12 +525,12 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { Label: string(byteLabel), } if loadProp { - entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), id, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetVertex: PageCache.Get( error: %v", err) return nil } - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetVertex: table.GetRow( error: %v", err) return nil @@ -552,7 +552,7 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element out := make(chan gdbi.ElementLookup, 100) go func() { defer close(out) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for id := range ids { if id.IsSignal() { out <- id @@ -568,12 +568,12 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element } v.Label = string(label) - entry, err := ggraph.bsonkv.PageCache.Get(ctx, id.ID, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, id.ID, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetVertexChannel: PageCache.Get( error: %v", err) continue } - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) continue @@ -602,11 +602,11 @@ type lookup struct { // GetOutChannel process requests of vertex ids and find the connected vertices on outgoing edges func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - // Todo: implement bulk cache get + bulk get row to try to make this faster + // Todo: implement bulk cache get + bulk get row to try to make this faster lookupChan := make(chan lookup, 1000) go func() { defer close(lookupChan) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { lookupChan <- lookup{req: req} @@ -643,25 +643,25 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen o <- req.req } else { if req.key != "" { - entry, err := ggraph.bsonkv.PageCache.Get(ctx, req.key, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, req.key, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetOutChannel: PageCache.Get( error: %v", err) continue } - vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] + vLabel, ok := ggraph.jsonkv.LabelLookup[entry.Label] if !ok { log.Errorf("GetOutChannel: Label not a string %s", vLabel) continue } v := &gdbi.Vertex{ID: req.key, Label: vLabel} if load { - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetOutChannel: GetRow on %s: %s error: %v", vLabel, req.key, err) continue } v.Loaded = true - }else { + } else { v.Data = map[string]any{} } req.req.Vertex = v @@ -681,7 +681,7 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req @@ -691,13 +691,13 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { _, sid, _, label := DstEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { - entry, err := ggraph.bsonkv.PageCache.Get(ctx, sid, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, sid, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetInChannel: PageCache.Get( error: %v", err) continue } - vLabel, ok := ggraph.bsonkv.LabelLookup[entry.Label] + vLabel, ok := ggraph.jsonkv.LabelLookup[entry.Label] if !ok { log.Errorf("GetInChannel Label lookup failed") continue @@ -705,7 +705,7 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element v := &gdbi.Vertex{ID: sid, Label: vLabel} if load { - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetInChannel: GetRow on %s: %s error: %v", vLabel, sid, err) continue @@ -737,7 +737,7 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req @@ -754,12 +754,12 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El ID: eid, } if load { - entry, err := ggraph.bsonkv.PageCache.Get(ctx, e.ID, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, e.ID, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetOutEdgeChannel: PageCache.Get( error: %v", err) continue } - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) continue @@ -791,7 +791,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req @@ -808,14 +808,14 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele Label: label, } if load { - entry, err := ggraph.bsonkv.PageCache.Get(ctx, e.ID, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, e.ID, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetInEdgeChannel: PageCache.Get( error: %v", err) continue } //log.Debugln("IN EDGE LABEL: ", e.Label, "ENTRY: ", entry, "ID: ", e.ID) - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetInEdgeChannel: GetRow error: %v", err) continue @@ -847,7 +847,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { ekeyPrefix := EdgeKeyPrefix(id) var e *gdbi.Edge - err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { eid, src, dst, label := EdgeKeyParse(it.Key()) e = &gdbi.Edge{ @@ -857,13 +857,13 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { Label: label, } if loadProp { - entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), e.ID, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), e.ID, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetEdge: PageCache.Get( error: %v", err) continue } - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) continue @@ -886,7 +886,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g o := make(chan *gdbi.Vertex, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { vPrefix := VertexListPrefix() for it.Seek(vPrefix); it.Valid() && bytes.HasPrefix(it.Key(), vPrefix); it.Next() { select { @@ -903,13 +903,13 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g Label: string(byteLabel), } if loadProp { - entry, err := ggraph.bsonkv.PageCache.Get(context.Background(), v.ID, ggraph.bsonkv.PageLoader) + entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), v.ID, ggraph.jsonkv.PageLoader) if err != nil { log.Errorf("GetVertexList: PageCache.Get on %s error: %s", v.ID, err) continue } - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { log.Errorf("GetVertexList: table.GetRow error: %s", err) continue @@ -929,7 +929,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g // ListVertexLabels returns a list of vertex types in the graph func (ggraph *Graph) ListVertexLabels() ([]string, error) { labels := []string{} - for i := range ggraph.bsonkv.GetLabels(false, true) { + for i := range ggraph.jsonkv.GetLabels(false, true) { labels = append(labels, i) } return labels, nil @@ -938,7 +938,7 @@ func (ggraph *Graph) ListVertexLabels() ([]string, error) { // ListEdgeLabels returns a list of edge types in the graph func (ggraph *Graph) ListEdgeLabels() ([]string, error) { labels := []string{} - for i := range ggraph.bsonkv.GetLabels(true, true) { + for i := range ggraph.jsonkv.GetLabels(true, true) { labels = append(labels, i) } return labels, nil diff --git a/grids/index.go b/grids/index.go index 8252dbb1..312b8ea7 100644 --- a/grids/index.go +++ b/grids/index.go @@ -11,13 +11,13 @@ import ( // AddVertexIndex add index to vertices func (ggraph *Graph) AddVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") - return ggraph.bsonkv.AddField(VTABLE_PREFIX+label, field) + return ggraph.jsonkv.AddField(VTABLE_PREFIX+label, field) } // DeleteVertexIndex delete index from vertices func (ggraph *Graph) DeleteVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") - return ggraph.bsonkv.RemoveField(VTABLE_PREFIX+label, field) + return ggraph.jsonkv.RemoveField(VTABLE_PREFIX+label, field) } // GetVertexIndexList lists out all the vertex indices for a graph @@ -26,7 +26,7 @@ func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { out := make(chan *gripql.IndexID) go func() { defer close(out) - for _, f := range ggraph.bsonkv.ListFields() { + for _, f := range ggraph.jsonkv.ListFields() { out <- &gripql.IndexID{Graph: ggraph.graphID, Label: f.Label, Field: f.Field} } }() @@ -40,19 +40,19 @@ func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan str label = VTABLE_PREFIX + label } log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") - return ggraph.bsonkv.GetIDsForLabel(label) + return ggraph.jsonkv.GetIDsForLabel(label) } func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error { - ggraph.bsonkv.Lock.Lock() - defer ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + defer ggraph.jsonkv.Lock.Unlock() var prefix string = "v_" if edgeFlag { prefix = "e_" } - err := ggraph.bsonkv.Tables[prefix+label].DeleteRow([]byte(id)) + err := ggraph.jsonkv.Tables[prefix+label].DeleteRow([]byte(id)) if err != nil { if err == pebble.ErrNotFound { log.Debugln("Pebble not Found: %s", err) @@ -60,7 +60,7 @@ func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error } return err } - ggraph.bsonkv.PageCache.Invalidate(id) + ggraph.jsonkv.PageCache.Invalidate(id) return nil } diff --git a/grids/new.go b/grids/new.go index 189c11e7..2e34351f 100644 --- a/grids/new.go +++ b/grids/new.go @@ -8,24 +8,24 @@ import ( "strings" "sync" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/timestamp" ) // Graph implements the GDB interface using a genertic key/value storage driver type Graph struct { - graphID string + graphID string - bsonkv *bsontable.BSONDriver - ts *timestamp.Timestamp + jsonkv *jsontable.JSONDriver + ts *timestamp.Timestamp tempDeletedEdges map[string]struct{} - edgesMutex sync.Mutex + edgesMutex sync.Mutex } // Close the connection func (g *Graph) Close() error { - g.bsonkv.Close() + g.jsonkv.Close() return nil } @@ -62,20 +62,20 @@ func newGraph(baseDir, name string) (*Graph, error) { //bsonkvPath := fmt.Sprintf("%s", dbPath) bsonkvPath := dbPath - tabledr, err := bsontable.NewBSONDriver(bsonkvPath) + tabledr, err := jsontable.NewJSONDriver(bsonkvPath) if err != nil { return nil, fmt.Errorf("failed to open bsonkv at %s: %v", bsonkvPath, err) } - bsonkv := tabledr.(*bsontable.BSONDriver) + bsonkv := tabledr.(*jsontable.JSONDriver) ts := timestamp.NewTimestamp() o := &Graph{ - bsonkv: bsonkv, - ts: &ts, - graphID: name, + jsonkv: bsonkv, + ts: &ts, + graphID: name, tempDeletedEdges: make(map[string]struct{}), - edgesMutex: sync.Mutex{}, + edgesMutex: sync.Mutex{}, } return o, nil } @@ -107,16 +107,16 @@ func getGraph(baseDir, name string) (*Graph, error) { //bsonkvPath := fmt.Sprintf("%s", dbPath) bsonkvPath := dbPath - tabledr, err := bsontable.LoadBSONDriver(bsonkvPath) + tabledr, err := jsontable.LoadBSONDriver(bsonkvPath) if err != nil { return nil, fmt.Errorf("failed to open bsonkv at %s: %v", bsonkvPath, err) } - bsonkv := tabledr.(*bsontable.BSONDriver) + bsonkv := tabledr.(*jsontable.JSONDriver) ts := timestamp.NewTimestamp() o := &Graph{ - bsonkv: bsonkv, + jsonkv: bsonkv, ts: &ts, graphID: name, } diff --git a/grids/processor.go b/grids/processor.go index bc8e5b73..5baa8cf7 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -44,10 +44,10 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi var exists = true // Here if one of l.labels doesn't exist then not going to be querying all the data so leave it like this. cond := l.expr.GetCondition() - exists = len(l.db.bsonkv.Fields) > 0 && cond != nil + exists = len(l.db.jsonkv.Fields) > 0 && cond != nil if exists { for _, iterLabel := range l.labels { - label, ok := l.db.bsonkv.Fields[iterLabel] + label, ok := l.db.jsonkv.Fields[iterLabel] if !ok { exists = false break @@ -65,7 +65,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi defer close(out) for t := range in { for _, label := range l.labels { - tableFound, ok := l.db.bsonkv.Tables[label] + tableFound, ok := l.db.jsonkv.Tables[label] if !ok { log.Debugf("BSONTable for label '%s' is nil. Cannot scan.", label) continue @@ -96,7 +96,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi for t := range in { cond := l.expr.GetCondition() for _, label := range l.labels { - for id := range l.db.bsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), cond.Condition) { + for id := range l.db.jsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), cond.Condition) { queryChan <- gdbi.ElementLookup{ID: id, Ref: t} } } @@ -146,10 +146,10 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter otherwise this lookup will not fetch everything that was asked for */ - allMatch := len(l.db.bsonkv.Fields) > 0 && cond != nil + allMatch := len(l.db.jsonkv.Fields) > 0 && cond != nil if allMatch { - for lbl := range l.db.bsonkv.GetLabels(false, false) { - if val, exists := l.db.bsonkv.Fields[lbl]; exists { + for lbl := range l.db.jsonkv.GetLabels(false, false) { + if val, exists := l.db.jsonkv.Fields[lbl]; exists { if _, ok := val[cond.Key]; !ok { allMatch = false break @@ -167,7 +167,7 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager go func() { defer close(queryChan) for t := range in { - for id := range l.db.bsonkv.RowIdsByHas( + for id := range l.db.jsonkv.RowIdsByHas( cond.Key, cond.Value.AsInterface(), cond.Condition, diff --git a/grids/schema.go b/grids/schema.go index 8a6636c6..0847236c 100644 --- a/grids/schema.go +++ b/grids/schema.go @@ -34,7 +34,7 @@ func (ma *GDB) BuildSchema(ctx context.Context, graph string, sampleN uint32, ra } func (gi *Graph) sampleSchema(ctx context.Context, n uint32, random bool) ([]*gripql.Vertex, []*gripql.Edge, error) { - labels := gi.bsonkv.List() + labels := gi.jsonkv.List() vertLabels := []string{} for _, label := range labels { if label[:2] == "v_" { From 705343bedd3d0c594338d4453932c5e9e41694bd Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 21 Aug 2025 16:19:01 -0700 Subject: [PATCH 28/30] update rest of names --- go.mod | 4 +--- go.sum | 2 ++ grids/graph.go | 4 ++-- grids/new.go | 20 ++++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 1857eab2..bd07ee28 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,13 @@ go 1.24 toolchain go1.24.2 -replace github.com/bmeg/benchtop v0.0.0-20250714172932-08f649a9f1c3 => ../benchtop - require ( github.com/IBM/sarama v1.45.1 github.com/Shopify/sarama v1.38.1 github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb + github.com/bmeg/benchtop v0.0.0-20250821231639-df57ed2bb713 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 3bb5dbac..9628e9d7 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311 h1:oEc44MjeTPX8DArCO github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb h1:k3md0eKLb80Ve/fYuNh8komOw9qD00NVho+n/G9ZQus= github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= +github.com/bmeg/benchtop v0.0.0-20250821231639-df57ed2bb713 h1:VsdsDhvfnagpEOBc6Hu8SNxYbMvyxkYGr1pVARmktEQ= +github.com/bmeg/benchtop v0.0.0-20250821231639-df57ed2bb713/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/graph.go b/grids/graph.go index 44190f7c..831dd39c 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -127,7 +127,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error log.Debugf("Creating new table %s for label %s on graph %s", edgeLabel, edge.Label, ggraph.graphID) newTable, err := ggraph.jsonkv.New(edgeLabel, nil) if err != nil { - return fmt.Errorf("indexEdge: bsonkv.New: %s", err) + return fmt.Errorf("indexEdge: jsonkv.New: %s", err) } ggraph.jsonkv.Lock.Lock() table = newTable.(*jsontable.JSONTable) @@ -290,7 +290,7 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { defer wg.Done() err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := ggraph.jsonkv.BulkLoad(indexStream, tx); err != nil { - return fmt.Errorf("bsonkv bulk load error: %v", err) + return fmt.Errorf("jsonkv bulk load error: %v", err) } ggraph.ts.Touch(ggraph.graphID) return nil diff --git a/grids/new.go b/grids/new.go index 2e34351f..ccc8da7a 100644 --- a/grids/new.go +++ b/grids/new.go @@ -61,17 +61,17 @@ func newGraph(baseDir, name string) (*Graph, error) { } //bsonkvPath := fmt.Sprintf("%s", dbPath) - bsonkvPath := dbPath - tabledr, err := jsontable.NewJSONDriver(bsonkvPath) + jsonkvPath := dbPath + tabledr, err := jsontable.NewJSONDriver(jsonkvPath) if err != nil { - return nil, fmt.Errorf("failed to open bsonkv at %s: %v", bsonkvPath, err) + return nil, fmt.Errorf("failed to open jsonkv at %s: %v", jsonkvPath, err) } - bsonkv := tabledr.(*jsontable.JSONDriver) + jsonkv := tabledr.(*jsontable.JSONDriver) ts := timestamp.NewTimestamp() o := &Graph{ - jsonkv: bsonkv, + jsonkv: jsonkv, ts: &ts, graphID: name, tempDeletedEdges: make(map[string]struct{}), @@ -106,17 +106,17 @@ func getGraph(baseDir, name string) (*Graph, error) { } //bsonkvPath := fmt.Sprintf("%s", dbPath) - bsonkvPath := dbPath - tabledr, err := jsontable.LoadBSONDriver(bsonkvPath) + jsonkvPath := dbPath + tabledr, err := jsontable.LoadJSONDriver(jsonkvPath) if err != nil { - return nil, fmt.Errorf("failed to open bsonkv at %s: %v", bsonkvPath, err) + return nil, fmt.Errorf("failed to open bsonkv at %s: %v", jsonkvPath, err) } - bsonkv := tabledr.(*jsontable.JSONDriver) + jsonkv := tabledr.(*jsontable.JSONDriver) ts := timestamp.NewTimestamp() o := &Graph{ - jsonkv: bsonkv, + jsonkv: jsonkv, ts: &ts, graphID: name, } From fd182359f1c2a4cc2453aa183b0e055e16f7fd08 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 26 Aug 2025 10:41:33 -0700 Subject: [PATCH 29/30] Fixes bugs with bulk delete function. Improves bulk delete function to be performant --- go.mod | 2 +- go.sum | 10 +- grids/graph.go | 525 ++++++++++++++++++++++++++++++++++++++++++++++--- grids/index.go | 40 +++- 4 files changed, 529 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index bd07ee28..a96e8a39 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250821231639-df57ed2bb713 + github.com/bmeg/benchtop v0.0.0-20250826173532-50ea19466c1c github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 9628e9d7..8e4476b1 100644 --- a/go.sum +++ b/go.sum @@ -35,14 +35,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36 h1:frNXEP0pgmYHu7WDgaXLrsf/pB4hv1SMT6SMK+bVk1g= -github.com/bmeg/benchtop v0.0.0-20250818194308-7b4d6a6bfa36/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= -github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311 h1:oEc44MjeTPX8DArCOrg/DBQDtT7xrfhz6q8ScfYmJj8= -github.com/bmeg/benchtop v0.0.0-20250818195147-fd718212c311/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= -github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb h1:k3md0eKLb80Ve/fYuNh8komOw9qD00NVho+n/G9ZQus= -github.com/bmeg/benchtop v0.0.0-20250821165736-18d0cca965bb/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= -github.com/bmeg/benchtop v0.0.0-20250821231639-df57ed2bb713 h1:VsdsDhvfnagpEOBc6Hu8SNxYbMvyxkYGr1pVARmktEQ= -github.com/bmeg/benchtop v0.0.0-20250821231639-df57ed2bb713/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= +github.com/bmeg/benchtop v0.0.0-20250826173532-50ea19466c1c h1:JjsqIMnoIihXqdf52UuK4MZOEa1RIppaNhK2ahuNqbQ= +github.com/bmeg/benchtop v0.0.0-20250826173532-50ea19466c1c/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= diff --git a/grids/graph.go b/grids/graph.go index 831dd39c..f8dd2dec 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -4,7 +4,11 @@ import ( "bytes" "context" "fmt" + "runtime" + "slices" + "sort" "sync" + "sync/atomic" "github.com/bmeg/benchtop" "github.com/bmeg/benchtop/jsontable" @@ -13,6 +17,8 @@ import ( "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/log" "github.com/bmeg/grip/util/setcmp" + "github.com/bytedance/sonic" + "github.com/cockroachdb/pebble" multierror "github.com/hashicorp/go-multierror" ) @@ -76,7 +82,18 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) if fieldsExist { for field := range ggraph.jsonkv.Fields[vertexLabel] { if val := jsontable.PathLookup(vertex.Data, field); val != nil { - tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) + err := tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) + if err != nil { + return err + } + Mval, err := sonic.ConfigFastest.Marshal(val) + if err != nil { + return err + } + err = tx.Set(benchtop.RFieldKey(vertexLabel, field, vertex.ID), Mval, nil) + if err != nil { + return err + } } } } @@ -150,7 +167,18 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error if fieldsExist { for field := range ggraph.jsonkv.Fields[edgeLabel] { if val := jsontable.PathLookup(edge.Data, field); val != nil { - tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) + err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) + if err != nil { + return err + } + eMarsh, err := sonic.ConfigFastest.Marshal(val) + if err != nil { + return err + } + err = tx.Set(benchtop.RFieldKey(edgeLabel, field, edge.ID), eMarsh, nil) + if err != nil { + return err + } } } } @@ -225,34 +253,6 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { } -func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { - var bulkErr *multierror.Error - ggraph.tempDeletedEdges = make(map[string]struct{}) - ggraph.edgesMutex.Lock() - - for _, val := range data.Vertices { - err := ggraph.DelVertex(val) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - - for _, val := range data.Edges { - if _, ok := ggraph.tempDeletedEdges[val]; ok { - log.Debugf("Skipping edge %s: already deleted during vertex deletion", val) - continue - } - err := ggraph.DelEdge(val) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - - ggraph.tempDeletedEdges = nil // Clean up - defer ggraph.edgesMutex.Unlock() - return bulkErr.ErrorOrNil() -} - func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { var errs *multierror.Error insertStream := make(chan *gdbi.GraphElement, 100) @@ -343,7 +343,6 @@ func (ggraph *Graph) DelVertex(id string) error { edgesToDelete := make(map[string]string) var bulkErr *multierror.Error - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() @@ -399,6 +398,19 @@ func (ggraph *Graph) DelVertex(id string) error { } } + loc, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + if err != nil { + return err + } + + label := ggraph.jsonkv.LabelLookup[loc.Label] + if label == "" { + bulkErr = multierror.Append(bulkErr, fmt.Errorf("Failed to lookup table label %d", loc.Label)) + } + if err := ggraph.DeleteAnyRow(id, label, false); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := tx.DeletePrefix(vid); err != nil { return err @@ -943,3 +955,454 @@ func (ggraph *Graph) ListEdgeLabels() ([]string, error) { } return labels, nil } + +// New Bulk Delete Function. Testing... +// BulkDel deletes vertices and edges in bulk. +func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { + type keyBatch struct { + singles [][]byte + ranges [][2][]byte + posKeys [][]byte + } + + type itemInfo struct { + id string + label string + isEdge bool + tbl string + } + + type fieldInfo struct { + rKey []byte + field string + tbl string + id []byte + } + + const shardSize = 64 + const bufferSize = 8192 + numCpus := runtime.NumCPU() + ctx := context.Background() + + var bulkErr *multierror.Error + addErr := func(err error) { + if err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + } + + // Sharded bitmap for edge deduplication (lock-free for reads) + type shard struct { + mu sync.Mutex + set map[string]struct{} + count uint32 // Atomic counter for seen edges + } + shards := make([]*shard, shardSize) + for i := range shards { + shards[i] = &shard{set: make(map[string]struct{}, bufferSize/shardSize)} + } + hasSeenEdge := func(eid string) bool { + h := fnv32a(eid) % uint32(shardSize) + shard := shards[h] + shard.mu.Lock() + defer shard.mu.Unlock() + if _, exists := shard.set[eid]; exists { + return true + } + shard.set[eid] = struct{}{} + atomic.AddUint32(&shard.count, 1) + return false + } + getSeenCount := func() uint64 { + var total uint64 + for _, shard := range shards { + total += uint64(atomic.LoadUint32(&shard.count)) + } + return total + } + + // Channels and wait groups + itemChan := make(chan itemInfo, bufferSize) + fieldChan := make(chan fieldInfo, bufferSize) + keyChan := make(chan keyBatch, bufferSize) + var prodWG, consWG, aggWG, fieldWG sync.WaitGroup + + // Aggregator for keys + var singles [][]byte + var ranges [][2][]byte + var posKeys [][]byte + aggWG.Add(1) + go func() { + defer aggWG.Done() + for batch := range keyChan { + select { + case <-ctx.Done(): + return + default: + singles = append(singles, batch.singles...) + ranges = append(ranges, batch.ranges...) + posKeys = append(posKeys, batch.posKeys...) + } + } + }() + + // Aggregator for fields + var allFields []fieldInfo + fieldWG.Add(1) + go func() { + defer fieldWG.Done() + for fi := range fieldChan { + allFields = append(allFields, fi) + } + }() + + // Workers for items + consWG.Add(numCpus) + for range numCpus { + go func() { + defer consWG.Done() + localBatch := keyBatch{posKeys: make([][]byte, 0, bufferSize)} + i := 0 + for item := range itemChan { + select { + case <-ctx.Done(): + return + default: + } + if i%100_000 == 0 && i != 0 { + log.Debugf("[BulkDel worker] processed %d items", i) + } + i++ + + // Fetch from page cache + loc, err := ggraph.jsonkv.PageCache.Get(ctx, item.id, ggraph.jsonkv.PageLoader) + if err != nil { + addErr(err) + continue + } + + // Mark table for deletion + table, ok := ggraph.jsonkv.Tables[item.tbl] + if !ok { + addErr(fmt.Errorf("table %s not found", item.tbl)) + continue + } + if err := table.MarkDeleteTable(loc); err != nil { + addErr(err) + } + + // Position key + localBatch.posKeys = append(localBatch.posKeys, benchtop.NewPosKey(loc.Label, []byte(item.id))) + ggraph.jsonkv.PageCache.Invalidate(item.id) + + // Send field infos + if fields, exists := ggraph.jsonkv.Fields[item.tbl]; exists { + for field := range fields { + rKey := benchtop.RFieldKey(item.tbl, field, item.id) + select { + case fieldChan <- fieldInfo{rKey: rKey, field: field, tbl: item.tbl, id: []byte(item.id)}: + case <-ctx.Done(): + return + } + } + } + + if len(localBatch.posKeys) >= 500_000 { + keyChan <- localBatch + localBatch = keyBatch{posKeys: make([][]byte, 0, bufferSize)} + } + } + + if len(localBatch.posKeys) > 0 { + keyChan <- localBatch + } + }() + } + + // Prepare vertex producers + slices.Sort(data.Vertices) + vertexSlices := make([][]string, numCpus) + for i, vid := range data.Vertices { + vertexSlices[i%numCpus] = append(vertexSlices[i%numCpus], vid) + } + for i := range vertexSlices { + slices.Sort(vertexSlices[i]) + } + + for _, slice := range vertexSlices { + if len(slice) == 0 { + continue + } + prodWG.Add(1) + go func(slice []string) { + defer prodWG.Done() + localBatch := keyBatch{singles: make([][]byte, 0, 256), ranges: make([][2][]byte, 0, 256)} + + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, vid := range slice { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + sPrefix := SrcEdgePrefix(vid) + if err := it.Seek(sPrefix); err != nil { + return err + } + if it.Valid() && bytes.HasPrefix(it.Key(), sPrefix) { + nextPrefix := upperBound(sPrefix) + if nextPrefix != nil { + localBatch.ranges = append(localBatch.ranges, [2][]byte{sPrefix, nextPrefix}) + } + for it.Valid() && bytes.HasPrefix(it.Key(), sPrefix) { + eid, sid, did, lbl := SrcEdgeKeyParse(it.Key()) + if !hasSeenEdge(eid) { + localBatch.singles = append(localBatch.singles, + EdgeKey(eid, sid, did, lbl), + bytes.Clone(it.Key()), + DstEdgeKey(eid, sid, did, lbl)) + select { + case itemChan <- itemInfo{id: eid, label: lbl, isEdge: true, tbl: ETABLE_PREFIX + lbl}: + case <-ctx.Done(): + return ctx.Err() + } + } + it.Next() + } + } + + dPrefix := DstEdgePrefix(vid) + if err := it.Seek(dPrefix); err != nil { + return err + } + if it.Valid() && bytes.HasPrefix(it.Key(), dPrefix) { + nextPrefix := upperBound(dPrefix) + if nextPrefix != nil { + localBatch.ranges = append(localBatch.ranges, [2][]byte{dPrefix, nextPrefix}) + } + for it.Valid() && bytes.HasPrefix(it.Key(), dPrefix) { + eid, sid, did, lbl := DstEdgeKeyParse(it.Key()) + if !hasSeenEdge(eid) { + localBatch.singles = append(localBatch.singles, + EdgeKey(eid, sid, did, lbl), + SrcEdgeKey(eid, sid, did, lbl), + bytes.Clone(it.Key())) + select { + case itemChan <- itemInfo{id: eid, label: lbl, isEdge: true, tbl: ETABLE_PREFIX + lbl}: + case <-ctx.Done(): + return ctx.Err() + } + } + it.Next() + } + } + + vkey := VertexKey(vid) + if err := it.Seek(vkey); err != nil { + return err + } + var label string + if it.Valid() && bytes.Equal(it.Key(), vkey) { + labelBytes, err := it.Value() + if err != nil { + return err + } + label = string(labelBytes) + } + localBatch.singles = append(localBatch.singles, vkey) + if label != "" { + select { + case itemChan <- itemInfo{id: vid, label: label, isEdge: false, tbl: VTABLE_PREFIX + label}: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil + }) + addErr(err) + + if len(localBatch.singles) > 0 || len(localBatch.ranges) > 0 { + select { + case keyChan <- localBatch: + case <-ctx.Done(): + } + } + }(slice) + } + + // Prepare edge producers + slices.Sort(data.Edges) + edgeSlices := make([][]string, numCpus) + for i, eid := range data.Edges { + edgeSlices[i%numCpus] = append(edgeSlices[i%numCpus], eid) + } + for i := range edgeSlices { + slices.Sort(edgeSlices[i]) + } + + for _, slice := range edgeSlices { + if len(slice) == 0 { + continue + } + prodWG.Add(1) + go func(slice []string) { + defer prodWG.Done() + localBatch := keyBatch{singles: make([][]byte, 0, 12), ranges: make([][2][]byte, 0, 12)} + + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, eid := range slice { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if hasSeenEdge(eid) { + continue + } + + prefix := EdgeKeyPrefix(eid) + if err := it.Seek(prefix); err != nil { + return err + } + if it.Valid() && bytes.HasPrefix(it.Key(), prefix) { + nextPrefix := upperBound(prefix) + if nextPrefix != nil { + localBatch.ranges = append(localBatch.ranges, [2][]byte{prefix, nextPrefix}) + } + var label string + for it.Valid() && bytes.HasPrefix(it.Key(), prefix) { + _, sid, did, lbl := EdgeKeyParse(it.Key()) + label = lbl + localBatch.singles = append(localBatch.singles, + SrcEdgeKey(eid, sid, did, lbl), + DstEdgeKey(eid, sid, did, lbl)) + it.Next() + } + if label != "" { + select { + case itemChan <- itemInfo{id: eid, label: label, isEdge: true, tbl: ETABLE_PREFIX + label}: + case <-ctx.Done(): + return ctx.Err() + } + } + } + } + return nil + }) + addErr(err) + + if len(localBatch.singles) > 0 || len(localBatch.ranges) > 0 { + select { + case keyChan <- localBatch: + case <-ctx.Done(): + } + } + }(slice) + } + + // Close channels and wait + go func() { + prodWG.Wait() + close(itemChan) + }() + consWG.Wait() + close(keyChan) + aggWG.Wait() + close(fieldChan) + fieldWG.Wait() + + // Process field indices with single iterator + var indexDelKeys [][]byte + if len(allFields) > 0 { + sort.Slice(allFields, func(i, j int) bool { + return bytes.Compare(allFields[i].rKey, allFields[j].rKey) < 0 + }) + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, fi := range allFields { + if err := it.Seek(fi.rKey); err != nil { + return err + } + if it.Valid() && bytes.Equal(it.Key(), fi.rKey) { + valueBytes, err := it.Value() + if err != nil { + return err + } + var fieldValue any + if err := sonic.ConfigFastest.Unmarshal(valueBytes, &fieldValue); err != nil { + return err + } + if fieldValue != nil { + fKey := benchtop.FieldKey(fi.field, fi.tbl, fieldValue, fi.id) + indexDelKeys = append(indexDelKeys, fKey, fi.rKey) + } + } + } + return nil + }) + addErr(err) + } + + // Chunked deletes with Pebble batch + chunked := func(singles [][]byte, ranges [][2][]byte, posKeys [][]byte, indexDelKeys [][]byte) error { + batch := ggraph.jsonkv.Pb.Db.NewBatch() + defer batch.Close() + for _, k := range singles { + if err := batch.Delete(k, nil); err != nil { + return err + } + } + for _, r := range ranges { + if err := batch.DeleteRange(r[0], r[1], nil); err != nil { + return err + } + } + for _, k := range posKeys { + if err := batch.Delete(k, nil); err != nil { + return err + } + } + for _, k := range indexDelKeys { + if err := batch.Delete(k, nil); err != nil { + return err + } + } + return batch.Commit(pebble.Sync) + } + + // Perform deletes + ggraph.jsonkv.PebbleLock.Lock() + if err := chunked(singles, ranges, posKeys, indexDelKeys); err != nil { + addErr(err) + } + ggraph.ts.Touch(ggraph.graphID) + ggraph.jsonkv.PebbleLock.Unlock() + + log.Debugf("Total edges seen: %d", getSeenCount()) + return bulkErr.ErrorOrNil() +} + +// upperBound computes the tight upper bound for range delete +func upperBound(prefix []byte) []byte { + ub := make([]byte, len(prefix)) + copy(ub, prefix) + for i := len(ub) - 1; i >= 0; i-- { + if ub[i] < 0xFF { + ub[i]++ + return ub[:i+1] + } + } + return nil +} + +// fnv32a computes FNV-1a 32-bit hash +func fnv32a(s string) uint32 { + var h uint32 = 2166136261 + for i := range s { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} diff --git a/grids/index.go b/grids/index.go index 312b8ea7..11d92811 100644 --- a/grids/index.go +++ b/grids/index.go @@ -2,10 +2,12 @@ package grids import ( "context" + "fmt" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/cockroachdb/pebble" + multierror "github.com/hashicorp/go-multierror" ) // AddVertexIndex add index to vertices @@ -16,6 +18,7 @@ func (ggraph *Graph) AddVertexIndex(label, field string) error { // DeleteVertexIndex delete index from vertices func (ggraph *Graph) DeleteVertexIndex(label, field string) error { + fmt.Println("HELLO WE HARE HERE") log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") return ggraph.jsonkv.RemoveField(VTABLE_PREFIX+label, field) } @@ -44,23 +47,44 @@ func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan str } func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error { - ggraph.jsonkv.Lock.Lock() - defer ggraph.jsonkv.Lock.Unlock() - var prefix string = "v_" if edgeFlag { prefix = "e_" } - err := ggraph.jsonkv.Tables[prefix+label].DeleteRow([]byte(id)) + loc, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + if err != nil { + return err + } + + tableLabel := prefix + label + var bulkErr *multierror.Error + if fields, exists := ggraph.jsonkv.Fields[tableLabel]; exists { + for field := range fields { + if err := ggraph.jsonkv.DeleteRowField(tableLabel, field, id); err != nil { + log.Errorf("Failed to delete index for field '%s' in table '%s' for row '%s': %v", field, tableLabel, id, err) + bulkErr = multierror.Append(bulkErr, err) + } + } + } + + ggraph.jsonkv.PebbleLock.Lock() + defer ggraph.jsonkv.PebbleLock.Unlock() + + table, ok := ggraph.jsonkv.Tables[prefix+label] + if !ok { + bulkErr = multierror.Append(bulkErr, fmt.Errorf("table %s not found in jsonkv.Tables: %#v", prefix+label, ggraph.jsonkv.Tables)) + return bulkErr.ErrorOrNil() + } + + err = table.DeleteRow(loc, []byte(id)) if err != nil { if err == pebble.ErrNotFound { - log.Debugln("Pebble not Found: %s", err) + log.Debugf("Pebble not Found: %s", err) return nil } - return err + bulkErr = multierror.Append(bulkErr, err) } ggraph.jsonkv.PageCache.Invalidate(id) - - return nil + return bulkErr.ErrorOrNil() } From 9a28feec2599cc25d455a2176105790f012dc8ae Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 27 Aug 2025 13:19:45 -0700 Subject: [PATCH 30/30] update benchtop version to merged version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a96e8a39..cbf8518f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250826173532-50ea19466c1c + github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a diff --git a/go.sum b/go.sum index 8e4476b1..e0ae85e3 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250826173532-50ea19466c1c h1:JjsqIMnoIihXqdf52UuK4MZOEa1RIppaNhK2ahuNqbQ= -github.com/bmeg/benchtop v0.0.0-20250826173532-50ea19466c1c/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= +github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 h1:sIgPwNZKv3pSuIm/hngszbBrg3pKlZcyJ+HTzeFyjNA= +github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU=