diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fd151e6..93ca2b22 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,7 @@ jobs: name: gripBin path: grip + unitTests: needs: build name: Unit tests @@ -65,7 +66,6 @@ jobs: sleep 5 python conformance/run_kafka.py - badgerTest: needs: build name: Badger Conformance @@ -86,7 +86,6 @@ jobs: sleep 5 make test-conformance - pebbleTest: needs: build name: Pebble Conformance @@ -107,7 +106,6 @@ jobs: sleep 5 make test-conformance - mongoTest: needs: build name: Mongo Test @@ -127,8 +125,7 @@ jobs: make start-mongo ./grip server --rpc-port 18202 --http-port 18201 --config ./test/mongo.yml & sleep 5 - make test-conformance - + python conformance/run_conformance.py http://localhost:18201 --exclude nested_index mongoCoreTest: needs: build @@ -171,8 +168,7 @@ jobs: sleep 15 ./grip server --rpc-port 18202 --http-port 18201 --config ./test/psql.yml & sleep 5 - python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations - + python conformance/run_conformance.py http://localhost:18201 --exclude aggregations sqliteTest: needs: build @@ -192,8 +188,7 @@ jobs: chmod +x grip ./grip server --rpc-port 18202 --http-port 18201 --config ./test/sqlite.yml & sleep 5 - python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations - + python conformance/run_conformance.py http://localhost:18201 --exclude aggregations gripperTest: needs: build @@ -218,8 +213,7 @@ jobs: sleep 5 ./grip server --rpc-port 18202 --http-port 18201 --config ./gripper/test-graph/config.yaml --er tableServer=localhost:50051 & sleep 5 - python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude pivot - + python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude index pivot authTest: needs: build @@ -251,8 +245,6 @@ jobs: # run specialized role based tests make test-authorization ARGS="--grip_config_file_path test/pebble-auth.yml" - - gridsTest: needs: build name: GRIDs Conformance diff --git a/accounts/basic.go b/accounts/basic.go index ffbf6b11..780ae28a 100644 --- a/accounts/basic.go +++ b/accounts/basic.go @@ -4,6 +4,8 @@ import ( "encoding/base64" "fmt" "strings" + + "github.com/bmeg/grip/log" ) // BasicCredential describes a username and password for use with Funnel's basic auth. @@ -18,7 +20,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { var auth []string var ok bool - fmt.Printf("Running BasicAuth: %#v\n", md) + log.Infof("Running BasicAuth: %#v\n", md) if auth, ok = md["Authorization"]; !ok { if auth, ok = md["authorization"]; !ok { @@ -28,7 +30,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { if len(auth) > 0 { user, password, ok := parseBasicAuth(auth[0]) - fmt.Printf("User: %s Password: %s OK: %s\n", user, password, ok) + log.Infof("Authenticating as User: %s OK: %t\n", user, ok) for _, c := range ba { if c.User == user && c.Password == password { return user, nil diff --git a/accounts/interface.go b/accounts/interface.go index 7f86438c..e75fbc3f 100644 --- a/accounts/interface.go +++ b/accounts/interface.go @@ -43,6 +43,7 @@ var MethodMap = map[string]Operation{ "/gripql.Edit/DeleteVertex": Write, "/gripql.Edit/DeleteEdge": Write, "/gripql.Edit/AddIndex": Write, + "/gripql.Edit/DeleteIndex": Write, "/gripql.Edit/AddSchema": Write, "/gripql.Edit/AddJsonSchema": Write, "/gripql.Edit/AddMapping": Write, diff --git a/conformance/run_util.py b/conformance/run_util.py index dbee58f1..1a32dc00 100644 --- a/conformance/run_util.py +++ b/conformance/run_util.py @@ -180,19 +180,22 @@ def setGraph(self, name): self._conn.addGraph(self.curGraph) G = self._conn.graph(self.curGraph) + bulk = G.bulkAdd() with open(os.path.join(BASE, "graphs", "%s.vertices" % (name))) as handle: for line in handle: data = json.loads(line) - G.addVertex(data["_id"], data["_label"], self.collect_fields_dict(data)) + bulk.addVertex(data["_id"], data["_label"], self.collect_fields_dict(data)) with open(os.path.join(BASE, "graphs", "%s.edges" % (name))) as handle: for line in handle: data = json.loads(line) - G.addEdge(src=data["_from"], dst=data["_to"], + bulk.addEdge(src=data["_from"], dst=data["_to"], id=data.get("_id", None), label=data["_label"], data=self.collect_fields_dict(data)) + + bulk.execute() self.curName = name return G diff --git a/conformance/tests/auth_basic.py b/conformance/tests/auth_basic.py index b22a1ee0..1f9a579f 100644 --- a/conformance/tests/auth_basic.py +++ b/conformance/tests/auth_basic.py @@ -1,7 +1,5 @@ from __future__ import absolute_import -import requests - def test_current_user_has_policy(manager): """Ensure current user has a policy defined.""" @@ -18,7 +16,7 @@ def test_current_user_can_query(manager): account = manager.current_user_account() assert account, f"Could not find account for {manager.user}" policies = account.policies - assert len(policies) > 0, f"Should have at least one policy" + assert len(policies) > 0, "Should have at least one policy" errors = [] if not account.is_admin: @@ -56,7 +54,7 @@ def test_current_user_can_read(manager): account = manager.current_user_account() assert account, f"Could not find account for {manager.user}" policies = account.policies - assert len(policies) > 0, f"Should have at least one policy" + assert len(policies) > 0, "Should have at least one policy" errors = [] # non admin user if not account.is_admin: @@ -92,7 +90,7 @@ def test_current_user_can_write(manager): account = manager.current_user_account() assert account, f"Could not find account for {manager.user}" policies = account.policies - assert len(policies) > 0, f"Should have at least one policy" + assert len(policies) > 0, "Should have at least one policy" errors = [] # non admin user if not account.is_admin: @@ -109,7 +107,7 @@ def test_current_user_can_write(manager): try: manager.test_write('dummy') errors.append(f"{manager.user} should not be able to write dummy graph") - except AssertionError as e: + except AssertionError: pass else: diff --git a/conformance/tests/ot_has.py b/conformance/tests/ot_has.py index 84469671..4b1f6b1a 100644 --- a/conformance/tests/ot_has.py +++ b/conformance/tests/ot_has.py @@ -137,7 +137,6 @@ def test_has_prev(man): q = q.has(gripql.neq("$1._id", "$._id")) count = 0 for i in q.render(["$1._id", "$._id"]): - print(i) if i[0] == i[1]: errors.append("History based filter failed: %s" % (i[0]) ) count += 1 @@ -171,6 +170,7 @@ def test_has_neq(man): "Fail: G.query().V().has(gripql.not_(gripql.eq(\"_label\", \"Character\"))) %s != %s" % (count, 21)) + count = 0 for i in G.query().V().hasLabel("Character").has(gripql.neq("eye_color", "brown")): count += 1 @@ -181,6 +181,7 @@ def test_has_neq(man): "Fail: G.query().V().has(gripql.not_(gripql.eq(\"eye_color\", \"brown\"))) %s != %s" % (count, 14)) + return errors diff --git a/conformance/tests/ot_index.py b/conformance/tests/ot_index.py index 991d42db..46e106f4 100644 --- a/conformance/tests/ot_index.py +++ b/conformance/tests/ot_index.py @@ -1,15 +1,40 @@ +import gripql + + +def test_index_create_and_delete(man): + errors = [] + G = man.writeTest() + G.addIndex("Person", "name") + found = False + indices = G.listIndices() + for i in indices: + if i["field"] == "name" and i["label"] == "Person": + found = True + if not found: + errors.append("Expected index to be found") + + G.deleteIndex("Person", "name") + indices = G.listIndices() + + found = False + for i in indices: + if i["field"] == "name" and i["label"] == "Person": + found = True + if found: + errors.append("Expected index not found") + + return errors def test_index(man): errors = [] G = man.writeTest() - G.addIndex("Person", "name") G.addVertex("1", "Person", {"name": "marko", "age": "29"}) G.addVertex("2", "Person", {"name": "vadas", "age": "27"}) - G.addVertex("3", "Software", {"name": "lop", "lang": "java"}) + G.addVertex("3", "Software", {"name": "lop", "lang": "java"}) G.addVertex("4", "Person", {"name": "josh", "age": "32"}) G.addVertex("5", "Software", {"name": "ripple", "lang": "java"}) G.addVertex("6", "Person", {"name": "peter", "age": "35"}) @@ -22,12 +47,201 @@ def test_index(man): G.addEdge("6", "3", "created", {"weight": 0.2}) G.addEdge("4", "5", "created", {"weight": 1.0}) - resp = G.listIndices() + count = 0 + for i in G.query().V().has(gripql.eq("name","marko")): + count += 1 + if "name" not in i: + errors.append("'name' field not found in vertex") + if i["name"] != "marko": + errors.append("Filtering on field name, value marko but got '%s' instead" % i["name"]) + if count != 2: + errors.append("Expecting 2 vertices returned but got %d instead" % (count)) + + return errors + + +def test_bulk_index(man): + errors = [] + + G = man.writeTest() + G.addIndex("Person", "age") + + bulk = G.bulkAdd() + + bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) + bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) + bulk.addVertex("4", "Person", {"name": "josh", "age": "32"}) + bulk.addVertex("6", "Person", {"name": "peter", "age": "35"}) + bulk.addVertex("7", "Person", {"name": "alice", "age": "31"}) + bulk.addVertex("8", "Person", {"name": "bob", "age": "32"}) + bulk.addVertex("9", "Person", {"name": "charlie", "age": "28"}) + bulk.addVertex("10", "Person", {"name": "diana", "age": "32"}) + bulk.addVertex("11", "Person", {"name": "eve", "age": "30"}) + bulk.addVertex("12", "Person", {"name": "frank", "age": "33"}) + bulk.addVertex("13", "Person", {"name": "grace", "age": "26"}) + bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"}) + bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"}) + bulk.addVertex("16", "Person", {"name": "judy", "age": "34"}) + res = bulk.execute() + if res["errorCount"] > 0: + errors.append("errorCount on bulk add > 0") + + count = 0 + resp3 = G.query().V().has(gripql.eq("age","32")) + for i in resp3: + count += 1 + if "age" not in i: + errors.append("field 'age' not found in vertex") + if "age" in i and i["age"] != "32": + errors.append("filtering on field age value '32' but got %s instead" % (i["age"])) + if count != 4: + errors.append("expected count 4 but got %d instead" % (count)) + + return errors + + +def test_index_after_write(man): + errors = [] + G = man.writeTest() + bulk = G.bulkAdd() + bulk.addVertex("1", "Person", {"name": "marko", "age": "29"}) + bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"}) + bulk.addVertex("4", "Person", {"name": "josh", "age": "32"}) + bulk.addVertex("6", "Person", {"name": "peter", "age": "35"}) + bulk.addVertex("7", "Person", {"name": "alice", "age": "31"}) + bulk.addVertex("8", "Person", {"name": "bob", "age": "32"}) + bulk.addVertex("9", "Person", {"name": "charlie", "age": "28"}) + bulk.addVertex("10", "Person", {"name": "diana", "age": "32"}) + bulk.addVertex("11", "Person", {"name": "eve", "age": "30"}) + bulk.addVertex("12", "Person", {"name": "frank", "age": "33"}) + bulk.addVertex("13", "Person", {"name": "grace", "age": "26"}) + bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"}) + bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"}) + bulk.addVertex("16", "Person", {"name": "judy", "age": "34"}) + res = bulk.execute() + if res["errorCount"] > 0: + errors.append("errorCount on bulk add > 0") + + G.addIndex("Person", "age") + + count = 0 + restwo = G.query().V().has(gripql.within("name", ["marko", "vadas", "eve", "ivan", "charlie", "nothere"])) + for i in restwo: + count += 1 + if count != 5: + errors.append("Expected 5 names from filter but got %d instead" % (count)) + + return errors + + +def test_index_filter(man): + errors = [] + G = man.setGraph("swapi") + G.addIndex("Starship", "cost_in_credits") + G.addIndex("Starship", "cargo_capacity") + + respthree = G.query().V().hasLabel("Starship").has(gripql.lt("cost_in_credits", 150000000)) + count = 0 + for i in respthree: + count += 1 + if i['cost_in_credits'] > 149999999: + errors.append("filtering on ships that cost less than 150000000, but %d > 149999999" % (i['cost_in_credits'])) + + if count != 5: + errors.append("Expected 5 results got %d instead" % (count)) + + + indices = G.listIndices() found = False - for i in resp: - if i["field"] == "name" and i["label"] == "Person": + count = 0 + for i in indices: + count +=1 + if i["field"] == "cost_in_credits" and i["label"] == "Starship": found = True if not found: errors.append("Expected index not found") + if count != 2: + errors.append("Expected to find 2 indices but found %d instead" % (count)) + + G.deleteIndex("Starship", "cost_in_credits") + count = 0 + indices_two = G.listIndices() + found = False + for i in indices_two: + count += 1 + if i["field"] == "cost_in_credits" and i["label"] == "Starship": + found = True + if found: + errors.append("Expected index not found, but it was found") + if count != 1: + errors.append("Expected to find 1 index but found %d instead" % (count)) return errors + + +def test_consistent_results(man): + errors = [] + G = man.setGraph("swapi") + + resp = G.query().V().has(gripql.contains("eye_colors", "yellow")) + count = 0 + for i in resp: + count += 1 + if count != 2: + errors.append("Expected 2 results but got %d instead" % (count)) + + G.addIndex("Species", "eye_colors") + resp = G.query().V().has(gripql.contains("eye_colors", "yellow")) + count = 0 + for i in resp: + count += 1 + if count != 2: + errors.append("Expected 2 results but got %d instead" % (count)) + + return errors + + +def test_hasLabel_contains(man): + # If using the grids driver + no indexing this test uses the optimized scan function pipeline + errors = [] + G = man.setGraph("swapi") + resp = G.query().V().hasLabel("Species").has(gripql.contains("eye_colors", "yellow")) + count = 0 + for i in resp: + count += 1 + if count != 2: + errors.append("Expected 2 results but got %d instead" % (count)) + + return errors + + +def test_multiple_labels_and_indices(man): + """ In this scenario both Starship:13 and Vehicle:6 and Vehicle:8 have the speed of 1200 + but since only one index has been declared, in the grids driver the general v.has() should be used""" + + errors = [] + G = man.setGraph("swapi") + G.addIndex("Vehicle", "max_atmosphering_speed") + + count = 0 + for i in G.query().V().has(gripql.eq("max_atmosphering_speed", 1200)): + count += 1 + if count != 3: + errors.append("Expected 3 results but got %d instead" % (count)) + + #Every vertex needs to be indexed for this "index" feature to work in grids driver + G.addIndex("Starship", "max_atmosphering_speed") + G.addIndex("Character", "max_atmosphering_speed") + G.addIndex("Planet", "max_atmosphering_speed") + G.addIndex("Species", "max_atmosphering_speed") + G.addIndex("Film", "max_atmosphering_speed") + + # Add the other index to verify that using the other execution pipline path won't change the end result + + count = 0 + for i in G.query().V().has(gripql.eq("max_atmosphering_speed", 1200)): + count += 1 + if count != 3: + errors.append("Expected 3 results but got %d instead" % (count)) + + return errors \ No newline at end of file diff --git a/conformance/tests/ot_job.py b/conformance/tests/ot_job.py index 7190bdd9..fc93c24c 100644 --- a/conformance/tests/ot_job.py +++ b/conformance/tests/ot_job.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -import gripql import time def test_job(man): diff --git a/conformance/tests/ot_nested_index.py b/conformance/tests/ot_nested_index.py new file mode 100644 index 00000000..7fb90150 --- /dev/null +++ b/conformance/tests/ot_nested_index.py @@ -0,0 +1,42 @@ +import gripql + +def test_cond_nested_field(man): + # Tests nested filters on optimized grids query driver + + errors = [] + G = man.writeTest() + + G.addIndex("Observation", "code.coding.[0].code") + + G.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + G.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + + count = 0 + for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): + count +=1 + + if count != 1: + errors.append("Expected count == 1 but got %d instead" % (count)) + + return errors + + +def test_bulk_cond_nested_field(man): + errors = [] + G = man.writeTest() + G.addIndex("Observation", "code.coding.[0].code") + + bulk = G.bulkAdd() + bulk.addVertex("1", "Observation", {"resourceType": "Observation", "id": "e87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-9", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + bulk.addVertex("2", "Observation", {"resourceType": "Observation", "id": "f87cecef-c91d-3861-a35e-6eaed41580c8", "status": "final", "category": [{"coding": [{"system": "http://terminology.hl7.org/CodeSystem/observation-category", "code": "laboratory", "display": "laboratory"}]}], "code": {"coding": [{"system": "http://loinc.org", "code": "81247-8", "display": "Master HL7 genetic variant reporting panel"}]}, "subject": {"reference": "Patient/ac0d7a82-82cb-4aec-b859-e37375f3de8b"}, "specimen": {"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}, "focus": [{"reference": "Specimen/dc48f578-193c-4740-93f3-61a78e3c6ba0"}], "effectiveDateTime": "2024-06-03T08:00:00+00:00", "valueString": "Sequencing parameters", "component": [{"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "concentration", "display": "concentration"}], "text": "concentration"}, "valueQuantity": {"value": 0.16}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_quantity", "display": "aliquot_quantity"}], "text": "aliquot_quantity"}, "valueQuantity": {"value": 2.13}}, {"code": {"coding": [{"system": "https://cadsr.cancer.gov/sample_laboratory_observation", "code": "aliquot_volume", "display": "aliquot_volume"}], "text": "aliquot_volume"}, "valueQuantity": {"value": 13.3}}]}) + + bulk.execute() + + count = 0 + for i in G.query().V().has(gripql.eq("code.coding.[0].code","81247-8")): + count +=1 + + if count != 1: + errors.append("Expected count == 1 but got %d instead" % (count)) + + return errors diff --git a/engine/core/optimize.go b/engine/core/optimize.go index d40d8af7..1840317a 100644 --- a/engine/core/optimize.go +++ b/engine/core/optimize.go @@ -1,142 +1,93 @@ package core import ( - "github.com/bmeg/grip/gdbi/tpath" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/util/protoutil" ) -// IndexStartOptimize looks at processor pipeline for queries like -// V().Has(Eq("$._label", "Person")) and V().Has(Eq("$._id", "1")), -// streamline into a single index lookup -func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { - optimized := []*gripql.GraphStatement{} +// OptimizationRule defines a structure for matching and replacing query pipeline patterns. +type OptimizationRule struct { + Match func(pipe []*gripql.GraphStatement) bool + Replace func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement +} - //var lookupV *gripql.GraphStatement_V - hasIDIdx := []int{} - hasLabelIdx := []int{} - isDone := false - for i, step := range pipe { - if isDone { - break - } - if i == 0 { - if v, ok := step.GetStatement().(*gripql.GraphStatement_V); ok { - if v.V != nil && len(v.V.Values) > 0 { - break - } - } else { - break +// startOptimizations is a list of rules to optimize the query pipeline. +var startOptimizations = []OptimizationRule{ + { + // Matches V().HasId(...) + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false } - continue - } - switch s := step.GetStatement().(type) { - case *gripql.GraphStatement_HasId: - hasIDIdx = append(hasIDIdx, i) - case *gripql.GraphStatement_HasLabel: - hasLabelIdx = append(hasLabelIdx, i) - case *gripql.GraphStatement_Has: - if and := s.Has.GetAnd(); and != nil { - stmts := and.GetExpressions() - newPipe := []*gripql.GraphStatement{} - newPipe = append(newPipe, pipe[:i]...) - for _, stmt := range stmts { - newPipe = append(newPipe, &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{Has: stmt}}) - } - newPipe = append(newPipe, pipe[i+1:]...) - return IndexStartOptimize(newPipe) + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false } - if cond := s.Has.GetCondition(); cond != nil { - path := tpath.NormalizePath(cond.Key) - switch path { - case "$_current._id": - hasIDIdx = append(hasIDIdx, i) - case "$_current._label": - hasLabelIdx = append(hasLabelIdx, i) - default: - // do nothing - } + if hasId, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasId); ok { + return len(hasId.HasId.Values) > 0 } - default: - isDone = true - } - } - - idOpt := false - if len(hasIDIdx) > 0 { - ids := []string{} - idx := hasIDIdx[0] - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { - ids = append(ids, extractHasVals(has)...) - } - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_HasId); ok { - ids = append(ids, protoutil.AsStringList(has.HasId)...) - } - if len(ids) > 0 { - idOpt = true - hIdx := &gripql.GraphStatement_V{V: protoutil.NewListFromStrings(ids)} - optimized = append(optimized, &gripql.GraphStatement{Statement: hIdx}) - } - } - - labelOpt := false - if len(hasLabelIdx) > 0 && !idOpt { - labels := []string{} - idx := hasLabelIdx[0] - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { - labels = append(labels, extractHasVals(has)...) - } - if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_HasLabel); ok { - labels = append(labels, protoutil.AsStringList(has.HasLabel)...) - } - if len(labels) > 0 { - labelOpt = true - hIdx := &gripql.GraphStatement_LookupVertsIndex{Labels: labels} - optimized = append(optimized, &gripql.GraphStatement{Statement: hIdx}) - } - } - - for i, step := range pipe { - if idOpt || labelOpt { - if i == 0 { - continue + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + ids := protoutil.AsStringList(pipe[1].GetHasId()) + optimized := []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_V{V: protoutil.NewListFromStrings(ids)}}, } - } else { - optimized = append(optimized, step) - } - if idOpt { - if i != hasIDIdx[0] { - optimized = append(optimized, step) + return append(optimized, pipe[2:]...) + }, + }, + { + // Matches V().HasLabel(...) + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false } - } - if labelOpt { - if i != hasLabelIdx[0] { - optimized = append(optimized, step) + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if hasLabel, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); ok { + return len(hasLabel.HasLabel.GetValues()) > 0 + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + optimized := []*gripql.GraphStatement{ + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: labels}}, + } + return append(optimized, pipe[2:]...) + }, + }, +} + +// expandHasAnd preprocesses the pipeline to split Has statements with And expressions. +func expandHasAnd(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + expanded := []*gripql.GraphStatement{} + for _, step := range pipe { + if has, ok := step.GetStatement().(*gripql.GraphStatement_Has); ok { + if and := has.Has.GetAnd(); and != nil { + for _, expr := range and.Expressions { + expanded = append(expanded, &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{Has: expr}}) + } + } else { + expanded = append(expanded, step) } + } else { + expanded = append(expanded, step) } } - - return optimized + return expanded } -func extractHasVals(h *gripql.GraphStatement_Has) []string { - vals := []string{} - if cond := h.Has.GetCondition(); cond != nil { - // path := jsonpath.GetJSONPath(cond.Key) - val := cond.Value.AsInterface() - switch cond.Condition { - case gripql.Condition_EQ: - if l, ok := val.(string); ok { - vals = []string{l} - } - case gripql.Condition_WITHIN: - v := val.([]interface{}) - for _, x := range v { - vals = append(vals, x.(string)) - } - default: - // do nothing +// IndexStartOptimize applies optimization rules to the query pipeline. +func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + // Preprocess to handle Has with And expressions + pipe = expandHasAnd(pipe) + // Apply the first matching optimization rule + for _, rule := range startOptimizations { + if rule.Match(pipe) { + return rule.Replace(pipe) } } - return vals + // Return the original pipeline if no optimizations apply + return pipe } diff --git a/engine/core/optimizer_test.go b/engine/core/optimizer_test.go index 527649f6..44bbe8a3 100644 --- a/engine/core/optimizer_test.go +++ b/engine/core/optimizer_test.go @@ -211,7 +211,7 @@ func TestIndexStartOptimize(t *testing.T) { } expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Out{}}, } @@ -266,7 +266,7 @@ func TestIndexStartOptimize(t *testing.T) { } expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Out{}}, } @@ -294,7 +294,7 @@ func TestIndexStartOptimize(t *testing.T) { barValue, _ := structpb.NewValue("bar") expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Has{ Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ Condition: &gripql.HasCondition{ @@ -339,7 +339,7 @@ func TestIndexStartOptimize(t *testing.T) { bazValue, _ := structpb.NewValue("baz") expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Has{ Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ Condition: &gripql.HasCondition{ @@ -440,7 +440,7 @@ func TestIndexStartOptimize(t *testing.T) { // handle 'and' statements expected = []*gripql.GraphStatement{ - {Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + {Statement: &gripql.GraphStatement_LookupVertsLabelIndex{Labels: []string{"foo", "bar"}}}, {Statement: &gripql.GraphStatement_Has{ Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ Condition: &gripql.HasCondition{ diff --git a/engine/core/processors.go b/engine/core/processors.go index 7c7108ed..b47e7cea 100644 --- a/engine/core/processors.go +++ b/engine/core/processors.go @@ -7,6 +7,7 @@ import ( "github.com/bmeg/grip/engine/logic" "github.com/bmeg/grip/gdbi" + //"github.com/bmeg/grip/log" "github.com/bmeg/grip/util/copy" "github.com/spf13/cast" ) @@ -59,14 +60,14 @@ func (l *LookupVerts) Process(ctx context.Context, man gdbi.Manager, in gdbi.InP //////////////////////////////////////////////////////////////////////////////// // LookupVertsIndex look up vertices by indexed based feature -type LookupVertsIndex struct { +type LookupVertsLabelIndex struct { db gdbi.GraphInterface labels []string loadData bool } // Process LookupVertsIndex -func (l *LookupVertsIndex) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { +func (l *LookupVertsLabelIndex) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { queryChan := make(chan gdbi.ElementLookup, 100) go func() { defer close(queryChan) @@ -226,6 +227,7 @@ func (r *Unwind) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, continue } v := gdbi.TravelerPathLookup(t, r.Field) + //log.Debugln("UNWIND V RES: ", v) if a, ok := v.([]interface{}); ok { cur := t.GetCurrent() if len(a) > 0 { diff --git a/engine/core/statement_compiler.go b/engine/core/statement_compiler.go index 7e2b1822..2709938a 100644 --- a/engine/core/statement_compiler.go +++ b/engine/core/statement_compiler.go @@ -242,10 +242,9 @@ func (sc *DefaultStmtCompiler) Custom(gs *gripql.GraphStatement, ps *gdbi.State) switch stmt := gs.GetStatement().(type) { //Custom graph statements - case *gripql.GraphStatement_LookupVertsIndex: + case *gripql.GraphStatement_LookupVertsLabelIndex: ps.LastType = gdbi.VertexData - return &LookupVertsIndex{db: sc.db, labels: stmt.Labels, loadData: ps.StepLoadData()}, nil - + return &LookupVertsLabelIndex{db: sc.db, labels: stmt.Labels, loadData: ps.StepLoadData()}, nil case *gripql.GraphStatement_EngineCustom: proc := stmt.Custom.(gdbi.CustomProcGen) ps.LastType = proc.GetType() diff --git a/engine/logic/match.go b/engine/logic/match.go index 4001a010..e02156fc 100644 --- a/engine/logic/match.go +++ b/engine/logic/match.go @@ -12,8 +12,8 @@ import ( ) func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { - var val interface{} - var condVal interface{} + var val any + var condVal any val = gdbi.TravelerPathLookup(trav, cond.Key) condVal = cond.Value.AsInterface() @@ -26,7 +26,6 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { //TODO: Add escape for $ user string } //If filtering on nil or no match was found on float64 casting operators return false - log.Debug("val: ", val, "condVal: ", condVal) if (val == nil || condVal == nil) && cond.Condition != gripql.Condition_EQ && cond.Condition != gripql.Condition_NEQ && @@ -36,7 +35,7 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { return false } - log.Debugf("match: %s %s %s", condVal, val, cond.Key) + //log.Debugf("match: %s %s %s", condVal, val, cond.Key) switch cond.Condition { case gripql.Condition_EQ: @@ -68,7 +67,6 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { return valN >= condN case gripql.Condition_LT: - //log.Debugf("match: %#v %#v %s", condVal, val, cond.Key) valN, err := cast.ToFloat64E(val) //log.Debugf("CAST: ", valN, "ERROR: ", err) if err != nil { @@ -194,7 +192,7 @@ func MatchesCondition(trav gdbi.Traveler, cond *gripql.HasCondition) bool { case gripql.Condition_WITHOUT: found := false switch condVal := condVal.(type) { - case []interface{}: + case []any: for _, v := range condVal { if reflect.DeepEqual(val, v) { found = true @@ -239,7 +237,6 @@ func MatchesHasExpression(trav gdbi.Traveler, stmt *gripql.HasExpression) bool { switch stmt.Expression.(type) { case *gripql.HasExpression_Condition: cond := stmt.GetCondition() - log.Debug("COND: ", cond) return MatchesCondition(trav, cond) case *gripql.HasExpression_And: diff --git a/engine/logic/sorter_kv.go b/engine/logic/sorter_kv.go index 8a4a97e3..31546e31 100644 --- a/engine/logic/sorter_kv.go +++ b/engine/logic/sorter_kv.go @@ -63,11 +63,11 @@ func (ks *KVSorter[T]) Sorted() chan T { func (ks *kvCompare[T]) compareEncoded(a, b []byte) int { aT, err := ks.conf.FromBytes(a) if err != nil { - log.Debug("error compareEncoded: %s\n", err) + log.Debugf("error compareEncoded: %s\n", err) } bT, err := ks.conf.FromBytes(b) if err != nil { - log.Debug("error compareEncoded: %s\n", err) + log.Debugf("error compareEncoded: %s\n", err) } return ks.conf.Compare(aT, bT) } diff --git a/gdbi/data_element.go b/gdbi/data_element.go index 31974d1a..1d643b3f 100644 --- a/gdbi/data_element.go +++ b/gdbi/data_element.go @@ -12,7 +12,7 @@ import ( func (elem *DataElement) ToVertex() *gripql.Vertex { sValue, err := structpb.NewStruct(elem.Data) if err != nil { - log.Errorf("Error: %s For elem.Data: '%#v'\n", err, elem.Data) + log.Errorf("ToVertex: %s For elem.Data: '%#v'\n", err, elem.Data) } return &gripql.Vertex{ Id: elem.ID, diff --git a/gdbi/pipeline.go b/gdbi/pipeline.go index ee406d2b..2a8ef056 100644 --- a/gdbi/pipeline.go +++ b/gdbi/pipeline.go @@ -9,6 +9,7 @@ import ( type PipelineState interface { GetLastType() DataType SetLastType(DataType) + StepLoadData() bool } type CustomProcGen interface { diff --git a/gdbi/traveler_doc.go b/gdbi/traveler_doc.go index 131c250d..4aab7c89 100644 --- a/gdbi/traveler_doc.go +++ b/gdbi/traveler_doc.go @@ -79,7 +79,7 @@ func TravelerPathLookup(traveler Traveler, path string) interface{} { return doc } res, err := jsonpath.JsonPathLookup(doc, jpath) - log.Debug("field: ", field, " jpath: ", jpath, " namespace: ", namespace, " doc: ", doc, " res: ", res) + //log.Debug("field: ", field, " jpath: ", jpath, " namespace: ", namespace, " doc: ", doc, " res: ", res) if err != nil { return nil diff --git a/go.mod b/go.mod index f1f9f6e1..cbf8518f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/bmeg/grip -go 1.23.6 +go 1.24 + +toolchain go1.24.2 require ( github.com/IBM/sarama v1.45.1 @@ -8,20 +10,20 @@ require ( github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250418205623-cbcef7cb3ad1 + github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 github.com/bmeg/jsonschemagraph v0.0.3-0.20250330060023-8f61d8bfec9a github.com/boltdb/bolt v1.3.1 + github.com/bytedance/sonic v1.13.3 github.com/casbin/casbin/v2 v2.97.0 github.com/cockroachdb/pebble v1.1.2 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.2007.4 - github.com/dgraph-io/ristretto/v2 v2.1.0 github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5 github.com/felixge/httpsnoop v1.0.4 github.com/go-sql-driver/mysql v1.8.1 - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.6.1 @@ -41,17 +43,17 @@ require ( github.com/robertkrimen/otto v0.4.0 github.com/segmentio/ksuid v1.0.4 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cast v1.6.0 + github.com/spf13/cast v1.9.2 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.0 go.mongodb.org/mongo-driver v1.17.0 - golang.org/x/crypto v0.33.0 - golang.org/x/net v0.35.0 - golang.org/x/sync v0.11.0 - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb - google.golang.org/grpc v1.70.0 - google.golang.org/protobuf v1.36.5 + golang.org/x/net v0.37.0 + golang.org/x/sync v0.12.0 + golang.org/x/term v0.30.0 + google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a + google.golang.org/grpc v1.71.0 + google.golang.org/protobuf v1.36.7 sigs.k8s.io/yaml v1.4.0 ) @@ -63,21 +65,24 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.11.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/edsrzf/mmap-go v1.2.0 // indirect github.com/fatih/color v1.17.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/getsentry/sentry-go v0.28.1 // indirect @@ -86,7 +91,6 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect - github.com/golang/glog v1.2.3 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect @@ -108,6 +112,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/maypok86/otter/v2 v2.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/montanaflynn/stats v0.7.1 // indirect @@ -128,16 +133,18 @@ require ( github.com/rs/xid v1.5.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.23.0 // indirect gonum.org/v1/gonum v0.8.2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index fd7a1d58..e0ae85e3 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= @@ -31,14 +32,11 @@ github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YT github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250407225705-b34b3f7eeae3 h1:R2kBnE790WVMJ7/cyOk5t/JdDbAhWfRm39R9scq8JKA= -github.com/bmeg/benchtop v0.0.0-20250407225705-b34b3f7eeae3/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= -github.com/bmeg/benchtop v0.0.0-20250417164945-eae1ff034cce h1:ffXdKMWrziCNnYiVrLwApRWr9iRR7GTVHYEEGYZfU/w= -github.com/bmeg/benchtop v0.0.0-20250417164945-eae1ff034cce/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= -github.com/bmeg/benchtop v0.0.0-20250418205623-cbcef7cb3ad1 h1:xKFGnYOIKul7kVHly2ov6mI5GzJWWOPlW45zURcUzC8= -github.com/bmeg/benchtop v0.0.0-20250418205623-cbcef7cb3ad1/go.mod h1:lq5bLUSwfHFghH0evzfknaBwl8YvTnrA1rBpqqtg2EM= +github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 h1:sIgPwNZKv3pSuIm/hngszbBrg3pKlZcyJ+HTzeFyjNA= +github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v5 v5.3.4-0.20241111204732-55db82022a92 h1:Myx/j+WxfEg+P3nDaizR1hBpjKSLgvr4ydzgp1/1pAU= @@ -49,16 +47,26 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= +github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/casbin/casbin/v2 v2.97.0 h1:FFHIzY+6fLIcoAB/DKcG5xvscUo9XqRpBniRYhlPWkg= github.com/casbin/casbin/v2 v2.97.0/go.mod h1:jX8uoN4veP85O/n2674r2qtfSXI6myvxW85f6TH50fw= github.com/casbin/govaluate v1.1.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.2.0 h1:wXCXFmqyY+1RwiKfYo3jMKyrtZmOL3kHwaqDyCPOYak= github.com/casbin/govaluate v1.2.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= @@ -85,10 +93,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I= -github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -107,6 +113,12 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= +github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= @@ -127,6 +139,8 @@ github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxI github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -135,6 +149,7 @@ github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TC github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -143,12 +158,13 @@ github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17w github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= -github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -176,8 +192,8 @@ github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25d github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -234,10 +250,13 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042 h1:Vzdm5hdlLdpJOKK+hKtkV5u7xGZmNW6aUBjGcTfwx84= github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042/go.mod h1:fYE0718xXI13XMYLc6iHtvXudfyCGMsZ9hxSM1Ommpg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -264,6 +283,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/maypok86/otter/v2 v2.1.0 h1:H+FO9NtLuSWYUlIUQ/kT6VNEpWSIF4w4GZJRDhxYb7k= +github.com/maypok86/otter/v2 v2.1.0/go.mod h1:jX2xEKz9PrNVbDqnk8JUuOt5kURK8h7jd1kDYI5QsZk= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo= @@ -295,6 +316,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -313,6 +335,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= @@ -337,6 +360,7 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPO github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= @@ -348,8 +372,8 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE= +github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -359,9 +383,11 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -374,6 +400,8 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -389,34 +417,50 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -428,19 +472,23 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -454,35 +502,40 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -499,21 +552,32 @@ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a h1:DMCgtIAIQGZqJXMVzJF4MV8BlWoJh2ZuFiRdAleyr58= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a/go.mod h1:y2yVLIE/CSMCPXaHnSKXxu1spLPnglFLegmgdY23uuE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -525,12 +589,17 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/grids/filters.go b/grids/filters.go new file mode 100644 index 00000000..b0e620ad --- /dev/null +++ b/grids/filters.go @@ -0,0 +1,140 @@ +package grids + +import ( + bFilters "github.com/bmeg/benchtop/filters" + "github.com/bmeg/benchtop/jsontable" + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" + "github.com/bytedance/sonic" + "github.com/bytedance/sonic/ast" +) + +type GripQLFilter struct { + Expression *gripql.HasExpression +} + +func (f *GripQLFilter) GetFilter() any { + return f.Expression +} +func (f *GripQLFilter) IsNoOp() bool { + // A GripQLFilter is a no-op if its Expression is nil + return f.Expression == nil +} + +func (f *GripQLFilter) Matches(row any) bool { + return MatchesHasExpression(row, f.Expression) +} + +func (f *GripQLFilter) RequiredFields() []string { + return extractKeys(f.Expression) +} + +func extractKeys(expr *gripql.HasExpression) []string { + keys := map[string]struct{}{} + + var recurse func(*gripql.HasExpression) + recurse = func(e *gripql.HasExpression) { + if e == nil { + return + } + switch st := e.Expression.(type) { + case *gripql.HasExpression_Condition: + keys[st.Condition.GetKey()] = struct{}{} + case *gripql.HasExpression_And: + for _, subExpr := range st.And.GetExpressions() { + recurse(subExpr) + } + case *gripql.HasExpression_Or: + for _, subExpr := range st.Or.GetExpressions() { + recurse(subExpr) + } + case *gripql.HasExpression_Not: + recurse(st.Not.GetNot()) + } + } + + recurse(expr) + + out := make([]string, 0, len(keys)) + for k := range keys { + out = append(out, k) + } + return out +} +func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { + switch stmt.Expression.(type) { + case *gripql.HasExpression_Condition: + cond := stmt.GetCondition() + var lookupVal any + + // Handle lookup based on input type + switch v := val.(type) { + case map[string]any: + lookupVal = jsontable.PathLookup(v, cond.Key) + case []byte: + pathArr, err := jsontable.ConvertJSONPathToArray(cond.Key) + if err != nil { + log.Errorf("Error converting JSON path: %v", err) + return false + } + node, err := sonic.Get(v, pathArr...) + if err != nil { + if err != ast.ErrNotExist { + log.Errorf("Sonic Fetch err for path: %s on doc %#v: %v", pathArr, string(v), err) + } + return false + } + lookupVal, err = node.Interface() + if err != nil { + log.Errorf("Error unmarshaling node: %v", err) + return false + } + default: + log.Errorf("Unsupported input type: %T", val) + return false + } + + return bFilters.ApplyFilterCondition( + lookupVal, + &bFilters.FieldFilter{ + Operator: cond.Condition, + Field: cond.Key, + Value: cond.Value.AsInterface(), + }, + ) + + case *gripql.HasExpression_And: + and := stmt.GetAnd() + andRes := []bool{} + for _, e := range and.Expressions { + andRes = append(andRes, MatchesHasExpression(val, e)) + } + for _, r := range andRes { + if !r { + return false + } + } + return true + + case *gripql.HasExpression_Or: + or := stmt.GetOr() + orRes := []bool{} + for _, e := range or.Expressions { + orRes = append(orRes, MatchesHasExpression(val, e)) + } + for _, r := range orRes { + if r { + return true + } + } + return false + + case *gripql.HasExpression_Not: + e := stmt.GetNot() + return !MatchesHasExpression(val, e) + + default: + log.Errorf("unknown where expression type: %T", stmt.Expression) + return false + } +} diff --git a/grids/graph.go b/grids/graph.go index f2c08b99..f8dd2dec 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -4,16 +4,21 @@ import ( "bytes" "context" "fmt" + "runtime" + "slices" + "sort" "sync" + "sync/atomic" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/benchtop/pebblebulk" "github.com/bmeg/grip/engine/core" "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/log" - "github.com/bmeg/grip/util/protoutil" "github.com/bmeg/grip/util/setcmp" + "github.com/bytedance/sonic" + "github.com/cockroachdb/pebble" multierror "github.com/hashicorp/go-multierror" ) @@ -27,105 +32,170 @@ func (ggraph *Graph) GetTimestamp() string { return ggraph.ts.Get(ggraph.graphID) } -func insertVertex(tx *pebblebulk.PebbleBulk, keyMap *KeyMap, vertex *gdbi.Vertex) error { +func insertVertex(tx *pebblebulk.PebbleBulk, vertex *gdbi.Vertex) error { if vertex.ID == "" { return fmt.Errorf("inserting null key vertex") } - vertexKey, _ := keyMap.GetsertVertexKeyLabel(vertex.ID, vertex.Label, tx) - key := VertexKey(vertexKey) - if err := tx.Set(key, nil, nil); err != nil { + if err := tx.Set(VertexKey(vertex.ID), []byte(vertex.Label), nil); err != nil { return fmt.Errorf("AddVertex Error %s", err) } return nil } -func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex) error { +func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) error { vertexLabel := VTABLE_PREFIX + vertex.Label - ggraph.bsonkv.Lock.Lock() - table, ok := ggraph.bsonkv.Tables[vertexLabel] - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table, ok := ggraph.jsonkv.Tables[vertexLabel] + ggraph.jsonkv.Lock.Unlock() if !ok { - log.Debugf("Creating new table for: %s on graph %s", vertex.Label, ggraph.graphID) - newTable, err := ggraph.bsonkv.New(vertexLabel, nil) + log.Debugf("Creating new table %s for label %s on graph %s", vertexLabel, vertex.Label, ggraph.graphID) + newTable, err := ggraph.jsonkv.New(vertexLabel, nil) if err != nil { return fmt.Errorf("indexVertex: %s", err) } - ggraph.bsonkv.Lock.Lock() - table = newTable.(*bsontable.BSONTable) - ggraph.bsonkv.Tables[vertexLabel] = table - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table = newTable.(*jsontable.JSONTable) + ggraph.jsonkv.Tables[vertexLabel] = table + ggraph.jsonkv.Lock.Unlock() } - if err := table.AddRow(benchtop.Row{Id: []byte(vertex.ID), TableName: vertexLabel, Data: vertex.Data}); err != nil { + + rowLoc, err := table.AddRow( + benchtop.Row{ + Id: []byte(vertex.ID), + TableName: vertexLabel, + Data: vertex.Data, + }, + ) + if err != nil { return fmt.Errorf("AddVertex Error %s", err) } - return nil -} + table.AddTableEntryInfo(tx, []byte(vertex.ID), *rowLoc) -func insertEdge(tx *pebblebulk.PebbleBulk, keyMap *KeyMap, edge *gdbi.Edge) error { - var err error - if edge.ID == "" { - return fmt.Errorf("inserting null key edge") + _, ok = ggraph.jsonkv.PageCache.Set(vertex.ID, *rowLoc) + if !ok { + ggraph.jsonkv.PageCache.Invalidate(vertex.ID) + ggraph.jsonkv.PageCache.Set(vertex.ID, *rowLoc) + //log.Debugln("Replaced vals: ", vertex.ID, oldVal, newVal) } - eid, lid := keyMap.GetsertEdgeKey(edge.ID, edge.Label, tx) - /* providing a label doesn't matter if not going to use the label key anyway. - It can get set in the insertvertex func later */ - src := keyMap.GetsertVertexKey(edge.From, tx) - dst := keyMap.GetsertVertexKey(edge.To, tx) + _, fieldsExist := ggraph.jsonkv.Fields[vertexLabel] + if fieldsExist { + for field := range ggraph.jsonkv.Fields[vertexLabel] { + if val := jsontable.PathLookup(vertex.Data, field); val != nil { + err := tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) + if err != nil { + return err + } + Mval, err := sonic.ConfigFastest.Marshal(val) + if err != nil { + return err + } + err = tx.Set(benchtop.RFieldKey(vertexLabel, field, vertex.ID), Mval, nil) + if err != nil { + return err + } + } + } + } - ekey := EdgeKey(eid, src, dst, lid) - skey := SrcEdgeKey(eid, src, dst, lid) - dkey := DstEdgeKey(eid, src, dst, lid) + return nil +} - err = tx.Set(ekey, nil, nil) +func insertEdge(tx *pebblebulk.PebbleBulk, edge *gdbi.Edge) error { + if edge.ID == "" || + edge.From == "" || + edge.To == "" || + edge.Label == "" { + log.Errorln("insertEdge Err: ", edge) + return fmt.Errorf("inserting null key edge") + } + err := tx.Set(EdgeKey(edge.ID, edge.From, edge.To, edge.Label), nil, nil) if err != nil { return err } - err = tx.Set(skey, []byte{}, nil) + err = tx.Set(DstEdgeKey( + edge.ID, + edge.From, + edge.To, + edge.Label, + ), []byte{}, nil) if err != nil { return err } - err = tx.Set(dkey, []byte{}, nil) + err = tx.Set(SrcEdgeKey( + edge.ID, + edge.From, + edge.To, + edge.Label, + ), []byte{}, nil) if err != nil { return err } return nil } -func (ggraph *Graph) indexEdge(edge *gdbi.Edge) error { +func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error { edgeLabel := ETABLE_PREFIX + edge.Label - ggraph.bsonkv.Lock.Lock() - table, ok := ggraph.bsonkv.Tables[edgeLabel] - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table, ok := ggraph.jsonkv.Tables[edgeLabel] + ggraph.jsonkv.Lock.Unlock() if !ok { - log.Debugf("Creating new table for: %s on graph %s", edge.Label, ggraph.graphID) - newTable, err := ggraph.bsonkv.New(edgeLabel, nil) + log.Debugf("Creating new table %s for label %s on graph %s", edgeLabel, edge.Label, ggraph.graphID) + newTable, err := ggraph.jsonkv.New(edgeLabel, nil) if err != nil { - return fmt.Errorf("indexEdge: bsonkv.New: %s", err) + return fmt.Errorf("indexEdge: jsonkv.New: %s", err) } - ggraph.bsonkv.Lock.Lock() - table = newTable.(*bsontable.BSONTable) - ggraph.bsonkv.Tables[edgeLabel] = table - ggraph.bsonkv.Lock.Unlock() + ggraph.jsonkv.Lock.Lock() + table = newTable.(*jsontable.JSONTable) + ggraph.jsonkv.Tables[edgeLabel] = table + ggraph.jsonkv.Lock.Unlock() } - if err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}); err != nil { + rowLoc, err := table.AddRow(benchtop.Row{Id: []byte(edge.ID), TableName: edgeLabel, Data: edge.Data}) + if err != nil { return fmt.Errorf("indexEdge: table.AddRow: %s", err) } + table.AddTableEntryInfo(tx, []byte(edge.ID), *rowLoc) + + _, ok = ggraph.jsonkv.PageCache.Set(edge.ID, *rowLoc) + if !ok { + ggraph.jsonkv.PageCache.Invalidate(edge.ID) + ggraph.jsonkv.PageCache.Set(edge.ID, *rowLoc) + } + + _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] + if fieldsExist { + for field := range ggraph.jsonkv.Fields[edgeLabel] { + if val := jsontable.PathLookup(edge.Data, field); val != nil { + err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) + if err != nil { + return err + } + eMarsh, err := sonic.ConfigFastest.Marshal(val) + if err != nil { + return err + } + err = tx.Set(benchtop.RFieldKey(edgeLabel, field, edge.ID), eMarsh, nil) + if err != nil { + return err + } + } + } + } return nil } func (ggraph *Graph) Compiler() gdbi.Compiler { - return core.NewCompiler(ggraph, core.IndexStartOptimize) + return core.NewCompiler(ggraph, GridsOptimizer, core.IndexStartOptimize) } // AddVertex adds an edge to the graph, if it already exists // in the graph, it is replaced func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { - if err := insertVertex(tx, ggraph.keyMap, vert); err != nil { + if err := insertVertex(tx, vert); err != nil { bulkErr = multierror.Append(bulkErr, err) log.Errorf("AddVertex Error %s", err) } @@ -134,10 +204,10 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { return bulkErr.ErrorOrNil() }) - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { - if err := ggraph.indexVertex(vert); err != nil { + if err := ggraph.indexVertex(vert, tx); err != nil { bulkErr = multierror.Append(bulkErr, err) log.Errorf("IndexVertex Error %s", err) } @@ -151,23 +221,28 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { // AddEdge adds an edge to the graph, if the id is not "" and in already exists // in the graph, it is replaced func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - for _, edge := range edges { - err := insertEdge(tx, ggraph.keyMap, edge) - if err != nil { - return err + var err error = nil + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, edge := range edges { + err = insertEdge(tx, edge) + if err != nil { + log.Errorln("Err insertEdge: ", err) + return err + } } - } + return err + }) ggraph.ts.Touch(ggraph.graphID) - return nil + return err }) if err != nil { return err } - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, edge := range edges { - if err := ggraph.indexEdge(edge); err != nil { + if err := ggraph.indexEdge(edge, tx); err != nil { bulkErr = multierror.Append(bulkErr, err) } } @@ -178,23 +253,6 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { } -func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { - var bulkErr *multierror.Error - for _, val := range data.Edges { - err := ggraph.DelEdge(val) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - for _, val := range data.Vertices { - err := ggraph.DelVertex(val) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - } - return bulkErr.ErrorOrNil() -} - func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { var errs *multierror.Error insertStream := make(chan *gdbi.GraphElement, 100) @@ -204,33 +262,35 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { var wg sync.WaitGroup wg.Add(2) - // Goroutine for inserting vertices and edges into graphkv go func() { defer wg.Done() - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { for elem := range insertStream { if elem.Vertex != nil { - if err := insertVertex(tx, ggraph.keyMap, elem.Vertex); err != nil { + if err := insertVertex(tx, elem.Vertex); err != nil { return fmt.Errorf("vertex insert error: %v", err) } } if elem.Edge != nil { - if err := insertEdge(tx, ggraph.keyMap, elem.Edge); err != nil { + if err := insertEdge(tx, elem.Edge); err != nil { return fmt.Errorf("edge insert error: %v", err) } } } - ggraph.ts.Touch(ggraph.graphID) return nil }) - errChan <- err + if err != nil { + log.Errorf("ERR in graph Bulk Add: %s", err) + return + } + }() go func() { defer wg.Done() - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := ggraph.bsonkv.BulkLoad(indexStream, tx); err != nil { - return fmt.Errorf("bsonkv bulk load error: %v", err) + err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + if err := ggraph.jsonkv.BulkLoad(indexStream, tx); err != nil { + return fmt.Errorf("jsonkv bulk load error: %v", err) } ggraph.ts.Touch(ggraph.graphID) return nil @@ -271,128 +331,93 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { } } - // Return any accumulated errors return errs.ErrorOrNil() } -func (ggraph *Graph) DelEdge(eid string) error { - edgeKey, ok := ggraph.keyMap.GetEdgeKey(eid, ggraph.bsonkv.Pb.Db) - if !ok { - return fmt.Errorf("edge not found") - } - ekeyPrefix := EdgeKeyPrefix(edgeKey) - var ekey []byte - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { - ekey = it.Key() - } - return nil - }) - if ekey == nil { - return fmt.Errorf("edge not found") - } - - eidParsed, sid, did, lbl := EdgeKeyParse(ekey) - - skey := SrcEdgeKey(eidParsed, sid, did, lbl) - dkey := DstEdgeKey(eidParsed, sid, did, lbl) - - var bulkErr *multierror.Error - err := ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := tx.Delete(ekey, nil); err != nil { - return err - } - if err := tx.Delete(skey, nil); err != nil { - return err - } - if err := tx.Delete(dkey, nil); err != nil { - return err - } - ggraph.ts.Touch(ggraph.graphID) - return nil - }) - if err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - - if err := ggraph.keyMap.DelEdgeKey(eid, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - if err := ggraph.bsonkv.DeleteAnyRow([]byte(eid)); err != nil { - bulkErr = multierror.Append(bulkErr, err) - } - - return bulkErr.ErrorOrNil() -} - -// DelVertex deletes vertex with id `key` func (ggraph *Graph) DelVertex(id string) error { - vertexKey, ok := ggraph.keyMap.GetVertexKey(id, ggraph.bsonkv.Pb.Db) - if !ok { - return fmt.Errorf("vertex %s not found", id) - } - vid := VertexKey(vertexKey) - skeyPrefix := SrcEdgePrefix(vertexKey) - dkeyPrefix := DstEdgePrefix(vertexKey) + vid := VertexKey(id) + skeyPrefix := SrcEdgePrefix(id) + dkeyPrefix := DstEdgePrefix(id) delKeys := make([][]byte, 0, 1000) + edgesToDelete := make(map[string]string) var bulkErr *multierror.Error - - err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { - var bulkErr *multierror.Error + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() - // get edge ID from key eid, sid, did, label := SrcEdgeKeyParse(skey) - ekey := EdgeKey(eid, sid, did, label) - dkey := DstEdgeKey(eid, sid, did, label) - delKeys = append(delKeys, ekey, skey, dkey) - edgeID, ok := ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - if ok { - if err := ggraph.keyMap.DelEdgeKey(edgeID, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) + if ggraph.tempDeletedEdges != nil { + if _, exists := ggraph.tempDeletedEdges[eid]; exists { + continue } } - if err := ggraph.bsonkv.DeleteAnyRow([]byte(edgeID)); err != nil { - bulkErr = multierror.Append(bulkErr, err) + if _, exists := edgesToDelete[eid]; exists { + continue } + + ekey := EdgeKey(eid, sid, did, label) + dkey := DstEdgeKey(eid, sid, did, label) + delKeys = append(delKeys, ekey, skey, dkey) + edgesToDelete[eid] = label } + for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { dkey := it.Key() - // get edge ID from key eid, sid, did, label := DstEdgeKeyParse(dkey) - ekey := EdgeKey(eid, sid, did, label) - skey := SrcEdgeKey(eid, sid, did, label) - delKeys = append(delKeys, ekey, skey, dkey) - edgeID, ok := ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - if ok { - if err := ggraph.keyMap.DelEdgeKey(edgeID, ggraph.bsonkv.Pb.Db); err != nil { - bulkErr = multierror.Append(bulkErr, err) + if ggraph.tempDeletedEdges != nil { + if _, exists := ggraph.tempDeletedEdges[eid]; exists { + continue } } - if err := ggraph.bsonkv.DeleteAnyRow([]byte(edgeID)); err != nil { - bulkErr = multierror.Append(bulkErr, err) + if _, exists := edgesToDelete[eid]; exists { + continue } + + ekey := EdgeKey(eid, sid, did, label) + skey := SrcEdgeKey(eid, sid, did, label) + delKeys = append(delKeys, ekey, skey, dkey) + edgesToDelete[eid] = label } - return bulkErr.ErrorOrNil() + return nil }) + if err != nil { - bulkErr = multierror.Append(bulkErr, err) + return err + } + + for eid, label := range edgesToDelete { + if err := ggraph.DeleteAnyRow(eid, label, true); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + + if ggraph.tempDeletedEdges != nil { + ggraph.tempDeletedEdges[eid] = struct{}{} + } + } + + loc, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + if err != nil { + return err } - if err := ggraph.keyMap.DelVertexKey(id, ggraph.bsonkv.Pb.Db); err != nil { + label := ggraph.jsonkv.LabelLookup[loc.Label] + if label == "" { + bulkErr = multierror.Append(bulkErr, fmt.Errorf("Failed to lookup table label %d", loc.Label)) + } + if err := ggraph.DeleteAnyRow(id, label, false); err != nil { bulkErr = multierror.Append(bulkErr, err) } - err = ggraph.bsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - if err := tx.Delete(vid, nil); err != nil { + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + if err := tx.DeletePrefix(vid); err != nil { return err } for _, k := range delKeys { - if err := tx.Delete(k, nil); err != nil { + if err := tx.DeletePrefix(k); err != nil { + log.Errorf("BulkWrite failed to delete key %s: %v", string(k), err) return err } } @@ -402,6 +427,55 @@ func (ggraph *Graph) DelVertex(id string) error { if err != nil { bulkErr = multierror.Append(bulkErr, err) } + + return bulkErr.ErrorOrNil() +} + +func (ggraph *Graph) DelEdge(eid string) error { + ekeyPrefix := EdgeKeyPrefix(eid) + var ekey []byte + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { + ekey = it.Key() + } + return nil + }) + if err != nil { + return err + } + + if ekey == nil { + log.Debugf("Edge %s not found", eid) + return nil + } + + _, sid, did, lbl := EdgeKeyParse(ekey) + skey := SrcEdgeKey(sid, did, eid, lbl) + dkey := DstEdgeKey(sid, did, eid, lbl) + + var bulkErr *multierror.Error + err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + if err := tx.Delete(ekey, nil); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + if err := tx.Delete(skey, nil); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + if err := tx.Delete(dkey, nil); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + ggraph.ts.Touch(ggraph.graphID) + return nil + }) + + if err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + + if err := ggraph.DeleteAnyRow(eid, lbl, true); err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + return bulkErr.ErrorOrNil() } @@ -410,7 +484,7 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb o := make(chan *gdbi.Edge, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { ePrefix := EdgeListPrefix() for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { select { @@ -418,16 +492,15 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb return nil default: } - keyValue := it.Key() - ekey, skey, dkey, label := EdgeKeyParse(keyValue) - labelID, _ := ggraph.keyMap.GetLabelID(label, ggraph.bsonkv.Pb.Db) - sid, _ := ggraph.keyMap.GetVertexID(skey, ggraph.bsonkv.Pb.Db) - did, _ := ggraph.keyMap.GetVertexID(dkey, ggraph.bsonkv.Pb.Db) - eid, _ := ggraph.keyMap.GetEdgeID(ekey, ggraph.bsonkv.Pb.Db) - e := &gdbi.Edge{ID: eid, Label: labelID, From: sid, To: did} + eid, sid, did, label := EdgeKeyParse(it.Key()) + e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} if loadProp { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+labelID].GetRow([]byte(eid)) + entry, err := ggraph.jsonkv.PageCache.Get(ctx, eid, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetEdgeList: PageCache.Get( error: %v", err) + continue + } + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+label].GetRow(entry) if err != nil { log.Errorf("GetEdgeList: GetRow error: %v", err) continue @@ -446,21 +519,32 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb // GetVertex loads a vertex given an id. It returns a nil if not found func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { - key, ok := ggraph.keyMap.GetVertexKey(id, ggraph.bsonkv.Pb.Db) - if !ok { + ekeyPrefix := VertexKey(id) + var byteLabel []byte = nil + var err error = nil + err = ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { + byteLabel, err = it.Value() + } + return nil + }) + if err != nil || byteLabel == nil { return nil } - var v *gdbi.Vertex - lKey := ggraph.keyMap.GetVertexLabel(key, ggraph.bsonkv.Pb.Db) - lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - v = &gdbi.Vertex{ + + v := &gdbi.Vertex{ ID: id, - Label: lID, + Label: string(byteLabel), } if loadProp { - var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(id)) + entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetVertex: PageCache.Get( error: %v", err) + return nil + } + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) if err != nil { + log.Errorf("GetVertex: table.GetRow( error: %v", err) return nil } v.Loaded = true @@ -471,108 +555,90 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { } type elementData struct { - key uint64 - req gdbi.ElementLookup - data []byte + label string + req gdbi.ElementLookup + data []byte } -// GetVertexChannel is passed a channel of vertex ids and it produces a channel -// of vertices func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { - data := make(chan elementData, 100) - go func() { - defer close(data) - for id := range ids { - if id.IsSignal() { - data <- elementData{req: id} - } else { - key, _ := ggraph.keyMap.GetVertexKey(id.ID, ggraph.bsonkv.Pb.Db) - ed := elementData{key: key, req: id} - if load { - lKey := ggraph.keyMap.GetVertexLabel(key, ggraph.bsonkv.Pb.Db) - lID, ok := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - if !ok || lID == "" { - log.Debugln("No LID for lkey: ", lKey) - continue - } - vData, err := ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(id.ID)) - if err != nil { - log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) - continue - } - vdataM, _ := protoutil.StructMarshal(vData) - ed.data = vdataM - } - data <- ed - } - } - }() - out := make(chan gdbi.ElementLookup, 100) go func() { defer close(out) - for d := range data { - if d.req.IsSignal() { - out <- d.req - } else { - lKey := ggraph.keyMap.GetVertexLabel(d.key, ggraph.bsonkv.Pb.Db) - lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - v := gdbi.Vertex{ID: d.req.ID, Label: lID} - var err error - v.Data, err = protoutil.StructUnMarshal(d.data) - v.Loaded = true - if err != nil { - log.Errorf("GetVertexChannel: unmarshal error: %v", err) - continue - } + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for id := range ids { + if id.IsSignal() { + out <- id + } else { + if load { + prefix := VertexKey(id.ID) + v := gdbi.Vertex{ID: id.ID} + for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { + label, err := it.Value() + if err != nil { + log.Errorln("GetVertexChannel it.Value() err: ", err) + continue + } + v.Label = string(label) - d.req.Vertex = &v - out <- d.req + entry, err := ggraph.jsonkv.PageCache.Get(ctx, id.ID, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetVertexChannel: PageCache.Get( error: %v", err) + continue + } + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + if err != nil { + log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) + continue + } + v.Loaded = true + } + id.Vertex = &v + out <- id + + } else { + id.Vertex = &gdbi.Vertex{ID: id.ID} + out <- id + } + } } - } + return nil + }) }() return out } +type lookup struct { + req gdbi.ElementLookup + key string +} + // GetOutChannel process requests of vertex ids and find the connected vertices on outgoing edges func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { - vertexChan := make(chan elementData, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } + // Todo: implement bulk cache get + bulk get row to try to make this faster + lookupChan := make(chan lookup, 1000) go func() { - defer close(vertexChan) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + defer close(lookupChan) + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { - if req.IsSignal() { - vertexChan <- elementData{req: req} + lookupChan <- lookup{req: req} } else { found := false - key, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) - if ok { - skeyPrefix := SrcEdgePrefix(key) - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - keyValue := it.Key() - _, _, dst, label := SrcEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - vkey := VertexKey(dst) - vertexChan <- elementData{ - data: vkey, - req: req, - } - found = true + skeyPrefix := SrcEdgePrefix(req.ID) + for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { + _, _, dst, label := SrcEdgeKeyParse(it.Key()) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + lookupChan <- lookup{ + key: dst, + req: req, } + found = true } } if !found && emitNull { - vertexChan <- elementData{ - data: nil, - req: req, + lookupChan <- lookup{ + req: req, + key: "", } } } @@ -584,34 +650,38 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - for req := range vertexChan { + for req := range lookupChan { if req.req.IsSignal() { o <- req.req } else { - if req.data == nil { + if req.key != "" { + entry, err := ggraph.jsonkv.PageCache.Get(ctx, req.key, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetOutChannel: PageCache.Get( error: %v", err) + continue + } + vLabel, ok := ggraph.jsonkv.LabelLookup[entry.Label] + if !ok { + log.Errorf("GetOutChannel: Label not a string %s", vLabel) + continue + } + v := &gdbi.Vertex{ID: req.key, Label: vLabel} + if load { + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + if err != nil { + log.Errorf("GetOutChannel: GetRow on %s: %s error: %v", vLabel, req.key, err) + continue + } + v.Loaded = true + } else { + v.Data = map[string]any{} + } + req.req.Vertex = v + o <- req.req + } else { req.req.Vertex = nil o <- req.req - continue } - vkey := VertexKeyParse(req.data) - id, _ := ggraph.keyMap.GetVertexID(vkey, ggraph.bsonkv.Pb.Db) - lkey := ggraph.keyMap.GetVertexLabel(vkey, ggraph.bsonkv.Pb.Db) - lid, ok := ggraph.keyMap.GetLabelID(lkey, ggraph.bsonkv.Pb.Db) - if !ok || lid == "" { - log.Debugln("No LID for lkey: ", lkey) - continue - } - v := &gdbi.Vertex{ID: id, Label: lid} - var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lid].GetRow([]byte(id)) - v.Loaded = true - if err != nil { - log.Errorf("GetOutChannel: GetRow error: %v", err) - continue - } - - req.req.Vertex = v - o <- req.req } } }() @@ -621,49 +691,47 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen // GetInChannel process requests of vertex ids and find the connected vertices on incoming edges func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req } else { found := false - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) - if ok { - dkeyPrefix := DstEdgePrefix(vkey) - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - _, src, _, label := DstEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - srcID, _ := ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - lKey := ggraph.keyMap.GetVertexLabel(src, ggraph.bsonkv.Pb.Db) - lID, _ := ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) - v := &gdbi.Vertex{ID: srcID, Label: lID} - if load { - var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+lID].GetRow([]byte(srcID)) - if err != nil { - log.Errorf("GetInChannel: GetRow error: %v", err) - continue - } - v.Loaded = true - } else { - v.Data = map[string]any{} + dkeyPrefix := DstEdgePrefix(req.ID) + for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { + _, sid, _, label := DstEdgeKeyParse(it.Key()) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + entry, err := ggraph.jsonkv.PageCache.Get(ctx, sid, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetInChannel: PageCache.Get( error: %v", err) + continue + } + + vLabel, ok := ggraph.jsonkv.LabelLookup[entry.Label] + if !ok { + log.Errorf("GetInChannel Label lookup failed") + continue + } + + v := &gdbi.Vertex{ID: sid, Label: vLabel} + if load { + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + if err != nil { + log.Errorf("GetInChannel: GetRow on %s: %s error: %v", vLabel, sid, err) + continue } - req.Vertex = v - o <- req - found = true + v.Loaded = true + } else { + v.Data = map[string]any{} } + req.Vertex = v + o <- req + found = true } } + if !found && emitNull { req.Vertex = nil o <- req @@ -679,50 +747,45 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element // GetOutEdgeChannel process requests of vertex ids and find the connected outgoing edges func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req } else { found := false - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) - if ok { - skeyPrefix := SrcEdgePrefix(vkey) - for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := SrcEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - e := gdbi.Edge{} - e.ID, _ = ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - e.From, _ = ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - e.To, _ = ggraph.keyMap.GetVertexID(dst, ggraph.bsonkv.Pb.Db) - e.Label, _ = ggraph.keyMap.GetLabelID(label, ggraph.bsonkv.Pb.Db) - if load { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow([]byte(e.ID)) - if err != nil { - log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) - continue - } - e.Loaded = true - } else { - e.Data = map[string]any{} + skeyPrefix := SrcEdgePrefix(req.ID) + for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { + eid, src, dst, label := SrcEdgeKeyParse(it.Key()) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + e := gdbi.Edge{ + From: src, + To: dst, + Label: label, + ID: eid, + } + if load { + entry, err := ggraph.jsonkv.PageCache.Get(ctx, e.ID, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetOutEdgeChannel: PageCache.Get( error: %v", err) + continue + } + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) + if err != nil { + log.Errorf("GetOutEdgeChannel: GetRow error: %v", err) + continue } - req.Edge = &e - o <- req - found = true + e.Loaded = true + } else { + e.Data = map[string]any{} } + req.Edge = &e + o <- req + found = true } } + if !found && emitNull { req.Edge = nil o <- req @@ -731,7 +794,6 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El } return nil }) - }() return o } @@ -739,51 +801,47 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El // GetInEdgeChannel process requests of vertex ids and find the connected incoming edges func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool, emitNull bool, edgeLabels []string) chan gdbi.ElementLookup { o := make(chan gdbi.ElementLookup, 100) - edgeLabelKeys := make([]uint64, 0, len(edgeLabels)) - for i := range edgeLabels { - el, ok := ggraph.keyMap.GetLabelKey(edgeLabels[i], ggraph.bsonkv.Pb.Db) - if ok { - edgeLabelKeys = append(edgeLabelKeys, el) - } - } go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req } else { - vkey, ok := ggraph.keyMap.GetVertexKey(req.ID, ggraph.bsonkv.Pb.Db) found := false - if ok { - dkeyPrefix := DstEdgePrefix(vkey) - for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { - keyValue := it.Key() - eid, src, dst, label := DstEdgeKeyParse(keyValue) - if len(edgeLabelKeys) == 0 || setcmp.ContainsUint(edgeLabelKeys, label) { - e := gdbi.Edge{} - e.ID, _ = ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - e.From, _ = ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - e.To, _ = ggraph.keyMap.GetVertexID(dst, ggraph.bsonkv.Pb.Db) - e.Label, _ = ggraph.keyMap.GetLabelID(label, ggraph.bsonkv.Pb.Db) - if load { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow([]byte(e.ID)) - if err != nil { - log.Errorf("GetInEdgeChannel: GetRow error: %v", err) - continue - } - e.Loaded = true - - } else { - e.Data = map[string]any{} + dkeyPrefix := DstEdgePrefix(req.ID) + for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { + eid, src, dst, label := DstEdgeKeyParse(it.Key()) + if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { + e := gdbi.Edge{ + ID: eid, + From: src, + To: dst, + Label: label, + } + if load { + entry, err := ggraph.jsonkv.PageCache.Get(ctx, e.ID, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetInEdgeChannel: PageCache.Get( error: %v", err) + continue } - req.Edge = &e - o <- req - found = true + //log.Debugln("IN EDGE LABEL: ", e.Label, "ENTRY: ", entry, "ID: ", e.ID) + + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) + if err != nil { + log.Errorf("GetInEdgeChannel: GetRow error: %v", err) + continue + } + e.Loaded = true + } else { + e.Data = map[string]any{} } + req.Edge = &e + o <- req + found = true } } + if !found && emitNull { req.Edge = nil o <- req @@ -799,29 +857,25 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele // GetEdge loads an edge given an id. It returns nil if not found func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { - ekey, ok := ggraph.keyMap.GetEdgeKey(id, ggraph.bsonkv.Pb.Db) - if !ok { - return nil - } - ekeyPrefix := EdgeKeyPrefix(ekey) - + ekeyPrefix := EdgeKeyPrefix(id) var e *gdbi.Edge - err := ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { - eid, src, dst, labelKey := EdgeKeyParse(it.Key()) - id, _ := ggraph.keyMap.GetEdgeID(eid, ggraph.bsonkv.Pb.Db) - from, _ := ggraph.keyMap.GetVertexID(src, ggraph.bsonkv.Pb.Db) - to, _ := ggraph.keyMap.GetVertexID(dst, ggraph.bsonkv.Pb.Db) - label, _ := ggraph.keyMap.GetLabelID(labelKey, ggraph.bsonkv.Pb.Db) + eid, src, dst, label := EdgeKeyParse(it.Key()) e = &gdbi.Edge{ - ID: id, - From: from, - To: to, + ID: eid, + From: src, + To: dst, Label: label, } if loadProp { - var err error - e.Data, err = ggraph.bsonkv.Tables[ETABLE_PREFIX+label].GetRow([]byte(id)) + entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), e.ID, ggraph.jsonkv.PageLoader) + if err != nil { + log.Errorf("GetEdge: PageCache.Get( error: %v", err) + continue + } + + e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+e.Label].GetRow(entry) if err != nil { log.Errorf("GetEdge: GetRow error: %v", err) continue @@ -844,7 +898,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g o := make(chan *gdbi.Vertex, 100) go func() { defer close(o) - ggraph.bsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { vPrefix := VertexListPrefix() for it.Seek(vPrefix); it.Valid() && bytes.HasPrefix(it.Key(), vPrefix); it.Next() { select { @@ -852,17 +906,24 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g return nil default: } - v := &gdbi.Vertex{} - keyValue := it.Key() - vKey := VertexKeyParse(keyValue) - lKey := ggraph.keyMap.GetVertexLabel(vKey, ggraph.bsonkv.Pb.Db) - v.ID, _ = ggraph.keyMap.GetVertexID(vKey, ggraph.bsonkv.Pb.Db) - v.Label, _ = ggraph.keyMap.GetLabelID(lKey, ggraph.bsonkv.Pb.Db) + byteLabel, err := it.Value() + if err != nil { + log.Errorf("GetVertexList it.Value() error: %s", err) + } + v := &gdbi.Vertex{ + ID: VertexKeyParse(it.Key()), + Label: string(byteLabel), + } if loadProp { - var err error - v.Data, err = ggraph.bsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow([]byte(v.ID)) + entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), v.ID, ggraph.jsonkv.PageLoader) if err != nil { - log.Errorf("GetVertexList: GetRow error: %v", err) + log.Errorf("GetVertexList: PageCache.Get on %s error: %s", v.ID, err) + continue + } + + v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) + if err != nil { + log.Errorf("GetVertexList: table.GetRow error: %s", err) continue } v.Loaded = true @@ -880,7 +941,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g // ListVertexLabels returns a list of vertex types in the graph func (ggraph *Graph) ListVertexLabels() ([]string, error) { labels := []string{} - for i := range ggraph.bsonkv.GetLabels(false) { + for i := range ggraph.jsonkv.GetLabels(false, true) { labels = append(labels, i) } return labels, nil @@ -889,8 +950,459 @@ func (ggraph *Graph) ListVertexLabels() ([]string, error) { // ListEdgeLabels returns a list of edge types in the graph func (ggraph *Graph) ListEdgeLabels() ([]string, error) { labels := []string{} - for i := range ggraph.bsonkv.GetLabels(true) { + for i := range ggraph.jsonkv.GetLabels(true, true) { labels = append(labels, i) } return labels, nil } + +// New Bulk Delete Function. Testing... +// BulkDel deletes vertices and edges in bulk. +func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { + type keyBatch struct { + singles [][]byte + ranges [][2][]byte + posKeys [][]byte + } + + type itemInfo struct { + id string + label string + isEdge bool + tbl string + } + + type fieldInfo struct { + rKey []byte + field string + tbl string + id []byte + } + + const shardSize = 64 + const bufferSize = 8192 + numCpus := runtime.NumCPU() + ctx := context.Background() + + var bulkErr *multierror.Error + addErr := func(err error) { + if err != nil { + bulkErr = multierror.Append(bulkErr, err) + } + } + + // Sharded bitmap for edge deduplication (lock-free for reads) + type shard struct { + mu sync.Mutex + set map[string]struct{} + count uint32 // Atomic counter for seen edges + } + shards := make([]*shard, shardSize) + for i := range shards { + shards[i] = &shard{set: make(map[string]struct{}, bufferSize/shardSize)} + } + hasSeenEdge := func(eid string) bool { + h := fnv32a(eid) % uint32(shardSize) + shard := shards[h] + shard.mu.Lock() + defer shard.mu.Unlock() + if _, exists := shard.set[eid]; exists { + return true + } + shard.set[eid] = struct{}{} + atomic.AddUint32(&shard.count, 1) + return false + } + getSeenCount := func() uint64 { + var total uint64 + for _, shard := range shards { + total += uint64(atomic.LoadUint32(&shard.count)) + } + return total + } + + // Channels and wait groups + itemChan := make(chan itemInfo, bufferSize) + fieldChan := make(chan fieldInfo, bufferSize) + keyChan := make(chan keyBatch, bufferSize) + var prodWG, consWG, aggWG, fieldWG sync.WaitGroup + + // Aggregator for keys + var singles [][]byte + var ranges [][2][]byte + var posKeys [][]byte + aggWG.Add(1) + go func() { + defer aggWG.Done() + for batch := range keyChan { + select { + case <-ctx.Done(): + return + default: + singles = append(singles, batch.singles...) + ranges = append(ranges, batch.ranges...) + posKeys = append(posKeys, batch.posKeys...) + } + } + }() + + // Aggregator for fields + var allFields []fieldInfo + fieldWG.Add(1) + go func() { + defer fieldWG.Done() + for fi := range fieldChan { + allFields = append(allFields, fi) + } + }() + + // Workers for items + consWG.Add(numCpus) + for range numCpus { + go func() { + defer consWG.Done() + localBatch := keyBatch{posKeys: make([][]byte, 0, bufferSize)} + i := 0 + for item := range itemChan { + select { + case <-ctx.Done(): + return + default: + } + if i%100_000 == 0 && i != 0 { + log.Debugf("[BulkDel worker] processed %d items", i) + } + i++ + + // Fetch from page cache + loc, err := ggraph.jsonkv.PageCache.Get(ctx, item.id, ggraph.jsonkv.PageLoader) + if err != nil { + addErr(err) + continue + } + + // Mark table for deletion + table, ok := ggraph.jsonkv.Tables[item.tbl] + if !ok { + addErr(fmt.Errorf("table %s not found", item.tbl)) + continue + } + if err := table.MarkDeleteTable(loc); err != nil { + addErr(err) + } + + // Position key + localBatch.posKeys = append(localBatch.posKeys, benchtop.NewPosKey(loc.Label, []byte(item.id))) + ggraph.jsonkv.PageCache.Invalidate(item.id) + + // Send field infos + if fields, exists := ggraph.jsonkv.Fields[item.tbl]; exists { + for field := range fields { + rKey := benchtop.RFieldKey(item.tbl, field, item.id) + select { + case fieldChan <- fieldInfo{rKey: rKey, field: field, tbl: item.tbl, id: []byte(item.id)}: + case <-ctx.Done(): + return + } + } + } + + if len(localBatch.posKeys) >= 500_000 { + keyChan <- localBatch + localBatch = keyBatch{posKeys: make([][]byte, 0, bufferSize)} + } + } + + if len(localBatch.posKeys) > 0 { + keyChan <- localBatch + } + }() + } + + // Prepare vertex producers + slices.Sort(data.Vertices) + vertexSlices := make([][]string, numCpus) + for i, vid := range data.Vertices { + vertexSlices[i%numCpus] = append(vertexSlices[i%numCpus], vid) + } + for i := range vertexSlices { + slices.Sort(vertexSlices[i]) + } + + for _, slice := range vertexSlices { + if len(slice) == 0 { + continue + } + prodWG.Add(1) + go func(slice []string) { + defer prodWG.Done() + localBatch := keyBatch{singles: make([][]byte, 0, 256), ranges: make([][2][]byte, 0, 256)} + + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, vid := range slice { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + sPrefix := SrcEdgePrefix(vid) + if err := it.Seek(sPrefix); err != nil { + return err + } + if it.Valid() && bytes.HasPrefix(it.Key(), sPrefix) { + nextPrefix := upperBound(sPrefix) + if nextPrefix != nil { + localBatch.ranges = append(localBatch.ranges, [2][]byte{sPrefix, nextPrefix}) + } + for it.Valid() && bytes.HasPrefix(it.Key(), sPrefix) { + eid, sid, did, lbl := SrcEdgeKeyParse(it.Key()) + if !hasSeenEdge(eid) { + localBatch.singles = append(localBatch.singles, + EdgeKey(eid, sid, did, lbl), + bytes.Clone(it.Key()), + DstEdgeKey(eid, sid, did, lbl)) + select { + case itemChan <- itemInfo{id: eid, label: lbl, isEdge: true, tbl: ETABLE_PREFIX + lbl}: + case <-ctx.Done(): + return ctx.Err() + } + } + it.Next() + } + } + + dPrefix := DstEdgePrefix(vid) + if err := it.Seek(dPrefix); err != nil { + return err + } + if it.Valid() && bytes.HasPrefix(it.Key(), dPrefix) { + nextPrefix := upperBound(dPrefix) + if nextPrefix != nil { + localBatch.ranges = append(localBatch.ranges, [2][]byte{dPrefix, nextPrefix}) + } + for it.Valid() && bytes.HasPrefix(it.Key(), dPrefix) { + eid, sid, did, lbl := DstEdgeKeyParse(it.Key()) + if !hasSeenEdge(eid) { + localBatch.singles = append(localBatch.singles, + EdgeKey(eid, sid, did, lbl), + SrcEdgeKey(eid, sid, did, lbl), + bytes.Clone(it.Key())) + select { + case itemChan <- itemInfo{id: eid, label: lbl, isEdge: true, tbl: ETABLE_PREFIX + lbl}: + case <-ctx.Done(): + return ctx.Err() + } + } + it.Next() + } + } + + vkey := VertexKey(vid) + if err := it.Seek(vkey); err != nil { + return err + } + var label string + if it.Valid() && bytes.Equal(it.Key(), vkey) { + labelBytes, err := it.Value() + if err != nil { + return err + } + label = string(labelBytes) + } + localBatch.singles = append(localBatch.singles, vkey) + if label != "" { + select { + case itemChan <- itemInfo{id: vid, label: label, isEdge: false, tbl: VTABLE_PREFIX + label}: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil + }) + addErr(err) + + if len(localBatch.singles) > 0 || len(localBatch.ranges) > 0 { + select { + case keyChan <- localBatch: + case <-ctx.Done(): + } + } + }(slice) + } + + // Prepare edge producers + slices.Sort(data.Edges) + edgeSlices := make([][]string, numCpus) + for i, eid := range data.Edges { + edgeSlices[i%numCpus] = append(edgeSlices[i%numCpus], eid) + } + for i := range edgeSlices { + slices.Sort(edgeSlices[i]) + } + + for _, slice := range edgeSlices { + if len(slice) == 0 { + continue + } + prodWG.Add(1) + go func(slice []string) { + defer prodWG.Done() + localBatch := keyBatch{singles: make([][]byte, 0, 12), ranges: make([][2][]byte, 0, 12)} + + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, eid := range slice { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if hasSeenEdge(eid) { + continue + } + + prefix := EdgeKeyPrefix(eid) + if err := it.Seek(prefix); err != nil { + return err + } + if it.Valid() && bytes.HasPrefix(it.Key(), prefix) { + nextPrefix := upperBound(prefix) + if nextPrefix != nil { + localBatch.ranges = append(localBatch.ranges, [2][]byte{prefix, nextPrefix}) + } + var label string + for it.Valid() && bytes.HasPrefix(it.Key(), prefix) { + _, sid, did, lbl := EdgeKeyParse(it.Key()) + label = lbl + localBatch.singles = append(localBatch.singles, + SrcEdgeKey(eid, sid, did, lbl), + DstEdgeKey(eid, sid, did, lbl)) + it.Next() + } + if label != "" { + select { + case itemChan <- itemInfo{id: eid, label: label, isEdge: true, tbl: ETABLE_PREFIX + label}: + case <-ctx.Done(): + return ctx.Err() + } + } + } + } + return nil + }) + addErr(err) + + if len(localBatch.singles) > 0 || len(localBatch.ranges) > 0 { + select { + case keyChan <- localBatch: + case <-ctx.Done(): + } + } + }(slice) + } + + // Close channels and wait + go func() { + prodWG.Wait() + close(itemChan) + }() + consWG.Wait() + close(keyChan) + aggWG.Wait() + close(fieldChan) + fieldWG.Wait() + + // Process field indices with single iterator + var indexDelKeys [][]byte + if len(allFields) > 0 { + sort.Slice(allFields, func(i, j int) bool { + return bytes.Compare(allFields[i].rKey, allFields[j].rKey) < 0 + }) + err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + for _, fi := range allFields { + if err := it.Seek(fi.rKey); err != nil { + return err + } + if it.Valid() && bytes.Equal(it.Key(), fi.rKey) { + valueBytes, err := it.Value() + if err != nil { + return err + } + var fieldValue any + if err := sonic.ConfigFastest.Unmarshal(valueBytes, &fieldValue); err != nil { + return err + } + if fieldValue != nil { + fKey := benchtop.FieldKey(fi.field, fi.tbl, fieldValue, fi.id) + indexDelKeys = append(indexDelKeys, fKey, fi.rKey) + } + } + } + return nil + }) + addErr(err) + } + + // Chunked deletes with Pebble batch + chunked := func(singles [][]byte, ranges [][2][]byte, posKeys [][]byte, indexDelKeys [][]byte) error { + batch := ggraph.jsonkv.Pb.Db.NewBatch() + defer batch.Close() + for _, k := range singles { + if err := batch.Delete(k, nil); err != nil { + return err + } + } + for _, r := range ranges { + if err := batch.DeleteRange(r[0], r[1], nil); err != nil { + return err + } + } + for _, k := range posKeys { + if err := batch.Delete(k, nil); err != nil { + return err + } + } + for _, k := range indexDelKeys { + if err := batch.Delete(k, nil); err != nil { + return err + } + } + return batch.Commit(pebble.Sync) + } + + // Perform deletes + ggraph.jsonkv.PebbleLock.Lock() + if err := chunked(singles, ranges, posKeys, indexDelKeys); err != nil { + addErr(err) + } + ggraph.ts.Touch(ggraph.graphID) + ggraph.jsonkv.PebbleLock.Unlock() + + log.Debugf("Total edges seen: %d", getSeenCount()) + return bulkErr.ErrorOrNil() +} + +// upperBound computes the tight upper bound for range delete +func upperBound(prefix []byte) []byte { + ub := make([]byte, len(prefix)) + copy(ub, prefix) + for i := len(ub) - 1; i >= 0; i-- { + if ub[i] < 0xFF { + ub[i]++ + return ub[:i+1] + } + } + return nil +} + +// fnv32a computes FNV-1a 32-bit hash +func fnv32a(s string) uint32 { + var h uint32 = 2166136261 + for i := range s { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} diff --git a/grids/graphdb.go b/grids/graphdb.go index 332e59ea..14b7a637 100644 --- a/grids/graphdb.go +++ b/grids/graphdb.go @@ -8,6 +8,7 @@ import ( "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" ) // GridsGDB implements the GripInterface using a generic key/value storage driver @@ -18,6 +19,7 @@ type GDB struct { // NewKVGraphDB intitalize a new grids graph driver func NewGraphDB(baseDir string) (gdbi.GraphDB, error) { + log.Redf("Disclaimer: the Grids driver is an experimental database driver. Use with caution.") _, err := os.Stat(baseDir) if os.IsNotExist(err) { os.Mkdir(baseDir, 0700) @@ -46,7 +48,10 @@ func (kgraph *GDB) Graph(graph string) (gdbi.GraphInterface, error) { if err != nil { return nil, err } + mu.Lock() kgraph.drivers[graph] = g + mu.Unlock() + return g, nil } return nil, fmt.Errorf("graph '%s' was not found", graph) diff --git a/grids/index.go b/grids/index.go index bb593854..11d92811 100644 --- a/grids/index.go +++ b/grids/index.go @@ -3,31 +3,24 @@ package grids import ( "context" "fmt" - "strings" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" + "github.com/cockroachdb/pebble" + multierror "github.com/hashicorp/go-multierror" ) -func normalizePath(path string) string { - path = strings.TrimPrefix(path, "$.") - path = strings.TrimPrefix(path, "data.") - return path -} - // AddVertexIndex add index to vertices -func (ggraph *Graph) AddVertexIndex(label string, field string) error { +func (ggraph *Graph) AddVertexIndex(label, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") - field = normalizePath(field) - //TODO kick off background process to reindex existing data - return ggraph.bsonkv.AddField(fmt.Sprintf("%s.v.%s.%s", ggraph.graphID, label, field)) + return ggraph.jsonkv.AddField(VTABLE_PREFIX+label, field) } // DeleteVertexIndex delete index from vertices -func (ggraph *Graph) DeleteVertexIndex(label string, field string) error { +func (ggraph *Graph) DeleteVertexIndex(label, field string) error { + fmt.Println("HELLO WE HARE HERE") log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") - field = normalizePath(field) - return ggraph.bsonkv.RemoveField(fmt.Sprintf("%s.v.%s.%s", ggraph.graphID, label, field)) + return ggraph.jsonkv.RemoveField(VTABLE_PREFIX+label, field) } // GetVertexIndexList lists out all the vertex indices for a graph @@ -36,12 +29,8 @@ func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { out := make(chan *gripql.IndexID) go func() { defer close(out) - fields := ggraph.bsonkv.ListFields() - for _, f := range fields { - t := strings.Split(f, ".") - if len(t) > 3 { - out <- &gripql.IndexID{Graph: ggraph.graphID, Label: t[2], Field: t[3]} - } + for _, f := range ggraph.jsonkv.ListFields() { + out <- &gripql.IndexID{Graph: ggraph.graphID, Label: f.Label, Field: f.Field} } }() return out @@ -50,18 +39,52 @@ func (ggraph *Graph) GetVertexIndexList() <-chan *gripql.IndexID { // VertexLabelScan produces a channel of all vertex ids in a graph // that match a given label func (ggraph *Graph) VertexLabelScan(ctx context.Context, label string) chan string { - log.WithFields(log.Fields{"label": label}).Debug("Running VertexLabelScan") - //TODO: Make this work better - out := make(chan string, 100) - if label[:2] != "v_" { - label = "v_" + label + if label[:2] != VTABLE_PREFIX { + label = VTABLE_PREFIX + label } - go func() { - defer close(out) - log.Infof("Searching %s %s", fmt.Sprintf("%s.label", ggraph.graphID), label) - for i := range ggraph.bsonkv.GetIDsForLabel(label) { - out <- i + log.WithFields(log.Fields{"label": label}).Info("Running VertexLabelScan") + return ggraph.jsonkv.GetIDsForLabel(label) +} + +func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error { + var prefix string = "v_" + if edgeFlag { + prefix = "e_" + } + + loc, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + if err != nil { + return err + } + + tableLabel := prefix + label + var bulkErr *multierror.Error + if fields, exists := ggraph.jsonkv.Fields[tableLabel]; exists { + for field := range fields { + if err := ggraph.jsonkv.DeleteRowField(tableLabel, field, id); err != nil { + log.Errorf("Failed to delete index for field '%s' in table '%s' for row '%s': %v", field, tableLabel, id, err) + bulkErr = multierror.Append(bulkErr, err) + } } - }() - return out + } + + ggraph.jsonkv.PebbleLock.Lock() + defer ggraph.jsonkv.PebbleLock.Unlock() + + table, ok := ggraph.jsonkv.Tables[prefix+label] + if !ok { + bulkErr = multierror.Append(bulkErr, fmt.Errorf("table %s not found in jsonkv.Tables: %#v", prefix+label, ggraph.jsonkv.Tables)) + return bulkErr.ErrorOrNil() + } + + err = table.DeleteRow(loc, []byte(id)) + if err != nil { + if err == pebble.ErrNotFound { + log.Debugf("Pebble not Found: %s", err) + return nil + } + bulkErr = multierror.Append(bulkErr, err) + } + ggraph.jsonkv.PageCache.Invalidate(id) + return bulkErr.ErrorOrNil() } diff --git a/grids/keyindex.go b/grids/keyindex.go index 4f06fb92..2eae94c0 100644 --- a/grids/keyindex.go +++ b/grids/keyindex.go @@ -1,7 +1,7 @@ package grids import ( - "encoding/binary" + "bytes" ) var vertexPrefix = []byte(".") @@ -12,155 +12,126 @@ var dstEdgePrefix = []byte(">") var intSize = 10 // VertexKey generates the key given a vertexId -func VertexKey(id uint64) []byte { - out := make([]byte, intSize+1) - out[0] = vertexPrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out +func VertexKey(id string) []byte { + return bytes.Join([][]byte{ + vertexPrefix, + []byte(id), + }, []byte{0}) } -// VertexKeyParse takes a byte array key and returns: -// `graphId`, `vertexId` -func VertexKeyParse(key []byte) uint64 { - id, _ := binary.Uvarint(key[1 : intSize+1]) - return id -} - -// EdgeKey takes the required components of an edge key and returns the byte array -func EdgeKey(id, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = edgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], dst) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out +func VertexKeyParse(key []byte) (id string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[1]) } // EdgeKeyPrefix returns the byte array prefix for a particular edge id -func EdgeKeyPrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = edgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out +func EdgeKeyPrefix(id string) []byte { + return bytes.Join([][]byte{ + edgePrefix, + []byte(id), + {}, + }, []byte{0}) } -// EdgeKeyParse takes a edge key and returns the elements encoded in it: -// `graph`, `edgeID`, `srcVertexId`, `dstVertexId`, `label` -func EdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - eid, _ := binary.Uvarint(key[1 : intSize+1]) - sid, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - did, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label +// SrcEdgePrefix returns a byte array prefix for all entries in the source +// edge index a particular vertex (the source vertex) +func SrcEdgePrefix(id string) []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + []byte(id), + {}, + }, []byte{0}) } -// VertexListPrefix returns a byte array prefix for all vertices in a graph -func VertexListPrefix() []byte { - out := make([]byte, 1) - out[0] = vertexPrefix[0] - return out +// DstEdgePrefix returns a byte array prefix for all entries in the dest +// edge index a particular vertex (the dest vertex) +func DstEdgePrefix(id string) []byte { + return bytes.Join([][]byte{ + dstEdgePrefix, + []byte(id), + {}, + }, []byte{0}) } -// EdgeListPrefix returns a byte array prefix for all edges in a graph -func EdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = edgePrefix[0] - return out +// EdgeKey takes the required components of an edge key and returns the byte array +func EdgeKey(id, src, dst, label string) []byte { + return bytes.Join([][]byte{ + edgePrefix, + []byte(id), + []byte(label), + []byte(src), + []byte(dst), + }, []byte{0}) } -// SrcEdgeListPrefix returns a byte array prefix for all entries in the source -// edge index for a graph -func SrcEdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = srcEdgePrefix[0] - return out +func EdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[1]), string(tmp[3]), string(tmp[4]), string(tmp[2]) } -// DstEdgeListPrefix returns a byte array prefix for all entries in the dest -// edge index for a graph -func DstEdgeListPrefix() []byte { - out := make([]byte, 1) - out[0] = dstEdgePrefix[0] - return out +// SrcEdgeKey creates a src edge index key +func SrcEdgeKey(eid, src, dst, label string) []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + []byte(src), + []byte(dst), + []byte(eid), + []byte(label), + }, []byte{0}) } -// SrcEdgeKey creates a src edge index key -func SrcEdgeKey(eid, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], src) - binary.PutUvarint(out[intSize+1:intSize*2+1], dst) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out +func SrcEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[3]), string(tmp[1]), string(tmp[2]), string(tmp[4]) } // DstEdgeKey creates a dest edge index key -func DstEdgeKey(eid, src, dst, label uint64) []byte { - out := make([]byte, 1+intSize*4) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], dst) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - binary.PutUvarint(out[intSize*3+1:intSize*4+1], label) - return out +func DstEdgeKey(eid, src, dst, label string) []byte { + return bytes.Join([][]byte{ + dstEdgePrefix, + []byte(dst), + []byte(src), + []byte(eid), + []byte(label), + }, []byte{0}) } -// SrcEdgeKeyParse takes a src index key entry and parses it into: -// `graph`, `edgeId`, `srcVertexId`, `dstVertexId`, `label`, `etype` -func SrcEdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - sid, _ := binary.Uvarint(key[1 : intSize+1]) - did, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - eid, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label +func DstEdgeKeyParse(key []byte) (eid string, sid string, did string, label string) { + tmp := bytes.Split(key, []byte{0}) + return string(tmp[3]), string(tmp[2]), string(tmp[1]), string(tmp[4]) } -// DstEdgeKeyParse takes a dest index key entry and parses it into: -// `graph`, `edgeId`, `dstVertexId`, `srcVertexId`, `label`, `etype` -func DstEdgeKeyParse(key []byte) (uint64, uint64, uint64, uint64) { - did, _ := binary.Uvarint(key[1 : intSize+1]) - sid, _ := binary.Uvarint(key[intSize+1 : intSize*2+1]) - eid, _ := binary.Uvarint(key[intSize*2+1 : intSize*3+1]) - label, _ := binary.Uvarint(key[intSize*3+1 : intSize*4+1]) - return eid, sid, did, label -} -// SrcEdgeKeyPrefix creates a byte array prefix for a src edge index entry -func SrcEdgeKeyPrefix(eid, src, dst uint64) []byte { - out := make([]byte, 1+intSize*3) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], src) - binary.PutUvarint(out[intSize+1:intSize*2+1], dst) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - return out +// VertexListPrefix returns a byte array prefix for all vertices in a graph +func VertexListPrefix() []byte { + return bytes.Join([][]byte{ + vertexPrefix, + {}, + }, []byte{0}) } -// DstEdgeKeyPrefix creates a byte array prefix for a dest edge index entry -func DstEdgeKeyPrefix(eid, src, dst uint64) []byte { - out := make([]byte, 1+intSize*3) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], dst) - binary.PutUvarint(out[intSize+1:intSize*2+1], src) - binary.PutUvarint(out[intSize*2+1:intSize*3+1], eid) - return out +// EdgeListPrefix returns a byte array prefix for all edges in a graph +func EdgeListPrefix() []byte { + return bytes.Join([][]byte{ + edgePrefix, + {}, + }, []byte{0}) } -// SrcEdgePrefix returns a byte array prefix for all entries in the source -// edge index a particular vertex (the source vertex) -func SrcEdgePrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = srcEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out +// SrcEdgeListPrefix returns a byte array prefix for all entries in the source +// edge index for a graph +func SrcEdgeListPrefix() []byte { + return bytes.Join([][]byte{ + srcEdgePrefix, + {}, + }, []byte{0}) } -// DstEdgePrefix returns a byte array prefix for all entries in the dest -// edge index a particular vertex (the dest vertex) -func DstEdgePrefix(id uint64) []byte { - out := make([]byte, 1+intSize) - out[0] = dstEdgePrefix[0] - binary.PutUvarint(out[1:intSize+1], id) - return out +// DstEdgeListPrefix returns a byte array prefix for all entries in the dest +// edge index for a graph +func DstEdgeListPrefix() []byte { + return bytes.Join([][]byte{ + dstEdgePrefix, + {}, + }, []byte{0}) } diff --git a/grids/keymap.go b/grids/keymap.go deleted file mode 100644 index a6aeb706..00000000 --- a/grids/keymap.go +++ /dev/null @@ -1,319 +0,0 @@ -package grids - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "sync" - - "github.com/cockroachdb/pebble" - - "github.com/bmeg/grip/log" - - ristretto "github.com/dgraph-io/ristretto/v2" -) - -type GetSet interface { - Get(key []byte) ([]byte, io.Closer, error) - Set(key, value []byte, _ *pebble.WriteOptions) error - Delete(key []byte, _ *pebble.WriteOptions) error -} - -type KeyMap struct { - cache ristretto.Cache[string, uint64] - - vIncCur uint64 - eIncCur uint64 - lIncCur uint64 - - vIncMut sync.Mutex - eIncMut sync.Mutex - lIncMut sync.Mutex -} - -var incMod uint64 = 1000 - -var vIDPrefix = []byte{'v'} -var eIDPrefix = []byte{'e'} -var lIDPrefix = []byte{'l'} - -var vKeyPrefix byte = 'V' -var eKeyPrefix byte = 'E' -var lKeyPrefix byte = 'L' - -var vLabelPrefix byte = 'x' -var eLabelPrefix byte = 'y' - -var vInc = []byte{'i', 'v'} -var eInc = []byte{'i', 'e'} -var lInc = []byte{'i', 'l'} - -func NewKeyMap() *KeyMap { - return &KeyMap{} -} - -func (km *KeyMap) Close() {} - -// GetsertVertexKey : Get or Insert Vertex Key -func (km *KeyMap) GetsertVertexKeyLabel(id, label string, db GetSet) (uint64, uint64) { - o, ok := getIDKey(vIDPrefix, id, db) - if !ok { - km.vIncMut.Lock() - var err error - o, err = dbInc(&km.vIncCur, vInc, db) - if err != nil { - log.Errorf("%s", err) - } - km.vIncMut.Unlock() - err = setKeyID(vKeyPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - err = setIDKey(vIDPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - } - lkey := km.GetsertLabelKey(label, db) - setIDLabel(vLabelPrefix, o, lkey, db) - return o, lkey -} - -func (km *KeyMap) GetsertVertexKey(id string, db GetSet) uint64 { - o, ok := getIDKey(vIDPrefix, id, db) - if !ok { - km.vIncMut.Lock() - var err error - o, err = dbInc(&km.vIncCur, vInc, db) - if err != nil { - log.Errorf("%s", err) - } - km.vIncMut.Unlock() - err = setKeyID(vKeyPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - err = setIDKey(vIDPrefix, id, o, db) - if err != nil { - log.Errorf("%s", err) - } - } - return o -} - -func (km *KeyMap) GetVertexKey(id string, db GetSet) (uint64, bool) { - return getIDKey(vIDPrefix, id, db) -} - -// GetVertexID -func (km *KeyMap) GetVertexID(key uint64, db GetSet) (string, bool) { - return getKeyID(vKeyPrefix, key, db) -} - -func (km *KeyMap) GetVertexLabel(key uint64, db GetSet) uint64 { - k, _ := getIDLabel(vLabelPrefix, key, db) - return k -} - -// GetsertEdgeKey gets or inserts a new uint64 id for a given edge GID string -func (km *KeyMap) GetsertEdgeKey(id, label string, db GetSet) (uint64, uint64) { - o, ok := getIDKey(eIDPrefix, id, db) - if !ok { - km.eIncMut.Lock() - o, _ = dbInc(&km.eIncCur, eInc, db) - km.eIncMut.Unlock() - if err := setKeyID(eKeyPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - if err := setIDKey(eIDPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - } - lkey := km.GetsertLabelKey(label, db) - if err := setIDLabel(eLabelPrefix, o, lkey, db); err != nil { - log.Errorf("%s", err) - } - return o, lkey -} - -// GetEdgeKey gets the uint64 key for a given GID string -func (km *KeyMap) GetEdgeKey(id string, db GetSet) (uint64, bool) { - return getIDKey(eIDPrefix, id, db) -} - -// GetEdgeID gets the GID string for a given edge id uint64 -func (km *KeyMap) GetEdgeID(key uint64, db GetSet) (string, bool) { - return getKeyID(eKeyPrefix, key, db) -} - -func (km *KeyMap) GetEdgeLabel(key uint64, db GetSet) uint64 { - k, _ := getIDLabel(eLabelPrefix, key, db) - return k -} - -// DelVertexKey -func (km *KeyMap) DelVertexKey(id string, db GetSet) error { - key, ok := km.GetVertexKey(id, db) - if !ok { - return fmt.Errorf("%s vertexKey not found", id) - } - if err := delKeyID(vKeyPrefix, key, db); err != nil { - return err - } - if err := delIDKey(vIDPrefix, id, db); err != nil { - return err - } - return nil -} - -// DelEdgeKey -func (km *KeyMap) DelEdgeKey(id string, db GetSet) error { - key, ok := km.GetEdgeKey(id, db) - if !ok { - return fmt.Errorf("%s edgeKey not found", id) - } - if err := delKeyID(eKeyPrefix, key, db); err != nil { - return err - } - if err := delIDKey(eIDPrefix, id, db); err != nil { - return err - } - return nil -} - -// GetsertLabelKey gets-or-inserts a new label key uint64 for a given string -func (km *KeyMap) GetsertLabelKey(id string, db GetSet) uint64 { - u, ok := getIDKey(lIDPrefix, id, db) - if ok { - return u - } - km.lIncMut.Lock() - o, _ := dbInc(&km.lIncCur, lInc, db) - km.lIncMut.Unlock() - if err := setKeyID(lKeyPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - if err := setIDKey(lIDPrefix, id, o, db); err != nil { - log.Errorf("%s", err) - } - return o -} - -func (km *KeyMap) GetLabelKey(id string, db GetSet) (uint64, bool) { - return getIDKey(lIDPrefix, id, db) -} - -// GetLabelID gets the GID for a given uint64 label key -func (km *KeyMap) GetLabelID(key uint64, db GetSet) (string, bool) { - return getKeyID(lKeyPrefix, key, db) -} - -func getIDKey(prefix []byte, id string, db GetSet) (uint64, bool) { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - v, closer, err := db.Get(k) - if v == nil || err != nil { - return 0, false - } - key, _ := binary.Uvarint(v) - closer.Close() - return key, true -} - -func setIDKey(prefix []byte, id string, key uint64, db GetSet) error { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - b := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(b, key) - return db.Set(k, b, nil) -} - -func delIDKey(prefix []byte, id string, db GetSet) error { - k := bytes.Join([][]byte{prefix, []byte(id)}, []byte{}) - return db.Delete(k, nil) -} - -func getIDLabel(prefix byte, key uint64, db GetSet) (uint64, bool) { - k := make([]byte, 1+binary.MaxVarintLen64) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - v, closer, err := db.Get(k) - if v == nil || err != nil { - return 0, false - } - label, _ := binary.Uvarint(v) - closer.Close() - return label, true -} - -func setIDLabel(prefix byte, key uint64, label uint64, db GetSet) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - - b := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(b, label) - - err := db.Set(k, b, nil) - return err -} - -func setKeyID(prefix byte, id string, key uint64, db GetSet) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - return db.Set(k, []byte(id), nil) -} - -func getKeyID(prefix byte, key uint64, db GetSet) (string, bool) { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - b, closer, err := db.Get(k) - if b == nil || err != nil { - return "", false - } - out := string(b) - closer.Close() - return out, true -} - -func delKeyID(prefix byte, key uint64, db GetSet) error { - k := make([]byte, binary.MaxVarintLen64+1) - k[0] = prefix - binary.PutUvarint(k[1:binary.MaxVarintLen64+1], key) - return db.Delete(k, nil) -} - -func dbInc(inc *uint64, k []byte, db GetSet) (uint64, error) { - b := make([]byte, binary.MaxVarintLen64) - if *inc == 0 { - v, closer, _ := db.Get(k) - if v == nil { - binary.PutUvarint(b, incMod) - if err := db.Set(k, b, nil); err != nil { - return 0, err - } - (*inc) += 2 - return 1, nil - } - closer.Close() - newInc, _ := binary.Uvarint(v) - *inc = newInc - binary.PutUvarint(b, (*inc)+incMod) - if err := db.Set(k, b, nil); err != nil { - return 0, err - } - o := (*inc) - (*inc)++ - return o, nil - } - o := *inc - (*inc)++ - if *inc%incMod == 0 { - binary.PutUvarint(b, *inc+incMod) - if err := db.Set(k, b, nil); err != nil { - return 0, err - } - } - return o, nil -} diff --git a/grids/new.go b/grids/new.go index 42c52441..ccc8da7a 100644 --- a/grids/new.go +++ b/grids/new.go @@ -6,8 +6,9 @@ import ( "os" "path/filepath" "strings" + "sync" - "github.com/bmeg/benchtop/bsontable" + "github.com/bmeg/benchtop/jsontable" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/timestamp" ) @@ -16,14 +17,15 @@ import ( type Graph struct { graphID string - keyMap *KeyMap - bsonkv *bsontable.BSONDriver - ts *timestamp.Timestamp + jsonkv *jsontable.JSONDriver + ts *timestamp.Timestamp + tempDeletedEdges map[string]struct{} + edgesMutex sync.Mutex } // Close the connection func (g *Graph) Close() error { - g.bsonkv.Close() + g.jsonkv.Close() return nil } @@ -40,6 +42,7 @@ func (kgraph *GDB) AddGraph(graph string) error { kgraph.drivers[graph] = g return nil } + func newGraph(baseDir, name string) (*Graph, error) { dbPath := filepath.Join(baseDir, name) fmt.Printf("Creating new GRIDS graph %s\n", name) @@ -58,20 +61,21 @@ func newGraph(baseDir, name string) (*Graph, error) { } //bsonkvPath := fmt.Sprintf("%s", dbPath) - bsonkvPath := dbPath - tabledr, err := bsontable.NewBSONDriver(bsonkvPath) + jsonkvPath := dbPath + tabledr, err := jsontable.NewJSONDriver(jsonkvPath) if err != nil { - return nil, fmt.Errorf("failed to open bsonkv at %s: %v", bsonkvPath, err) + return nil, fmt.Errorf("failed to open jsonkv at %s: %v", jsonkvPath, err) } - bsonkv := tabledr.(*bsontable.BSONDriver) + jsonkv := tabledr.(*jsontable.JSONDriver) ts := timestamp.NewTimestamp() o := &Graph{ - keyMap: NewKeyMap(), - bsonkv: bsonkv, - ts: &ts, - graphID: name, + jsonkv: jsonkv, + ts: &ts, + graphID: name, + tempDeletedEdges: make(map[string]struct{}), + edgesMutex: sync.Mutex{}, } return o, nil } @@ -102,18 +106,17 @@ func getGraph(baseDir, name string) (*Graph, error) { } //bsonkvPath := fmt.Sprintf("%s", dbPath) - bsonkvPath := dbPath - tabledr, err := bsontable.LoadBSONDriver(bsonkvPath) + jsonkvPath := dbPath + tabledr, err := jsontable.LoadJSONDriver(jsonkvPath) if err != nil { - return nil, fmt.Errorf("failed to open bsonkv at %s: %v", bsonkvPath, err) + return nil, fmt.Errorf("failed to open bsonkv at %s: %v", jsonkvPath, err) } - bsonkv := tabledr.(*bsontable.BSONDriver) + jsonkv := tabledr.(*jsontable.JSONDriver) ts := timestamp.NewTimestamp() o := &Graph{ - keyMap: NewKeyMap(), - bsonkv: bsonkv, + jsonkv: jsonkv, ts: &ts, graphID: name, } diff --git a/grids/optimizer.go b/grids/optimizer.go new file mode 100644 index 00000000..16912398 --- /dev/null +++ b/grids/optimizer.go @@ -0,0 +1,127 @@ +package grids + +import ( + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/util/protoutil" +) + +type OptimizationRule struct { + Match func(pipe []*gripql.GraphStatement) bool + Replace func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement +} + +func GridsOptimizer(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + // Preprocess to handle Has with And expressions + //pipe = expandHasAnd(pipe) + // Apply the first matching optimization rule + for _, rule := range startOptimizations { + if rule.Match(pipe) { + return rule.Replace(pipe) + } + } + // Return the original pipeline if no optimizations apply + return pipe +} + +var startOptimizations = []OptimizationRule{ + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_Has); ok { + return true + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[1].GetHas() + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids V.Has() indexing", + Custom: lookupVertsCondIndexStep{ + expr: has, + }, + }, + }, + } + return append(optimized, pipe[2:]...) + }, + }, + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 3 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); !ok { + return false + } + if _, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { + return true + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + has := pipe[2].GetHas() + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + for i, label := range labels { + if label[:2] != VTABLE_PREFIX { + labels[i] = VTABLE_PREFIX + label + } + } + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids V().HasLabel().Has() indexing", + Custom: lookupVertsHasLabelCondIndexStep{ + expr: has, + labels: labels, + }, + }, + }, + } + return append(optimized, pipe[3:]...) + }, + }, + { + Match: func(pipe []*gripql.GraphStatement) bool { + if len(pipe) < 2 { + return false + } + if _, ok := pipe[0].GetStatement().(*gripql.GraphStatement_V); !ok { + return false + } + if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); ok { + return true + } + return false + }, + Replace: func(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + labels := protoutil.AsStringList(pipe[1].GetHasLabel()) + for i, label := range labels { + if label[:2] != VTABLE_PREFIX { + labels[i] = VTABLE_PREFIX + label + } + } + var optimized = []*gripql.GraphStatement{ + { + Statement: &gripql.GraphStatement_EngineCustom{ + Desc: "Grids V().HasLabel()", + Custom: lookupVertsHasLabelCondIndexStep{ + expr: nil, + labels: labels, + }, + }, + }, + } + return append(optimized, pipe[2:]...) + }, + }, +} diff --git a/grids/processor.go b/grids/processor.go new file mode 100644 index 00000000..5baa8cf7 --- /dev/null +++ b/grids/processor.go @@ -0,0 +1,214 @@ +package grids + +import ( + "context" + + "github.com/bmeg/grip/gdbi" + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" +) + +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertexHasLabelCondIndex look up vertices has label + +type lookupVertsHasLabelCondIndexStep struct { + labels []string + expr *gripql.HasExpression + loadData bool +} + +func (t lookupVertsHasLabelCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { + graph := db.(*Graph) + return &lookupVertsHasLabelCondIndexProc{ + db: graph, + expr: t.expr, + labels: t.labels, + loadData: ps.StepLoadData(), + }, nil + +} + +func (t lookupVertsHasLabelCondIndexStep) GetType() gdbi.DataType { + return gdbi.VertexData +} + +type lookupVertsHasLabelCondIndexProc struct { + db *Graph + labels []string + expr *gripql.HasExpression + loadData bool +} + +func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor", l.loadData) + var exists = true + // Here if one of l.labels doesn't exist then not going to be querying all the data so leave it like this. + cond := l.expr.GetCondition() + exists = len(l.db.jsonkv.Fields) > 0 && cond != nil + if exists { + for _, iterLabel := range l.labels { + label, ok := l.db.jsonkv.Fields[iterLabel] + if !ok { + exists = false + break + } + _, exists = label[cond.Key] + if !exists { + break + } + } + } + + count := 0 + if !exists || (l.expr == nil && cond == nil) { + go func() { + defer close(out) + for t := range in { + for _, label := range l.labels { + tableFound, ok := l.db.jsonkv.Tables[label] + if !ok { + log.Debugf("BSONTable for label '%s' is nil. Cannot scan.", label) + continue + } + for roMaps := range tableFound.Scan(l.loadData, &GripQLFilter{Expression: l.expr}) { + v := gdbi.Vertex{ + Label: label[2:], + Loaded: l.loadData, + } + if l.loadData { + v.ID = roMaps.(map[string]any)["_id"].(string) + delete(roMaps.(map[string]any), "_id") + v.Data = roMaps.(map[string]any) + } else { + v.ID = roMaps.(string) + v.Data = map[string]any{} + } + count += 1 + out <- t.AddCurrent(v.Copy()) + } + } + } + }() + } else { + queryChan := make(chan gdbi.ElementLookup, 100) + go func() { + defer close(queryChan) + for t := range in { + cond := l.expr.GetCondition() + for _, label := range l.labels { + for id := range l.db.jsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), cond.Condition) { + queryChan <- gdbi.ElementLookup{ID: id, Ref: t} + } + } + } + }() + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + } + + return ctx +} + +// ////////////////////////////////////////////////////////////////////////////// +// LookupVertsCondIndex look up vertices by indexed +type lookupVertsCondIndexStep struct { + expr *gripql.HasExpression +} + +func (t lookupVertsCondIndexStep) GetProcessor(db gdbi.GraphInterface, ps gdbi.PipelineState) (gdbi.Processor, error) { + graph := db.(*Graph) + return &lookupVertsCondIndexProc{ + db: graph, + expr: t.expr, + loadData: ps.StepLoadData()}, nil +} + +func (t lookupVertsCondIndexStep) GetType() gdbi.DataType { + return gdbi.VertexData +} + +type lookupVertsCondIndexProc struct { + db *Graph + expr *gripql.HasExpression + loadData bool + fallback bool +} + +func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { + log.Debugln("Entering lookupVertsCondIndexProc custom processor") + queryChan := make(chan gdbi.ElementLookup, 100) + cond := l.expr.GetCondition() + + /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter + otherwise this lookup will not fetch everything that was asked for */ + allMatch := len(l.db.jsonkv.Fields) > 0 && cond != nil + if allMatch { + for lbl := range l.db.jsonkv.GetLabels(false, false) { + if val, exists := l.db.jsonkv.Fields[lbl]; exists { + if _, ok := val[cond.Key]; !ok { + allMatch = false + break + } + } else { + allMatch = false + break + } + } + } + /* Optimized indexing only works for Simple filters. / + / If compound filter or index doesn't exist use backup method */ + if cond != nil && allMatch { + log.Debugln("Chose index optimized V().Has() statement path") + go func() { + defer close(queryChan) + for t := range in { + for id := range l.db.jsonkv.RowIdsByHas( + cond.Key, + cond.Value.AsInterface(), + cond.Condition, + ) { + queryChan <- gdbi.ElementLookup{ + ID: id, + Ref: t, + } + } + } + }() + } else { + log.Debugf("Base case GetVertexList is used. No indexing") + go func() { + defer close(queryChan) + for t := range in { + for v := range l.db.GetVertexList(ctx, true) { + if MatchesHasExpression( + AddSpecialFields(v), + l.expr, + ) { + queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} + } + + } + } + }() + } + + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() + return ctx +} + +func AddSpecialFields(v *gdbi.Vertex) any { + v.Data["_label"] = v.Label + v.Data["_id"] = v.ID + return v.Data +} diff --git a/grids/schema.go b/grids/schema.go index 8a6636c6..0847236c 100644 --- a/grids/schema.go +++ b/grids/schema.go @@ -34,7 +34,7 @@ func (ma *GDB) BuildSchema(ctx context.Context, graph string, sampleN uint32, ra } func (gi *Graph) sampleSchema(ctx context.Context, n uint32, random bool) ([]*gripql.Vertex, []*gripql.Edge, error) { - labels := gi.bsonkv.List() + labels := gi.jsonkv.List() vertLabels := []string{} for _, label := range labels { if label[:2] == "v_" { diff --git a/gripql/custom_statements.go b/gripql/custom_statements.go index c65bcb8f..e9cbc792 100644 --- a/gripql/custom_statements.go +++ b/gripql/custom_statements.go @@ -7,11 +7,11 @@ package gripql //in the traversal that the optimizer may add in, but can't be coded by a //serialized user request -type GraphStatement_LookupVertsIndex struct { +type GraphStatement_LookupVertsLabelIndex struct { Labels []string `protobuf:"bytes,1,rep,name=labels" json:"labels,omitempty"` } -func (*GraphStatement_LookupVertsIndex) isGraphStatement_Statement() {} +func (*GraphStatement_LookupVertsLabelIndex) isGraphStatement_Statement() {} type GraphStatement_EngineCustom struct { Desc string `protobuf:"bytes,1,opt,name=desc" json:"desc,omitempty"` diff --git a/gripql/inspect/inspect.go b/gripql/inspect/inspect.go index da4911d9..5dcc1938 100644 --- a/gripql/inspect/inspect.go +++ b/gripql/inspect/inspect.go @@ -45,7 +45,8 @@ func PipelineSteps(stmts []*gripql.GraphStatement) []string { *gripql.GraphStatement_Set, *gripql.GraphStatement_Increment, *gripql.GraphStatement_Mark, *gripql.GraphStatement_Jump, *gripql.GraphStatement_Sort, *gripql.GraphStatement_Pivot, *gripql.GraphStatement_Group, *gripql.GraphStatement_Totype: - case *gripql.GraphStatement_LookupVertsIndex, *gripql.GraphStatement_EngineCustom: + case *gripql.GraphStatement_LookupVertsLabelIndex, + *gripql.GraphStatement_EngineCustom: default: log.Errorf("Unknown Graph Statement: %T", gs.GetStatement()) } @@ -114,7 +115,9 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st onLast = false case *gripql.GraphStatement_Pivot: - //TODO: figure out which fields are referenced + if onLast { + out[steps[i]] = []string{"*"} + } onLast = false case *gripql.GraphStatement_Group: @@ -145,7 +148,15 @@ func PipelineStepOutputs(stmts []*gripql.GraphStatement, storeMarks bool) map[st out[steps[i]] = []string{"*"} } onLast = false - case *gripql.GraphStatement_LookupVertsIndex: + + case *gripql.GraphStatement_Sort, *gripql.GraphStatement_Totype, + *gripql.GraphStatement_Unwind, *gripql.GraphStatement_Aggregate: + if onLast { + out[steps[i]] = []string{"*"} + } + onLast = false + + case *gripql.GraphStatement_LookupVertsLabelIndex: if onLast { out[steps[i]] = []string{"*"} } diff --git a/gripql/python/gripql/graph.py b/gripql/python/gripql/graph.py index 6ba38e30..6f1c8e9d 100644 --- a/gripql/python/gripql/graph.py +++ b/gripql/python/gripql/graph.py @@ -174,6 +174,14 @@ def addIndex(self, label, field): raise_for_status(response) return response.json() + def deleteIndex(self, label, field): + url = self.url + f"/index/{label}/{field}" + response = self.session.delete( + url, + ) + raise_for_status(response) + return response.json() + def listIndices(self): url = self.url + "/index" response = self.session.get( @@ -307,7 +315,7 @@ def addEdge(self, src, dst, label, data={}, id=None): } } if id is not None: - payload["id"] = id + payload["edge"]["id"] = id self.elements.append(json.dumps(payload)) def execute(self): @@ -325,14 +333,13 @@ def __init__(self, url, graph, extraArgs=None, user=None, password=None, token=N super(BulkAddRaw, self).__init__(url, user, password, token, credential_file) self.url = self.base_url + "/v1/rawJson" self.graph = graph - self.extraArgs = {"auth_resource_path": "test-data"} self.elements = [] - def addJson(self, data={}): + def addJson(self, data={}, extra_args={}): payload = { "graph": self.graph, - "extra_args": self.extraArgs, + "extra_args": extra_args, "data": data } self.elements.append(json.dumps(payload)) diff --git a/gripql/python/gripql/util.py b/gripql/python/gripql/util.py index f1cb34de..6ab0cad4 100644 --- a/gripql/python/gripql/util.py +++ b/gripql/python/gripql/util.py @@ -64,6 +64,10 @@ def close(self): if self.i == 0: return + if self.start is None: + self.logger.error("Rate.close() called but Rate.init() was never called. self.start is None.") + return + now = datetime.now() dt = now - self.start rate = self.i / dt.total_seconds() diff --git a/log/logger.go b/log/logger.go index 9bb84edf..a316bf8d 100644 --- a/log/logger.go +++ b/log/logger.go @@ -11,13 +11,13 @@ import ( "strings" "time" + "golang.org/x/term" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "github.com/kr/pretty" "github.com/logrusorgru/aurora" "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh/terminal" ) var PanicLevel = logrus.PanicLevel @@ -110,7 +110,7 @@ type textFormatter struct { func checkIfTerminal(w io.Writer) bool { switch v := w.(type) { case *os.File: - return terminal.IsTerminal(int(v.Fd())) + return term.IsTerminal(int(v.Fd())) default: return false } @@ -141,7 +141,7 @@ func (f *textFormatter) Format(entry *logrus.Entry) ([]byte, error) { case logrus.DebugLevel: levelColor = aurora.MagentaFg case logrus.WarnLevel: - levelColor = aurora.BrownFg + levelColor = aurora.YellowFg case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel: levelColor = aurora.RedFg default: @@ -276,67 +276,67 @@ func ConfigureLogger(conf Logger) { } // Debug log message -func Debug(args ...interface{}) { +func Debug(args ...any) { logger.Debug(args...) } // Debugln log message -func Debugln(args ...interface{}) { +func Debugln(args ...any) { logger.Debugln(args...) } // Debugf log message -func Debugf(format string, args ...interface{}) { +func Debugf(format string, args ...any) { logger.Debugf(format, args...) } // Info log message -func Info(args ...interface{}) { +func Info(args ...any) { logger.Info(args...) } // Infoln log message -func Infoln(args ...interface{}) { +func Infoln(args ...any) { logger.Infoln(args...) } // Infof log message -func Infof(format string, args ...interface{}) { +func Infof(format string, args ...any) { logger.Infof(format, args...) } // Warning log message -func Warning(args ...interface{}) { +func Warning(args ...any) { logger.Warning(args...) } // Warningln log message -func Warningln(args ...interface{}) { +func Warningln(args ...any) { logger.Warningln(fmt.Sprint(args...)) } // Warningf log message -func Warningf(format string, args ...interface{}) { +func Warningf(format string, args ...any) { logger.Warningf(format, args...) } // Error log message -func Error(args ...interface{}) { +func Error(args ...any) { logger.Error(args...) } // Errorln log message -func Errorln(args ...interface{}) { +func Errorln(args ...any) { logger.Errorln(args...) } // Errorf log message -func Errorf(format string, args ...interface{}) { +func Errorf(format string, args ...any) { logger.Errorf(format, args...) } // WithField creates an entry from the standard logger and adds a field to it. -func WithField(key string, value interface{}) *Entry { +func WithField(key string, value any) *Entry { return logger.WithField(key, value) } @@ -358,3 +358,16 @@ func GetLogger() *logrus.Logger { func Sub(ns string) *Entry { return logger.WithFields(Fields{"namespace": ns}) } + +func Redf(format string, args ...any) { + if tf, ok := logger.Formatter.(*textFormatter); ok { + isColored := (tf.ForceColors || isColorTerminal(logger.Out)) && !tf.DisableColors + if isColored { + msg := fmt.Sprintf(format, args...) + redMsg := aurora.Red(msg).String() + fmt.Fprintln(logger.Out, redMsg) + return + } + } + logger.Errorf(format, args...) +} diff --git a/mongo/compile.go b/mongo/compile.go index 26454f2c..6331cea9 100644 --- a/mongo/compile.go +++ b/mongo/compile.go @@ -1005,7 +1005,7 @@ func (comp *Compiler) Compile(stmts []*gripql.GraphStatement, opts *gdbi.Compile } } - bsonDoc, err := bson.Marshal(bson.D{{"doc", query}}) + bsonDoc, err := bson.Marshal(bson.D{{Key: "doc", Value: query}}) if err != nil { fmt.Printf("Error: %s\n", err) } diff --git a/mongo/graph.go b/mongo/graph.go index 7ba33ec2..e94b9cd6 100644 --- a/mongo/graph.go +++ b/mongo/graph.go @@ -4,9 +4,6 @@ import ( "context" "fmt" - //"io" - //"strings" - "time" "github.com/bmeg/grip/engine/core" @@ -45,13 +42,13 @@ func (mg *Graph) GetTimestamp() string { func (mg *Graph) GetVertex(id string, load bool) *gdbi.Vertex { opts := options.FindOne() if !load { - opts.SetProjection(map[string]interface{}{FIELD_ID: 1, FIELD_LABEL: 1}) + opts.SetProjection(map[string]any{FIELD_ID: 1, FIELD_LABEL: 1}) } result := mg.ar.VertexCollection(mg.graph).FindOne(context.Background(), bson.M{FIELD_ID: id}, opts) if result.Err() != nil { return nil } - d := map[string]interface{}{} + d := map[string]any{} if nil == result.Decode(d) { v := UnpackVertex(d) return v @@ -63,13 +60,13 @@ func (mg *Graph) GetVertex(id string, load bool) *gdbi.Vertex { func (mg *Graph) GetEdge(id string, load bool) *gdbi.Edge { opts := options.FindOne() if !load { - opts.SetProjection(map[string]interface{}{FIELD_ID: 1, FIELD_LABEL: 1, FIELD_FROM: 1, FIELD_TO: 1}) + opts.SetProjection(map[string]any{FIELD_ID: 1, FIELD_LABEL: 1, FIELD_FROM: 1, FIELD_TO: 1}) } result := mg.ar.EdgeCollection(mg.graph).FindOne(context.TODO(), bson.M{FIELD_ID: id}, opts) if result.Err() != nil { return nil } - d := map[string]interface{}{} + d := map[string]any{} if nil == result.Decode(d) { v := UnpackEdge(d) return v @@ -268,7 +265,7 @@ func (mg *Graph) GetVertexList(ctx context.Context, load bool) <-chan *gdbi.Vert return default: } - result := map[string]interface{}{} + result := map[string]any{} if err := query.Decode(&result); err == nil { v := UnpackVertex(result) o <- v @@ -303,7 +300,7 @@ func (mg *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdbi.Ed return default: } - result := map[string]interface{}{} + result := map[string]any{} if err := query.Decode(&result); err == nil { if _, ok := result[FIELD_TO]; ok { e := UnpackEdge(result) @@ -349,7 +346,7 @@ func (mg *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLook } chunk := map[string]*gdbi.Vertex{} for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { v := UnpackVertex(result) chunk[v.ID] = v @@ -412,9 +409,9 @@ func (mg *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLoo cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { - if dst, ok := result["dst"].(map[string]interface{}); ok { + if dst, ok := result["dst"].(map[string]any); ok { v := UnpackVertex(dst) fromID := result[FIELD_FROM].(string) r := batchMap[fromID] @@ -424,7 +421,7 @@ func (mg *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.ElementLoo o <- ri } } else { - log.WithFields(log.Fields{"result": result["dst"]}).Error("GetOutChannel: unable to cast result to map[string]interface{}") + log.WithFields(log.Fields{"result": result["dst"]}).Error("GetOutChannel: unable to cast result to map[string]any") } } else { log.WithFields(log.Fields{"result": result, "error": err}).Error("GetOutChannel: decode error") @@ -491,9 +488,9 @@ func (mg *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLook cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { - if src, ok := result["src"].(map[string]interface{}); ok { + if src, ok := result["src"].(map[string]any); ok { v := UnpackVertex(src) toID := result[FIELD_TO].(string) r := batchMap[toID] @@ -503,7 +500,7 @@ func (mg *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.ElementLook o <- ri } } else { - log.WithFields(log.Fields{"result": result["src"]}).Error("GetInChannel: unable to cast result to map[string]interface{}") + log.WithFields(log.Fields{"result": result["src"]}).Error("GetInChannel: unable to cast result to map[string]any") } } else { log.WithFields(log.Fields{"error": err}).Error("Decode") @@ -561,7 +558,7 @@ func (mg *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.Elemen cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { e := UnpackEdge(result) fromID := result[FIELD_FROM].(string) @@ -628,7 +625,7 @@ func (mg *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Element cursor, err := eCol.Aggregate(context.TODO(), query) if err == nil { for cursor.Next(context.TODO()) { - result := map[string]interface{}{} + result := map[string]any{} if err := cursor.Decode(&result); err == nil { e := UnpackEdge(result) toID := result[FIELD_TO].(string) diff --git a/mongo/index.go b/mongo/index.go index 3077d255..d06b4e8e 100644 --- a/mongo/index.go +++ b/mongo/index.go @@ -13,7 +13,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -// AddVertexIndex add index to vertices +// AddVertexIndex adds an index to vertices for a specific label and field func (mg *Graph) AddVertexIndex(label string, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Adding vertex index") field = tpath.NormalizePath(field) @@ -21,38 +21,59 @@ func (mg *Graph) AddVertexIndex(label string, field string) error { field = strings.TrimPrefix(field, "$.") idx := mg.ar.VertexCollection(mg.graph).Indexes() - + indexName := fmt.Sprintf("label_%s_%s_idx", label, field) + + // Create a compound index on _label and the specified field, filtered by the specific label _, err := idx.CreateOne( context.Background(), mongo.IndexModel{ - Keys: bson.D{{"label", 1}, {field, 1}}, - Options: options.Index().SetUnique(false).SetSparse(true).SetBackground(true), + Keys: bson.D{ + {Key: FIELD_LABEL, Value: 1}, + {Key: field, Value: 1}, + }, + Options: options.Index(). + SetName(indexName). + SetUnique(false). + SetBackground(true). + SetPartialFilterExpression(bson.M{FIELD_LABEL: label}), }) if err != nil { - return fmt.Errorf("failed create index %s %s %s", label, field, err) + return fmt.Errorf("failed to create index for label %s on field %s: %s", label, field, err) } return nil } -// DeleteVertexIndex delete index from vertices +// DeleteVertexIndex deletes an index from vertices for a specific label and field func (mg *Graph) DeleteVertexIndex(label string, field string) error { log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") field = tpath.NormalizePath(field) field = tpath.ToLocalPath(field) - field = strings.TrimPrefix(field, "$.") //FIXME + field = strings.TrimPrefix(field, "$.") idx := mg.ar.VertexCollection(mg.graph).Indexes() cursor, err := idx.List(context.TODO()) + if err != nil { + return fmt.Errorf("failed to list indices: %s", err) + } + var results []bson.M if err = cursor.All(context.TODO(), &results); err != nil { - return err + return fmt.Errorf("failed to retrieve index list: %s", err) } + for _, rec := range results { - recKeys := rec["key"].(bson.M) - if _, ok := recKeys["label"]; ok { - if _, ok := recKeys[field]; ok { - if _, err := idx.DropOne(context.TODO(), rec["name"].(string)); err != nil { - return err + if recKeys, ok := rec["key"].(bson.M); ok { + if _, hasLabel := recKeys[FIELD_LABEL]; hasLabel { + if _, hasField := recKeys[field]; hasField { + if partialFilter, ok := rec["partialFilterExpression"].(bson.M); ok { + if partialLabel, ok := partialFilter[FIELD_LABEL].(string); ok && partialLabel == label { + log.Debugln("HELLO ", partialLabel) + if _, err := idx.DropOne(context.TODO(), rec["name"].(string)); err != nil { + return fmt.Errorf("failed to delete index for label %s on field %s: %s", label, field, err) + } + return nil + } + } } } } @@ -67,34 +88,31 @@ func (mg *Graph) GetVertexIndexList() <-chan *gripql.IndexID { go func() { defer close(out) - c := mg.ar.VertexCollection(mg.graph) - - labels, err := mg.ListVertexLabels() + idx := mg.ar.VertexCollection(mg.graph).Indexes() + cursor, err := idx.List(context.TODO()) if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: finding distinct labels") + log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: failed to list indices") + return } - // list indexed fields - idx := c.Indexes() - cursor, err := idx.List(context.TODO()) var idxList []bson.M if err = cursor.All(context.TODO(), &idxList); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: finding indexed fields") + log.WithFields(log.Fields{"error": err}).Error("GetVertexIndexList: failed to retrieve index list") + return } + for _, rec := range idxList { - recKeys := rec["key"].(bson.M) - if len(recKeys) > 1 { - if _, ok := recKeys["label"]; ok { - key := "" - for k := range recKeys { - if k != "label" { - key = k - } - } - if len(key) > 0 { - f := strings.TrimPrefix(key, "data.") - for _, l := range labels { - out <- &gripql.IndexID{Graph: mg.graph, Label: l, Field: f} + if recKeys, ok := rec["key"].(bson.M); ok { + if _, hasLabelKey := recKeys[FIELD_LABEL]; hasLabelKey { + for key := range recKeys { + if key != FIELD_LABEL { + f := strings.TrimPrefix(key, "data.") + if partialFilter, ok := rec["partialFilterExpression"].(bson.M); ok { + log.Debugln("HELLO2 ", partialFilter) + if label, ok := partialFilter[FIELD_LABEL].(string); ok { + out <- &gripql.IndexID{Graph: mg.graph, Label: label, Field: f} + } + } } } } diff --git a/psql/index.go b/psql/index.go index 23cf5fec..1d6eff01 100644 --- a/psql/index.go +++ b/psql/index.go @@ -1,24 +1,102 @@ package psql import ( - "errors" + "fmt" + "strings" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" + "github.com/lib/pq" ) -// AddVertexIndex add index to vertices -func (g *Graph) AddVertexIndex(label string, field string) error { - return errors.New("not implemented") +func (g *Graph) AddVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf( + "CREATE INDEX IF NOT EXISTS %s ON %s ((data->>'%s')) WHERE label = '%s'", + pq.QuoteIdentifier(indexName), + pq.QuoteIdentifier(g.v), + field, + vertexLabel, + ) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error adding index %s: %v", indexName, err) + return fmt.Errorf("failed to add index %s: %w", indexName, err) + } + log.Infof("Successfully added index %s for vertex label %s field %s", indexName, vertexLabel, field) + return nil } -// DeleteVertexIndex delete index from vertices -func (g *Graph) DeleteVertexIndex(label string, field string) error { - return errors.New("not implemented") +// DeleteVertexIndex drops the specified index. +func (g *Graph) DeleteVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf("DROP INDEX IF EXISTS %s", pq.QuoteIdentifier(indexName)) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error deleting index %s: %v", indexName, err) + return fmt.Errorf("failed to delete index %s: %w", indexName, err) + } + log.Infof("Successfully deleted index %s", indexName) + return nil } // GetVertexIndexList lists indices func (g *Graph) GetVertexIndexList() <-chan *gripql.IndexID { - o := make(chan *gripql.IndexID) - defer close(o) + o := make(chan *gripql.IndexID, 100) + go func() { + defer close(o) + // Query indices only on the vertex table (g.v) + query := ` + SELECT i.relname AS index_name + FROM pg_class t, + pg_class i, + pg_index ix + WHERE t.oid = ix.indrelid + AND i.oid = ix.indexrelid + AND t.relkind = 'r' + AND i.relkind = 'i' + AND NOT ix.indisprimary + AND NOT ix.indisunique + AND t.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = current_schema()) + AND t.relname = $1 + ` + rows, err := g.db.Queryx(query, g.v) + if err != nil { + log.Errorf("Error getting index list: %v", err) + return + } + defer rows.Close() + + prefix := fmt.Sprintf("%s_", g.graph) + for rows.Next() { + var indexName string + if err := rows.Scan(&indexName); err != nil { + log.Errorf("Error scanning index row: %v", err) + continue + } + if !strings.HasPrefix(indexName, prefix) { + log.Infof("Skipping index '%s': does not match graph '%s'", indexName, g.graph) + continue + } + rest := strings.TrimPrefix(indexName, prefix) + if strings.HasPrefix(rest, "vertices_") { + continue + } else { + parts := strings.SplitN(rest, "_", 2) + if len(parts) == 2 { + o <- &gripql.IndexID{ + Graph: g.graph, + Label: parts[0], // Vertex label, e.g., "Person" + Field: parts[1], // Field, e.g., "age" + } + } else { + log.Infof("Skipping index '%s': unrecognized format", indexName) + } + } + } + if err := rows.Err(); err != nil { + log.Errorf("Error during index row iteration: %v", err) + } + }() return o } diff --git a/server/marshaler.go b/server/marshaler.go index ee564c87..a25c89f9 100644 --- a/server/marshaler.go +++ b/server/marshaler.go @@ -21,8 +21,8 @@ type MarshalClean struct { func NewMarshaler() runtime.Marshaler { return &MarshalClean{ m: &runtime.JSONPb{ - protojson.MarshalOptions{EmitUnpopulated: true}, - protojson.UnmarshalOptions{}, + MarshalOptions: protojson.MarshalOptions{EmitUnpopulated: true}, + UnmarshalOptions: protojson.UnmarshalOptions{}, //EnumsAsInts: false, //EmitDefaults: true, //OrigName: true, diff --git a/server/server.go b/server/server.go index 7abc8462..4f1ab14c 100644 --- a/server/server.go +++ b/server/server.go @@ -366,7 +366,7 @@ func (server *GripServer) Serve(pctx context.Context) error { } partition, offset, err := server.kafkaProducer.SendMessage(msg) if err != nil { - log.Errorf("Failed to send Kafka message to topic %s: %v", *&server.conf.Kafka.Topic, err) + log.Errorf("Failed to send Kafka message to topic %#v: %v", *&server.conf.Kafka.Topic, err) } else { log.Infof("Message sent to Kafka topic %s [partition %d, offset %d]", *server.conf.Kafka.Topic, partition, offset) } @@ -495,7 +495,9 @@ func (server *GripServer) Serve(pctx context.Context) error { if isSchema(graph) { log.WithFields(log.Fields{"graph": graph}).Debug("Loading existing schema into cache") schema, err := server.getGraph(graph) - if err == nil { + if err != nil { + log.Errorln("Error in server.getGraph: ", err) + } else { server.schemas[strings.TrimSuffix(graph, schemaSuffix)] = schema } } else if isMapping(graph) { diff --git a/sqlite/index.go b/sqlite/index.go index 2862fd58..ca2f0530 100644 --- a/sqlite/index.go +++ b/sqlite/index.go @@ -1,24 +1,99 @@ package sqlite import ( - "errors" + "fmt" + "strings" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" + "github.com/lib/pq" ) -// AddVertexIndex add index to vertices -func (g *Graph) AddVertexIndex(label string, field string) error { - return errors.New("not implemented") +func (g *Graph) AddVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf( + "CREATE INDEX IF NOT EXISTS %s ON %s ((data->>'%s')) WHERE label = '%s'", + pq.QuoteIdentifier(indexName), + pq.QuoteIdentifier(g.v), + field, + vertexLabel, + ) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error adding index %s: %v", indexName, err) + return fmt.Errorf("failed to add index %s: %w", indexName, err) + } + log.Infof("Successfully added index %s for vertex label %s field %s", indexName, vertexLabel, field) + return nil } -// DeleteVertexIndex delete index from vertices -func (g *Graph) DeleteVertexIndex(label string, field string) error { - return errors.New("not implemented") +// DeleteVertexIndex drops the specified index. +func (g *Graph) DeleteVertexIndex(vertexLabel string, field string) error { + indexName := fmt.Sprintf("%s_%s_%s", g.graph, vertexLabel, field) + query := fmt.Sprintf("DROP INDEX IF EXISTS %s", pq.QuoteIdentifier(indexName)) + _, err := g.db.Exec(query) + if err != nil { + log.Errorf("Error deleting index %s: %v", indexName, err) + return fmt.Errorf("failed to delete index %s: %w", indexName, err) + } + log.Infof("Successfully deleted index %s", indexName) + return nil } // GetVertexIndexList lists indices func (g *Graph) GetVertexIndexList() <-chan *gripql.IndexID { - o := make(chan *gripql.IndexID) - defer close(o) + o := make(chan *gripql.IndexID, 100) + go func() { + defer close(o) + // Query indices only on the vertex table (g.v) + query := ` + SELECT name + FROM sqlite_master + WHERE type = 'index' AND tbl_name = ? + AND name NOT LIKE 'sqlite_autoindex_%' -- Exclude auto-generated primary key indices + AND name IN ( + SELECT name + FROM pragma_index_list(?) + WHERE "unique" = 0 -- Exclude unique indices + ) + ` + rows, err := g.db.Queryx(query, g.v, g.v) + if err != nil { + log.Errorf("Error getting index list: %v", err) + return + } + defer rows.Close() + + prefix := fmt.Sprintf("%s_", g.graph) + for rows.Next() { + var indexName string + if err := rows.Scan(&indexName); err != nil { + log.Errorf("Error scanning index row: %v", err) + continue + } + if !strings.HasPrefix(indexName, prefix) { + log.Infof("Skipping index '%s': does not match graph '%s'", indexName, g.graph) + continue + } + rest := strings.TrimPrefix(indexName, prefix) + if strings.HasPrefix(rest, "vertices_") { + continue + } else { + parts := strings.SplitN(rest, "_", 2) + if len(parts) == 2 { + o <- &gripql.IndexID{ + Graph: g.graph, + Label: parts[0], // Vertex label, e.g., "Person" + Field: parts[1], // Field, e.g., "age" + } + } else { + log.Infof("Skipping index '%s': unrecognized format", indexName) + } + } + } + if err := rows.Err(); err != nil { + log.Errorf("Error during index row iteration: %v", err) + } + }() return o } diff --git a/test/main_test.go b/test/main_test.go index 2a79d619..a725f5fb 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -78,6 +78,7 @@ func TestMain(m *testing.M) { panic(err) } for e := range edgeChan { + fmt.Printf("Adding edge: %s %#v\n", e.Id, e.Data.AsMap()) edges = append(edges, e) } diff --git a/test/processors_test.go b/test/processors_test.go index 584c9c44..b199f692 100644 --- a/test/processors_test.go +++ b/test/processors_test.go @@ -419,15 +419,15 @@ func compare(expect []*gripql.QueryResult) checker { sort.Strings(expectS) if !reflect.DeepEqual(actualS, expectS) { - for _, s := range actualS { - t.Log("actual", s) - } - for _, s := range expectS { - t.Log("expect", s) - } if len(expectS) != len(actualS) { t.Logf("expected # results: %d actual # results: %d", len(expectS), len(actualS)) + } else { + for i, s := range actualS { + t.Log("actual", s) + t.Log("expect", expectS[i]) + } } + t.Errorf("not equal") } }