Feature/indexing

.github/workflows/tests.yml

@@ -29,6 +29,7 @@ jobs:
           name: gripBin
           path: grip
 
+
   unitTests:
     needs: build
     name: Unit tests
@@ -65,7 +66,6 @@ jobs:
           sleep 5
           python conformance/run_kafka.py
 
-
   badgerTest:
     needs: build
     name: Badger Conformance
@@ -86,7 +86,6 @@ jobs:
           sleep 5
           make test-conformance
 
-
   pebbleTest:
     needs: build
     name: Pebble Conformance
@@ -107,7 +106,6 @@ jobs:
           sleep 5
           make test-conformance
 
-
   mongoTest:
     needs: build
     name: Mongo Test
@@ -129,7 +127,6 @@ jobs:
           sleep 5
           make test-conformance
 
-
   mongoCoreTest:
     needs: build
     name: Mongo Core Test
@@ -171,8 +168,7 @@ jobs:
           sleep 15
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/psql.yml &
           sleep 5
-          python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations
-
+          python conformance/run_conformance.py http://localhost:18201 --exclude aggregations
 
   sqliteTest:
     needs: build
@@ -192,8 +188,7 @@ jobs:
           chmod +x grip
           ./grip server --rpc-port 18202 --http-port 18201 --config ./test/sqlite.yml &
           sleep 5
-          python conformance/run_conformance.py http://localhost:18201 --exclude index aggregations
-
+          python conformance/run_conformance.py http://localhost:18201 --exclude aggregations
 
   gripperTest:
     needs: build
@@ -218,8 +213,7 @@ jobs:
           sleep 5
           ./grip server --rpc-port 18202 --http-port 18201 --config ./gripper/test-graph/config.yaml --er tableServer=localhost:50051 &
           sleep 5
-          python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude pivot
-
+          python conformance/run_conformance.py http://localhost:18201 --readOnly swapi --exclude index pivot
 
   authTest:
     needs: build
@@ -251,8 +245,6 @@ jobs:
           # run specialized role based tests
           make test-authorization ARGS="--grip_config_file_path test/pebble-auth.yml"
 
-
-
   gridsTest:
     needs: build
     name: GRIDs Conformance

accounts/basic.go

@@ -4,6 +4,8 @@ import (
 	"encoding/base64"
 	"fmt"
 	"strings"
+
+	"github.com/bmeg/grip/log"
 )
 
 // BasicCredential describes a username and password for use with Funnel's basic auth.
@@ -18,7 +20,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) {
 	var auth []string
 	var ok bool
 
-	fmt.Printf("Running BasicAuth: %#v\n", md)
+	log.Infof("Running BasicAuth: %#v\n", md)
 
 	if auth, ok = md["Authorization"]; !ok {
 		if auth, ok = md["authorization"]; !ok {
@@ -28,7 +30,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) {
 
 	if len(auth) > 0 {
 		user, password, ok := parseBasicAuth(auth[0])
-		fmt.Printf("User: %s Password: %s OK: %s\n", user, password, ok)
+		log.Infof("User: %s Password: %s OK: %t\n", user, password, ok)
 		for _, c := range ba {
 			if c.User == user && c.Password == password {
 				return user, nil

accounts/interface.go

@@ -43,6 +43,7 @@ var MethodMap = map[string]Operation{
 	"/gripql.Edit/DeleteVertex":  Write,
 	"/gripql.Edit/DeleteEdge":    Write,
 	"/gripql.Edit/AddIndex":      Write,
+	"/gripql.Edit/DeleteIndex":   Write,
 	"/gripql.Edit/AddSchema":     Write,
 	"/gripql.Edit/AddJsonSchema": Write,
 	"/gripql.Edit/AddMapping":    Write,

conformance/tests/auth_basic.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-import requests
-
 
 def test_current_user_has_policy(manager):
     """Ensure current user has a policy defined."""
@@ -18,7 +16,7 @@ def test_current_user_can_query(manager):
     account = manager.current_user_account()
     assert account, f"Could not find account for {manager.user}"
     policies = account.policies
-    assert len(policies) > 0, f"Should have at least one policy"
+    assert len(policies) > 0, "Should have at least one policy"
     errors = []
 
     if not account.is_admin:
@@ -56,7 +54,7 @@ def test_current_user_can_read(manager):
     account = manager.current_user_account()
     assert account, f"Could not find account for {manager.user}"
     policies = account.policies
-    assert len(policies) > 0, f"Should have at least one policy"
+    assert len(policies) > 0, "Should have at least one policy"
     errors = []
     # non admin user
     if not account.is_admin:
@@ -92,7 +90,7 @@ def test_current_user_can_write(manager):
     account = manager.current_user_account()
     assert account, f"Could not find account for {manager.user}"
     policies = account.policies
-    assert len(policies) > 0, f"Should have at least one policy"
+    assert len(policies) > 0, "Should have at least one policy"
     errors = []
     # non admin user
     if not account.is_admin:
@@ -109,7 +107,7 @@ def test_current_user_can_write(manager):
         try:
             manager.test_write('dummy')
             errors.append(f"{manager.user} should not be able to write dummy graph")
-        except AssertionError as e:
+        except AssertionError:
             pass
 
     else:

conformance/tests/ot_has.py

@@ -137,7 +137,6 @@ def test_has_prev(man):
     q = q.has(gripql.neq("$1._id", "$._id"))
     count = 0
     for i in q.render(["$1._id", "$._id"]):
-        print(i)
         if i[0] == i[1]:
             errors.append("History based filter failed: %s" % (i[0]) )
         count += 1
@@ -171,6 +170,7 @@ def test_has_neq(man):
             "Fail: G.query().V().has(gripql.not_(gripql.eq(\"_label\", \"Character\"))) %s != %s" %
             (count, 21))
 
+
     count = 0
     for i in G.query().V().hasLabel("Character").has(gripql.neq("eye_color", "brown")):
         count += 1
@@ -181,6 +181,7 @@ def test_has_neq(man):
             "Fail: G.query().V().has(gripql.not_(gripql.eq(\"eye_color\", \"brown\"))) %s != %s" %
             (count, 14))
 
+
     return errors
 
 

conformance/tests/ot_index.py

@@ -1,15 +1,40 @@
+import gripql
+
+
+def test_index_create_and_delete(man):
+    errors = []
+    G = man.writeTest()
+    G.addIndex("Person", "name")
+    found = False
+    indices = G.listIndices()
+    for i in indices:
+        if i["field"] == "name" and i["label"] == "Person":
+            found = True
+    if not found:
+        errors.append("Expected index to be found")
+
+    G.deleteIndex("Person", "name")
+    indices = G.listIndices()
+
+    found = False
+    for i in indices:
+        if i["field"] == "name" and i["label"] == "Person":
+            found = True
+    if found:
+        errors.append("Expected index not found")
+
+    return errors
 
 
 def test_index(man):
     errors = []
 
     G = man.writeTest()
-
     G.addIndex("Person", "name")
 
     G.addVertex("1", "Person", {"name": "marko", "age": "29"})
     G.addVertex("2", "Person", {"name": "vadas", "age": "27"})
-    G.addVertex("3", "Software", {"name": "lop", "lang": "java"})
+    G.addVertex("3",  "Software", {"name": "lop", "lang": "java"})
     G.addVertex("4", "Person", {"name": "josh", "age": "32"})
     G.addVertex("5", "Software", {"name": "ripple", "lang": "java"})
     G.addVertex("6", "Person", {"name": "peter", "age": "35"})
@@ -22,12 +47,169 @@ def test_index(man):
     G.addEdge("6", "3", "created", {"weight": 0.2})
     G.addEdge("4", "5", "created", {"weight": 1.0})
 
-    resp = G.listIndices()
+    count = 0
+    for i in G.query().V().has(gripql.eq("name","marko")):
+        count += 1
+        if "name" not in i:
+            errors.append("'name' field not found in vertex")
+        if i["name"] != "marko":
+            errors.append("Filtering on field name, value marko but got '%s' instead" % i["name"])
+    if count != 2:
+        errors.append("Expecting 2 vertices returned but got %d instead" % (count))
+
+    return errors
+
+
+def test_bulk_index(man):
+    errors = []
+
+    G = man.writeTest()
+    G.addIndex("Person", "age")
+
+    bulk = G.bulkAdd()
+
+    bulk.addVertex("1", "Person", {"name": "marko", "age": "29"})
+    bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"})
+    bulk.addVertex("4", "Person", {"name": "josh", "age": "32"})
+    bulk.addVertex("6", "Person", {"name": "peter", "age": "35"})
+    bulk.addVertex("7", "Person", {"name": "alice", "age": "31"})
+    bulk.addVertex("8", "Person", {"name": "bob", "age": "32"})
+    bulk.addVertex("9", "Person", {"name": "charlie", "age": "28"})
+    bulk.addVertex("10", "Person", {"name": "diana", "age": "32"})
+    bulk.addVertex("11", "Person", {"name": "eve", "age": "30"})
+    bulk.addVertex("12", "Person", {"name": "frank", "age": "33"})
+    bulk.addVertex("13", "Person", {"name": "grace", "age": "26"})
+    bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"})
+    bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"})
+    bulk.addVertex("16", "Person", {"name": "judy", "age": "34"})
+    res = bulk.execute()
+    if res["errorCount"] > 0:
+        errors.append("errorCount on bulk add > 0")
+
+    count = 0
+    resp3 = G.query().V().has(gripql.eq("age","32"))
+    for i in resp3:
+        count += 1
+        if "age" not in i:
+            errors.append("field 'age' not found in vertex")
+        if "age" in i and i["age"] != "32":
+            errors.append("filtering on field age value '32' but got %s instead" % (i["age"]))
+    if count != 4:
+        errors.append("expected count 4 but got %d instead" % (count))
+
+    return errors
+
+
+def test_index_after_write(man):
+    errors = []
+    G = man.writeTest()
+    bulk = G.bulkAdd()
+    bulk.addVertex("1", "Person", {"name": "marko", "age": "29"})
+    bulk.addVertex("2", "Person", {"name": "vadas", "age": "27"})
+    bulk.addVertex("4", "Person", {"name": "josh", "age": "32"})
+    bulk.addVertex("6", "Person", {"name": "peter", "age": "35"})
+    bulk.addVertex("7", "Person", {"name": "alice", "age": "31"})
+    bulk.addVertex("8", "Person", {"name": "bob", "age": "32"})
+    bulk.addVertex("9", "Person", {"name": "charlie", "age": "28"})
+    bulk.addVertex("10", "Person", {"name": "diana", "age": "32"})
+    bulk.addVertex("11", "Person", {"name": "eve", "age": "30"})
+    bulk.addVertex("12", "Person", {"name": "frank", "age": "33"})
+    bulk.addVertex("13", "Person", {"name": "grace", "age": "26"})
+    bulk.addVertex("14", "Person", {"name": "heidi", "age": "32"})
+    bulk.addVertex("15", "Person", {"name": "ivan", "age": "29"})
+    bulk.addVertex("16", "Person", {"name": "judy", "age": "34"})
+    res = bulk.execute()
+    if res["errorCount"] > 0:
+        errors.append("errorCount on bulk add > 0")
+
+    G.addIndex("Person", "age")
+
+    count = 0
+    restwo = G.query().V().has(gripql.within("name", ["marko", "vadas", "eve", "ivan", "charlie", "nothere"]))
+    for i in restwo:
+        count += 1
+    if count != 5:
+        errors.append("Expected 5 names from filter but got %d instead" % (count))
+
+    return errors
+
+
+def test_index_filter(man):
+    errors = []
+    G = man.setGraph("swapi")
+    G.addIndex("Starship", "cost_in_credits")
+    G.addIndex("Starship", "cargo_capacity")
+
+    respthree = G.query().V().hasLabel("Starship").has(gripql.lt("cost_in_credits", 150000000))
+    count = 0
+    for i in respthree:
+        count += 1
+        if i['cost_in_credits'] > 149999999:
+            errors.append("filtering on ships that cost less than 150000000, but %d > 149999999" % (i['cost_in_credits']))
+
+    if count != 5:
+        errors.append("Expected 5 results got %d instead" % (count))
+
+
+    indices = G.listIndices()
     found = False
-    for i in resp:
-        if i["field"] == "name" and i["label"] == "Person":
+    count = 0
+    for i in indices:
+        count +=1
+        if i["field"] == "cost_in_credits" and i["label"] == "Starship":
             found = True
     if not found:
         errors.append("Expected index not found")
+    if count != 2:
+        errors.append("Expected to find 2 indices but found %d instead" % (count))
+
+    G.deleteIndex("Starship", "cost_in_credits")
+    count = 0
+    indices_two = G.listIndices()
+    found = False
+    for i in indices_two:
+        count += 1
+        if i["field"] == "cost_in_credits" and i["label"] == "Starship":
+            found = True
+    if found:
+        errors.append("Expected index not found, but it was found")
+    if count != 1:
+        errors.append("Expected to find 1 index but found %d instead" % (count))
+
+    return errors
+
+
+def test_consistent_results(man):
+    errors = []
+    G = man.setGraph("swapi")
+
+    resp = G.query().V().has(gripql.contains("eye_colors", "yellow"))
+    count = 0
+    for i in resp:
+        count += 1
+    if count != 2:
+        errors.append("Expected 2 results but got %d instead" % (count))
+
+    G.addIndex("Species", "eye_colors")
+    resp = G.query().V().has(gripql.contains("eye_colors", "yellow"))
+    count = 0
+    for i in resp:
+        count += 1
+    if count != 2:
+        errors.append("Expected 2 results but got %d instead" % (count))
+
+    return errors
+
+
+def test_hasLabel_contains(man):
+    # If using the grids driver + no indexing this test uses the optimized scan function pipeline
+    errors = []
+    G = man.setGraph("swapi")
+    resp = G.query().V().hasLabel("Species").has(gripql.contains("eye_colors", "yellow"))
+    count = 0
+    for i in resp:
+        count += 1
+    if count != 2:
+        errors.append("Expected 2 results but got %d instead" % (count))
 
     return errors