From 95dad1ab079730865be7b7efde940d82c9e9be47 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Sat, 26 Oct 2019 11:58:19 -0400 Subject: [PATCH 01/20] create and update command parsing --- tinytalk/grammar.py | 105 ++++++++++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 621560e..863e319 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -1,11 +1,16 @@ +import pprint + from parsimonious.grammar import Grammar, NodeVisitor from parsimonious.nodes import RegexNode -import pprint grammar = Grammar( r""" - query = "when" condition (";" condition)* ws? - condition = adjectives? tags data? (ws alias)? + program = query write + query = "when" entry (";" entry)* + write = (create / update) (";" (create / update))* + create = ws "create" entry + update = ws "update" ws name data + entry = adjectives? tags data? (ws alias)? adjectives = (ws adjective)+ tags = (ws tag)+ data = (ws datum)+ @@ -27,44 +32,76 @@ def not_whitespace(o): return not (isinstance(o, RegexNode) and o.expr_name == "ws") -class Condition: - - def __init__(self, *_, adjectives, tags, data, alias): +class Entry: + def __init__(self, *_, adjectives, tags, data, alias=None): self.adjectives = adjectives self.tags = tags self.data = data self.alias = alias - def __repr__(self): - return f"{self.__class__.__name__}(adjectives={self.adjectives}, tags={self.tags}, data={self.data}, alias={self.alias})" - def __str__(self): return pprint.pformat(self) + def __repr__(self): + return f"{self.__class__.__name__}(adjectives={self.adjectives}, " \ + f"tags={self.tags}, " \ + f"data={self.data}, " \ + f"alias={self.alias})" + + +class Update: + + def __init__(self, *_, alias, data): + self.data = data + self.alias = alias + + def __repr__(self): + return f"{self.__class__.__name__}(alias={self.alias}, " \ + f"data={self.data})" + class TinyTalkVisitor(NodeVisitor): + def visit_program(self, _node, visited_children): + query, write = visited_children + return {"query": query, "write": write} + + def visit_write(self, _node, visited_children): + first, rest = visited_children + write = [first[0]] + if isinstance(rest, list): + write += [pair[1][0] for pair in rest] + return write + + def visit_create(self, _node, visited_children): + ws1, _create, entry = visited_children + return entry + + def visit_update(self, _node, visited_children): + _ws1, _update, _ws2, alias, data = visited_children + return Update(alias=alias, data=data) + def visit_query(self, _node, visited_children): """ Returns the overall output. """ - _when, first, rest, _ws = visited_children + _when, first, rest = visited_children query = [first] if isinstance(rest, list): query += [pair[1] for pair in rest] return query - def visit_condition(self, _node, visited_children): + def visit_entry(self, _node, visited_children): adjectives_opt, tags, data_opt, alias_opt = visited_children adjectives = set(adjectives_opt[0]) if isinstance(adjectives_opt, list) else None - data = dict(data_opt[0]) if isinstance(data_opt, list) else None + data = data_opt[0] if isinstance(data_opt, list) else None alias = alias_opt[0][1] if isinstance(alias_opt, list) else None - cond = Condition(adjectives=adjectives, - tags=set(tags), - data=data, - alias=alias) + cond = Entry(adjectives=adjectives, + tags=set(tags), + data=data, + alias=alias) return cond def visit_alias(self, _node, visited_children): _as, _ws, name = visited_children - return name.text + return name def visit_adjectives(self, _node, visited_children): adjs = [child[1] for child in visited_children] @@ -79,11 +116,11 @@ def visit_tags(self, _node, visited_children): def visit_tag(self, _node, visited_children): _pound, name = visited_children - return name.text + return name def visit_data(self, _node, visited_children): data = [child[1] for child in visited_children] - return data + return dict(data) def visit_datum(self, _node, visited_children): _as, name, val_opt = visited_children @@ -91,19 +128,13 @@ def visit_datum(self, _node, visited_children): if isinstance(val_opt, list): _ws, _colon, val = val_opt[0] val = val[0] - return name.text, val - - def visit_number(self, _node, visited_children): - sign_opt, integer, decimal_opt = visited_children - number = integer[0] - if isinstance(decimal_opt, list): - point, decimal = decimal_opt[0] - number += point.text - number += decimal[0] - if isinstance(sign_opt, list): - sign = sign_opt[0][0].text - number = sign + number - return float(number) + return name, val + + def visit_number(self, node, _visited_children): + return float(node.text) + + def visit_name(self, node, _visited_children): + return node.text def visit_digit(self, node, _visited_children): return node.text @@ -134,14 +165,14 @@ def generic_visit(self, node, visited_children): # TODO fix rules to block protected namespaces? # * namespace Pong -# when #aruco x where 0 < x < 100, y [create friend #paddle x: 100, y] +# when [#aruco x where 0 < x < 100, y] create [friend #paddle x: 100, y] # ** app #paddle x y -# when friend #aruco y as marker [update my y: marker y] -# when global nearby #ball +# when [friend #aruco y] as marker update [my y: marker y] +# when [global nearby #ball # x where 90 < x < 110, # velocity_x, -# y where (my y - 50) < y < (my y + 50) +# y where (my y - 50) < y < (my y + 50)] # as ball -# [update ball velocity_x: (ball velocity_x * -1)] +# update [ball velocity_x: (ball velocity_x * -1)] # implies that the #paddle was declared in the Pong namespace From fc8f6bcfc1a100a741cd4e721313eb689eb8895b Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Sat, 26 Oct 2019 17:16:36 -0400 Subject: [PATCH 02/20] Updates TinyTalk grammar --- tinytalk/grammar.py | 74 +++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 863e319..2d41ded 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -5,26 +5,41 @@ grammar = Grammar( r""" - program = query write - query = "when" entry (";" entry)* + start = expr + app = read write + read = "when" match (";" match)* write = (create / update) (";" (create / update))* - create = ws "create" entry - update = ws "update" ws name data - entry = adjectives? tags data? (ws alias)? + match = adjectives? relation? begin tags ws data_condition? end (ws alias)? + create = ws "create" (ws relation)? begin tags ws data? end + update = ws "update" ws name begin data end adjectives = (ws adjective)+ - tags = (ws tag)+ - data = (ws datum)+ - alias = "as" ws name + tags = ws? tag (ws tag)* + data = ws? datum (ws datum)* + data_condition = ws? (condition / datum) (ws (condition / datum))* + alias = "as" ws name_with_pronouns ws = ~"([ \t\n,])+" - adjective = "one" / "only" / "global" / "nearest" / "friend" + adjective = "one" / "only" / "global" + relation = ws? "friend" tag = "#" name - datum = !"as" name (":" ws value)? + datum = name (":" ws expr)? + condition = name ws "where" ws truthy + truthy = boolean / inequality + inequality = (subexpr / name / value) ws? comparison ws? expr + comparison = ">" / "<" / "is" / "not" + expr = multiplication / addition / subexpr / inequality / name / value + addition = (subexpr / multiplication / name / value) ws? ("+" / "-") ws? expr + multiplication = (subexpr / value / name) ws? "*" ws? expr + subexpr = "(" ws? expr ws? ")" value = number / string / boolean boolean = "true" / "false" string = ~'"[ ^\"]*"' number = ("+" / "-")? digit+ ("." digit+)? - digit = ~"[0-9]+" - name = ~"[a-z][a-z_-]*" + digit = ~"[0-9]" + name_with_pronouns = name ("/" name)* + name = !reserved_word ~"[a-z][a-z_-]*" + begin = ws? "[" + end = ws? "]" + reserved_word = "as" / "where" """) @@ -152,27 +167,28 @@ def generic_visit(self, node, visited_children): -# TODO data with ranges (eg "when #paddle x where 0 < x < 100, y") +# DONE data with ranges (eg "when #paddle x where 0 < x < 100, y") # TODO explore alternative (shorthand?) query string options, ## eg "mfw #success [epicwin.gif]" ## equivalent to "when #success x, y [#image x y src: epicwin.gif]" ## (macro system??) -# QUESTION who is "me" in a given query?? (wrt "friend" or "nearest") -# TODO namespace declaration syntax (eg "* namespace Pong") -# TODO commit syntax -# TODO maybe locus declaration syntax (eg "** app #paddle") +# DONE pronouns (via QUESTION who is "me" in a given query?? (wrt "friend" or "nearest")) +# TODO DESIGNED namespace declaration syntax (eg "* namespace Pong") +# DONE commit syntax +# CANCELLED maybe locus declaration syntax (eg "** app #paddle") # TODO paramaterizable adjectives? -# TODO fix rules to block protected namespaces? +# DONE fix rules to block protected namespaces & reserved words? +# DONE expression syntax # * namespace Pong -# when [#aruco x where 0 < x < 100, y] create [friend #paddle x: 100, y] -# ** app #paddle x y -# when [friend #aruco y] as marker update [my y: marker y] -# when [global nearby #ball -# x where 90 < x < 110, -# velocity_x, -# y where (my y - 50) < y < (my y + 50)] -# as ball -# update [ball velocity_x: (ball velocity_x * -1)] - -# implies that the #paddle was declared in the Pong namespace +# when [#aruco x where 0 < x < 100, y] create friend [#paddle x: 100, y] +# +# when [#paddle y] as me/my; friend [#aruco y] as tag/its update [my y: its y] +# +# when [#paddle x y] as me/my; +# global [#ball +# x where (my x - 10) < x < (my x + 10), +# velocity_x, +# y where (my y - 50) < y < (my y + 50)] +# as ball +# update ball [velocity_x: (ball velocity_x * -1)] From e2575018f0bfed764a1938ff3931b0cbc68a2e24 Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Sun, 27 Oct 2019 18:32:43 -0400 Subject: [PATCH 03/20] Adds property based testing for some grammars Also fixes some grammar bugs revealed during testing :3 Black is added as a dependency & used on grammar.py for formatting --- requirements.txt | 3 ++ tinytalk/grammar.py | 86 +++++++++++++++++++++------------------- tinytalk/test_grammar.py | 86 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 40 deletions(-) create mode 100644 tinytalk/test_grammar.py diff --git a/requirements.txt b/requirements.txt index d954ead..d572a93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,6 @@ opencv-contrib-python==4.1.1.26 toml==0.10.0 deep_merge==0.0.4 parsimonious==0.8.1 +black==19.3b0 +hypothesis==4.41.3 +pytest==5.2.2 diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 2d41ded..b59b414 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -24,59 +24,52 @@ datum = name (":" ws expr)? condition = name ws "where" ws truthy truthy = boolean / inequality - inequality = (subexpr / name / value) ws? comparison ws? expr - comparison = ">" / "<" / "is" / "not" - expr = multiplication / addition / subexpr / inequality / name / value - addition = (subexpr / multiplication / name / value) ws? ("+" / "-") ws? expr - multiplication = (subexpr / value / name) ws? "*" ws? expr + inequality = (subexpr / name / value) + comparison + (subexpr / (ws name) / value) + (comparison (subexpr / (ws name) /value))? + comparison = (ws? (">" / "<") ws?) / (ws ("is" / "not") ws) + expr = addition / multiplication / subexpr / inequality / value + addition = (multiplication / subexpr / value) ws? ("+" / "-") ws? expr + multiplication = (subexpr / value) ws? "*" ws? expr subexpr = "(" ws? expr ws? ")" - value = number / string / boolean + value = number / string / boolean / name boolean = "true" / "false" - string = ~'"[ ^\"]*"' - number = ("+" / "-")? digit+ ("." digit+)? + string = ~'"[^\"]*"' + number = ("+" / "-")? digit+ ("." digit+)? ("e" ("+" / "-") digit+)? digit = ~"[0-9]" name_with_pronouns = name ("/" name)* name = !reserved_word ~"[a-z][a-z_-]*" begin = ws? "[" end = ws? "]" - reserved_word = "as" / "where" - """) + reserved_word = ("as" / "where") &ws + """ +) def not_whitespace(o): return not (isinstance(o, RegexNode) and o.expr_name == "ws") +def is_wrapped(value): + return hasattr(value, "__iter__") and hasattr(value, "__len__") and len(value) is 1 -class Entry: - def __init__(self, *_, adjectives, tags, data, alias=None): - self.adjectives = adjectives - self.tags = tags - self.data = data - self.alias = alias - - def __str__(self): - return pprint.pformat(self) - - def __repr__(self): - return f"{self.__class__.__name__}(adjectives={self.adjectives}, " \ - f"tags={self.tags}, " \ - f"data={self.data}, " \ - f"alias={self.alias})" +def unwrap(value): + while is_wrapped(value): + value = value[0] + return value class Update: - - def __init__(self, *_, alias, data): + def __init__(self, *_, name, data): self.data = data - self.alias = alias + self.name = name def __repr__(self): - return f"{self.__class__.__name__}(alias={self.alias}, " \ - f"data={self.data})" + return f"{self.__class__.__name__}(name={self.name}, data={self.data})" class TinyTalkVisitor(NodeVisitor): - def visit_program(self, _node, visited_children): + def visit_app(self, _node, visited_children): query, write = visited_children return {"query": query, "write": write} @@ -95,8 +88,7 @@ def visit_update(self, _node, visited_children): _ws1, _update, _ws2, alias, data = visited_children return Update(alias=alias, data=data) - def visit_query(self, _node, visited_children): - """ Returns the overall output. """ + def visit_read(self, _node, visited_children): _when, first, rest = visited_children query = [first] if isinstance(rest, list): @@ -105,13 +97,12 @@ def visit_query(self, _node, visited_children): def visit_entry(self, _node, visited_children): adjectives_opt, tags, data_opt, alias_opt = visited_children - adjectives = set(adjectives_opt[0]) if isinstance(adjectives_opt, list) else None + adjectives = ( + set(adjectives_opt[0]) if isinstance(adjectives_opt, list) else None + ) data = data_opt[0] if isinstance(data_opt, list) else None alias = alias_opt[0][1] if isinstance(alias_opt, list) else None - cond = Entry(adjectives=adjectives, - tags=set(tags), - data=data, - alias=alias) + cond = Entry(adjectives=adjectives, tags=set(tags), data=data, alias=alias) return cond def visit_alias(self, _node, visited_children): @@ -145,6 +136,23 @@ def visit_datum(self, _node, visited_children): val = val[0] return name, val + def visit_expr(self, _node, visited_children): + return unwrap(visited_children) + + def visit_multiplication(self, _node, visited_children): + left, _ws, op, _ws, right = [unwrap(child) for child in visited_children] + return (op.text, left, right) + + def visit_addition(self, _node, visited_children): + left, _ws, op, _ws, right = [ + unwrap(child) for child in unwrap(visited_children) + ] + return (op.text, left, right) + + def visit_subexpr(self, _node, visited_children): + _paren, _ws, expr, _ws, _paren = unwrap(visited_children) + return unwrap(expr) + def visit_number(self, node, _visited_children): return float(node.text) @@ -165,8 +173,6 @@ def generic_visit(self, node, visited_children): return visited_children or node - - # DONE data with ranges (eg "when #paddle x where 0 < x < 100, y") # TODO explore alternative (shorthand?) query string options, ## eg "mfw #success [epicwin.gif]" diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py new file mode 100644 index 0000000..f01eba2 --- /dev/null +++ b/tinytalk/test_grammar.py @@ -0,0 +1,86 @@ +from . import grammar +from hypothesis import given, reject +from hypothesis.strategies import composite, floats, text, sampled_from +from math import isnan +from random import choice + + +@composite +def names(draw): + alpha_chars = "abcdefghijklmnopqrstuvwxyz" + name_chars = alpha_chars + "-_" + return draw(text(alphabet=alpha_chars, min_size=1)) + draw( + text(alphabet=name_chars) + ) + + +@composite +def names_or_floats(draw): + return choice([draw(floats(allow_infinity=False, allow_nan=False)), draw(names())]) + + +def whitespaces(min_size=0): + return text(alphabet=" ,\n", min_size=min_size) + + +@given(floats(allow_infinity=False, allow_nan=False)) +def test_number_grammar(n): + grammar["number"].parse(str(n)) + + +@given(text()) +def test_string(t): + if '"' in t: + reject() + grammar["string"].parse('"' + t + '"') + + +@given(names()) +def test_name(n): + if n in ["as", "where"]: + reject() + grammar["name"].parse(n) + + +@given(names(), names()) +def test_name_with_pronouns(n, pn): + if len(set([n, pn]).intersection(set(["as", "where"]))) > 0: + reject() + grammar["name_with_pronouns"].parse(n + "/" + pn) + + +@given(text(alphabet=" \n,", min_size=1)) +def test_whitespace(t): + grammar["ws"].parse(t) + + +@given( + floats(allow_infinity=False, allow_nan=False), + floats(allow_infinity=False, allow_nan=False), + whitespaces(), +) +def test_addition(n, m, ws): + grammar["addition"].parse(f"{n}{ws}+{ws}{m}") + + +@given( + floats(allow_infinity=False, allow_nan=False), + floats(allow_infinity=False, allow_nan=False), + whitespaces(), +) +def test_multiplication(n, m, ws): + grammar["multiplication"].parse(f"{n}{ws}*{ws}{m}") + + +@given( + names_or_floats(), + sampled_from(["<", ">", " is ", " not "]), + names_or_floats(), + sampled_from(["<", ">", " is ", " not "]), + names_or_floats(), + whitespaces(), +) +def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): + grammar["inequality"].parse( + f"{val_a}{ws}{comp_a}{ws}{val_b}{ws}{comp_b}{ws}{val_c}" + ) From b5866aa0630f593e7ab4eb21c13e391ad78db0e2 Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Sun, 27 Oct 2019 20:04:45 -0400 Subject: [PATCH 04/20] Adds tests for exprs, fixes some grammar rules --- tinytalk/grammar.py | 12 ++++++------ tinytalk/test_grammar.py | 41 ++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index b59b414..05cef06 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -24,12 +24,12 @@ datum = name (":" ws expr)? condition = name ws "where" ws truthy truthy = boolean / inequality - inequality = (subexpr / name / value) - comparison - (subexpr / (ws name) / value) - (comparison (subexpr / (ws name) /value))? + expr = inequality / addition / multiplication / subexpr / value + inequality = (addition / multiplication / subexpr / name / value) + comparison ws? + expr + (comparison ws? expr)? comparison = (ws? (">" / "<") ws?) / (ws ("is" / "not") ws) - expr = addition / multiplication / subexpr / inequality / value addition = (multiplication / subexpr / value) ws? ("+" / "-") ws? expr multiplication = (subexpr / value) ws? "*" ws? expr subexpr = "(" ws? expr ws? ")" @@ -42,7 +42,7 @@ name = !reserved_word ~"[a-z][a-z_-]*" begin = ws? "[" end = ws? "]" - reserved_word = ("as" / "where") &ws + reserved_word = ("as" / "where" / "true" / "false") &ws """ ) diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index f01eba2..3b349aa 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -2,13 +2,12 @@ from hypothesis import given, reject from hypothesis.strategies import composite, floats, text, sampled_from from math import isnan -from random import choice @composite def names(draw): - alpha_chars = "abcdefghijklmnopqrstuvwxyz" - name_chars = alpha_chars + "-_" + alpha_chars = "bcdeghijklmnopqrsuvxyz" # no [awtf] at start to avoid reserved words + name_chars = alpha_chars + "awtf-_" return draw(text(alphabet=alpha_chars, min_size=1)) + draw( text(alphabet=name_chars) ) @@ -16,13 +15,36 @@ def names(draw): @composite def names_or_floats(draw): - return choice([draw(floats(allow_infinity=False, allow_nan=False)), draw(names())]) + return draw( + sampled_from( + [draw(floats(allow_infinity=False, allow_nan=False)), draw(names())] + ) + ) + + +def operators(): + return sampled_from(["+", " -", "*", "<", ">", " is ", " not "]) + + +@composite +def exprs(draw): + length=draw(sampled_from([1, 2, 3])) + result = str(draw(names_or_floats())) + for _ in range(0, length): + result += ws_between( + draw(whitespaces()), draw(operators()), draw(names_or_floats()) + ) + return result def whitespaces(min_size=0): return text(alphabet=" ,\n", min_size=min_size) +def ws_between(ws, *enum): + return ws.join(map(str, enum)) + + @given(floats(allow_infinity=False, allow_nan=False)) def test_number_grammar(n): grammar["number"].parse(str(n)) @@ -69,7 +91,7 @@ def test_addition(n, m, ws): whitespaces(), ) def test_multiplication(n, m, ws): - grammar["multiplication"].parse(f"{n}{ws}*{ws}{m}") + grammar["multiplication"].parse(ws_between(ws, n, "*", m)) @given( @@ -81,6 +103,9 @@ def test_multiplication(n, m, ws): whitespaces(), ) def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): - grammar["inequality"].parse( - f"{val_a}{ws}{comp_a}{ws}{val_b}{ws}{comp_b}{ws}{val_c}" - ) + grammar["inequality"].parse(ws_between(ws, val_a, comp_a, val_b, comp_b, val_c)) + + +@given(exprs()) +def test_expr(e): + grammar["expr"].parse(e) From 491e802e6c661eba49ff24904e54f9b47d911a6f Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Sun, 27 Oct 2019 20:25:29 -0400 Subject: [PATCH 05/20] Adds comments for structure and clarity --- tinytalk/test_grammar.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index 3b349aa..665d3d1 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -1,7 +1,18 @@ from . import grammar from hypothesis import given, reject from hypothesis.strategies import composite, floats, text, sampled_from -from math import isnan + +## Tests for TinyTalk's grammar +# +# To run the tests: +# +# $ pip install -r requirements.txt +# $ pytest +# ... +# ====== n passed in 1.97s ====== + + +## Helper functions @composite @@ -28,7 +39,7 @@ def operators(): @composite def exprs(draw): - length=draw(sampled_from([1, 2, 3])) + length = draw(sampled_from([1, 2, 3])) result = str(draw(names_or_floats())) for _ in range(0, length): result += ws_between( @@ -45,6 +56,9 @@ def ws_between(ws, *enum): return ws.join(map(str, enum)) +## Tests + + @given(floats(allow_infinity=False, allow_nan=False)) def test_number_grammar(n): grammar["number"].parse(str(n)) @@ -109,3 +123,6 @@ def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): @given(exprs()) def test_expr(e): grammar["expr"].parse(e) + + +## end test_grammar.py From 178d61a33b553ea25e1856c17f40e82669b07e4c Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Sun, 27 Oct 2019 22:53:12 -0400 Subject: [PATCH 06/20] Adds tests for match, create, update --- tinytalk/grammar.py | 13 ++++----- tinytalk/test_grammar.py | 58 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 05cef06..e62bb78 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -5,14 +5,13 @@ grammar = Grammar( r""" - start = expr - app = read write - read = "when" match (";" match)* + app = read ws? write + read = "when" ws match (";" ws match)* write = (create / update) (";" (create / update))* match = adjectives? relation? begin tags ws data_condition? end (ws alias)? - create = ws "create" (ws relation)? begin tags ws data? end - update = ws "update" ws name begin data end - adjectives = (ws adjective)+ + create = "create" (ws relation)? begin tags ws data? end + update = "update" ws name begin data end + adjectives = adjective (ws adjective)* tags = ws? tag (ws tag)* data = ws? datum (ws datum)* data_condition = ws? (condition / datum) (ws (condition / datum))* @@ -50,9 +49,11 @@ def not_whitespace(o): return not (isinstance(o, RegexNode) and o.expr_name == "ws") + def is_wrapped(value): return hasattr(value, "__iter__") and hasattr(value, "__len__") and len(value) is 1 + def unwrap(value): while is_wrapped(value): value = value[0] diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index 665d3d1..dbebae4 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -33,6 +33,17 @@ def names_or_floats(draw): ) +@composite +def data(draw, n=1): + result = [] + for i in range(0, n): + name = draw(names()) + ws = draw(whitespaces(1)) + val = draw(exprs()) + result.append(f"{name}:{ws}{val}") + return ws_between(ws, *result) + + def operators(): return sampled_from(["+", " -", "*", "<", ">", " is ", " not "]) @@ -60,7 +71,7 @@ def ws_between(ws, *enum): @given(floats(allow_infinity=False, allow_nan=False)) -def test_number_grammar(n): +def test_number(n): grammar["number"].parse(str(n)) @@ -85,7 +96,7 @@ def test_name_with_pronouns(n, pn): grammar["name_with_pronouns"].parse(n + "/" + pn) -@given(text(alphabet=" \n,", min_size=1)) +@given(whitespaces(1)) def test_whitespace(t): grammar["ws"].parse(t) @@ -125,4 +136,47 @@ def test_expr(e): grammar["expr"].parse(e) +@given(names(), whitespaces(1), names_or_floats(), sampled_from(["<", ">"])) +def test_condition(name, ws, val, comp): + grammar["condition"].parse(ws_between(ws, name, "where", name, comp, val)) + + +@given(data()) +def test_datum(d): + grammar["datum"].parse(d) + + +@given(names()) +def test_tag(name): + grammar["tag"].parse(f"#{name}") + + +@given(names(), data(2), whitespaces()) +def test_update(name, data, ws): + grammar["update"].parse(ws_between(ws, "update ", name, " [", data, "]")) + + +@given(sampled_from(["", "friend"]), names(), data(2), whitespaces()) +def test_create(relation, tag, data, ws): + grammar["create"].parse( + ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data, "]") + ) + + +@given( + sampled_from(["", "one", "only", "global"]), + sampled_from(["", "friend"]), + names(), + data(2), + names(), + whitespaces(), +) +def test_match(adjective, relation, tag, data, alias, ws): + grammar["match"].parse( + ws_between( + ws, adjective, " ", relation, " [", f"#{tag} #{tag} ", data, "] as ", alias + ) + ) + + ## end test_grammar.py From e00eee83a47b38a194ea2562ce911544c8da781a Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Sun, 27 Oct 2019 23:12:22 -0400 Subject: [PATCH 07/20] Ignores .hypothesis cache directory --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0429fe7..93eb0b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ config.toml -__pycache__/ \ No newline at end of file +__pycache__/ +tinytalk/.hypothesis From b7bf53df447800f0ead78480d30d40f04b5961f8 Mon Sep 17 00:00:00 2001 From: Ryan Prior Date: Mon, 28 Oct 2019 00:57:05 -0400 Subject: [PATCH 08/20] Tests visitors for a few types, adds inequality visitor --- tinytalk/grammar.py | 27 +++++++++++++++++++--- tinytalk/test_grammar.py | 50 ++++++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index e62bb78..c550d62 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -3,6 +3,8 @@ from parsimonious.grammar import Grammar, NodeVisitor from parsimonious.nodes import RegexNode +whitespace_chars = " \n," + grammar = Grammar( r""" app = read ws? write @@ -26,8 +28,8 @@ expr = inequality / addition / multiplication / subexpr / value inequality = (addition / multiplication / subexpr / name / value) comparison ws? - expr - (comparison ws? expr)? + (addition / multiplication / subexpr / name / value) + (comparison ws? (addition / multiplication / subexpr / name / value))? comparison = (ws? (">" / "<") ws?) / (ws ("is" / "not") ws) addition = (multiplication / subexpr / value) ws? ("+" / "-") ws? expr multiplication = (subexpr / value) ws? "*" ws? expr @@ -51,7 +53,12 @@ def not_whitespace(o): def is_wrapped(value): - return hasattr(value, "__iter__") and hasattr(value, "__len__") and len(value) is 1 + return ( + hasattr(value, "__iter__") + and hasattr(value, "__len__") + and len(value) is 1 + and value != value[0] + ) def unwrap(value): @@ -140,6 +147,17 @@ def visit_datum(self, _node, visited_children): def visit_expr(self, _node, visited_children): return unwrap(visited_children) + def visit_inequality(self, _node, visited_children): + left, comp_a, _ws, middle, rest = [unwrap(child) for child in visited_children] + if not hasattr(rest, 'children'): + comp_b, _ws, right = [unwrap(child) for child in rest] + return ("and", (comp_a, middle, left), (comp_b, middle, right)) + else: + return (comp_a, left, middle) + + def visit_comparison(self, node, _visited_children): + return node.text.strip(whitespace_chars) + def visit_multiplication(self, _node, visited_children): left, _ws, op, _ws, right = [unwrap(child) for child in visited_children] return (op.text, left, right) @@ -157,6 +175,9 @@ def visit_subexpr(self, _node, visited_children): def visit_number(self, node, _visited_children): return float(node.text) + def visit_name_with_pronouns(self, node, _visited_children): + return node.text.split("/") + def visit_name(self, node, _visited_children): return node.text diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index dbebae4..ebae4b5 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -1,4 +1,4 @@ -from . import grammar +from . import grammar, TinyTalkVisitor from hypothesis import given, reject from hypothesis.strategies import composite, floats, text, sampled_from @@ -14,6 +14,12 @@ ## Helper functions +visitor = TinyTalkVisitor() + + +def visit(tree): + return visitor.visit(tree) + @composite def names(draw): @@ -34,9 +40,10 @@ def names_or_floats(draw): @composite -def data(draw, n=1): +def data(draw, max_n=1): result = [] - for i in range(0, n): + n = draw(sampled_from(list(range(1, max_n + 1)))) + for _ in range(0, n): name = draw(names()) ws = draw(whitespaces(1)) val = draw(exprs()) @@ -49,10 +56,10 @@ def operators(): @composite -def exprs(draw): - length = draw(sampled_from([1, 2, 3])) +def exprs(draw, max_n=1): result = str(draw(names_or_floats())) - for _ in range(0, length): + n = draw(sampled_from(list(range(1, max_n + 1)))) + for _ in range(0, n): result += ws_between( draw(whitespaces()), draw(operators()), draw(names_or_floats()) ) @@ -72,28 +79,34 @@ def ws_between(ws, *enum): @given(floats(allow_infinity=False, allow_nan=False)) def test_number(n): - grammar["number"].parse(str(n)) - + result = grammar["number"].parse(str(n)) + assert visit(result) == n @given(text()) def test_string(t): if '"' in t: reject() - grammar["string"].parse('"' + t + '"') + result = grammar["string"].parse('"' + t + '"') + assert visit(result) == f'"{t}"' @given(names()) def test_name(n): if n in ["as", "where"]: reject() - grammar["name"].parse(n) + result = grammar["name"].parse(n) + assert visit(result) == n @given(names(), names()) def test_name_with_pronouns(n, pn): if len(set([n, pn]).intersection(set(["as", "where"]))) > 0: reject() - grammar["name_with_pronouns"].parse(n + "/" + pn) + result = grammar["name_with_pronouns"].parse(n) + assert visit(result) == [n] + + result = grammar["name_with_pronouns"].parse(n + "/" + pn + "/" + pn) + assert visit(result) == [n, pn, pn] @given(whitespaces(1)) @@ -107,7 +120,8 @@ def test_whitespace(t): whitespaces(), ) def test_addition(n, m, ws): - grammar["addition"].parse(f"{n}{ws}+{ws}{m}") + result = grammar["addition"].parse(f"{n}{ws}+{ws}{m}") + assert visit(result) == ("+", n, m) @given( @@ -116,7 +130,8 @@ def test_addition(n, m, ws): whitespaces(), ) def test_multiplication(n, m, ws): - grammar["multiplication"].parse(ws_between(ws, n, "*", m)) + result = grammar["multiplication"].parse(ws_between(ws, n, "*", m)) + assert visit(result) == ("*", n, m) @given( @@ -128,7 +143,8 @@ def test_multiplication(n, m, ws): whitespaces(), ) def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): - grammar["inequality"].parse(ws_between(ws, val_a, comp_a, val_b, comp_b, val_c)) + result = grammar["inequality"].parse(ws_between(ws, val_a, comp_a, val_b, comp_b, val_c)) + assert visit(result) == ('and', (comp_a.strip(), val_b, val_a), (comp_b.strip(), val_b, val_c)) @given(exprs()) @@ -151,12 +167,12 @@ def test_tag(name): grammar["tag"].parse(f"#{name}") -@given(names(), data(2), whitespaces()) +@given(names(), data(3), whitespaces()) def test_update(name, data, ws): grammar["update"].parse(ws_between(ws, "update ", name, " [", data, "]")) -@given(sampled_from(["", "friend"]), names(), data(2), whitespaces()) +@given(sampled_from(["", "friend"]), names(), data(3), whitespaces()) def test_create(relation, tag, data, ws): grammar["create"].parse( ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data, "]") @@ -167,7 +183,7 @@ def test_create(relation, tag, data, ws): sampled_from(["", "one", "only", "global"]), sampled_from(["", "friend"]), names(), - data(2), + data(3), names(), whitespaces(), ) From d43a7afe979950fef3878a1cf077851ec62f6299 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Mon, 28 Oct 2019 23:24:31 -0400 Subject: [PATCH 09/20] add pytests for rules --- tinytalk/__init__.py | 2 +- tinytalk/grammar.py | 11 +++--- tinytalk/test_grammar.py | 72 ++++++++++++++++++++++++++++------------ 3 files changed, 59 insertions(+), 26 deletions(-) diff --git a/tinytalk/__init__.py b/tinytalk/__init__.py index 59bf542..a6fade1 100644 --- a/tinytalk/__init__.py +++ b/tinytalk/__init__.py @@ -1,5 +1,5 @@ from . import data -from .grammar import grammar, TinyTalkVisitor +from .grammar import grammar, TinyTalkVisitor, whitespace_chars def dump(): print("database dump:") diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index c550d62..cdd5ca7 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -103,6 +103,10 @@ def visit_read(self, _node, visited_children): query += [pair[1] for pair in rest] return query + def visit_condition(self, _node, visited_children): + name, _ws, _where, _ws, truthy = [unwrap(child) for child in visited_children] + return name, ("cond", truthy) + def visit_entry(self, _node, visited_children): adjectives_opt, tags, data_opt, alias_opt = visited_children adjectives = ( @@ -137,11 +141,10 @@ def visit_data(self, _node, visited_children): return dict(data) def visit_datum(self, _node, visited_children): - _as, name, val_opt = visited_children + name, val_opt = [unwrap(child) for child in visited_children] val = "ANY" - if isinstance(val_opt, list): - _ws, _colon, val = val_opt[0] - val = val[0] + if not hasattr(val_opt, "children"): + _ws, _colon, val = [unwrap(child) for child in val_opt] return name, val def visit_expr(self, _node, visited_children): diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index ebae4b5..5759171 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -1,4 +1,4 @@ -from . import grammar, TinyTalkVisitor +from . import grammar, TinyTalkVisitor, whitespace_chars from hypothesis import given, reject from hypothesis.strategies import composite, floats, text, sampled_from @@ -41,28 +41,47 @@ def names_or_floats(draw): @composite def data(draw, max_n=1): - result = [] + result = {"data": [], + "text": ""} n = draw(sampled_from(list(range(1, max_n + 1)))) for _ in range(0, n): name = draw(names()) ws = draw(whitespaces(1)) val = draw(exprs()) - result.append(f"{name}:{ws}{val}") - return ws_between(ws, *result) + result["data"].append((name, val["data"])) + result["text"] += f"{ws}{name}: {val['text']}" + result["text"] = result["text"].strip(whitespace_chars) + return result -def operators(): - return sampled_from(["+", " -", "*", "<", ">", " is ", " not "]) +def operators(allow_inequalities=True): + ops = ["+", " -", "*"] + if allow_inequalities: + ops += ["<", ">", " is ", " not "] + return sampled_from(ops) @composite -def exprs(draw, max_n=1): - result = str(draw(names_or_floats())) +def exprs(draw, max_n=1, allow_inequalities=True): + result = {} + init_val = draw(names_or_floats()) + vals = [init_val] + ops = [] + result["text"] = f"{init_val}" n = draw(sampled_from(list(range(1, max_n + 1)))) for _ in range(0, n): - result += ws_between( - draw(whitespaces()), draw(operators()), draw(names_or_floats()) - ) + op = draw(operators(allow_inequalities)) + val = draw(names_or_floats()) + ops.append(op) + vals.append(val) + result["text"] += draw(whitespaces()).join(["", op, str(val)]) + last2_vals = vals[-2:] + vals = vals[:-2] + op = ops.pop(-1).strip() + data = op, *last2_vals + while vals: + data = ops.pop(-1).strip(), vals.pop(-1), data + result["data"] = data return result @@ -82,6 +101,7 @@ def test_number(n): result = grammar["number"].parse(str(n)) assert visit(result) == n + @given(text()) def test_string(t): if '"' in t: @@ -143,39 +163,49 @@ def test_multiplication(n, m, ws): whitespaces(), ) def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): - result = grammar["inequality"].parse(ws_between(ws, val_a, comp_a, val_b, comp_b, val_c)) - assert visit(result) == ('and', (comp_a.strip(), val_b, val_a), (comp_b.strip(), val_b, val_c)) + result = grammar["inequality"].parse( + ws_between(ws, val_a, comp_a, val_b, comp_b, val_c) + ) + assert visit(result) == ( + "and", + (comp_a.strip(), val_b, val_a), + (comp_b.strip(), val_b, val_c), + ) -@given(exprs()) +@given(exprs(13, allow_inequalities=False)) def test_expr(e): - grammar["expr"].parse(e) + result = grammar["expr"].parse(e["text"]) + assert visit(result) == e["data"] @given(names(), whitespaces(1), names_or_floats(), sampled_from(["<", ">"])) def test_condition(name, ws, val, comp): - grammar["condition"].parse(ws_between(ws, name, "where", name, comp, val)) + result = grammar["condition"].parse(ws_between(ws, name, "where", name, comp, val)) + assert visit(result) == (name, ("cond", (comp, name, val))) @given(data()) def test_datum(d): - grammar["datum"].parse(d) + result = grammar["datum"].parse(d["text"]) + assert visit(result) == d["data"][0] @given(names()) def test_tag(name): - grammar["tag"].parse(f"#{name}") + result = grammar["tag"].parse(f"#{name}") + assert visit(result) == name @given(names(), data(3), whitespaces()) def test_update(name, data, ws): - grammar["update"].parse(ws_between(ws, "update ", name, " [", data, "]")) + grammar["update"].parse(ws_between(ws, "update ", name, " [", data["text"], "]")) @given(sampled_from(["", "friend"]), names(), data(3), whitespaces()) def test_create(relation, tag, data, ws): grammar["create"].parse( - ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data, "]") + ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data["text"], "]") ) @@ -190,7 +220,7 @@ def test_create(relation, tag, data, ws): def test_match(adjective, relation, tag, data, alias, ws): grammar["match"].parse( ws_between( - ws, adjective, " ", relation, " [", f"#{tag} #{tag} ", data, "] as ", alias + ws, adjective, " ", relation, " [", f"#{tag} #{tag} ", data["text"], "] as ", alias ) ) From 9d3eb12d4de093a88097de939f341f164994028f Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Wed, 30 Oct 2019 19:49:27 -0400 Subject: [PATCH 10/20] Add tests for parsing create, update, app rules and more --- tinytalk/__init__.py | 2 +- tinytalk/grammar.py | 95 ++++++++++++++++++++++----------------- tinytalk/test_grammar.py | 97 ++++++++++++++++++++++++++++++++++------ 3 files changed, 138 insertions(+), 56 deletions(-) diff --git a/tinytalk/__init__.py b/tinytalk/__init__.py index a6fade1..b911581 100644 --- a/tinytalk/__init__.py +++ b/tinytalk/__init__.py @@ -1,5 +1,5 @@ from . import data -from .grammar import grammar, TinyTalkVisitor, whitespace_chars +from .grammar import Command, grammar, TinyTalkVisitor, whitespace_chars def dump(): print("database dump:") diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index cdd5ca7..0f83635 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -1,19 +1,27 @@ +from enum import Enum import pprint from parsimonious.grammar import Grammar, NodeVisitor from parsimonious.nodes import RegexNode +class Command(Enum): + MATCH = 0 + CREATE = 1 + UPDATE = 2 + COND = 3 + AND = 4 + whitespace_chars = " \n," grammar = Grammar( r""" app = read ws? write - read = "when" ws match (";" ws match)* - write = (create / update) (";" (create / update))* - match = adjectives? relation? begin tags ws data_condition? end (ws alias)? - create = "create" (ws relation)? begin tags ws data? end + read = "when" ws match (";" ws? match)* + write = (create / update) (";" ws? (create / update))* + match = adjectives? relation? begin ws? tags ws data_condition? end (ws alias)? + create = "create" (ws relation)? begin ws? tags ws data? end update = "update" ws name begin data end - adjectives = adjective (ws adjective)* + adjectives = ws? adjective (ws adjective)* tags = ws? tag (ws tag)* data = ws? datum (ws datum)* data_condition = ws? (condition / datum) (ws (condition / datum))* @@ -54,8 +62,7 @@ def not_whitespace(o): def is_wrapped(value): return ( - hasattr(value, "__iter__") - and hasattr(value, "__len__") + isinstance(value, list) and len(value) is 1 and value != value[0] ) @@ -67,77 +74,81 @@ def unwrap(value): return value -class Update: - def __init__(self, *_, name, data): - self.data = data - self.name = name - - def __repr__(self): - return f"{self.__class__.__name__}(name={self.name}, data={self.data})" - - class TinyTalkVisitor(NodeVisitor): def visit_app(self, _node, visited_children): - query, write = visited_children - return {"query": query, "write": write} + read, _ws, write = visited_children + return read, write #TODO command string? + + def visit_match(self, node, visited_children): + [adjectives_opt, relation_opt, _begin, _ws, tags, _ws, data_condition_opt, _end, + ws_alias_opt] = visited_children + adjectives = unwrap(adjectives_opt) if isinstance(adjectives_opt, list) else None + relation = unwrap(relation_opt) if isinstance(relation_opt, list) else None + data_condition = unwrap(data_condition_opt) if isinstance(data_condition_opt, list) else None + alias = unwrap(ws_alias_opt)[1] if isinstance(ws_alias_opt, list) else None + return Command.MATCH.name, adjectives, relation, tags, data_condition, unwrap(alias) + + def visit_data_condition(self, node, visited_children): + _ws, first, rest = visited_children + data_conds = [unwrap(first)] + [unwrap(data_cond) for [_ws, data_cond] in rest] + return dict(data_conds) def visit_write(self, _node, visited_children): first, rest = visited_children - write = [first[0]] + write = [unwrap(first)] if isinstance(rest, list): - write += [pair[1][0] for pair in rest] + write += [unwrap(pair[2]) for pair in rest] return write def visit_create(self, _node, visited_children): - ws1, _create, entry = visited_children - return entry + _create, relation_opt, _begin, _ws, tags, _ws, data_opt, _end = visited_children + relation = unwrap(relation_opt)[1] if isinstance(relation_opt, list) else None + data = unwrap(data_opt) if isinstance(data_opt, list) else None + return Command.CREATE.name, tags, relation, data def visit_update(self, _node, visited_children): - _ws1, _update, _ws2, alias, data = visited_children - return Update(alias=alias, data=data) + _update, _ws, name, _begin, data, _end = visited_children + return Command.UPDATE.name, name, data def visit_read(self, _node, visited_children): - _when, first, rest = visited_children + _when, _ws, first, rest = visited_children query = [first] if isinstance(rest, list): - query += [pair[1] for pair in rest] + query += [pair[2] for pair in rest] return query def visit_condition(self, _node, visited_children): name, _ws, _where, _ws, truthy = [unwrap(child) for child in visited_children] - return name, ("cond", truthy) - - def visit_entry(self, _node, visited_children): - adjectives_opt, tags, data_opt, alias_opt = visited_children - adjectives = ( - set(adjectives_opt[0]) if isinstance(adjectives_opt, list) else None - ) - data = data_opt[0] if isinstance(data_opt, list) else None - alias = alias_opt[0][1] if isinstance(alias_opt, list) else None - cond = Entry(adjectives=adjectives, tags=set(tags), data=data, alias=alias) - return cond + return name, (Command.COND.name, truthy) def visit_alias(self, _node, visited_children): _as, _ws, name = visited_children return name def visit_adjectives(self, _node, visited_children): - adjs = [child[1] for child in visited_children] - return adjs + _ws, first, rest = visited_children + adjectives = [unwrap(first)] + [unwrap(adjective) for [_ws, adjective] in rest] + return adjectives def visit_adjective(self, _node, visited_children): return visited_children[0].text def visit_tags(self, _node, visited_children): - tags = [child[1] for child in visited_children] + _ws, first, rest = visited_children + tags = [unwrap(first)] + [unwrap(tag) for [_ws, tag] in rest] return tags def visit_tag(self, _node, visited_children): _pound, name = visited_children return name + + def visit_relation(self, _node, visited_children): + _ws, relation = visited_children + return relation.text def visit_data(self, _node, visited_children): - data = [child[1] for child in visited_children] + _ws, first, rest = visited_children + data = [unwrap(first)] + [unwrap(datum) for [_ws, datum] in rest] return dict(data) def visit_datum(self, _node, visited_children): @@ -154,7 +165,7 @@ def visit_inequality(self, _node, visited_children): left, comp_a, _ws, middle, rest = [unwrap(child) for child in visited_children] if not hasattr(rest, 'children'): comp_b, _ws, right = [unwrap(child) for child in rest] - return ("and", (comp_a, middle, left), (comp_b, middle, right)) + return (Command.AND.name, (comp_a, middle, left), (comp_b, middle, right)) else: return (comp_a, left, middle) diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index 5759171..3aa7c0e 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -1,7 +1,9 @@ -from . import grammar, TinyTalkVisitor, whitespace_chars +from . import grammar, TinyTalkVisitor, whitespace_chars, Command from hypothesis import given, reject from hypothesis.strategies import composite, floats, text, sampled_from +import random + ## Tests for TinyTalk's grammar # # To run the tests: @@ -39,6 +41,20 @@ def names_or_floats(draw): ) +@composite +def conditions(draw, max_n=1): + result = {"data": [], + "text": ""} + n = draw(sampled_from(list(range(1, max_n + 1)))) + for _ in range(n): + name = draw(names()) + ws = draw(whitespaces(0)) + val = draw(names_or_floats()) + op = draw(sampled_from([">", "<", " is "])) + result["data"].append((name, (Command.COND.name, (op.strip(), name, val)))) + result["text"] = ws_between(draw(whitespaces(1)), result["text"], f"{name}{ws} where {ws}{name}{ws}{op}{ws}{val}") + return result + @composite def data(draw, max_n=1): result = {"data": [], @@ -167,7 +183,7 @@ def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): ws_between(ws, val_a, comp_a, val_b, comp_b, val_c) ) assert visit(result) == ( - "and", + Command.AND.name, (comp_a.strip(), val_b, val_a), (comp_b.strip(), val_b, val_c), ) @@ -182,7 +198,7 @@ def test_expr(e): @given(names(), whitespaces(1), names_or_floats(), sampled_from(["<", ">"])) def test_condition(name, ws, val, comp): result = grammar["condition"].parse(ws_between(ws, name, "where", name, comp, val)) - assert visit(result) == (name, ("cond", (comp, name, val))) + assert visit(result) == (name, (Command.COND.name, (comp, name, val))) @given(data()) @@ -191,38 +207,93 @@ def test_datum(d): assert visit(result) == d["data"][0] +@given(data(10)) +def test_data(d): + result = grammar["data"].parse(d["text"]) + assert visit(result) == dict(d["data"]) + + @given(names()) def test_tag(name): result = grammar["tag"].parse(f"#{name}") assert visit(result) == name +@given(whitespaces(), whitespaces(1), names(), names()) +def test_tags(lead_ws, ws, tag1, tag2): + tags = [f"#{tag}" for tag in [tag1, tag2]] + result = grammar["tags"].parse(ws_between(ws, lead_ws, *tags)) + assert visit(result) == [tag1, tag2] + + @given(names(), data(3), whitespaces()) def test_update(name, data, ws): - grammar["update"].parse(ws_between(ws, "update ", name, " [", data["text"], "]")) + result = grammar["update"].parse(ws_between(ws, "update ", name, " [", data["text"], "]")) + assert visit(result) == (Command.UPDATE.name, name, dict(data["data"])) @given(sampled_from(["", "friend"]), names(), data(3), whitespaces()) def test_create(relation, tag, data, ws): - grammar["create"].parse( - ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data["text"], "]") - ) + parse_string = ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data["text"], "]") + result = grammar["create"].parse(parse_string) + assert visit(result) == (Command.CREATE.name, [tag, tag], relation or None, dict(data["data"]) if data else None) @given( sampled_from(["", "one", "only", "global"]), sampled_from(["", "friend"]), names(), - data(3), + conditions(3), names(), whitespaces(), ) -def test_match(adjective, relation, tag, data, alias, ws): - grammar["match"].parse( +def test_match(adjective, relation, tag, conditions, alias, ws): + result = grammar["match"].parse( ws_between( - ws, adjective, " ", relation, " [", f"#{tag} #{tag} ", data["text"], "] as ", alias - ) - ) + ws, adjective, relation, " [", f"#{tag} #{tag} ", conditions["text"], "] as ", alias)) + assert visit(result) == (Command.MATCH.name, + adjective or None, + relation or None, + [tag, tag], + dict(conditions["data"]), + alias) + + +@given( + sampled_from(list(range(1, 3))), + sampled_from(list(range(0, 3))), + sampled_from(list(range(0, 3))), + whitespaces(1) +) +def test_app(num_matches, num_creates, num_updates, ws): + if num_creates + num_updates == 0: + reject() + match_texts = ["global friend [ #a x where 0 < x < 50 ] as f", "[ #x ]"] + match_data = [ + (Command.MATCH.name, + "global", + "friend", + ["a"], + {"x": (Command.COND.name, (Command.AND.name, ("<", "x", 0.0), ("<", "x", 50.0)))}, + "f"), + (Command.MATCH.name, None, None, ["x"], None, None)] + create_texts = ["create friend [ #a #b x: 50 ]"] + create_data = [(Command.CREATE.name, ["a", "b"], "friend", {"x": 50.0})] + update_texts = ["update paddle [ x: y ]"] + update_data = [(Command.UPDATE.name, "paddle", {"x": "y"})] + + test_match_data = [match_data[i % len(match_data)] for i in range(num_matches)] + test_create_update_texts = [create_texts[i % len(create_texts)] for i in range(num_creates)] + \ + [update_texts[i % len(update_texts)] for i in range(num_updates)] + test_create_update_data = [create_data[i % len(create_data)] for i in range(num_creates)] + \ + [update_data[i % len(update_data)] for i in range(num_updates)] + + test_match_text = "; ".join([match_texts[i % len(match_texts)] for i in range(num_matches)]) + test_write_text = "; ".join(test_create_update_texts) + + app_text = ws_between(" ", "when", test_match_text, test_write_text) + result = grammar["app"].parse(app_text) + assert visit(result) == (test_match_data, test_create_update_data) ## end test_grammar.py From f6261193559a78560013ddd127843fe16219aabd Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Wed, 13 Nov 2019 16:50:16 -0500 Subject: [PATCH 11/20] Condition and match interpreter implementation. The following assumptions are made in current implementation: - Database is a dict mapping tag strings to sets of row tuples: the same row may appear for multiple tags, but identical rows cannot appear in a single tag set. - Rows are named tuples with column values that can be accessed using the gettatr function. --- tinytalk/data.py | 79 ++++++++++++++++++++++++++++++++++++++++++- tinytalk/grammar.py | 2 +- tinytalk/test_data.py | 43 +++++++++++++++++++++++ 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 tinytalk/test_data.py diff --git a/tinytalk/data.py b/tinytalk/data.py index 03a2db5..f24eca9 100644 --- a/tinytalk/data.py +++ b/tinytalk/data.py @@ -1,9 +1,86 @@ -from collections import defaultdict +from collections import defaultdict, namedtuple from deep_merge import merge +from .grammar import Command + data = defaultdict(list) +def lookup(name: str, namespace: namedtuple): + if hasattr(namespace, str(name)): + return getattr(namespace, name) + return None + + +def condition(var_name, cond, row: namedtuple) -> bool: + if not hasattr(row, var_name): + return False + elif cond == "ANY": + return True + elif cond[0] == Command.AND.name: + return condition(var_name, cond[1], row) and condition(var_name, cond[2], row) + else: + [operator, left, right] = cond + + # Check if left and right are variable names and access their values if so + left = lookup(left, row) or left + right = lookup(right, row) or right + + if operator in ["is"]: + return left == right + elif operator in ["<"]: + return left < right + elif operator in [">"]: + return left > right + elif operator in ["not"]: + return left != right + + +def match(mat: list) -> None: + """Retrieve each matching entry in the database. + + Assumes database var, `data`, is of the form: + { + (str): [ + (namedtuple), + (namedtuple), + ... + ], + (str): [...] + } + + Args: + mat (list): [ + _command: "MATCH" + adjectives: list of adjectives + relation: relation to previous matched items + tags: list of tags + data_conditions: dict mapping var names to condition (list or "ANY") + alias: str alias for entries that match above criteria + ] + Returns: + alias: str alias for the matches + matches (list): database rows that match criteria + """ + [_command, adjectives, relation, tags, data_conditions, alias] = mat + + # Filter by tags, there must be at least one. + matches = set(data[tags[0]]) + for tag in tags[1:]: + matches &= set(data[tag]) + + #TODO: Filter by adjectives + #TODO: Filter by relation (requires previous match contexts?) + + # Filter by data_conditions + for var_name, cond in data_conditions.items(): + if cond[0] == Command.COND.name: + cond = cond[1] + matches = filter(lambda row: condition(var_name, cond, row), matches) + + return alias, matches + + def commit(name, *args): global data names = [name] + list(args)[:-1] diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 0f83635..85ace26 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -165,7 +165,7 @@ def visit_inequality(self, _node, visited_children): left, comp_a, _ws, middle, rest = [unwrap(child) for child in visited_children] if not hasattr(rest, 'children'): comp_b, _ws, right = [unwrap(child) for child in rest] - return (Command.AND.name, (comp_a, middle, left), (comp_b, middle, right)) + return (Command.AND.name, (comp_a, left, middle), (comp_b, middle, right)) else: return (comp_a, left, middle) diff --git a/tinytalk/test_data.py b/tinytalk/test_data.py new file mode 100644 index 0000000..aea310e --- /dev/null +++ b/tinytalk/test_data.py @@ -0,0 +1,43 @@ +from . import data, Command +from collections import namedtuple + + +def test_variable_exists(): + rowtup = namedtuple("row", ["x"]) + row = rowtup(x=100) + var = "x" + cond = "ANY" + assert data.condition(var, cond, row) + + +def test_variable_does_not_exist(): + rowtup = namedtuple("row", ["x"]) + row = rowtup(x=100) + var = "y" + cond = "ANY" + assert not data.condition(var, cond, row) + + +def test_single_condition_true(): + rowtup = namedtuple("row", ["x"]) + row = rowtup(x=50) + var = "x" + cond = ["is", "x", 50.0] + assert data.condition(var, cond, row) + + +def test_single_condition_false(): + rowtup = namedtuple("row", ["x"]) + row = rowtup(x=50) + var = "x" + cond = ["not", "x", 50.0] + assert not data.condition(var, cond, row) + + +def test_multiple_condition(): + rowtup = namedtuple("row", ["x"]) + row = rowtup(x=50) + var = "x" + cond = [Command.AND.name, ["<", 0.0, "x"], ["<", "x", 100.0]] + assert data.condition(var, cond, row) + From 7463986d84403b1a28f28f697f04d78987251f56 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Fri, 15 Nov 2019 21:46:00 -0500 Subject: [PATCH 12/20] App scene manipulation based on dict scene --- tinytalk/data.py | 93 ----------------------- tinytalk/interpreter.py | 159 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 93 deletions(-) delete mode 100644 tinytalk/data.py create mode 100644 tinytalk/interpreter.py diff --git a/tinytalk/data.py b/tinytalk/data.py deleted file mode 100644 index f24eca9..0000000 --- a/tinytalk/data.py +++ /dev/null @@ -1,93 +0,0 @@ -from collections import defaultdict, namedtuple -from deep_merge import merge - -from .grammar import Command - -data = defaultdict(list) - - -def lookup(name: str, namespace: namedtuple): - if hasattr(namespace, str(name)): - return getattr(namespace, name) - return None - - -def condition(var_name, cond, row: namedtuple) -> bool: - if not hasattr(row, var_name): - return False - elif cond == "ANY": - return True - elif cond[0] == Command.AND.name: - return condition(var_name, cond[1], row) and condition(var_name, cond[2], row) - else: - [operator, left, right] = cond - - # Check if left and right are variable names and access their values if so - left = lookup(left, row) or left - right = lookup(right, row) or right - - if operator in ["is"]: - return left == right - elif operator in ["<"]: - return left < right - elif operator in [">"]: - return left > right - elif operator in ["not"]: - return left != right - - -def match(mat: list) -> None: - """Retrieve each matching entry in the database. - - Assumes database var, `data`, is of the form: - { - (str): [ - (namedtuple), - (namedtuple), - ... - ], - (str): [...] - } - - Args: - mat (list): [ - _command: "MATCH" - adjectives: list of adjectives - relation: relation to previous matched items - tags: list of tags - data_conditions: dict mapping var names to condition (list or "ANY") - alias: str alias for entries that match above criteria - ] - Returns: - alias: str alias for the matches - matches (list): database rows that match criteria - """ - [_command, adjectives, relation, tags, data_conditions, alias] = mat - - # Filter by tags, there must be at least one. - matches = set(data[tags[0]]) - for tag in tags[1:]: - matches &= set(data[tag]) - - #TODO: Filter by adjectives - #TODO: Filter by relation (requires previous match contexts?) - - # Filter by data_conditions - for var_name, cond in data_conditions.items(): - if cond[0] == Command.COND.name: - cond = cond[1] - matches = filter(lambda row: condition(var_name, cond, row), matches) - - return alias, matches - - -def commit(name, *args): - global data - names = [name] + list(args)[:-1] - commit_data = list(args)[-1] - for name in names: - data = merge(data, {name: merge({"names": names}, commit_data)}) - - -def query(search): - return data[search] diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py new file mode 100644 index 0000000..b4a2e75 --- /dev/null +++ b/tinytalk/interpreter.py @@ -0,0 +1,159 @@ +from collections import defaultdict, namedtuple +from deep_merge import merge +import uuid + +from .grammar import Command + + +adjectives = { + "only": lambda l: return l if len(l) == 1 else [] +} + + +def tup(**kwargs): + tup = namedtuple("tinyTuple", list(kwargs)) + return tup(**kwargs) + + +def lookup(name: str, namespace: namedtuple): + if hasattr(namespace, str(name)): + return getattr(namespace, name) + return None + + +def condition(var_name, context, cond, row: namedtuple) -> bool: + if not hasattr(row, var_name): + return False + elif cond == "ANY": + return True + elif cond[0] == Command.AND.name: + return (condition(var_name, context, cond[1], row) and + condition(var_name, context, cond[2], row)) + else: + [operator, left, right] = cond + + if isinstance(left, str) and "." in left: + alias, attribute = left.split(".") + left = lookup(context[alias], attribute) + else: + left = lookup(left, row) or left + + if isinstance(right, str) and "." in right: + alias, attribute = right.split(".") + right = lookup(context[alias], attribute) + else: + right = lookup(right, row) or right + + if operator in ["is"]: + return left == right + elif operator in ["<"]: + return left < right + elif operator in [">"]: + return left > right + elif operator in ["not"]: + return left != right + + +def match(m: list, context: dict, scene: dict) -> tuple: + [_command, adjectives, relation, tags, data_conditions, alias] = m + + + # Filter by tags, there must be at least one. + matches = [] + for key, thing in {**scene["allMarkers"], **scene["virtualObjects"]}.items(): + thing_tup = tup(id=key, **thing) + if set(thing_tup.type.split(" ")) == set(tags): + matches.append(thing_tup) + + for adjective in adjectives: + matches = adjectives[adjective](matches) + + #TODO: Filter by relation (requires previous match contexts?) + + # Filter by data_conditions + for var_name, cond in data_conditions.items(): + if cond[0] == Command.COND.name: + cond = cond[1] + matches = filter(lambda row: condition(var_name, context, cond, row), matches) + + return [(alias, m) for m in matches] + + +def create(c: list, context: dict, scene: dict) -> dict: + [_command, tags, relation, data] = c + + parsed_data = {} + for var, value in data.items(): + if isinstance(value, str) and "." in value: + alias, attribute = value.split(".") + parsed_data[var] = lookup(context[alias], attribute) + else: + parsed_data[var] = value + + new_thing = { + "type": " ".join(tags), + relation: [tup.id for tup in context], + **parsed_data + } + + new_scene = { + "allMarkers": **scene["allMarkers"], + "virtualObjects": { + uuid.UUID(): new_thing, + **scene["virtualObjects"] + } + } + + #TODO: implement relation stuff + + return new_scene + + +def update(u: list, context: dict, scene: dict) -> dict: + [_command, alias, data] = u + + parsed_data = {} + for var, value in data.items(): + if isinstance(value, str) and "." in value: + alias, attribute = value.split(".") + parsed_data[var] = lookup(context[alias], attribute) + else: + parsed_data[var] = value + + update_id = context[alias].id + updated_thing = scene["virtualObjects"][update_id] + updated_thing.update(parsed_data) + + new_scene = scene.copy() + new_scene["virtualObjects"][update_id] = updated_thing + + return new_scene + + +def run(app_json: list, scene: dict) -> dict: + [reads, writes] = app_json + context = {} + + new_scene = scene.copy() + + contexts = [{}] + + for match_json in reads: + new_contexts = [] + for context in contexts: + matches = match(match_json, context, scene) + for alias, tup in matches: + new_contexts.append({alias: tup, **context}) + contexts = new_contexts + + for context in contexts: + for write in writes: + if write[0] == Command.CREATE.name: + new_scene = create(write, context, new_scene) + elif write[0] == Command.UPDATE.name: + new_scene = update(write, context, new_scene) + + return new_scene + + + From e4e5a7425d0a597bd593e81e570d9d55bcddd299 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Fri, 15 Nov 2019 21:46:00 -0500 Subject: [PATCH 13/20] App scene manipulation based on dict scene --- tinytalk/data.py | 93 --------------- tinytalk/interpreter.py | 249 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 93 deletions(-) delete mode 100644 tinytalk/data.py create mode 100644 tinytalk/interpreter.py diff --git a/tinytalk/data.py b/tinytalk/data.py deleted file mode 100644 index f24eca9..0000000 --- a/tinytalk/data.py +++ /dev/null @@ -1,93 +0,0 @@ -from collections import defaultdict, namedtuple -from deep_merge import merge - -from .grammar import Command - -data = defaultdict(list) - - -def lookup(name: str, namespace: namedtuple): - if hasattr(namespace, str(name)): - return getattr(namespace, name) - return None - - -def condition(var_name, cond, row: namedtuple) -> bool: - if not hasattr(row, var_name): - return False - elif cond == "ANY": - return True - elif cond[0] == Command.AND.name: - return condition(var_name, cond[1], row) and condition(var_name, cond[2], row) - else: - [operator, left, right] = cond - - # Check if left and right are variable names and access their values if so - left = lookup(left, row) or left - right = lookup(right, row) or right - - if operator in ["is"]: - return left == right - elif operator in ["<"]: - return left < right - elif operator in [">"]: - return left > right - elif operator in ["not"]: - return left != right - - -def match(mat: list) -> None: - """Retrieve each matching entry in the database. - - Assumes database var, `data`, is of the form: - { - (str): [ - (namedtuple), - (namedtuple), - ... - ], - (str): [...] - } - - Args: - mat (list): [ - _command: "MATCH" - adjectives: list of adjectives - relation: relation to previous matched items - tags: list of tags - data_conditions: dict mapping var names to condition (list or "ANY") - alias: str alias for entries that match above criteria - ] - Returns: - alias: str alias for the matches - matches (list): database rows that match criteria - """ - [_command, adjectives, relation, tags, data_conditions, alias] = mat - - # Filter by tags, there must be at least one. - matches = set(data[tags[0]]) - for tag in tags[1:]: - matches &= set(data[tag]) - - #TODO: Filter by adjectives - #TODO: Filter by relation (requires previous match contexts?) - - # Filter by data_conditions - for var_name, cond in data_conditions.items(): - if cond[0] == Command.COND.name: - cond = cond[1] - matches = filter(lambda row: condition(var_name, cond, row), matches) - - return alias, matches - - -def commit(name, *args): - global data - names = [name] + list(args)[:-1] - commit_data = list(args)[-1] - for name in names: - data = merge(data, {name: merge({"names": names}, commit_data)}) - - -def query(search): - return data[search] diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py new file mode 100644 index 0000000..f16a92f --- /dev/null +++ b/tinytalk/interpreter.py @@ -0,0 +1,249 @@ +"""Functions for running compiled tinytalk apps on a scene.""" + +from collections import defaultdict, namedtuple +from deep_merge import merge +import uuid + +from .grammar import Command + + +# Dict mapping adjective names in tinytalk to functions that filter a list +# of tinyland things. +adjectives = { + "only": lambda l: return l if len(l) == 1 else [] +} + + +def tup(**kwargs) -> namedtuple: + """Convert an arbitrary number of kwargs to a namedtuple.""" + tup = namedtuple("tinyTuple", list(kwargs)) + return tup(**kwargs) + + +def lookup(name: str, namespace: namedtuple): + if hasattr(namespace, str(name)): + return getattr(namespace, name) + return None + + +def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: + """Check if condition holds true for the given parameters. + + Specifically, if the `var_name` attribute of `row` satisfies `cond_json` given + `context`. + + Args: + var_name: the attribute in row to which the condition will be applied. + context: in apps with multiple match statements, this maps previous + matched objects to their aliases so that the current match can use + that data. + cond_json: json object for data_condition. + row: the tinyland object to which the condition is being applied. + + Returns: + bool: Whether the condition is satisfied. + """ + if not hasattr(row, var_name): + return False + elif cond_json == "ANY": + return True + elif cond_json[0] == Command.AND.name: + return (condition(var_name, context, cond_json[1], row) and + condition(var_name, context, cond_json[2], row)) + else: + [operator, left, right] = cond_json + + if isinstance(left, str) and "." in left: + alias, attribute = left.split(".") + left = lookup(context[alias], attribute) + else: + left = lookup(left, row) or left + + if isinstance(right, str) and "." in right: + alias, attribute = right.split(".") + right = lookup(context[alias], attribute) + else: + right = lookup(right, row) or right + + if operator in ["is"]: + return left == right + elif operator in ["<"]: + return left < right + elif operator in [">"]: + return left > right + elif operator in ["not"]: + return left != right + + +def match(match_json: list, context: dict, scene: dict) -> list: + """Search the tinyland scene for objects that match statement. + + Specifically, return each object in `scene` that satisfies `match_json` + given `context`. + + Args: + match_json: json for match. + context: in apps with multiple match statements, this maps previous + matched objects to their aliases so that the current match can use + that data. + scene: "database" of all that exists in tinyland. The current implementation + assumes the following structure: + { + "appMarkers": { + : { + "type": "marker", + "id": , + ... + }, + ... + }, + "virtualObjects": { + : { + "type": , + ... + }, + ... + } + } + """ + [_command, adjectives, relation, tags, data_conditions, alias] = match_json + + #TODO: match first appearance of something + + # Filter by tags. + matches = [] + for key, thing in {**scene["allMarkers"], **scene["virtualObjects"]}.items(): + thing_tup = tup(id=key, **thing) + if set(thing_tup.type.split(" ")) == set(tags): + matches.append(thing_tup) + + # Filter by adjectives. + for adjective in adjectives: + matches = adjectives[adjective](matches) + + #TODO: Filter by relation (requires previous match contexts?) + + # Filter by data_conditions. + for var_name, cond in data_conditions.items(): + if cond[0] == Command.COND.name: + # This is currently necessary because condition can be "ANY". + cond = cond[1] + matches = filter(lambda row: condition(var_name, context, cond, row), matches) + + # The following format makes it easier for match combinations + # to be turned into context dictionaries. + return [(alias, m) for m in matches] + + +def create(create_json: list, context: dict, scene: dict) -> dict: + """Create a new tinyland object in the scene. + + Args: + create_json: json for create statement. + context: aliases mapped to their match statements. + scene: "database" of all that exists in tinyland. + See match docstring for schema. + Returns: + new_scene: `scene` with the new object added. + """ + [_command, tags, relation, data] = create_json + + parsed_data = {} + for var, value in data.items(): + if isinstance(value, str) and "." in value: + alias, attribute = value.split(".") + parsed_data[var] = lookup(context[alias], attribute) + else: + parsed_data[var] = value + + new_thing = { + "type": " ".join(tags), + relation: [tup.id for tup in context], + **parsed_data + } + + new_scene = { + "allMarkers": **scene["allMarkers"], + "virtualObjects": { + uuid.UUID(): new_thing, + **scene["virtualObjects"] + } + } + + #TODO: implement relation stuff + + return new_scene + + +def update(update_json: list, context: dict, scene: dict) -> dict: + """Update the desired object in tinyland scene. + + Args: + update_json: json for update statement. + context: aliases mapped to their match statements. + scene: "database" of all that exists in tinyland. + See match docstring for schema. + Returns: + new_scene: `scene` with the object updated. + """ + [_command, alias, data] = update_json + + parsed_data = {} + for var, value in data.items(): + if isinstance(value, str) and "." in value: + alias, attribute = value.split(".") + parsed_data[var] = lookup(context[alias], attribute) + else: + parsed_data[var] = value + + update_id = context[alias].id + updated_thing = scene["virtualObjects"][update_id] + updated_thing.update(parsed_data) + + new_scene = scene.copy() + new_scene["virtualObjects"][update_id] = updated_thing + + return new_scene + + +def run(app_json: list, scene: dict) -> dict: + """Run tinyland app on the tinyland scene. + + Args: + app_json: json for tinyland app. + scene: "database" of all that exists in tinyland. + See match docstring for schema. + Returns: + new_scene: the state of tinyland after running the app on `scene`. + """ + [reads, writes] = app_json + + new_scene = scene.copy() + + # We need to find all combinations of our match statements. + # contexts will hold one context for each time we need to run the + # write portion of our app. + contexts = [{}] + + # Populate contexts. + for match_json in reads: + new_contexts = [] + for context in contexts: + # For each context holding n-1 matches, + # find matches and create an new context with the nth match. + matches = match(match_json, context, scene) + for alias, tup in matches: + new_contexts.append({alias: tup, **context}) + contexts = new_contexts + + for context in contexts: + for write in writes: + if write[0] == Command.CREATE.name: + new_scene = create(write, context, new_scene) + elif write[0] == Command.UPDATE.name: + new_scene = update(write, context, new_scene) + + return new_scene + + + From 26a91d545c847fbcbd96149e6a30a5666fba8b4e Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Tue, 19 Nov 2019 11:47:10 -0500 Subject: [PATCH 14/20] basic test passes --- tinytalk/_test.py | 27 ++++++++++++ tinytalk/grammar.py | 4 +- tinytalk/interpreter.py | 94 +++++++---------------------------------- tinytalk/test_data.py | 43 ------------------- 4 files changed, 44 insertions(+), 124 deletions(-) create mode 100644 tinytalk/_test.py delete mode 100644 tinytalk/test_data.py diff --git a/tinytalk/_test.py b/tinytalk/_test.py new file mode 100644 index 0000000..6630af0 --- /dev/null +++ b/tinytalk/_test.py @@ -0,0 +1,27 @@ +import grammar +import interpreter + +app = "when [#marker x y] as m create [#ball x: m.y, y: m.x]" + +tree = grammar.grammar.parse(app) +visitor = grammar.TinyTalkVisitor() +app_json = visitor.visit(tree) + +scene = { + "appMarkers": { + "1": { + "type": "marker", + "x": 50, + "y": 0, + } + }, + "virtualObjects": { + } +} + +print(app_json) + +new_scene = interpreter.run(app_json, scene) + +print(scene) +print(new_scene) \ No newline at end of file diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 85ace26..4393de2 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -42,13 +42,13 @@ class Command(Enum): addition = (multiplication / subexpr / value) ws? ("+" / "-") ws? expr multiplication = (subexpr / value) ws? "*" ws? expr subexpr = "(" ws? expr ws? ")" - value = number / string / boolean / name + value = number / boolean / name / string boolean = "true" / "false" string = ~'"[^\"]*"' number = ("+" / "-")? digit+ ("." digit+)? ("e" ("+" / "-") digit+)? digit = ~"[0-9]" name_with_pronouns = name ("/" name)* - name = !reserved_word ~"[a-z][a-z_-]*" + name = !reserved_word ~"[a-z][a-z_-]*(\.[a-z][a-z_-]*)?" begin = ws? "[" end = ws? "]" reserved_word = ("as" / "where" / "true" / "false") &ws diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py index 18d297e..e4113b7 100644 --- a/tinytalk/interpreter.py +++ b/tinytalk/interpreter.py @@ -1,31 +1,20 @@ -<<<<<<< HEAD """Functions for running compiled tinytalk apps on a scene.""" -======= ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 from collections import defaultdict, namedtuple -from deep_merge import merge import uuid -from .grammar import Command +from grammar import Command -<<<<<<< HEAD # Dict mapping adjective names in tinytalk to functions that filter a list # of tinyland things. -======= ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 adjectives = { - "only": lambda l: return l if len(l) == 1 else [] + "only": lambda l: l if len(l) == 1 else [] } -<<<<<<< HEAD def tup(**kwargs) -> namedtuple: """Convert an arbitrary number of kwargs to a namedtuple.""" -======= -def tup(**kwargs): ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 tup = namedtuple("tinyTuple", list(kwargs)) return tup(**kwargs) @@ -36,7 +25,6 @@ def lookup(name: str, namespace: namedtuple): return None -<<<<<<< HEAD def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: """Check if condition holds true for the given parameters. @@ -63,28 +51,16 @@ def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: condition(var_name, context, cond_json[2], row)) else: [operator, left, right] = cond_json -======= -def condition(var_name, context, cond, row: namedtuple) -> bool: - if not hasattr(row, var_name): - return False - elif cond == "ANY": - return True - elif cond[0] == Command.AND.name: - return (condition(var_name, context, cond[1], row) and - condition(var_name, context, cond[2], row)) - else: - [operator, left, right] = cond ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 if isinstance(left, str) and "." in left: alias, attribute = left.split(".") - left = lookup(context[alias], attribute) + left = lookup(attribute, context[alias]) else: left = lookup(left, row) or left if isinstance(right, str) and "." in right: alias, attribute = right.split(".") - right = lookup(context[alias], attribute) + right = lookup(attribute, context[alias]) else: right = lookup(right, row) or right @@ -98,7 +74,6 @@ def condition(var_name, context, cond, row: namedtuple) -> bool: return left != right -<<<<<<< HEAD def match(match_json: list, context: dict, scene: dict) -> list: """Search the tinyland scene for objects that match statement. @@ -135,29 +110,19 @@ def match(match_json: list, context: dict, scene: dict) -> list: #TODO: match first appearance of something # Filter by tags. -======= -def match(m: list, context: dict, scene: dict) -> tuple: - [_command, adjectives, relation, tags, data_conditions, alias] = m - - - # Filter by tags, there must be at least one. ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 matches = [] - for key, thing in {**scene["allMarkers"], **scene["virtualObjects"]}.items(): + for key, thing in {**scene["appMarkers"], **scene["virtualObjects"]}.items(): thing_tup = tup(id=key, **thing) if set(thing_tup.type.split(" ")) == set(tags): matches.append(thing_tup) -<<<<<<< HEAD # Filter by adjectives. -======= ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 - for adjective in adjectives: - matches = adjectives[adjective](matches) + if adjectives: + for adjective in adjectives: + matches = adjectives[adjective](matches) #TODO: Filter by relation (requires previous match contexts?) -<<<<<<< HEAD # Filter by data_conditions. for var_name, cond in data_conditions.items(): if cond[0] == Command.COND.name: @@ -182,38 +147,27 @@ def create(create_json: list, context: dict, scene: dict) -> dict: new_scene: `scene` with the new object added. """ [_command, tags, relation, data] = create_json -======= - # Filter by data_conditions - for var_name, cond in data_conditions.items(): - if cond[0] == Command.COND.name: - cond = cond[1] - matches = filter(lambda row: condition(var_name, context, cond, row), matches) - - return [(alias, m) for m in matches] - - -def create(c: list, context: dict, scene: dict) -> dict: - [_command, tags, relation, data] = c ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 parsed_data = {} for var, value in data.items(): if isinstance(value, str) and "." in value: alias, attribute = value.split(".") - parsed_data[var] = lookup(context[alias], attribute) + parsed_data[var] = lookup(attribute, context[alias]) else: parsed_data[var] = value new_thing = { "type": " ".join(tags), - relation: [tup.id for tup in context], **parsed_data } + if relation: + new_thing[relation] = [tup.id for tup in context.values()], + new_scene = { - "allMarkers": **scene["allMarkers"], + "appMarkers": scene["appMarkers"], "virtualObjects": { - uuid.UUID(): new_thing, + uuid.uuid4(): new_thing, **scene["virtualObjects"] } } @@ -223,7 +177,6 @@ def create(c: list, context: dict, scene: dict) -> dict: return new_scene -<<<<<<< HEAD def update(update_json: list, context: dict, scene: dict) -> dict: """Update the desired object in tinyland scene. @@ -236,16 +189,12 @@ def update(update_json: list, context: dict, scene: dict) -> dict: new_scene: `scene` with the object updated. """ [_command, alias, data] = update_json -======= -def update(u: list, context: dict, scene: dict) -> dict: - [_command, alias, data] = u ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 parsed_data = {} for var, value in data.items(): if isinstance(value, str) and "." in value: alias, attribute = value.split(".") - parsed_data[var] = lookup(context[alias], attribute) + parsed_data[var] = lookup(attribute, context[alias]) else: parsed_data[var] = value @@ -260,7 +209,6 @@ def update(u: list, context: dict, scene: dict) -> dict: def run(app_json: list, scene: dict) -> dict: -<<<<<<< HEAD """Run tinyland app on the tinyland scene. Args: @@ -285,18 +233,6 @@ def run(app_json: list, scene: dict) -> dict: for context in contexts: # For each context holding n-1 matches, # find matches and create an new context with the nth match. -======= - [reads, writes] = app_json - context = {} - - new_scene = scene.copy() - - contexts = [{}] - - for match_json in reads: - new_contexts = [] - for context in contexts: ->>>>>>> 7463986d84403b1a28f28f697f04d78987251f56 matches = match(match_json, context, scene) for alias, tup in matches: new_contexts.append({alias: tup, **context}) diff --git a/tinytalk/test_data.py b/tinytalk/test_data.py deleted file mode 100644 index aea310e..0000000 --- a/tinytalk/test_data.py +++ /dev/null @@ -1,43 +0,0 @@ -from . import data, Command -from collections import namedtuple - - -def test_variable_exists(): - rowtup = namedtuple("row", ["x"]) - row = rowtup(x=100) - var = "x" - cond = "ANY" - assert data.condition(var, cond, row) - - -def test_variable_does_not_exist(): - rowtup = namedtuple("row", ["x"]) - row = rowtup(x=100) - var = "y" - cond = "ANY" - assert not data.condition(var, cond, row) - - -def test_single_condition_true(): - rowtup = namedtuple("row", ["x"]) - row = rowtup(x=50) - var = "x" - cond = ["is", "x", 50.0] - assert data.condition(var, cond, row) - - -def test_single_condition_false(): - rowtup = namedtuple("row", ["x"]) - row = rowtup(x=50) - var = "x" - cond = ["not", "x", 50.0] - assert not data.condition(var, cond, row) - - -def test_multiple_condition(): - rowtup = namedtuple("row", ["x"]) - row = rowtup(x=50) - var = "x" - cond = [Command.AND.name, ["<", 0.0, "x"], ["<", "x", 100.0]] - assert data.condition(var, cond, row) - From 3ad33e12f6f7f6c33213b9e88f078d0eedf9c44d Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Wed, 20 Nov 2019 18:15:15 -0500 Subject: [PATCH 15/20] add basic websocket server --- tinytalk/app.txt | 3 + tinytalk/grammar.py | 2 +- tinytalk/interpreter.py | 132 ++++++++++++++++++----------------- tinytalk/websocket_server.py | 55 +++++++++++++++ 4 files changed, 127 insertions(+), 65 deletions(-) create mode 100644 tinytalk/app.txt create mode 100644 tinytalk/websocket_server.py diff --git a/tinytalk/app.txt b/tinytalk/app.txt new file mode 100644 index 0000000..1db02a6 --- /dev/null +++ b/tinytalk/app.txt @@ -0,0 +1,3 @@ +when [] create [] + +when [] update [] \ No newline at end of file diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index 4393de2..e5c2f30 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -18,7 +18,7 @@ class Command(Enum): app = read ws? write read = "when" ws match (";" ws? match)* write = (create / update) (";" ws? (create / update))* - match = adjectives? relation? begin ws? tags ws data_condition? end (ws alias)? + match = adjectives? relation? begin ws? tags ws? data_condition? end (ws alias)? create = "create" (ws relation)? begin ws? tags ws data? end update = "update" ws name begin data end adjectives = ws? adjective (ws adjective)* diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py index e4113b7..b0c1910 100644 --- a/tinytalk/interpreter.py +++ b/tinytalk/interpreter.py @@ -1,11 +1,33 @@ """Functions for running compiled tinytalk apps on a scene.""" from collections import defaultdict, namedtuple +from functools import reduce import uuid from grammar import Command +create_triggers = defaultdict(set) +update_triggers = defaultdict(set) + +apps_by_id = {} # id integer -> app JSON + + +def load_app(app_json): + [reads, writes] = app_json + app_id = len(apps_by_id) + if app_json not in apps_by_id.values(): + apps_by_id[app_id] = app_json + for match_json in reads: + [_command, adjectives, relation, tags, data_conditions, alias] = match_json + if data_conditions is None: + for tag in tags: + create_triggers[tag].add(app_id) + else: + for tag in tags: + update_triggers[tag].add(app_id) + + # Dict mapping adjective names in tinytalk to functions that filter a list # of tinyland things. adjectives = { @@ -14,6 +36,7 @@ def tup(**kwargs) -> namedtuple: + print(kwargs) """Convert an arbitrary number of kwargs to a namedtuple.""" tup = namedtuple("tinyTuple", list(kwargs)) return tup(**kwargs) @@ -25,6 +48,25 @@ def lookup(name: str, namespace: namedtuple): return None +def create(id_string: str, thing: dict, scene: dict): + tags = thing["tags"] + scene[id_string] = thing + + apps_triggered = reduce((lambda x, y: x | y), [create_triggers[tag] for tag in tags]) + for app_id in apps_triggered: + run(apps_by_id[app_id], scene, trigger_id=id_string) + + +def update(id_string: str, data: dict, scene: dict): + thing = scene[id_string] + tags = thing["tags"] + scene[id_string].update(data) + + apps_triggered = reduce((lambda x, y: x | y), [update_triggers[tag] for tag in tags]) + for app_id in apps_triggered: + run(apps_by_id[app_id], scene, trigger_id=id_string) + + def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: """Check if condition holds true for the given parameters. @@ -85,25 +127,7 @@ def match(match_json: list, context: dict, scene: dict) -> list: context: in apps with multiple match statements, this maps previous matched objects to their aliases so that the current match can use that data. - scene: "database" of all that exists in tinyland. The current implementation - assumes the following structure: - { - "appMarkers": { - : { - "type": "marker", - "id": , - ... - }, - ... - }, - "virtualObjects": { - : { - "type": , - ... - }, - ... - } - } + scene: "database" of all that exists in tinyland. """ [_command, adjectives, relation, tags, data_conditions, alias] = match_json @@ -111,31 +135,32 @@ def match(match_json: list, context: dict, scene: dict) -> list: # Filter by tags. matches = [] - for key, thing in {**scene["appMarkers"], **scene["virtualObjects"]}.items(): + for key, thing in scene.items(): thing_tup = tup(id=key, **thing) if set(thing_tup.type.split(" ")) == set(tags): matches.append(thing_tup) # Filter by adjectives. - if adjectives: + if adjectives is not None: for adjective in adjectives: matches = adjectives[adjective](matches) #TODO: Filter by relation (requires previous match contexts?) # Filter by data_conditions. - for var_name, cond in data_conditions.items(): - if cond[0] == Command.COND.name: - # This is currently necessary because condition can be "ANY". - cond = cond[1] - matches = filter(lambda row: condition(var_name, context, cond, row), matches) + if data_conditions is not None: + for var_name, cond in data_conditions.items(): + if cond[0] == Command.COND.name: + # This is currently necessary because condition can be "ANY". + cond = cond[1] + matches = filter(lambda row: condition(var_name, context, cond, row), matches) # The following format makes it easier for match combinations # to be turned into context dictionaries. return [(alias, m) for m in matches] -def create(create_json: list, context: dict, scene: dict) -> dict: +def create_from_json(create_json: list, context: dict, scene: dict, create_id: str=None): """Create a new tinyland object in the scene. Args: @@ -143,8 +168,6 @@ def create(create_json: list, context: dict, scene: dict) -> dict: context: aliases mapped to their match statements. scene: "database" of all that exists in tinyland. See match docstring for schema. - Returns: - new_scene: `scene` with the new object added. """ [_command, tags, relation, data] = create_json @@ -157,6 +180,7 @@ def create(create_json: list, context: dict, scene: dict) -> dict: parsed_data[var] = value new_thing = { + "tags": tags, "type": " ".join(tags), **parsed_data } @@ -164,20 +188,11 @@ def create(create_json: list, context: dict, scene: dict) -> dict: if relation: new_thing[relation] = [tup.id for tup in context.values()], - new_scene = { - "appMarkers": scene["appMarkers"], - "virtualObjects": { - uuid.uuid4(): new_thing, - **scene["virtualObjects"] - } - } - - #TODO: implement relation stuff + create_id = create_id or uuid.uuid4() + create(create_id, new_thing, scene) - return new_scene - -def update(update_json: list, context: dict, scene: dict) -> dict: +def update_from_json(update_json: list, context: dict, scene: dict) -> dict: """Update the desired object in tinyland scene. Args: @@ -185,9 +200,7 @@ def update(update_json: list, context: dict, scene: dict) -> dict: context: aliases mapped to their match statements. scene: "database" of all that exists in tinyland. See match docstring for schema. - Returns: - new_scene: `scene` with the object updated. - """ + """ [_command, alias, data] = update_json parsed_data = {} @@ -199,29 +212,19 @@ def update(update_json: list, context: dict, scene: dict) -> dict: parsed_data[var] = value update_id = context[alias].id - updated_thing = scene["virtualObjects"][update_id] - updated_thing.update(parsed_data) + update(update_id, parsed_data, scene) - new_scene = scene.copy() - new_scene["virtualObjects"][update_id] = updated_thing - return new_scene - - -def run(app_json: list, scene: dict) -> dict: +def run(app_json: list, scene: dict, trigger_id: str=None) -> dict: """Run tinyland app on the tinyland scene. Args: app_json: json for tinyland app. scene: "database" of all that exists in tinyland. See match docstring for schema. - Returns: - new_scene: the state of tinyland after running the app on `scene`. """ [reads, writes] = app_json - new_scene = scene.copy() - # We need to find all combinations of our match statements. # contexts will hold one context for each time we need to run the # write portion of our app. @@ -235,17 +238,18 @@ def run(app_json: list, scene: dict) -> dict: # find matches and create an new context with the nth match. matches = match(match_json, context, scene) for alias, tup in matches: - new_contexts.append({alias: tup, **context}) + if tup.id not in [t.id for t in context.values()]: + new_contexts.append({alias: tup, **context}) contexts = new_contexts for context in contexts: - for write in writes: - if write[0] == Command.CREATE.name: - new_scene = create(write, context, new_scene) - elif write[0] == Command.UPDATE.name: - new_scene = update(write, context, new_scene) - - return new_scene + if trigger_id is None or trigger_id in [tup.id for tup in context.values()]: + for write in writes: + if write[0] == Command.CREATE.name: + create_from_json(write, context, scene) + elif write[0] == Command.UPDATE.name: + update_from_json(write, context, scene) + diff --git a/tinytalk/websocket_server.py b/tinytalk/websocket_server.py new file mode 100644 index 0000000..4d871ac --- /dev/null +++ b/tinytalk/websocket_server.py @@ -0,0 +1,55 @@ +import asyncio +import os +import sys +import websockets + +from grammar import grammar, TinyTalkVisitor +import interpreter + +IP_ADDRESS = "localhost" +PORT = 8765 +APP_FILE = "app.txt" +last_upload = -sys.maxsize + +visitor = TinyTalkVisitor() +scene = {} + + +def format_scene(scene): + formatted_scene = { + "appMarkers": {}, + "virtualObjects": {}, + } + for key, val in scene.items(): + if "marker" in val["tags"]: + formatted_scene["appMarkers"][key] = val + else: + formatted_scene["virtualObjects"][key] = val + return formatted_scene + + +async def tinyland_loop(websocket, path): + new_object = await websocket.recv() + + global last_upload + if os.path.getmtime(APP_FILE) > last_upload: + with open(APP_FILE, "r") as f: + apps = f.read().split("\n\n") + for app in apps: + app_json = visitor.visit(grammar.parse(app)) + interpreter.load_app(app_json) + + last_upload = os.path.getmtime(APP_FILE) + + if new_object["id"] in scene: + interpreter.update(new_object["id"], new_object, scene) + else: + interpreter.create(new_object["id"], new_object, scene) + + await websocket.send(format_scene(scene)) + + +start_server = websockets.serve(tinyland_loop, IP_ADDRESS, PORT) + +asyncio.get_event_loop().run_until_complete(start_server) +asyncio.get_event_loop().run_forever() \ No newline at end of file From 59bb8407fdf6c9da458d1d6a67e9bbf3cdba7af5 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Thu, 21 Nov 2019 19:35:46 -0500 Subject: [PATCH 16/20] Create server that listens for tinyland changes and serves the tinyland scene. Using asyncio create 2 coroutines: 1) listens for UDP messages and adds them to a queue 2) reads UDP data from queue, decodes it, and adds the data to the scene. Also loads apps from app.txt to be triggered. Then serves the scene as JSON on a websocket. --- tinytalk/app.txt | 4 +--- tinytalk/interpreter.py | 6 ++++-- tinytalk/websocket_server.py | 41 ++++++++++++++++++++++++++++++------ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/tinytalk/app.txt b/tinytalk/app.txt index 1db02a6..3494385 100644 --- a/tinytalk/app.txt +++ b/tinytalk/app.txt @@ -1,3 +1 @@ -when [] create [] - -when [] update [] \ No newline at end of file +when [#marker ] as m create [#ball x: m.x y: m.y] \ No newline at end of file diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py index b0c1910..8743d0c 100644 --- a/tinytalk/interpreter.py +++ b/tinytalk/interpreter.py @@ -136,7 +136,9 @@ def match(match_json: list, context: dict, scene: dict) -> list: # Filter by tags. matches = [] for key, thing in scene.items(): - thing_tup = tup(id=key, **thing) + if "id" not in thing: + thing["id"] = key + thing_tup = tup(**thing) if set(thing_tup.type.split(" ")) == set(tags): matches.append(thing_tup) @@ -188,7 +190,7 @@ def create_from_json(create_json: list, context: dict, scene: dict, create_id: s if relation: new_thing[relation] = [tup.id for tup in context.values()], - create_id = create_id or uuid.uuid4() + create_id = create_id or str(uuid.uuid4()) create(create_id, new_thing, scene) diff --git a/tinytalk/websocket_server.py b/tinytalk/websocket_server.py index 4d871ac..be15eca 100644 --- a/tinytalk/websocket_server.py +++ b/tinytalk/websocket_server.py @@ -1,18 +1,25 @@ import asyncio +import json import os +import parsimonious +import queue +import socket import sys import websockets from grammar import grammar, TinyTalkVisitor import interpreter -IP_ADDRESS = "localhost" -PORT = 8765 +BUFFER_SIZE = 1024 +IP_ADDRESS = "127.0.0.1" +UDP_PORT = 8766 +WEBSOCKET_PORT = 8765 APP_FILE = "app.txt" last_upload = -sys.maxsize visitor = TinyTalkVisitor() scene = {} +q = queue.Queue(64) def format_scene(scene): @@ -29,15 +36,24 @@ def format_scene(scene): async def tinyland_loop(websocket, path): - new_object = await websocket.recv() + # global sock + data = q.get(True) + new_object = json.loads(data.decode("utf-8")) global last_upload + global scene if os.path.getmtime(APP_FILE) > last_upload: with open(APP_FILE, "r") as f: apps = f.read().split("\n\n") for app in apps: - app_json = visitor.visit(grammar.parse(app)) - interpreter.load_app(app_json) + try: + app_json = visitor.visit(grammar.parse(app)) + except parsimonious.exceptions.ParseError as e: + #TODO: add bad app error handling + pass + else: + print(f"Loading app {app_json}") + interpreter.load_app(app_json) last_upload = os.path.getmtime(APP_FILE) @@ -46,10 +62,21 @@ async def tinyland_loop(websocket, path): else: interpreter.create(new_object["id"], new_object, scene) - await websocket.send(format_scene(scene)) + await websocket.send(json.dumps(format_scene(scene))) -start_server = websockets.serve(tinyland_loop, IP_ADDRESS, PORT) +async def udp_listener(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind((IP_ADDRESS, UDP_PORT)) + while True: + data, _addr = sock.recvfrom(1024) + print(data) + q.put(data, True) + await asyncio.sleep(0.5) + + +start_server = websockets.serve(tinyland_loop, IP_ADDRESS, WEBSOCKET_PORT) asyncio.get_event_loop().run_until_complete(start_server) +asyncio.get_event_loop().create_task(udp_listener()) asyncio.get_event_loop().run_forever() \ No newline at end of file From f70a51c87c1e574de431ab77356b867aa12a64cc Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Sun, 8 Dec 2019 17:25:01 -0500 Subject: [PATCH 17/20] Improved and refactored Websocket Server and tinyland runtime. - Moved websocket server into a class and added command line arguments to websocket_server.py - Refactored interpreter.py - Kept static methods for parsing compiled (JSON) tinyland apps as functions. - Moved state-based logic into scene.TinylandScene class, such as - state of scene - app execution logic & triggers --- tinytalk/app.txt | 6 +- tinytalk/interpreter.py | 175 +++++++++++------------------------ tinytalk/scene.py | 122 ++++++++++++++++++++++++ tinytalk/websocket_server.py | 138 +++++++++++++++++---------- 4 files changed, 270 insertions(+), 171 deletions(-) create mode 100644 tinytalk/scene.py diff --git a/tinytalk/app.txt b/tinytalk/app.txt index 3494385..f6b171f 100644 --- a/tinytalk/app.txt +++ b/tinytalk/app.txt @@ -1 +1,5 @@ -when [#marker ] as m create [#ball x: m.x y: m.y] \ No newline at end of file +when [#marker ] as m create [#blob x: m.x y: m.y size: 10] + +when [#blob x where x > 1.0 y] as b update b [x: 0.0] + +when [#blob x y] as b update b [x: b.x + 0.001 y: b.y] \ No newline at end of file diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py index 8743d0c..4e8cb95 100644 --- a/tinytalk/interpreter.py +++ b/tinytalk/interpreter.py @@ -1,42 +1,14 @@ -"""Functions for running compiled tinytalk apps on a scene.""" - -from collections import defaultdict, namedtuple -from functools import reduce +from collections import namedtuple import uuid from grammar import Command - -create_triggers = defaultdict(set) -update_triggers = defaultdict(set) - -apps_by_id = {} # id integer -> app JSON - - -def load_app(app_json): - [reads, writes] = app_json - app_id = len(apps_by_id) - if app_json not in apps_by_id.values(): - apps_by_id[app_id] = app_json - for match_json in reads: - [_command, adjectives, relation, tags, data_conditions, alias] = match_json - if data_conditions is None: - for tag in tags: - create_triggers[tag].add(app_id) - else: - for tag in tags: - update_triggers[tag].add(app_id) - - # Dict mapping adjective names in tinytalk to functions that filter a list # of tinyland things. -adjectives = { - "only": lambda l: l if len(l) == 1 else [] -} +adjectives = {"only": lambda l: l if len(l) == 1 else []} def tup(**kwargs) -> namedtuple: - print(kwargs) """Convert an arbitrary number of kwargs to a namedtuple.""" tup = namedtuple("tinyTuple", list(kwargs)) return tup(**kwargs) @@ -48,25 +20,6 @@ def lookup(name: str, namespace: namedtuple): return None -def create(id_string: str, thing: dict, scene: dict): - tags = thing["tags"] - scene[id_string] = thing - - apps_triggered = reduce((lambda x, y: x | y), [create_triggers[tag] for tag in tags]) - for app_id in apps_triggered: - run(apps_by_id[app_id], scene, trigger_id=id_string) - - -def update(id_string: str, data: dict, scene: dict): - thing = scene[id_string] - tags = thing["tags"] - scene[id_string].update(data) - - apps_triggered = reduce((lambda x, y: x | y), [update_triggers[tag] for tag in tags]) - for app_id in apps_triggered: - run(apps_by_id[app_id], scene, trigger_id=id_string) - - def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: """Check if condition holds true for the given parameters. @@ -89,8 +42,9 @@ def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: elif cond_json == "ANY": return True elif cond_json[0] == Command.AND.name: - return (condition(var_name, context, cond_json[1], row) and - condition(var_name, context, cond_json[2], row)) + return condition(var_name, context, cond_json[1], row) and condition( + var_name, context, cond_json[2], row + ) else: [operator, left, right] = cond_json @@ -99,7 +53,7 @@ def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: left = lookup(attribute, context[alias]) else: left = lookup(left, row) or left - + if isinstance(right, str) and "." in right: alias, attribute = right.split(".") right = lookup(attribute, context[alias]) @@ -116,6 +70,35 @@ def condition(var_name: str, context: dict, cond_json, row: namedtuple) -> bool: return left != right +def expression(expr_json: list, var_name: str, context: dict, row: namedtuple): + """Parse compiled app JSON expression and return value. + + Args: + expr_json: json representing visited expr node. + var_name: name of the variable set to this expression. + context: dict for looking up aliases and names used in expression. + row: named tuple for looking up attributes referenced in expression. + """ + if type(expr_json) not in [list, tuple]: + if isinstance(expr_json, str) and "." in expr_json: + alias, attribute = expr_json.split(".") + result = lookup(attribute, context[alias]) + else: + result = lookup(expr_json, row) or expr_json + # print(f"\n\n{result}\n\n") + return result + else: + [operator, left, right] = expr_json + left = expression(left, var_name, context, row) + right = expression(right, var_name, context, row) + if operator in ["*"]: + return left * right + elif operator in ["+"]: + return left + right + elif operator in ["-"]: + return left - right + + def match(match_json: list, context: dict, scene: dict) -> list: """Search the tinyland scene for objects that match statement. @@ -131,7 +114,7 @@ def match(match_json: list, context: dict, scene: dict) -> list: """ [_command, adjectives, relation, tags, data_conditions, alias] = match_json - #TODO: match first appearance of something + # TODO: match first appearance of something # Filter by tags. matches = [] @@ -141,13 +124,13 @@ def match(match_json: list, context: dict, scene: dict) -> list: thing_tup = tup(**thing) if set(thing_tup.type.split(" ")) == set(tags): matches.append(thing_tup) - + # Filter by adjectives. if adjectives is not None: for adjective in adjectives: matches = adjectives[adjective](matches) - #TODO: Filter by relation (requires previous match contexts?) + # TODO: Filter by relation (requires previous match contexts?) # Filter by data_conditions. if data_conditions is not None: @@ -155,103 +138,49 @@ def match(match_json: list, context: dict, scene: dict) -> list: if cond[0] == Command.COND.name: # This is currently necessary because condition can be "ANY". cond = cond[1] - matches = filter(lambda row: condition(var_name, context, cond, row), matches) + matches = list( + filter(lambda row: condition(var_name, context, cond, row), matches) + ) # The following format makes it easier for match combinations # to be turned into context dictionaries. return [(alias, m) for m in matches] -def create_from_json(create_json: list, context: dict, scene: dict, create_id: str=None): +def create_from_json(create_json: list, context: dict, create_id: str = None): """Create a new tinyland object in the scene. Args: create_json: json for create statement. context: aliases mapped to their match statements. - scene: "database" of all that exists in tinyland. - See match docstring for schema. """ [_command, tags, relation, data] = create_json parsed_data = {} for var, value in data.items(): - if isinstance(value, str) and "." in value: - alias, attribute = value.split(".") - parsed_data[var] = lookup(attribute, context[alias]) - else: - parsed_data[var] = value + parsed_data[var] = expression(value, var, context, ()) + + create_id = create_id or str(uuid.uuid4()) - new_thing = { - "tags": tags, - "type": " ".join(tags), - **parsed_data - } + new_thing = {"tags": tags, "type": " ".join(tags), "id": create_id, **parsed_data} if relation: - new_thing[relation] = [tup.id for tup in context.values()], + new_thing[relation] = ([tup.id for tup in context.values()],) - create_id = create_id or str(uuid.uuid4()) - create(create_id, new_thing, scene) + return create_id, new_thing -def update_from_json(update_json: list, context: dict, scene: dict) -> dict: +def update_from_json(update_json: list, context: dict) -> dict: + # print("UPDATE TRIGGERED BY APP") """Update the desired object in tinyland scene. Args: update_json: json for update statement. context: aliases mapped to their match statements. - scene: "database" of all that exists in tinyland. - See match docstring for schema. """ [_command, alias, data] = update_json parsed_data = {} for var, value in data.items(): - if isinstance(value, str) and "." in value: - alias, attribute = value.split(".") - parsed_data[var] = lookup(attribute, context[alias]) - else: - parsed_data[var] = value - - update_id = context[alias].id - update(update_id, parsed_data, scene) - - -def run(app_json: list, scene: dict, trigger_id: str=None) -> dict: - """Run tinyland app on the tinyland scene. - - Args: - app_json: json for tinyland app. - scene: "database" of all that exists in tinyland. - See match docstring for schema. - """ - [reads, writes] = app_json - - # We need to find all combinations of our match statements. - # contexts will hold one context for each time we need to run the - # write portion of our app. - contexts = [{}] - - # Populate contexts. - for match_json in reads: - new_contexts = [] - for context in contexts: - # For each context holding n-1 matches, - # find matches and create an new context with the nth match. - matches = match(match_json, context, scene) - for alias, tup in matches: - if tup.id not in [t.id for t in context.values()]: - new_contexts.append({alias: tup, **context}) - contexts = new_contexts - - for context in contexts: - if trigger_id is None or trigger_id in [tup.id for tup in context.values()]: - for write in writes: - if write[0] == Command.CREATE.name: - create_from_json(write, context, scene) - elif write[0] == Command.UPDATE.name: - update_from_json(write, context, scene) - - - - + parsed_data[var] = expression(value, var, context, ()) + return context[alias].id, parsed_data diff --git a/tinytalk/scene.py b/tinytalk/scene.py new file mode 100644 index 0000000..70fdbe6 --- /dev/null +++ b/tinytalk/scene.py @@ -0,0 +1,122 @@ +from collections import defaultdict +from functools import reduce + +from interpreter import * + + +class TinylandScene: + def __init__(self): + self.scene = {} + + self.create_triggers = defaultdict(set) + self.update_triggers = defaultdict(set) + + self.apps_by_id = {} # id integer -> app JSON + + self.cur_loop = set() + self.next_loop = set() + self.executed = set() + self.app_runs = defaultdict(lambda: 0) + + def load_app(self, app_json): + """Process app and store in self.create_triggers or self.update_triggers.""" + [reads, writes] = app_json + app_id = len(self.apps_by_id) + if app_json not in self.apps_by_id.values(): + self.apps_by_id[app_id] = app_json + for match_json in reads: + [ + _command, + adjectives, + relation, + tags, + data_conditions, + alias, + ] = match_json + if data_conditions is None: + for tag in tags: + self.create_triggers[tag].add(app_id) + else: + for tag in tags: + self.update_triggers[tag].add(app_id) + self.create_triggers[tag].add(app_id) + + def cascade(self, triggers, id_string, tags): + """Trigger apps based on tags. + + Make sure an app is only triggered once this loop for this id. + + Args: + triggers: self.create_triggers or self.update_triggers. + id_string: the id of the object that triggered this cascade. + tags: the tags of that same object. + """ + apps_triggered = reduce((lambda x, y: x | y), [triggers[tag] for tag in tags]) + for app_id in apps_triggered: + execute_args = (app_id, id_string) + if execute_args not in self.executed: + self.executed.add(execute_args) + self.run(*execute_args) + else: + self.next_loop.add(execute_args) + + def create(self, id_string: str, thing: dict): + """Create object in the scene and trigger apps.""" + self.scene[id_string] = thing + self.cascade(self.create_triggers, id_string, thing["tags"]) + + def update(self, id_string: str, data: dict): + """Update object in the scene and trigger apps.""" + thing = self.scene[id_string] + self.scene[id_string].update(data) + self.cascade(self.update_triggers, id_string, thing["tags"]) + + def run(self, app_id: int, trigger_id: str) -> dict: + """Run tinyland app on the tinyland scene. + + Args: + app_id: id for tinyland app. (key in self.apps_by_id) + """ + app_json = self.apps_by_id[app_id] + [reads, writes] = app_json + + # We need to find all combinations of our match statements. + # contexts will hold one context for each time we need to run the + # write portion of our app. + contexts = [{}] + + # Populate contexts. + for match_json in reads: + new_contexts = [] + for context in contexts: + # For each context holding n-1 matches, + # find matches and create an new context with the nth match. + matches = match(match_json, context, self.scene) + for alias, tup in matches: + if tup.id not in [t.id for t in context.values()]: + new_contexts.append({alias: tup, **context}) + contexts = new_contexts + + for context in contexts: + if trigger_id is None or trigger_id in [tup.id for tup in context.values()]: + for write in writes: + if write[0] == Command.CREATE.name: + create_id, new_thing = create_from_json(write, context) + self.create(create_id, new_thing) + elif write[0] == Command.UPDATE.name: + update_id, update_data = update_from_json(write, context) + self.update(update_id, update_data) + + def execute_loop(self): + """Run all apps currently in self.cur_loop and prepare next loop. + + Returns: + apps_run (bool): if any apps were run this loop. + """ + apps_run = False + while self.cur_loop: + apps_run = True + self.run(*self.cur_loop.pop()) + self.executed = set() + self.cur_loop, self.next_loop = self.next_loop, set() + return apps_run diff --git a/tinytalk/websocket_server.py b/tinytalk/websocket_server.py index be15eca..c161950 100644 --- a/tinytalk/websocket_server.py +++ b/tinytalk/websocket_server.py @@ -1,5 +1,7 @@ +import argparse import asyncio import json +import logging import os import parsimonious import queue @@ -8,75 +10,117 @@ import websockets from grammar import grammar, TinyTalkVisitor -import interpreter +from scene import TinylandScene + -BUFFER_SIZE = 1024 IP_ADDRESS = "127.0.0.1" UDP_PORT = 8766 -WEBSOCKET_PORT = 8765 +WEBSOCKET_PORT = 1234 APP_FILE = "app.txt" last_upload = -sys.maxsize visitor = TinyTalkVisitor() -scene = {} -q = queue.Queue(64) +udp_q = queue.Queue(64) def format_scene(scene): - formatted_scene = { - "appMarkers": {}, - "virtualObjects": {}, - } - for key, val in scene.items(): - if "marker" in val["tags"]: - formatted_scene["appMarkers"][key] = val - else: - formatted_scene["virtualObjects"][key] = val - return formatted_scene - - -async def tinyland_loop(websocket, path): - # global sock - data = q.get(True) - new_object = json.loads(data.decode("utf-8")) - - global last_upload - global scene - if os.path.getmtime(APP_FILE) > last_upload: - with open(APP_FILE, "r") as f: + for k, v in scene.items(): + if "x" in v and "y" in v: + scene[k].update({"point": [v["x"], v["y"]]}) + return {"type": "render", "payload": scene} + + +class Server: + def __init__(self, ip_address, port): + self.ip_address = ip_address + self.port = port + self.scene = TinylandScene() + + def reload_apps(self, filepath): + with open(filepath, "r") as f: apps = f.read().split("\n\n") for app in apps: try: - app_json = visitor.visit(grammar.parse(app)) + app_json = visitor.visit(grammar.parse(app.strip())) except parsimonious.exceptions.ParseError as e: - #TODO: add bad app error handling - pass + logging.error(f"Could not parse app {app_json}: {e}") else: - print(f"Loading app {app_json}") - interpreter.load_app(app_json) + logging.info(f"Loading app {app_json} from {filepath}") + self.scene.load_app(app_json) - last_upload = os.path.getmtime(APP_FILE) + def start(self): + logging.info(f"Starting WebSocket Server on {self.ip_address}:{self.port}") + return websockets.serve(self.tinyland_loop, self.ip_address, self.port) - if new_object["id"] in scene: - interpreter.update(new_object["id"], new_object, scene) - else: - interpreter.create(new_object["id"], new_object, scene) + async def tinyland_loop(self, websocket, path): + while True: + global last_upload - await websocket.send(json.dumps(format_scene(scene))) + # See if we've gotten an update over UDP + if not udp_q.empty(): + data = udp_q.get(True) + new_object = json.loads(data.decode("utf-8")) + if new_object["id"] in self.scene.scene: + self.scene.update(new_object["id"], new_object) + else: + self.scene.create(new_object["id"], new_object) + if os.path.getmtime(APP_FILE) > last_upload: + self.reload_apps(APP_FILE) + last_upload = os.path.getmtime(APP_FILE) -async def udp_listener(): + if self.scene.execute_loop(): + await websocket.send(json.dumps(format_scene(self.scene.scene))) + await asyncio.sleep(0.01) + + +async def udp_listener(ip_address, port): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind((IP_ADDRESS, UDP_PORT)) + logging.info(f"Started UDP listener on {ip_address}:{port}") + sock.settimeout(0.01) + sock.bind((ip_address, port)) while True: - data, _addr = sock.recvfrom(1024) - print(data) - q.put(data, True) - await asyncio.sleep(0.5) + try: + data, _addr = sock.recvfrom(1024) + except socket.timeout: + await asyncio.sleep(0.01) + else: + logging.debug(f"UDP received: {data}") + udp_q.put(data, True) + await asyncio.sleep(0.01) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-i", "--ip_address", default=IP_ADDRESS, type=str, help="IP address to run on." + ) + parser.add_argument( + "-p", + "--websocket_port", + default=WEBSOCKET_PORT, + type=int, + help="Port to send websocket packets to.", + ) + parser.add_argument( + "-u", + "--udp_port", + default=UDP_PORT, + type=int, + help="Port to receive UDP messages from.", + ) + parser.add_argument( + "-v", "--verbose", action="count", default=0, help="Verbosity level." + ) + args, unknown = parser.parse_known_args(sys.argv[1:]) -start_server = websockets.serve(tinyland_loop, IP_ADDRESS, WEBSOCKET_PORT) + if args.verbose == 1: + logging.getLogger().setLevel(logging.INFO) + elif args.verbose > 1: + logging.getLogger().setLevel(logging.DEBUG) -asyncio.get_event_loop().run_until_complete(start_server) -asyncio.get_event_loop().create_task(udp_listener()) -asyncio.get_event_loop().run_forever() \ No newline at end of file + ws = Server(args.ip_address, args.websocket_port) + asyncio.get_event_loop().run_until_complete(ws.start()) + asyncio.get_event_loop().create_task(udp_listener(args.ip_address, args.udp_port)) + asyncio.get_event_loop().run_forever() From c5fbd14ccda76d5c444d5ea92de24ce8d24ac796 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Tue, 10 Dec 2019 16:41:47 -0500 Subject: [PATCH 18/20] Add more grammar! - Add comments - Add arrays (change data read/write syntax to parens and give brackets to arrays) --- requirements.txt | 3 +++ tinytalk/__init__.py | 29 ---------------------------- tinytalk/app.txt | 32 ++++++++++++++++++++++++++++--- tinytalk/grammar.py | 27 +++++++++++++++++++------- tinytalk/interpreter.py | 21 +++++++++++--------- tinytalk/test_grammar.py | 37 ++++++++++++++++++++++++++---------- tinytalk/websocket_server.py | 22 ++++++++++----------- 7 files changed, 101 insertions(+), 70 deletions(-) diff --git a/requirements.txt b/requirements.txt index d572a93..bb4de55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,6 @@ parsimonious==0.8.1 black==19.3b0 hypothesis==4.41.3 pytest==5.2.2 +pytest-xdist==1.30.0 +mysql-connector-python==8.0.17 +websockets==8.1 diff --git a/tinytalk/__init__.py b/tinytalk/__init__.py index b911581..96a7461 100644 --- a/tinytalk/__init__.py +++ b/tinytalk/__init__.py @@ -1,32 +1,3 @@ -from . import data -from .grammar import Command, grammar, TinyTalkVisitor, whitespace_chars - -def dump(): - print("database dump:") - print(names) - -def read_tiny_talk(s): - print(f"***** reading {s}") - tree = grammar.parse(s) - print("***** the parse tree") - print(tree) - visitor = TinyTalkVisitor() - output = visitor.visit(tree) - print("***** output") - print(output) - -def init_database(): - data.commit("aruco", "visible", {"id": 111, "x": 0, "y": 0}) - - -def run(): - print("running simulation") - init_database() - data.commit("aruco", "visible", {"x": 10, "y": 1}) - print(data.query('aruco')) - print("done") - - # sample tinytalk programs # syntax for data: diff --git a/tinytalk/app.txt b/tinytalk/app.txt index f6b171f..2ac7f3f 100644 --- a/tinytalk/app.txt +++ b/tinytalk/app.txt @@ -1,5 +1,31 @@ -when [#marker ] as m create [#blob x: m.x y: m.y size: 10] +// ======= BALL APP ======= +// Create ball when we see ball +when (#ball ) as m create (#blob x: m.x, y: m.y, point: [m.x, m.y], size: 10, vy: 0.005, vx: 0.005) -when [#blob x where x > 1.0 y] as b update b [x: 0.0] -when [#blob x y] as b update b [x: b.x + 0.001 y: b.y] \ No newline at end of file +// Change ball velocity when we hit a wall +when (#blob y where y > 1.0) as b update b (y: 0.99, vy: b.vy * -1, point: [b.x, 0.95]) + +when (#blob y where y < 0.0) as b update b (y: 0.01, vy: b.vy * -1, point: [b.x, 0.05]) + +when (#blob x where x > 1.0) as b update b (x: 0.99, vx: b.vx * -1, point: [0.95, b.y]) + +when (#blob x where x < 0.0) as b update b (x: 0.01, vx: b.vx * -1, point: [0.05, b.y]) + + +// Update ball position based on velocity +when (#blob x y) as b update b (x: b.x + b.vx y: b.y + b.vy, point: [b.x + b.vx, b.y + b.vy]) + + +// ======= PADDLE APP ======= +// Create paddle when we see marker +when (#marker ) as m create (#line x: m.x, ya: m.y + 0.05, yb: m.y - 0.05, points: [m.x, ya, m.x, yb]) + + +// Update paddle line based on marker +when (#marker x y) as m; + (#line x ya yb) as l +update l (x: m.x, + ya: m.y + 0.05, + yb: m.y - 0.05, + points: [[l.x, l.ya], [l.x, l.yb]]) \ No newline at end of file diff --git a/tinytalk/grammar.py b/tinytalk/grammar.py index e5c2f30..16974d1 100644 --- a/tinytalk/grammar.py +++ b/tinytalk/grammar.py @@ -10,12 +10,14 @@ class Command(Enum): UPDATE = 2 COND = 3 AND = 4 + ARRAY = 5 whitespace_chars = " \n," grammar = Grammar( r""" - app = read ws? write + apps = app* + app = ws* read ws* write ws* read = "when" ws match (";" ws? match)* write = (create / update) (";" ws? (create / update))* match = adjectives? relation? begin ws? tags ws? data_condition? end (ws alias)? @@ -26,7 +28,9 @@ class Command(Enum): data = ws? datum (ws datum)* data_condition = ws? (condition / datum) (ws (condition / datum))* alias = "as" ws name_with_pronouns - ws = ~"([ \t\n,])+" + ws = blank / comment + blank = ~"([ \t\n,])+" + comment = ~"//[^\n]*\n" adjective = "one" / "only" / "global" relation = ws? "friend" tag = "#" name @@ -41,16 +45,17 @@ class Command(Enum): comparison = (ws? (">" / "<") ws?) / (ws ("is" / "not") ws) addition = (multiplication / subexpr / value) ws? ("+" / "-") ws? expr multiplication = (subexpr / value) ws? "*" ws? expr - subexpr = "(" ws? expr ws? ")" - value = number / boolean / name / string + subexpr = "{" ws? expr ws? "}" + array = "[" expr (ws expr)* "]" + value = array / number / boolean / name / string boolean = "true" / "false" string = ~'"[^\"]*"' number = ("+" / "-")? digit+ ("." digit+)? ("e" ("+" / "-") digit+)? digit = ~"[0-9]" name_with_pronouns = name ("/" name)* name = !reserved_word ~"[a-z][a-z_-]*(\.[a-z][a-z_-]*)?" - begin = ws? "[" - end = ws? "]" + begin = ws? "(" + end = ws? ")" reserved_word = ("as" / "where" / "true" / "false") &ws """ ) @@ -75,8 +80,11 @@ def unwrap(value): class TinyTalkVisitor(NodeVisitor): + def visit_apps(self, _node, visited_children): + return visited_children + def visit_app(self, _node, visited_children): - read, _ws, write = visited_children + _ws, read, _ws, write, _ws= visited_children return read, write #TODO command string? def visit_match(self, node, visited_children): @@ -145,6 +153,11 @@ def visit_tag(self, _node, visited_children): def visit_relation(self, _node, visited_children): _ws, relation = visited_children return relation.text + + def visit_array(self, node, visited_children): + _open, val, rest, _close = visited_children + data = [Command.ARRAY.name, unwrap(val)] + [unwrap(v) for [_ws, v] in rest] + return data def visit_data(self, _node, visited_children): _ws, first, rest = visited_children diff --git a/tinytalk/interpreter.py b/tinytalk/interpreter.py index 4e8cb95..87aaef7 100644 --- a/tinytalk/interpreter.py +++ b/tinytalk/interpreter.py @@ -88,15 +88,18 @@ def expression(expr_json: list, var_name: str, context: dict, row: namedtuple): # print(f"\n\n{result}\n\n") return result else: - [operator, left, right] = expr_json - left = expression(left, var_name, context, row) - right = expression(right, var_name, context, row) - if operator in ["*"]: - return left * right - elif operator in ["+"]: - return left + right - elif operator in ["-"]: - return left - right + if Command.ARRAY.name in expr_json: + return [expression(item, var_name, context, row) for item in expr_json[1:]] + else: + [operator, left, right] = expr_json + left = expression(left, var_name, context, row) + right = expression(right, var_name, context, row) + if operator in ["*"]: + return left * right + elif operator in ["+"]: + return left + right + elif operator in ["-"]: + return left - right def match(match_json: list, context: dict, scene: dict) -> list: diff --git a/tinytalk/test_grammar.py b/tinytalk/test_grammar.py index 3aa7c0e..dc01958 100644 --- a/tinytalk/test_grammar.py +++ b/tinytalk/test_grammar.py @@ -1,8 +1,10 @@ -from . import grammar, TinyTalkVisitor, whitespace_chars, Command +import random + from hypothesis import given, reject from hypothesis.strategies import composite, floats, text, sampled_from -import random +from .grammar import grammar, TinyTalkVisitor, whitespace_chars, Command + ## Tests for TinyTalk's grammar # @@ -41,6 +43,11 @@ def names_or_floats(draw): ) +@composite +def names_and_floats(draw, max_n=1): + return [draw(names_or_floats()) for _ in range(draw(sampled_from(list(range(1, max_n + 1)))))] + + @composite def conditions(draw, max_n=1): result = {"data": [], @@ -170,6 +177,16 @@ def test_multiplication(n, m, ws): assert visit(result) == ("*", n, m) +@given( + names_and_floats(), + whitespaces() +) +def test_array(a, ws): + to_parse = f"[{ws_between(ws, *a)}]" + result = grammar["array"].parse(to_parse) + assert visit(result) == [Command.ARRAY.name] + a + + @given( names_or_floats(), sampled_from(["<", ">", " is ", " not "]), @@ -184,7 +201,7 @@ def test_inequality(val_a, comp_a, val_b, comp_b, val_c, ws): ) assert visit(result) == ( Command.AND.name, - (comp_a.strip(), val_b, val_a), + (comp_a.strip(), val_a, val_b), (comp_b.strip(), val_b, val_c), ) @@ -228,13 +245,13 @@ def test_tags(lead_ws, ws, tag1, tag2): @given(names(), data(3), whitespaces()) def test_update(name, data, ws): - result = grammar["update"].parse(ws_between(ws, "update ", name, " [", data["text"], "]")) + result = grammar["update"].parse(ws_between(ws, "update ", name, " (", data["text"], ")")) assert visit(result) == (Command.UPDATE.name, name, dict(data["data"])) @given(sampled_from(["", "friend"]), names(), data(3), whitespaces()) def test_create(relation, tag, data, ws): - parse_string = ws_between(ws, "create ", relation, " [", f"#{tag} #{tag} ", data["text"], "]") + parse_string = ws_between(ws, "create ", relation, " (", f"#{tag} #{tag} ", data["text"], ")") result = grammar["create"].parse(parse_string) assert visit(result) == (Command.CREATE.name, [tag, tag], relation or None, dict(data["data"]) if data else None) @@ -250,7 +267,7 @@ def test_create(relation, tag, data, ws): def test_match(adjective, relation, tag, conditions, alias, ws): result = grammar["match"].parse( ws_between( - ws, adjective, relation, " [", f"#{tag} #{tag} ", conditions["text"], "] as ", alias)) + ws, adjective, relation, " (", f"#{tag} #{tag} ", conditions["text"], ") as ", alias)) assert visit(result) == (Command.MATCH.name, adjective or None, relation or None, @@ -268,18 +285,18 @@ def test_match(adjective, relation, tag, conditions, alias, ws): def test_app(num_matches, num_creates, num_updates, ws): if num_creates + num_updates == 0: reject() - match_texts = ["global friend [ #a x where 0 < x < 50 ] as f", "[ #x ]"] + match_texts = ["global friend ( #a x where 0 < x < 50 ) as f", "( #x )"] match_data = [ (Command.MATCH.name, "global", "friend", ["a"], - {"x": (Command.COND.name, (Command.AND.name, ("<", "x", 0.0), ("<", "x", 50.0)))}, + {"x": (Command.COND.name, (Command.AND.name, ("<", 0.0, "x"), ("<", "x", 50.0)))}, "f"), (Command.MATCH.name, None, None, ["x"], None, None)] - create_texts = ["create friend [ #a #b x: 50 ]"] + create_texts = ["create friend ( #a #b x: 50 )"] create_data = [(Command.CREATE.name, ["a", "b"], "friend", {"x": 50.0})] - update_texts = ["update paddle [ x: y ]"] + update_texts = ["update paddle ( x: y )"] update_data = [(Command.UPDATE.name, "paddle", {"x": "y"})] test_match_data = [match_data[i % len(match_data)] for i in range(num_matches)] diff --git a/tinytalk/websocket_server.py b/tinytalk/websocket_server.py index c161950..a471a09 100644 --- a/tinytalk/websocket_server.py +++ b/tinytalk/websocket_server.py @@ -24,9 +24,6 @@ def format_scene(scene): - for k, v in scene.items(): - if "x" in v and "y" in v: - scene[k].update({"point": [v["x"], v["y"]]}) return {"type": "render", "payload": scene} @@ -38,15 +35,15 @@ def __init__(self, ip_address, port): def reload_apps(self, filepath): with open(filepath, "r") as f: - apps = f.read().split("\n\n") - for app in apps: - try: - app_json = visitor.visit(grammar.parse(app.strip())) - except parsimonious.exceptions.ParseError as e: - logging.error(f"Could not parse app {app_json}: {e}") - else: - logging.info(f"Loading app {app_json} from {filepath}") - self.scene.load_app(app_json) + apps_string = f.read() + try: + apps_json = visitor.visit(grammar.parse(apps_string.strip())) + except parsimonious.exceptions.ParseError as e: + logging.error(f"Could not parse app {apps_string}: {e}") + else: + for app in apps_json: + logging.info(f"Loading app {app} from {filepath}") + self.scene.load_app(app) def start(self): logging.info(f"Starting WebSocket Server on {self.ip_address}:{self.port}") @@ -70,6 +67,7 @@ async def tinyland_loop(self, websocket, path): last_upload = os.path.getmtime(APP_FILE) if self.scene.execute_loop(): + logging.debug(f"Scene: {self.scene.scene}") await websocket.send(json.dumps(format_scene(self.scene.scene))) await asyncio.sleep(0.01) From caa163af093e429d8a5ff722cd5225aeb8489b93 Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Wed, 11 Dec 2019 20:05:17 -0500 Subject: [PATCH 19/20] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20Work=20in=20progress?= =?UTF-8?q?=20on=20ReacTIVision=20+=20Tinytalk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tinytalk/websocket_server.py | 57 ++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/tinytalk/websocket_server.py b/tinytalk/websocket_server.py index a471a09..704f08e 100644 --- a/tinytalk/websocket_server.py +++ b/tinytalk/websocket_server.py @@ -9,6 +9,9 @@ import sys import websockets +from pythonosc import osc_server +from pythonosc import dispatcher + from grammar import grammar, TinyTalkVisitor from scene import TinylandScene @@ -27,7 +30,7 @@ def format_scene(scene): return {"type": "render", "payload": scene} -class Server: +class WebsocketServer: def __init__(self, ip_address, port): self.ip_address = ip_address self.port = port @@ -55,8 +58,8 @@ async def tinyland_loop(self, websocket, path): # See if we've gotten an update over UDP if not udp_q.empty(): - data = udp_q.get(True) - new_object = json.loads(data.decode("utf-8")) + new_object = udp_q.get(True) + print(new_object) if new_object["id"] in self.scene.scene: self.scene.update(new_object["id"], new_object) else: @@ -72,22 +75,35 @@ async def tinyland_loop(self, websocket, path): await asyncio.sleep(0.01) -async def udp_listener(ip_address, port): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - logging.info(f"Started UDP listener on {ip_address}:{port}") - sock.settimeout(0.01) - sock.bind((ip_address, port)) - while True: - try: - data, _addr = sock.recvfrom(1024) - except socket.timeout: - await asyncio.sleep(0.01) - else: - logging.debug(f"UDP received: {data}") - udp_q.put(data, True) - await asyncio.sleep(0.01) +def get_udp_server(ip_address, port): + def unknown_handler(addr, *args): + logging.debug("Unknown stuff: {} {}".format(addr, args)) + + def marker_handler(addr, *args): + global udp_q + if args[0] == "set": + marker = { + "tags": ["marker"], + "type": "marker", + "id": args[2], + "x": args[3], + "y": args[4], + "a": args[5] + } + udp_q.put(marker) + disp = dispatcher.Dispatcher() + disp.map('/tuio/2Dobj', marker_handler) + disp.set_default_handler(unknown_handler) + server = osc_server.AsyncIOOSCUDPServer( + (args.ip_address, args.udp_port), + disp, + asyncio.get_event_loop()) + return server + + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( @@ -118,7 +134,10 @@ async def udp_listener(ip_address, port): elif args.verbose > 1: logging.getLogger().setLevel(logging.DEBUG) - ws = Server(args.ip_address, args.websocket_port) + ws = WebsocketServer(args.ip_address, args.websocket_port) + udp = get_udp_server(args.ip_address, args.udp_port) + asyncio.get_event_loop().run_until_complete(ws.start()) - asyncio.get_event_loop().create_task(udp_listener(args.ip_address, args.udp_port)) + asyncio.get_event_loop().create_task(udp.create_serve_endpoint()) asyncio.get_event_loop().run_forever() + From 275a17e754c6c778c5d69865f6c3fe57d468be98 Mon Sep 17 00:00:00 2001 From: Aidan Holloway-Bidwell Date: Wed, 11 Dec 2019 20:23:35 -0500 Subject: [PATCH 20/20] Integrate with actual OSC messages over UDP --- requirements.txt | 1 + tinytalk/app.txt | 2 +- tinytalk/websocket_server.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index bb4de55..f37813e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ pytest==5.2.2 pytest-xdist==1.30.0 mysql-connector-python==8.0.17 websockets==8.1 +python-osc==1.7.4 diff --git a/tinytalk/app.txt b/tinytalk/app.txt index 2ac7f3f..6791917 100644 --- a/tinytalk/app.txt +++ b/tinytalk/app.txt @@ -19,7 +19,7 @@ when (#blob x y) as b update b (x: b.x + b.vx y: b.y + b.vy, point: [b.x + b.vx, // ======= PADDLE APP ======= // Create paddle when we see marker -when (#marker ) as m create (#line x: m.x, ya: m.y + 0.05, yb: m.y - 0.05, points: [m.x, ya, m.x, yb]) +when (#marker ) as m create (#line x: m.x, ya: m.y + 0.05, yb: m.y - 0.05, points: [[m.x, ya], [m.x, yb]]) // Update paddle line based on marker diff --git a/tinytalk/websocket_server.py b/tinytalk/websocket_server.py index 704f08e..697f41b 100644 --- a/tinytalk/websocket_server.py +++ b/tinytalk/websocket_server.py @@ -23,7 +23,7 @@ last_upload = -sys.maxsize visitor = TinyTalkVisitor() -udp_q = queue.Queue(64) +udp_msg = None def format_scene(scene): @@ -55,15 +55,17 @@ def start(self): async def tinyland_loop(self, websocket, path): while True: global last_upload + global udp_msg # See if we've gotten an update over UDP - if not udp_q.empty(): - new_object = udp_q.get(True) + if udp_msg is not None: + new_object = udp_msg print(new_object) if new_object["id"] in self.scene.scene: self.scene.update(new_object["id"], new_object) else: self.scene.create(new_object["id"], new_object) + udp_msg = None if os.path.getmtime(APP_FILE) > last_upload: self.reload_apps(APP_FILE) @@ -80,7 +82,7 @@ def unknown_handler(addr, *args): logging.debug("Unknown stuff: {} {}".format(addr, args)) def marker_handler(addr, *args): - global udp_q + global udp_msg if args[0] == "set": marker = { "tags": ["marker"], @@ -90,7 +92,8 @@ def marker_handler(addr, *args): "y": args[4], "a": args[5] } - udp_q.put(marker) + udp_msg = marker + logging.debug(f"Marker: {marker}") disp = dispatcher.Dispatcher() disp.map('/tuio/2Dobj', marker_handler) @@ -102,7 +105,6 @@ def marker_handler(addr, *args): asyncio.get_event_loop()) return server - if __name__ == "__main__": parser = argparse.ArgumentParser()