Commit: 8a2d8be44debb6113b2229ef5cdccb567c5a2a74 - CCastle2 (hg)

Castle: The best Real-Time/Embedded/HighTech language EVER. Attempt 2

Commit MetaInfo

• Revision: 8a2d8be44debb6113b2229ef5cdccb567c5a2a74 (tree)
• Zeit: 2022-02-12 02:26:37
• Autor: Albert Mietus < albert AT mietus DOT nl >
• Commiter: Albert Mietus < albert AT mietus DOT nl >

Log Message

We can now parse alternativies (OrderedChoice)

Ändern Zusammenfassung

• modified: castle/ast/peg.py
• modified: castle/readers/parser/grammar.py
• modified: castle/readers/parser/visitor.py
• modified: pytst/readers/parser/d1_parse_tree/test_4_rule.py
• modified: pytst/readers/parser/d2_ast/test_3_Seq.py
• add: pytst/readers/parser/d1_parse_tree/test_3_expr.py
• delete: pytst/readers/parser/d1_parse_tree/test_3_expressions.py

Diff

diff -r 27e007fd92d5 -r 8a2d8be44deb castle/ast/peg.py
--- a/castle/ast/peg.py	Tue Feb 08 22:55:16 2022 +0100
+++ b/castle/ast/peg.py	Fri Feb 11 18:26:37 2022 +0100
@@ -13,13 +13,14 @@
     """With this MixIn PEG-classes get the ``.value`` property"""
 
     def __init__(self, *, value=None, **kwargs):
+        logger.debug(f'{self._typeName(self)}.MixIn_value_attribute:: value:=' +
+                     ('[[' +', '.join(f'{v}:{type(v).__name__}' for v in value) + ']]') if isinstance(value, list) else f's>>{value}<<')
         super().__init__(**kwargs)
-        logger.debug(f'{self._typeName(self)}:: value:={value}:{self._typeName(value)}')
         self._value=value
 
     @property
     def value(self):
-        logger.debug(f'{self._typeName(self)}:: @property={self._value}')
+        logger.debug(f'{self._typeName(self)}:: @value={self._value}')
         return self._value
 
 
@@ -27,17 +28,21 @@
     """With this MixIn PEG-classes get the ``.expr`` property"""
 
     def __init__(self, *, expr=None, **kwargs):
+        logger.debug(f'{self._typeName(self)}.MixIn_expr_attribute:: expr:={self._valType(expr)}')
         super().__init__(**kwargs)
         self._expr = expr
 
     @property
     def expr(self):
+        logger.debug(f'{self._typeName(self)}:: @expr={self._expr}')
         return self._expr
 
 
 class MixIn_children_tuple:
     """With this MixIn PEG-class get the ``.children`` property; and sequence-alike methods"""
     def __init__(self, *, children, **kwargs):
+        logger.debug(f'{self._typeName(self)}.MixIn_children_tuple:: children[{len(children)}]:=' +
+                     ('[[' +', '.join(f'{c}:{type(c).__name__}' for c in children) + ']]') if isinstance(children, list) else f's>>{children}<<')
         super().__init__(**kwargs)
         self._childeren = tuple(children)
 
@@ -125,8 +130,14 @@
     def __len__(self):       	return len(self._value)
     def __getitem__(self, n):	return self._value[n]
 
+    def __str__(self): # mostly for debugging
+        return "Seq{{" + " ; ".join(f"{c}" for c in self._value) + "}}" # XXX ToDo: _value -> children
 
-class OrderedChoice(Expression):pass                                    # It a an set of alternatives
+class OrderedChoice(MixIn_children_tuple, Expression):                  # A | B | C | ...  the order is relevant
+    """OC: A _tuple_ of alternative expressions"""
+
+    def __str__(self): # mostly for debugging
+        return "OC{{" + " | ".join(f"{c}" for c in self._childeren) + "}}"
 
 class Optional(Quantity):pass
 class ZeroOrMore(Quantity):pass

diff -r 27e007fd92d5 -r 8a2d8be44deb castle/readers/parser/grammar.py
--- a/castle/readers/parser/grammar.py	Tue Feb 08 22:55:16 2022 +0100
+++ b/castle/readers/parser/grammar.py	Fri Feb 11 18:26:37 2022 +0100
@@ -4,14 +4,17 @@
 
 def peg_grammar(): 	return rules, EOF
 def rules(): 		return OneOrMore(rule)
-def rule():		return rule_name, '<-', expressions, ";"
+def rule():		return rule_name, '<-', expression, ";"
 
-def expressions():	return ( OneOrMore(single_expr), Optional( '|' , expressions ) )
-def single_expr():	return ( [ rule_crossref, term, group, predicate ], op_quantity)
+def expression():	return expressions, op_alternative
+def expressions():	return OneOrMore(single_expr)
+def single_expr():	return [ rule_crossref, term, group, predicate ], op_quantity
 
+def op_alternative():   return Optional( '|' , expression )
 def op_quantity(): 	return Optional([ '?' , '*' , '+' , '#' ])
+
 def term():		return [ str_term, regex_term ]
-def group():		return '(', expressions, ')'
+def group():		return '(', expression, ')'
 def predicate():	return ['&','!'], single_expr
 
 def str_term():		return [  (S3,   str_no_s3,  S3),

diff -r 27e007fd92d5 -r 8a2d8be44deb castle/readers/parser/visitor.py
--- a/castle/readers/parser/visitor.py	Tue Feb 08 22:55:16 2022 +0100
+++ b/castle/readers/parser/visitor.py	Fri Feb 11 18:26:37 2022 +0100
@@ -17,9 +17,11 @@
 #NO_VISITOR_NEEDED: visit_term
 #NO_VISITOR_NEEDED: visit_re_no_slash
 #NO_VISITOR_NEEDED: visit_group
-#NO_VISITOR_NEEDED: visit_op_quantity (before: vist_expr_quantity)
+#NO_VISITOR_NEEDED: visit_op_quantity -- handle in visit_single_expr
 
 class PegVisitor(arpeggio.PTNodeVisitor):
+    def _logstr_node_children(self, node, children):
+        return f'>>{node}<< children[{len(children)}] >>' + ", ".join(f'{c}:{type(c).__name__}' for c in children) + '<<'
 
     def visit_str_term(self, node, children):
         return peg.StrTerm(value=node[1], parse_tree=node)
@@ -33,7 +35,7 @@
     def visit_rule_crossref(self, node, children):
         return peg.ID(name=str(node), parse_tree=node)
 
-    def visit_rule(self, node, children):                               #  Name '<-' expressions ';'
+    def visit_rule(self, node, children):                               #  Name '<-' expression ';'
         return peg.Rule(name=children[0],expr=children[1], parse_tree=node)
 
 
@@ -62,11 +64,32 @@
             raise  NotImplementedError("visit_single_expr, len>2")      # XXX -- Is this possible?
 
 
-    def visit_expressions(self, node, children):                        # OneOrMore(single_expr), Optional( '|' , expressions )
-        logger.debug(f'visit_expressions:: >>{node}<< #children={len(children)} children={children}:{type(children)}')
+    def visit_expression(self, node, children):                  # ( expressions, op_alternatives )
+        logger.debug('visit_expression::' + self._logstr_node_children(node, children))
+        if len(children) == 1: #Only expressions
+            return children[0]
+        elif len(children) == 2: # So, having with alternatives
+            # The 1st kid is a peg.Sequence ``expr``
+            # The 2nd kid can be a
+            # - peg.Sequence  `` | expr ``, OR
+            # - peg.OrderedChoice  `` | expr | expr``
+            # In all cased a (single) OrderedChoice with a list of alternatives should be returned.
+#           return peg.OrderedChoice(children = ((children[0].value,) +
+#                                                children[1]._childeren if isinstance(children[1], peg.OrderedChoice) else children[1].value), #XXX HACK
+#                                    parse_tree=node)
+            if isinstance(children[1], peg.OrderedChoice):
+                alternatives = [children[0]] + [alt for alt in children[1]]
+            else:
+                alternatives = children
+            return peg.OrderedChoice(children = alternatives, parse_tree=node)
+        else:
+            raise NotImplementedError("visit_expression, len>2")
+
+
+    def visit_expressions(self, node, children):                 # OneOrMore(single_expr)
+        logger.debug(f'visit_expressions::{self._logstr_node_children(node, children)}')
         return peg.Sequence(value=children, parse_tree=node)
 
-        raise NotImplementedError("visit_expressions, len>1 :: peg.OrderedChoice")
 
     def visit_predicate(self, node, children):
         token_2_predicate = {'&': peg.AndPredicate,

diff -r 27e007fd92d5 -r 8a2d8be44deb pytst/readers/parser/d1_parse_tree/test_3_expr.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pytst/readers/parser/d1_parse_tree/test_3_expr.py	Fri Feb 11 18:26:37 2022 +0100
@@ -0,0 +1,65 @@
+import pytest
+import logging;logger = logging.getLogger(__name__)
+
+from castle.readers.parser import grammar
+
+import arpeggio
+
+R, S, X = grammar.regex_term.__name__, grammar.str_term.__name__, grammar.rule_crossref.__name__  # shortcut in grammar
+P = grammar.predicate.__name__
+G = grammar.group.__name__
+
+def parse_expression(txt, pattern=None):
+    parser = arpeggio.ParserPython(grammar.expression)
+    parse_tree = parser.parse(txt)
+    logger.debug("\nPARSE-TREE\n" + parse_tree.tree_str()+'\n')
+
+    assert parse_tree.position_end == len(txt) , f"Not parsed whole input; Only: >>{txt[parse_tree.position: parse_tree.position_end]}<<; Not: >>{txt[parse_tree.position_end:]}<<."
+    assert parse_tree.rule_name == "expression"
+
+    if pattern: validate_pattern(parse_tree, pattern=pattern)
+
+    return parse_tree
+
+def validate_pattern(pt, pattern=None):
+    expressions = pt[0]
+    assert len(expressions) == len(pattern), f"Not correct number-of-element"
+
+    for p, s in zip(pattern, expressions):
+        if p is None: continue
+        if p == X:
+            assert s[0].rule_name == p
+        elif p in (S,R):
+            assert s[0][0].rule_name == p  # S => T => str/regex
+        elif isinstance(p, tuple):         # Group: '(' ... ')'
+            assert s[0].rule_name == G
+            validate_pattern(s[0][1:-1][0], pattern=p) # G=>E=>
+        elif p == P:
+            assert False, "To Do: Predicate"
+        else:
+            assert False, "To Do: More"
+
+
+def test_simple_1():	parse_expression(r"abc",	pattern=[X])
+def test_simple_2():	parse_expression(r'A  Bc',	pattern=[X, X])
+
+def test_string_1():	parse_expression(r"'abc'",	pattern=[S])
+def test_regexp_1():	parse_expression(r"/re/",	pattern=[R])
+
+def test_mix():		parse_expression(r'/regex/ "string" crossref crossref',	pattern=[R,S, X, X])
+
+def test_sub():		parse_expression(r'( A  B )',	pattern=[(X, X)])
+def test_mix_nosub():	parse_expression(r'/regex/ "string" ( A  B ) crossref',	pattern=[R,S, None, X])
+def test_mix_sub():	parse_expression(r'/regex/ "string" ( A  B ) crossref',	pattern=[R,S, (X, X), X])
+
+def test_sub_sub():	parse_expression(r'level0 ( level1_1  (level2a level2b ) level1_2) level0', pattern=[X, (X, (X,X), X), X])
+
+
+def test_bug1():	parse_expression(r"""( rule_crossref | term | group | predicate ) ( '?' | '*' | '+' | '#' )?""")
+def test_bug1a():	parse_expression(r"""( rule_crossref | term | group | predicate )""")
+def test_bug1a1():	parse_expression(r"""A | B | C | D""")
+def test_bug1a2():	parse_expression(r"""(A | B | C | D)""")
+def test_bug1b():	parse_expression(r"""( rule_crossref | term | group | predicate ) ( '?' | '*' | '+' | '#' )""")
+def test_bug1c():	parse_expression(r"""( '?' | '*' | '+' | '#' )""")
+
+

diff -r 27e007fd92d5 -r 8a2d8be44deb pytst/readers/parser/d1_parse_tree/test_3_expressions.py
--- a/pytst/readers/parser/d1_parse_tree/test_3_expressions.py	Tue Feb 08 22:55:16 2022 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,59 +0,0 @@
-import pytest
-import logging;logger = logging.getLogger(__name__)
-
-from castle.readers.parser import grammar
-
-import arpeggio
-
-R, S, X = grammar.regex_term.__name__, grammar.str_term.__name__, grammar.rule_crossref.__name__  # shortcut in grammar
-P = grammar.predicate.__name__
-G = grammar.group.__name__
-
-def parse_expressions(txt, pattern=None):
-    parser = arpeggio.ParserPython(grammar.expressions)
-    parse_tree = parser.parse(txt)
-    logger.info("\nPARSE-TREE\n" + parse_tree.tree_str()+'\n')
-
-    assert parse_tree.position_end == len(txt) , f"Not parsed whole input; Only: >>{txt[parse_tree.position: parse_tree.position_end]}<<; Not: >>{txt[parse_tree.position_end:]}<<."
-    assert parse_tree.rule_name == "expressions"
-
-    if pattern: validate_pattern(parse_tree, pattern=pattern)
-
-    return parse_tree
-
-def validate_pattern(pt, pattern=None):
-    assert len(pt) == len(pattern), f"Not correct number-of-element"
-
-    for p, s in zip(pattern, pt): # E <- S* (| E)?
-        if p is None: continue
-        if p == X:
-            assert s[0].rule_name == p
-        elif p in (S,R):
-            assert s[0][0].rule_name == p  # S => T => str/regex
-        elif isinstance(p, tuple):         # Group: '(' ... ')'
-            assert s[0].rule_name == G
-            validate_pattern(s[0][1:-1][0], pattern=p) # G=>E=>
-        elif p == P:
-            assert False, "To Do: Predicate"
-        else:
-            assert False, "To Do: More"
-
-
-def test_simple_1():	parse_expressions(r"abc",	pattern=[X])
-def test_simple_2():	parse_expressions(r'A  Bc',	pattern=[X, X])
-
-def test_string_1():	parse_expressions(r"'abc'",	pattern=[S])
-def test_regexp_1():	parse_expressions(r"/re/",	pattern=[R])
-
-def test_mix():		parse_expressions(r'/regex/ "string" crossref crossref',	pattern=[R,S, X, X])
-
-def test_sub():		parse_expressions(r'( A  B )',	pattern=[(X, X)])
-def test_mix_nosub():	parse_expressions(r'/regex/ "string" ( A  B ) crossref',	pattern=[R,S, None, X])
-def test_mix_sub():	parse_expressions(r'/regex/ "string" ( A  B ) crossref',	pattern=[R,S, (X, X), X])
-
-def test_sub_sub():	parse_expressions(r'level0 ( level1_1  (level2a level2b ) level1_2) level0', pattern=[X, (X, (X,X), X), X])
-
-
-
-
-

diff -r 27e007fd92d5 -r 8a2d8be44deb pytst/readers/parser/d1_parse_tree/test_4_rule.py
--- a/pytst/readers/parser/d1_parse_tree/test_4_rule.py	Tue Feb 08 22:55:16 2022 +0100
+++ b/pytst/readers/parser/d1_parse_tree/test_4_rule.py	Fri Feb 11 18:26:37 2022 +0100
@@ -9,22 +9,22 @@
 def parse_rule(txt, pattern=None):
     parser = arpeggio.ParserPython(grammar.rule)
     tree = parser.parse(txt)
-    logger.info(f'\nTREE\n{tree.tree_str()}')
+    logger.debug(f'\nTREE\n{tree.tree_str()}')
 
     assert tree.position_end == len(txt) , f"Not parsed whole input; Only: >>{txt[tree.position: tree.position_end]}<<; Not: >>{txt[tree.position_end:]}<<."
     assert len(tree) == 4, 		      		"A rule should have length=4; ..."
     assert tree[0].rule_name == "rule_name",  		"  at [0], the name of the rule"
     assert str(tree[1]) == '<-',                        "  then a arrow"
-    assert tree[2].rule_name == "expressions",		"  at [2] an ordered_choice"
+    assert tree[2].rule_name == "expression",		"  at [2] an expression" 
     assert str(tree[3]) == ';',                         "  and the the closing ':'"
 
     return tree
 
 
-def test_simple():	parse_rule(r"R <- A  B C ;")
+def test_simple():	parse_rule(r"R <- A B C ;")
 def test_OC():		parse_rule(r"Alts <-  This | That | Or So ;")
 
-def test_rule_rule():	parse_rule(r"""RULE	           <-  RULE_NAME '<-' ORDERED_CHOICE ';' ;""")
+def test_rule_rule():	parse_rule(r"""RULE  <-  RULE_NAME '<-' ORDERED_CHOICE ';' ;""")
 def test_expression_rule():	parse_rule(r"""
    expression	   <- regex_term
    		   |  rule_crossref

diff -r 27e007fd92d5 -r 8a2d8be44deb pytst/readers/parser/d2_ast/test_3_Seq.py
--- a/pytst/readers/parser/d2_ast/test_3_Seq.py	Tue Feb 08 22:55:16 2022 +0100
+++ b/pytst/readers/parser/d2_ast/test_3_Seq.py	Fri Feb 11 18:26:37 2022 +0100
@@ -1,6 +1,5 @@
 """Test that a sequence of expressions is an Expression()
-
-   Note: the value of Expression() is a list-subclass; which is fine. But use it as list!!"""
+"""
 
 import pytest
 import logging; logger = logging.getLogger(__name__)
@@ -23,17 +22,24 @@
         ast = parse(txt, grammar.single_expr)
 
 
-def test_seq_of_two_as_expressions():
+def test_seq_of_two_as_expression():
     txt = "A B"
-    ast = parse(txt, grammar.expressions)
+    ast = parse(txt, grammar.expression)
 
     assert_Seq(ast, 2, ids=('A', 'B'))
     assert isinstance(ast.value, list),		"It will be an `arpeggio.SemanticActionResult` which is a subclass of list"
 
+def test_seq_of_three_as_expression():
+    txt = "A B C"
+    ast = parse(txt, grammar.expression)
+
+    assert_Seq(ast, 3, ids=('A', 'B', 'C'))
+    assert isinstance(ast.value, list),		"It will be an `arpeggio.SemanticActionResult` which is a subclass of list"
+
 
 def test_seq_of_three_with_quantification():
     txt = "A? B+ C*"
-    ast = parse(txt, grammar.expressions)
+    ast = parse(txt, grammar.expression)
 
     assert_Seq(ast, 3)
 
@@ -46,7 +52,31 @@
     assert_ID(ast[2].expr, 'C'),	"The 3th one is a 'C'"
 
 
+def assert_OC(ast, length_pattern):
+    assert isinstance(ast, peg.OrderedChoice)
+    assert isinstance(ast, peg.Expression), "An OrderedChoice is also a Expression"
+    assert len(ast) == len(length_pattern), "Not the correct number of alternatives"
+    for i, (alt, l) in enumerate(zip(ast, length_pattern)):
+        if l is  not None:
+            assert len(alt) == l, f'The {i}th alternative does not match the specified length -- {alt}'
+        logger.debug(f'OC-alt[{i}] ==> {alt}')
+
 def test_OrderedChoice_of_two_alternatives():
     txt = "A | B"
-    ast = parse(txt, grammar.expressions)
-    assert False # XXX ToBoDone
+    ast = parse(txt, grammar.expression)
+    logger.debug(f"OC.2:: {ast}")
+    assert_OC(ast, length_pattern=[1,1])
+
+def test_OrderedChoice_of_three_alternatives():
+    txt = "A | B | C"
+    ast = parse(txt, grammar.expression)
+    logger.debug(f"OC.3:: {ast}")
+    assert_OC(ast, length_pattern=[1,1,1])
+
+
+def test_OrderedChoice_of_long_alternatives():
+    txt = "A | b1 b2 | C"
+    ast = parse(txt, grammar.expression)
+    logger.debug(f"OC.long:: {ast}")
+
+    assert_OC(ast, length_pattern=[1,2,1])