Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhance rule success strategy #82

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Enhance rule success strategy #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fb9d6ad
grammar.py: add Rule.is_loop1()
0dminnimda Feb 19, 2023
1bd7b06
parser.py: add last rule matching
0dminnimda Feb 19, 2023
22af813
python_generator.py: now the success is based on matching
0dminnimda Feb 19, 2023
cde3283
python_generator.py: extract RuleProperties into a class
0dminnimda Feb 26, 2023
d536365
Revert the changes adding `_matched`
0dminnimda Feb 26, 2023
420e6b2
python_generator.py: let call maker decide for None check
0dminnimda Feb 26, 2023
b06bfdf
Tests: carry_repeat0, None_termination, False_return_value
0dminnimda Feb 26, 2023
832d732
metagrammar.gram: sort imports
0dminnimda Feb 27, 2023
7b4d493
Update grammar_parser.py
0dminnimda Feb 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/pegen/grammar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ def __init__(self, name: str, type: Optional[str], rhs: Rhs, memo: Optional[obje
def is_loop(self) -> bool:
return self.name.startswith("_loop")

def is_loop1(self) -> bool:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this can be removes as it's not used anywhere

return self.name.startswith("_loop1")

def is_gather(self) -> bool:
return self.name.startswith("_gather")

Expand Down
131 changes: 78 additions & 53 deletions src/pegen/grammar_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
Rule,
RuleList,
RuleName,
StringLeaf
StringLeaf,
)
from pegen.parser import Parser, logger, memoize, memoize_left_rec

Expand All @@ -43,7 +43,7 @@ class GeneratedParser(Parser):
def start(self) -> Optional[Grammar]:
# start: grammar $
mark = self._mark()
if (grammar := self.grammar()) and (self.expect("ENDMARKER")):
if (grammar := self.grammar()) is not None and (self.expect("ENDMARKER")):
return grammar
self._reset(mark)
return None
Expand All @@ -52,10 +52,10 @@ def start(self) -> Optional[Grammar]:
def grammar(self) -> Optional[Grammar]:
# grammar: metas rules | rules
mark = self._mark()
if (metas := self.metas()) and (rules := self.rules()):
if (metas := self.metas()) is not None and (rules := self.rules()) is not None:
return Grammar(rules, metas)
self._reset(mark)
if rules := self.rules():
if (rules := self.rules()) is not None:
return Grammar(rules, [])
self._reset(mark)
return None
Expand All @@ -64,10 +64,10 @@ def grammar(self) -> Optional[Grammar]:
def metas(self) -> Optional[MetaList]:
# metas: meta metas | meta
mark = self._mark()
if (meta := self.meta()) and (metas := self.metas()):
if (meta := self.meta()) is not None and (metas := self.metas()) is not None:
return [meta] + metas
self._reset(mark)
if meta := self.meta():
if (meta := self.meta()) is not None:
return [meta]
self._reset(mark)
return None
Expand Down Expand Up @@ -101,10 +101,10 @@ def meta(self) -> Optional[MetaTuple]:
def rules(self) -> Optional[RuleList]:
# rules: rule rules | rule
mark = self._mark()
if (rule := self.rule()) and (rules := self.rules()):
if (rule := self.rule()) is not None and (rules := self.rules()) is not None:
return [rule] + rules
self._reset(mark)
if rule := self.rule():
if (rule := self.rule()) is not None:
return [rule]
self._reset(mark)
return None
Expand All @@ -114,33 +114,33 @@ def rule(self) -> Optional[Rule]:
# rule: rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" alts NEWLINE
mark = self._mark()
if (
(rulename := self.rulename())
and (opt := self.memoflag(),)
(rulename := self.rulename()) is not None
and (opt := self.memoflag(),) is not None
and (self.expect(":"))
and (alts := self.alts())
and (alts := self.alts()) is not None
and (self.expect("NEWLINE"))
and (self.expect("INDENT"))
and (more_alts := self.more_alts())
and (more_alts := self.more_alts()) is not None
and (self.expect("DEDENT"))
):
return Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt)
self._reset(mark)
if (
(rulename := self.rulename())
and (opt := self.memoflag(),)
(rulename := self.rulename()) is not None
and (opt := self.memoflag(),) is not None
and (self.expect(":"))
and (self.expect("NEWLINE"))
and (self.expect("INDENT"))
and (more_alts := self.more_alts())
and (more_alts := self.more_alts()) is not None
and (self.expect("DEDENT"))
):
return Rule(rulename[0], rulename[1], more_alts, memo=opt)
self._reset(mark)
if (
(rulename := self.rulename())
and (opt := self.memoflag(),)
(rulename := self.rulename()) is not None
and (opt := self.memoflag(),) is not None
and (self.expect(":"))
and (alts := self.alts())
and (alts := self.alts()) is not None
and (self.expect("NEWLINE"))
):
return Rule(rulename[0], rulename[1], alts, memo=opt)
Expand All @@ -151,7 +151,7 @@ def rule(self) -> Optional[Rule]:
def rulename(self) -> Optional[RuleName]:
# rulename: NAME annotation | NAME
mark = self._mark()
if (name := self.name()) and (annotation := self.annotation()):
if (name := self.name()) and (annotation := self.annotation()) is not None:
return (name.string, annotation)
self._reset(mark)
if name := self.name():
Expand All @@ -172,10 +172,14 @@ def memoflag(self) -> Optional[str]:
def alts(self) -> Optional[Rhs]:
# alts: alt "|" alts | alt
mark = self._mark()
if (alt := self.alt()) and (self.expect("|")) and (alts := self.alts()):
if (
(alt := self.alt()) is not None
and (self.expect("|"))
and (alts := self.alts()) is not None
):
return Rhs([alt] + alts.alts)
self._reset(mark)
if alt := self.alt():
if (alt := self.alt()) is not None:
return Rhs([alt])
self._reset(mark)
return None
Expand All @@ -186,13 +190,13 @@ def more_alts(self) -> Optional[Rhs]:
mark = self._mark()
if (
(self.expect("|"))
and (alts := self.alts())
and (alts := self.alts()) is not None
and (self.expect("NEWLINE"))
and (more_alts := self.more_alts())
and (more_alts := self.more_alts()) is not None
):
return Rhs(alts.alts + more_alts.alts)
self._reset(mark)
if (self.expect("|")) and (alts := self.alts()) and (self.expect("NEWLINE")):
if (self.expect("|")) and (alts := self.alts()) is not None and (self.expect("NEWLINE")):
return Rhs(alts.alts)
self._reset(mark)
return None
Expand All @@ -201,16 +205,20 @@ def more_alts(self) -> Optional[Rhs]:
def alt(self) -> Optional[Alt]:
# alt: items '$' action | items '$' | items action | items
mark = self._mark()
if (items := self.items()) and (self.expect("$")) and (action := self.action()):
if (
(items := self.items()) is not None
and (self.expect("$"))
and (action := self.action()) is not None
):
return Alt(items + [NamedItem(None, NameLeaf("ENDMARKER"))], action=action)
self._reset(mark)
if (items := self.items()) and (self.expect("$")):
if (items := self.items()) is not None and (self.expect("$")):
return Alt(items + [NamedItem(None, NameLeaf("ENDMARKER"))], action=None)
self._reset(mark)
if (items := self.items()) and (action := self.action()):
if (items := self.items()) is not None and (action := self.action()) is not None:
return Alt(items, action=action)
self._reset(mark)
if items := self.items():
if (items := self.items()) is not None:
return Alt(items, action=None)
self._reset(mark)
return None
Expand All @@ -219,10 +227,10 @@ def alt(self) -> Optional[Alt]:
def items(self) -> Optional[NamedItemList]:
# items: named_item items | named_item
mark = self._mark()
if (named_item := self.named_item()) and (items := self.items()):
if (named_item := self.named_item()) is not None and (items := self.items()) is not None:
return [named_item] + items
self._reset(mark)
if named_item := self.named_item():
if (named_item := self.named_item()) is not None:
return [named_item]
self._reset(mark)
return None
Expand All @@ -234,10 +242,10 @@ def named_item(self) -> Optional[NamedItem]:
cut = False
if (
(name := self.name())
and (annotation := self.annotation())
and (annotation := self.annotation()) is not None
and (self.expect("="))
and (cut := True)
and (item := self.item())
and (item := self.item()) is not None
):
return NamedItem(name.string, item, annotation)
self._reset(mark)
Expand All @@ -248,19 +256,19 @@ def named_item(self) -> Optional[NamedItem]:
(name := self.name())
and (self.expect("="))
and (cut := True)
and (item := self.item())
and (item := self.item()) is not None
):
return NamedItem(name.string, item)
self._reset(mark)
if cut:
return None
if item := self.item():
if (item := self.item()) is not None:
return NamedItem(None, item)
self._reset(mark)
if it := self.forced_atom():
if (it := self.forced_atom()) is not None:
return NamedItem(None, it)
self._reset(mark)
if it := self.lookahead():
if (it := self.lookahead()) is not None:
return NamedItem(None, it)
self._reset(mark)
return None
Expand All @@ -270,7 +278,12 @@ def forced_atom(self) -> Optional[LookaheadOrCut]:
# forced_atom: '&' '&' ~ atom
mark = self._mark()
cut = False
if (self.expect("&")) and (self.expect("&")) and (cut := True) and (atom := self.atom()):
if (
(self.expect("&"))
and (self.expect("&"))
and (cut := True)
and (atom := self.atom()) is not None
):
return Forced(atom)
self._reset(mark)
if cut:
Expand All @@ -282,13 +295,13 @@ def lookahead(self) -> Optional[LookaheadOrCut]:
# lookahead: '&' ~ atom | '!' ~ atom | '~'
mark = self._mark()
cut = False
if (self.expect("&")) and (cut := True) and (atom := self.atom()):
if (self.expect("&")) and (cut := True) and (atom := self.atom()) is not None:
return PositiveLookahead(atom)
self._reset(mark)
if cut:
return None
cut = False
if (self.expect("!")) and (cut := True) and (atom := self.atom()):
if (self.expect("!")) and (cut := True) and (atom := self.atom()) is not None:
return NegativeLookahead(atom)
self._reset(mark)
if cut:
Expand All @@ -303,29 +316,34 @@ def item(self) -> Optional[Item]:
# item: '[' ~ alts ']' | atom '?' | atom '*' | atom '+' | atom '.' atom '+' | atom
mark = self._mark()
cut = False
if (self.expect("[")) and (cut := True) and (alts := self.alts()) and (self.expect("]")):
if (
(self.expect("["))
and (cut := True)
and (alts := self.alts()) is not None
and (self.expect("]"))
):
return Opt(alts)
self._reset(mark)
if cut:
return None
if (atom := self.atom()) and (self.expect("?")):
if (atom := self.atom()) is not None and (self.expect("?")):
return Opt(atom)
self._reset(mark)
if (atom := self.atom()) and (self.expect("*")):
if (atom := self.atom()) is not None and (self.expect("*")):
return Repeat0(atom)
self._reset(mark)
if (atom := self.atom()) and (self.expect("+")):
if (atom := self.atom()) is not None and (self.expect("+")):
return Repeat1(atom)
self._reset(mark)
if (
(sep := self.atom())
(sep := self.atom()) is not None
and (self.expect("."))
and (node := self.atom())
and (node := self.atom()) is not None
and (self.expect("+"))
):
return Gather(sep, node)
self._reset(mark)
if atom := self.atom():
if (atom := self.atom()) is not None:
return atom
self._reset(mark)
return None
Expand All @@ -335,7 +353,12 @@ def atom(self) -> Optional[Plain]:
# atom: '(' ~ alts ')' | NAME | STRING
mark = self._mark()
cut = False
if (self.expect("(")) and (cut := True) and (alts := self.alts()) and (self.expect(")")):
if (
(self.expect("("))
and (cut := True)
and (alts := self.alts()) is not None
and (self.expect(")"))
):
return Group(alts)
self._reset(mark)
if cut:
Expand All @@ -356,7 +379,7 @@ def action(self) -> Optional[str]:
if (
(self.expect("{"))
and (cut := True)
and (target_atoms := self.target_atoms())
and (target_atoms := self.target_atoms()) is not None
and (self.expect("}"))
):
return target_atoms
Expand All @@ -373,7 +396,7 @@ def annotation(self) -> Optional[str]:
if (
(self.expect("["))
and (cut := True)
and (target_atoms := self.target_atoms())
and (target_atoms := self.target_atoms()) is not None
and (self.expect("]"))
):
return target_atoms
Expand All @@ -386,10 +409,12 @@ def annotation(self) -> Optional[str]:
def target_atoms(self) -> Optional[str]:
# target_atoms: target_atom target_atoms | target_atom
mark = self._mark()
if (target_atom := self.target_atom()) and (target_atoms := self.target_atoms()):
if (target_atom := self.target_atom()) is not None and (
target_atoms := self.target_atoms()
) is not None:
return target_atom + " " + target_atoms
self._reset(mark)
if target_atom := self.target_atom():
if (target_atom := self.target_atom()) is not None:
return target_atom
self._reset(mark)
return None
Expand All @@ -402,7 +427,7 @@ def target_atom(self) -> Optional[str]:
if (
(self.expect("{"))
and (cut := True)
and (atoms := self.target_atoms(),)
and (atoms := self.target_atoms(),) is not None
and (self.expect("}"))
):
return "{" + (atoms or "") + "}"
Expand All @@ -413,7 +438,7 @@ def target_atom(self) -> Optional[str]:
if (
(self.expect("["))
and (cut := True)
and (atoms := self.target_atoms(),)
and (atoms := self.target_atoms(),) is not None
and (self.expect("]"))
):
return "[" + (atoms or "") + "]"
Expand Down
6 changes: 3 additions & 3 deletions src/pegen/metagrammar.gram
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,16 @@ from pegen.grammar import (
Cut,
Forced,
Gather,
Grammar,
Group,
Item,
Lookahead,
LookaheadOrCut,
MetaTuple,
MetaList,
NameLeaf,
MetaTuple,
NamedItem,
NamedItemList,
NameLeaf,
NegativeLookahead,
Opt,
Plain,
Expand All @@ -25,7 +26,6 @@ from pegen.grammar import (
Rule,
RuleList,
RuleName,
Grammar,
StringLeaf,
)
"""
Expand Down
Loading