From 74a2d98e71e92287c258e0a09efa3a1af45add0b Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Sat, 24 Jun 2023 09:12:57 -0300 Subject: [PATCH] wip --- .../parser/grammar310/PythonGrammar310.java | 146 +++++++++++------- .../parser/grammar310/TreeBuilder310.java | 68 +++++--- .../python/pydev/parser/grammar310/python.jjt | 2 +- .../parser/grammar310/python.jjt_template | 2 +- .../parser/grammar311/PythonGrammar311.java | 146 +++++++++++------- .../parser/grammar311/TreeBuilder311.java | 68 +++++--- .../python/pydev/parser/grammar311/python.jjt | 2 +- .../parser/grammar311/python.jjt_template | 2 +- .../grammarcommon/AbstractTreeBuilder.java | 2 +- .../AbstractTreeBuilderHelpers.java | 2 +- .../parser/grammarcommon/ITreeConstants.java | 1 + .../parser/jython/ast/ISimpleNodeSwitch.java | 2 +- .../{MatchKeyword.java => MatchKeyVal.java} | 34 ++-- .../parser/jython/ast/MatchKeyValType.java | 99 ++++++++++++ .../pydev/parser/jython/ast/MatchMapping.java | 69 ++++----- .../pydev/parser/jython/ast/Python.asdl | 7 +- .../pydev/parser/jython/ast/VisitorBase.java | 2 +- .../pydev/parser/jython/ast/VisitorIF.java | 2 +- .../parser/jython/ast/name_contextType.java | 2 + .../MakeAstValidForPrettyPrintingVisitor.java | 4 +- .../PrettyPrinterVisitorV2.java | 14 +- .../python/pydev/parser/PyParser310Test.java | 46 ++++++ .../python/pydev/parser/PyParser311Test.java | 9 -- .../ast/visitors/FindDuplicatesVisitor.java | 4 +- 24 files changed, 489 insertions(+), 246 deletions(-) rename plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/{MatchKeyword.java => MatchKeyVal.java} (72%) create mode 100644 plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyValType.java diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/PythonGrammar310.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/PythonGrammar310.java index c6f01ba084..9b0b2ff968 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/PythonGrammar310.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/PythonGrammar310.java @@ -4476,29 +4476,35 @@ final public void closed_pattern() throws ParseException { } final public void literal_pattern() throws ParseException { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { - case PLUS: - case MINUS: - case DECNUMBER: - case HEXNUMBER: - case OCTNUMBER: - case BINNUMBER: - case FLOAT: - case COMPLEX: - signed_number(); + /*@bgen(jjtree) literal_pattern */ + SimpleNode jjtn000 = builder.openNode( JJTLITERAL_PATTERN); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); + jjtreeOpenNodeScope(jjtn000); + try { switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { case PLUS: case MINUS: + case DECNUMBER: + case HEXNUMBER: + case OCTNUMBER: + case BINNUMBER: + case FLOAT: + case COMPLEX: + signed_number(); switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { case PLUS: - jj_consume_token(PLUS); + case MINUS: + switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + case PLUS: + jj_consume_token(PLUS); SimpleNode jjtn001 = builder.openNode( JJTADD_2OP); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); - try { - signed_number(); - } catch (Throwable jjte001) { + try { + signed_number(); + } catch (Throwable jjte001) { if (jjtc001) { jjtree.clearNodeScope(jjtn001); jjtc001 = false; @@ -4512,22 +4518,22 @@ final public void literal_pattern() throws ParseException { {if (true) throw (ParseException)jjte001;} } {if (true) throw (Error)jjte001;} - } finally { + } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } - } - break; - case MINUS: - jj_consume_token(MINUS); + } + break; + case MINUS: + jj_consume_token(MINUS); SimpleNode jjtn002 = builder.openNode( JJTSUB_2OP); boolean jjtc002 = true; jjtree.openNodeScope(jjtn002); jjtreeOpenNodeScope(jjtn002); - try { - signed_number(); - } catch (Throwable jjte002) { + try { + signed_number(); + } catch (Throwable jjte002) { if (jjtc002) { jjtree.clearNodeScope(jjtn002); jjtc002 = false; @@ -4541,55 +4547,75 @@ final public void literal_pattern() throws ParseException { {if (true) throw (ParseException)jjte002;} } {if (true) throw (Error)jjte002;} - } finally { + } finally { if (jjtc002) { jjtree.closeNodeScope(jjtn002, 2); jjtreeCloseNodeScope(jjtn002); } + } + break; + default: + jj_la1[95] = jj_gen; + jj_consume_token(-1); + throw new ParseException(); } break; default: - jj_la1[95] = jj_gen; - jj_consume_token(-1); - throw new ParseException(); + jj_la1[96] = jj_gen; + ; } break; + case SINGLE_STRING: + case SINGLE_STRING2: + case TRIPLE_STRING: + case TRIPLE_STRING2: + case SINGLE_BSTRING: + case SINGLE_BSTRING2: + case TRIPLE_BSTRING: + case TRIPLE_BSTRING2: + case SINGLE_FSTRING: + case SINGLE_FSTRING2: + case TRIPLE_FSTRING: + case TRIPLE_FSTRING2: + case SINGLE_USTRING: + case SINGLE_USTRING2: + case TRIPLE_USTRING: + case TRIPLE_USTRING2: + String(); + break; + case NONE: + jj_consume_token(NONE); + break; + case TRUE: + jj_consume_token(TRUE); + break; + case FALSE: + jj_consume_token(FALSE); + break; default: - jj_la1[96] = jj_gen; - ; + jj_la1[97] = jj_gen; + jj_consume_token(-1); + throw new ParseException(); + } + } catch (Throwable jjte000) { + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + {if (true) throw (RuntimeException)jjte000;} + } + if (jjte000 instanceof ParseException) { + {if (true) throw (ParseException)jjte000;} + } + {if (true) throw (Error)jjte000;} + } finally { + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + jjtreeCloseNodeScope(jjtn000); } - break; - case SINGLE_STRING: - case SINGLE_STRING2: - case TRIPLE_STRING: - case TRIPLE_STRING2: - case SINGLE_BSTRING: - case SINGLE_BSTRING2: - case TRIPLE_BSTRING: - case TRIPLE_BSTRING2: - case SINGLE_FSTRING: - case SINGLE_FSTRING2: - case TRIPLE_FSTRING: - case TRIPLE_FSTRING2: - case SINGLE_USTRING: - case SINGLE_USTRING2: - case TRIPLE_USTRING: - case TRIPLE_USTRING2: - String(); - break; - case NONE: - jj_consume_token(NONE); - break; - case TRUE: - jj_consume_token(TRUE); - break; - case FALSE: - jj_consume_token(FALSE); - break; - default: - jj_la1[97] = jj_gen; - jj_consume_token(-1); - throw new ParseException(); } } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/TreeBuilder310.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/TreeBuilder310.java index 844131b1a6..02f7123730 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/TreeBuilder310.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/TreeBuilder310.java @@ -35,13 +35,14 @@ import org.python.pydev.parser.jython.ast.Match; import org.python.pydev.parser.jython.ast.MatchAs; import org.python.pydev.parser.jython.ast.MatchClass; -import org.python.pydev.parser.jython.ast.MatchKeyword; +import org.python.pydev.parser.jython.ast.MatchKeyVal; import org.python.pydev.parser.jython.ast.MatchMapping; import org.python.pydev.parser.jython.ast.MatchOr; import org.python.pydev.parser.jython.ast.MatchSequence; import org.python.pydev.parser.jython.ast.MatchValue; import org.python.pydev.parser.jython.ast.Name; import org.python.pydev.parser.jython.ast.NameTok; +import org.python.pydev.parser.jython.ast.NameTokType; import org.python.pydev.parser.jython.ast.NamedExpr; import org.python.pydev.parser.jython.ast.NonLocal; import org.python.pydev.parser.jython.ast.Pass; @@ -617,41 +618,69 @@ public final SimpleNode onCloseNode(SimpleNode n, int arity) throws Exception { addAndReportException(patternType.class.getName()); return getDefaultInvalidPattern(); + case JJTLITERAL_PATTERN: + return new MatchValue((exprType) securePop(exprType.class)); + case JJTCLOSED_PATTERN: popSurplus(arity, 2); if (arity == 1) { - return stack.popNode(); + return securePop(patternType.class); } else if (arity == 2) { MatchClass classPattern = (MatchClass) securePop(MatchClass.class); - exprType cls = (exprType) securePop(exprType.class); - return new MatchClass(cls, classPattern.args); + SimpleNode cls = stack.popNode(); + if (cls instanceof exprType) { + classPattern.cls = (exprType) cls; + return classPattern; + } else if (cls instanceof MatchValue) { + MatchValue matchValue = (MatchValue) cls; + classPattern.cls = matchValue.value; + return classPattern; + } } addAndReportException(exprType.class.getName()); return getDefaultInvalidExpr(); case JJTKEY_VALUE_PATTERN: popSurplus(arity, 2); + if (arity == 1) { + // This means we got into the double_star_pattern() + // and we just have a Name there. This is the '**rest' + Name doubleStarName = (Name) securePop(Name.class); + return makeNameTok(NameTok.PatternMappingRest, doubleStarName); + } + if (arity == 2) { patternType pattern = (patternType) securePop(patternType.class); - exprType arg = (exprType) securePop(exprType.class); - return new MatchKeyword(arg, pattern); + MatchValue arg = (MatchValue) securePop(MatchValue.class); + + // We want the expr in the MatchValue. As we're removing the MatchValue itself + // we need to copy what it had before. + this.addSpecialsAndClearOriginal(arg, arg.value); + + MatchKeyVal matchKeyword = new MatchKeyVal(arg.value, pattern); + return matchKeyword; } addAndReportException(exprType.class.getName()); return getDefaultInvalidExpr(); case JJTMAPPING_PATTERN: if (arity > 0) { - exprType[] keys = new exprType[arity]; - patternType[] patterns = new patternType[arity]; + java.util.List lst = new ArrayList<>(); + NameTokType rest = null; for (int i = arity - 1; i >= 0; i--) { - MatchKeyword pattern = (MatchKeyword) securePop(MatchKeyword.class); - keys[i] = pattern.arg; - patterns[i] = pattern.value; + SimpleNode found = stack.popNode(); + if (found instanceof MatchKeyVal) { + lst.add((MatchKeyVal) found); + } else { + if (found instanceof NameTok) { + rest = (NameTokType) found; + } + } } - return new MatchMapping(keys, patterns); + return new MatchMapping(lst.toArray(new patternType[0]), rest); } - addAndReportException(MatchKeyword.class.getName()); - return new MatchMapping(new exprType[0], new patternType[0]); + addAndReportException(MatchKeyVal.class.getName()); + return new MatchMapping(new patternType[0], null); case JJTLIST_PATTERN: return createMatchSequence(arity, enclosingType.LIST); @@ -664,16 +693,17 @@ public final SimpleNode onCloseNode(SimpleNode n, int arity) throws Exception { if (isPeekedNodeWildcardPattern()) { return new MatchValue(makeName(Name.Pattern)); } - return stack.popNode(); + return new MatchValue((exprType) securePop(exprType.class)); } else if (arity >= 2) { SimpleNode peekedNode = stack.peekNode(); if (peekedNode instanceof patternType) { popSurplus(arity, 2); patternType pattern = (patternType) securePop(patternType.class); Name arg = makeName(Name.Artificial); - return new MatchKeyword(arg, pattern); + return new MatchKeyVal(arg, pattern); } else { - return popMatchAttributeInMatch(arity); + Attribute attr = popMatchAttributeInMatch(arity); + return new MatchValue(attr); } } addAndReportException(Attribute.class.getName()); @@ -808,8 +838,8 @@ private SimpleNode getDefaultInvalidNode(Class cls) throws return new Suite(getDefaultBody()); } else if (MatchClass.class.equals(cls)) { return new MatchClass(getDefaultInvalidExpr(), null); - } else if (MatchKeyword.class.equals(cls)) { - return new MatchKeyword(getDefaultInvalidExpr(), getDefaultInvalidPattern()); + } else if (MatchKeyVal.class.equals(cls)) { + return new MatchKeyVal(getDefaultInvalidExpr(), getDefaultInvalidPattern()); } throw new Exception("Could not find any invalid default node for " + cls.getName()); } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt index 326e0a4f71..da21441b83 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt @@ -1162,7 +1162,7 @@ void closed_pattern() : {} // | wildcard_pattern() "_" is matched as a regular Name } -void literal_pattern() #void : {} +void literal_pattern() : {} { signed_number() ( signed_number() #add_2op(2) diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt_template b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt_template index 97f89ede09..2558c65896 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt_template +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar310/python.jjt_template @@ -994,7 +994,7 @@ void closed_pattern() : {} // | wildcard_pattern() "_" is matched as a regular Name } -void literal_pattern() #void : {} +void literal_pattern() : {} { signed_number() ( signed_number() #add_2op(2) diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/PythonGrammar311.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/PythonGrammar311.java index 6827e8c609..e19dbd6ff9 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/PythonGrammar311.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/PythonGrammar311.java @@ -4476,29 +4476,35 @@ final public void closed_pattern() throws ParseException { } final public void literal_pattern() throws ParseException { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { - case PLUS: - case MINUS: - case DECNUMBER: - case HEXNUMBER: - case OCTNUMBER: - case BINNUMBER: - case FLOAT: - case COMPLEX: - signed_number(); + /*@bgen(jjtree) literal_pattern */ + SimpleNode jjtn000 = builder.openNode( JJTLITERAL_PATTERN); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); + jjtreeOpenNodeScope(jjtn000); + try { switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { case PLUS: case MINUS: + case DECNUMBER: + case HEXNUMBER: + case OCTNUMBER: + case BINNUMBER: + case FLOAT: + case COMPLEX: + signed_number(); switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { case PLUS: - jj_consume_token(PLUS); + case MINUS: + switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + case PLUS: + jj_consume_token(PLUS); SimpleNode jjtn001 = builder.openNode( JJTADD_2OP); boolean jjtc001 = true; jjtree.openNodeScope(jjtn001); jjtreeOpenNodeScope(jjtn001); - try { - signed_number(); - } catch (Throwable jjte001) { + try { + signed_number(); + } catch (Throwable jjte001) { if (jjtc001) { jjtree.clearNodeScope(jjtn001); jjtc001 = false; @@ -4512,22 +4518,22 @@ final public void literal_pattern() throws ParseException { {if (true) throw (ParseException)jjte001;} } {if (true) throw (Error)jjte001;} - } finally { + } finally { if (jjtc001) { jjtree.closeNodeScope(jjtn001, 2); jjtreeCloseNodeScope(jjtn001); } - } - break; - case MINUS: - jj_consume_token(MINUS); + } + break; + case MINUS: + jj_consume_token(MINUS); SimpleNode jjtn002 = builder.openNode( JJTSUB_2OP); boolean jjtc002 = true; jjtree.openNodeScope(jjtn002); jjtreeOpenNodeScope(jjtn002); - try { - signed_number(); - } catch (Throwable jjte002) { + try { + signed_number(); + } catch (Throwable jjte002) { if (jjtc002) { jjtree.clearNodeScope(jjtn002); jjtc002 = false; @@ -4541,55 +4547,75 @@ final public void literal_pattern() throws ParseException { {if (true) throw (ParseException)jjte002;} } {if (true) throw (Error)jjte002;} - } finally { + } finally { if (jjtc002) { jjtree.closeNodeScope(jjtn002, 2); jjtreeCloseNodeScope(jjtn002); } + } + break; + default: + jj_la1[95] = jj_gen; + jj_consume_token(-1); + throw new ParseException(); } break; default: - jj_la1[95] = jj_gen; - jj_consume_token(-1); - throw new ParseException(); + jj_la1[96] = jj_gen; + ; } break; + case SINGLE_STRING: + case SINGLE_STRING2: + case TRIPLE_STRING: + case TRIPLE_STRING2: + case SINGLE_BSTRING: + case SINGLE_BSTRING2: + case TRIPLE_BSTRING: + case TRIPLE_BSTRING2: + case SINGLE_FSTRING: + case SINGLE_FSTRING2: + case TRIPLE_FSTRING: + case TRIPLE_FSTRING2: + case SINGLE_USTRING: + case SINGLE_USTRING2: + case TRIPLE_USTRING: + case TRIPLE_USTRING2: + String(); + break; + case NONE: + jj_consume_token(NONE); + break; + case TRUE: + jj_consume_token(TRUE); + break; + case FALSE: + jj_consume_token(FALSE); + break; default: - jj_la1[96] = jj_gen; - ; + jj_la1[97] = jj_gen; + jj_consume_token(-1); + throw new ParseException(); + } + } catch (Throwable jjte000) { + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + {if (true) throw (RuntimeException)jjte000;} + } + if (jjte000 instanceof ParseException) { + {if (true) throw (ParseException)jjte000;} + } + {if (true) throw (Error)jjte000;} + } finally { + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + jjtreeCloseNodeScope(jjtn000); } - break; - case SINGLE_STRING: - case SINGLE_STRING2: - case TRIPLE_STRING: - case TRIPLE_STRING2: - case SINGLE_BSTRING: - case SINGLE_BSTRING2: - case TRIPLE_BSTRING: - case TRIPLE_BSTRING2: - case SINGLE_FSTRING: - case SINGLE_FSTRING2: - case TRIPLE_FSTRING: - case TRIPLE_FSTRING2: - case SINGLE_USTRING: - case SINGLE_USTRING2: - case TRIPLE_USTRING: - case TRIPLE_USTRING2: - String(); - break; - case NONE: - jj_consume_token(NONE); - break; - case TRUE: - jj_consume_token(TRUE); - break; - case FALSE: - jj_consume_token(FALSE); - break; - default: - jj_la1[97] = jj_gen; - jj_consume_token(-1); - throw new ParseException(); } } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/TreeBuilder311.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/TreeBuilder311.java index c1c5625b4b..4ba7bf7251 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/TreeBuilder311.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/TreeBuilder311.java @@ -35,13 +35,14 @@ import org.python.pydev.parser.jython.ast.Match; import org.python.pydev.parser.jython.ast.MatchAs; import org.python.pydev.parser.jython.ast.MatchClass; -import org.python.pydev.parser.jython.ast.MatchKeyword; +import org.python.pydev.parser.jython.ast.MatchKeyVal; import org.python.pydev.parser.jython.ast.MatchMapping; import org.python.pydev.parser.jython.ast.MatchOr; import org.python.pydev.parser.jython.ast.MatchSequence; import org.python.pydev.parser.jython.ast.MatchValue; import org.python.pydev.parser.jython.ast.Name; import org.python.pydev.parser.jython.ast.NameTok; +import org.python.pydev.parser.jython.ast.NameTokType; import org.python.pydev.parser.jython.ast.NamedExpr; import org.python.pydev.parser.jython.ast.NonLocal; import org.python.pydev.parser.jython.ast.Pass; @@ -619,41 +620,69 @@ public final SimpleNode onCloseNode(SimpleNode n, int arity) throws Exception { addAndReportException(patternType.class.getName()); return getDefaultInvalidPattern(); + case JJTLITERAL_PATTERN: + return new MatchValue((exprType) securePop(exprType.class)); + case JJTCLOSED_PATTERN: popSurplus(arity, 2); if (arity == 1) { - return stack.popNode(); + return securePop(patternType.class); } else if (arity == 2) { MatchClass classPattern = (MatchClass) securePop(MatchClass.class); - exprType cls = (exprType) securePop(exprType.class); - return new MatchClass(cls, classPattern.args); + SimpleNode cls = stack.popNode(); + if (cls instanceof exprType) { + classPattern.cls = (exprType) cls; + return classPattern; + } else if (cls instanceof MatchValue) { + MatchValue matchValue = (MatchValue) cls; + classPattern.cls = matchValue.value; + return classPattern; + } } addAndReportException(exprType.class.getName()); return getDefaultInvalidExpr(); case JJTKEY_VALUE_PATTERN: popSurplus(arity, 2); + if (arity == 1) { + // This means we got into the double_star_pattern() + // and we just have a Name there. This is the '**rest' + Name doubleStarName = (Name) securePop(Name.class); + return makeNameTok(NameTok.PatternMappingRest, doubleStarName); + } + if (arity == 2) { patternType pattern = (patternType) securePop(patternType.class); - exprType arg = (exprType) securePop(exprType.class); - return new MatchKeyword(arg, pattern); + MatchValue arg = (MatchValue) securePop(MatchValue.class); + + // We want the expr in the MatchValue. As we're removing the MatchValue itself + // we need to copy what it had before. + this.addSpecialsAndClearOriginal(arg, arg.value); + + MatchKeyVal matchKeyword = new MatchKeyVal(arg.value, pattern); + return matchKeyword; } addAndReportException(exprType.class.getName()); return getDefaultInvalidExpr(); case JJTMAPPING_PATTERN: if (arity > 0) { - exprType[] keys = new exprType[arity]; - patternType[] patterns = new patternType[arity]; + java.util.List lst = new ArrayList<>(); + NameTokType rest = null; for (int i = arity - 1; i >= 0; i--) { - MatchKeyword pattern = (MatchKeyword) securePop(MatchKeyword.class); - keys[i] = pattern.arg; - patterns[i] = pattern.value; + SimpleNode found = stack.popNode(); + if (found instanceof MatchKeyVal) { + lst.add((MatchKeyVal) found); + } else { + if (found instanceof NameTok) { + rest = (NameTokType) found; + } + } } - return new MatchMapping(keys, patterns); + return new MatchMapping(lst.toArray(new patternType[0]), rest); } - addAndReportException(MatchKeyword.class.getName()); - return new MatchMapping(new exprType[0], new patternType[0]); + addAndReportException(MatchKeyVal.class.getName()); + return new MatchMapping(new patternType[0], null); case JJTLIST_PATTERN: return createMatchSequence(arity, enclosingType.LIST); @@ -666,16 +695,17 @@ public final SimpleNode onCloseNode(SimpleNode n, int arity) throws Exception { if (isPeekedNodeWildcardPattern()) { return new MatchValue(makeName(Name.Pattern)); } - return stack.popNode(); + return new MatchValue((exprType) securePop(exprType.class)); } else if (arity >= 2) { SimpleNode peekedNode = stack.peekNode(); if (peekedNode instanceof patternType) { popSurplus(arity, 2); patternType pattern = (patternType) securePop(patternType.class); Name arg = makeName(Name.Artificial); - return new MatchKeyword(arg, pattern); + return new MatchKeyVal(arg, pattern); } else { - return popMatchAttributeInMatch(arity); + Attribute attr = popMatchAttributeInMatch(arity); + return new MatchValue(attr); } } addAndReportException(Attribute.class.getName()); @@ -810,8 +840,8 @@ private SimpleNode getDefaultInvalidNode(Class cls) throws return new Suite(getDefaultBody()); } else if (MatchClass.class.equals(cls)) { return new MatchClass(getDefaultInvalidExpr(), null); - } else if (MatchKeyword.class.equals(cls)) { - return new MatchKeyword(getDefaultInvalidExpr(), getDefaultInvalidPattern()); + } else if (MatchKeyVal.class.equals(cls)) { + return new MatchKeyVal(getDefaultInvalidExpr(), getDefaultInvalidPattern()); } throw new Exception("Could not find any invalid default node for " + cls.getName()); } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt index 73ac95ebca..2b7701a0aa 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt @@ -1163,7 +1163,7 @@ void closed_pattern() : {} // | wildcard_pattern() "_" is matched as a regular Name } -void literal_pattern() #void : {} +void literal_pattern(): {} { signed_number() ( signed_number() #add_2op(2) diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt_template b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt_template index 67a4ac520c..227263a18c 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt_template +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammar311/python.jjt_template @@ -995,7 +995,7 @@ void closed_pattern() : {} // | wildcard_pattern() "_" is matched as a regular Name } -void literal_pattern() #void : {} +void literal_pattern(): {} { signed_number() ( signed_number() #add_2op(2) diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilder.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilder.java index 025d94e381..29447dd061 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilder.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilder.java @@ -965,7 +965,7 @@ private argumentsType __makeArguments(DefaultArg[] def, NameTok varg, NameTok kw } - protected SimpleNode popMatchAttributeInMatch(int arity) throws ParseException { + protected Attribute popMatchAttributeInMatch(int arity) throws ParseException { Attribute attr = null; for (int i = 0; i < arity - 1; i++) { if (attr == null) { diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilderHelpers.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilderHelpers.java index 3e2b16682d..fcdd8b09e0 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilderHelpers.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/AbstractTreeBuilderHelpers.java @@ -175,7 +175,7 @@ protected final void addAndReportException(String cls) throws ParseException { protected final void addAndReportException(String cls, SimpleNode nodeFound) throws ParseException { this.stack.getGrammar().addAndReport( - new ParseException("Syntax error. Expected " + cls + ", found: " + nodeFound.getClass().getName(), + new ParseException("Syntax error. Expected " + cls + ", found: " + nodeFound, nodeFound), "Treated class cast exception making " + cls); } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/ITreeConstants.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/ITreeConstants.java index 2727d91b1b..ed74ec4e96 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/ITreeConstants.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/grammarcommon/ITreeConstants.java @@ -167,4 +167,5 @@ public interface ITreeConstants { public static final int JJTLIST_PATTERN = 659; public static final int JJTTUPLE_PATTERN = 660; public static final int JJTBEGIN_EXCEPT_MULTIPLY_CLAUSE = 661; + public static final int JJTLITERAL_PATTERN = 662; } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/ISimpleNodeSwitch.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/ISimpleNodeSwitch.java index ccddccf88a..9771d09fad 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/ISimpleNodeSwitch.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/ISimpleNodeSwitch.java @@ -69,7 +69,7 @@ public interface ISimpleNodeSwitch { public void visit(MatchSequence node); public void visit(MatchMapping node); public void visit(MatchClass node); - public void visit(MatchKeyword node); + public void visit(MatchKeyVal node); public void visit(MatchAs node); public void visit(MatchOr node); } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyword.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyVal.java similarity index 72% rename from plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyword.java rename to plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyVal.java index fc2ddc49fd..7112cd53a5 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyword.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyVal.java @@ -4,12 +4,12 @@ import org.python.pydev.parser.jython.SimpleNode; import java.util.Arrays; -public final class MatchKeyword extends patternType { - public exprType arg; +public final class MatchKeyVal extends patternType { + public exprType key; public patternType value; - public MatchKeyword(exprType arg, patternType value) { - this.arg = arg; + public MatchKeyVal(exprType key, patternType value) { + this.key = key; this.value = value; } @@ -17,7 +17,7 @@ public MatchKeyword(exprType arg, patternType value) { public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((arg == null) ? 0 : arg.hashCode()); + result = prime * result + ((key == null) ? 0 : key.hashCode()); result = prime * result + ((value == null) ? 0 : value.hashCode()); return result; } @@ -27,20 +27,20 @@ public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; - MatchKeyword other = (MatchKeyword) obj; - if (arg == null) { if (other.arg != null) return false;} - else if (!arg.equals(other.arg)) return false; + MatchKeyVal other = (MatchKeyVal) obj; + if (key == null) { if (other.key != null) return false;} + else if (!key.equals(other.key)) return false; if (value == null) { if (other.value != null) return false;} else if (!value.equals(other.value)) return false; return true; } @Override - public MatchKeyword createCopy() { + public MatchKeyVal createCopy() { return createCopy(true); } @Override - public MatchKeyword createCopy(boolean copyComments) { - MatchKeyword temp = new MatchKeyword(arg!=null?(exprType)arg.createCopy(copyComments):null, + public MatchKeyVal createCopy(boolean copyComments) { + MatchKeyVal temp = new MatchKeyVal(key!=null?(exprType)key.createCopy(copyComments):null, value!=null?(patternType)value.createCopy(copyComments):null); temp.beginLine = this.beginLine; temp.beginColumn = this.beginColumn; @@ -65,9 +65,9 @@ public MatchKeyword createCopy(boolean copyComments) { @Override public String toString() { - StringBuffer sb = new StringBuffer("MatchKeyword["); - sb.append("arg="); - sb.append(dumpThis(this.arg)); + StringBuffer sb = new StringBuffer("MatchKeyVal["); + sb.append("key="); + sb.append(dumpThis(this.key)); sb.append(", "); sb.append("value="); sb.append(dumpThis(this.value)); @@ -77,13 +77,13 @@ public String toString() { @Override public Object accept(VisitorIF visitor) throws Exception { - return visitor.visitMatchKeyword(this); + return visitor.visitMatchKeyVal(this); } @Override public void traverse(VisitorIF visitor) throws Exception { - if (arg != null) { - arg.accept(visitor); + if (key != null) { + key.accept(visitor); } if (value != null) { value.accept(visitor); diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyValType.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyValType.java new file mode 100644 index 0000000000..bad09d3cef --- /dev/null +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchKeyValType.java @@ -0,0 +1,99 @@ +// Autogenerated AST node +package org.python.pydev.parser.jython.ast; + +import org.python.pydev.parser.jython.SimpleNode; +import java.util.Arrays; + +public final class MatchKeyValType extends SimpleNode { + public exprType key; + public patternType value; + + public MatchKeyValType(exprType key, patternType value) { + this.key = key; + this.value = value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((key == null) ? 0 : key.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + MatchKeyValType other = (MatchKeyValType) obj; + if (key == null) { if (other.key != null) return false;} + else if (!key.equals(other.key)) return false; + if (value == null) { if (other.value != null) return false;} + else if (!value.equals(other.value)) return false; + return true; + } + @Override + public MatchKeyValType createCopy() { + return createCopy(true); + } + @Override + public MatchKeyValType createCopy(boolean copyComments) { + MatchKeyValType temp = new + MatchKeyValType(key!=null?(exprType)key.createCopy(copyComments):null, + value!=null?(patternType)value.createCopy(copyComments):null); + temp.beginLine = this.beginLine; + temp.beginColumn = this.beginColumn; + if(this.specialsBefore != null && copyComments){ + for(Object o:this.specialsBefore){ + if(o instanceof commentType){ + commentType commentType = (commentType) o; + temp.getSpecialsBefore().add(commentType.createCopy(copyComments)); + } + } + } + if(this.specialsAfter != null && copyComments){ + for(Object o:this.specialsAfter){ + if(o instanceof commentType){ + commentType commentType = (commentType) o; + temp.getSpecialsAfter().add(commentType.createCopy(copyComments)); + } + } + } + return temp; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer("MatchKeyVal["); + sb.append("key="); + sb.append(dumpThis(this.key)); + sb.append(", "); + sb.append("value="); + sb.append(dumpThis(this.value)); + sb.append("]"); + return sb.toString(); + } + + @Override + public Object accept(VisitorIF visitor) throws Exception { + if (visitor instanceof VisitorBase) { + ((VisitorBase) visitor).traverse(this); + } else { + traverse(visitor); + } + return null; + } + + @Override + public void traverse(VisitorIF visitor) throws Exception { + if (key != null) { + key.accept(visitor); + } + if (value != null) { + value.accept(visitor); + } + } + +} diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchMapping.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchMapping.java index befd4a16aa..20d64dcaaa 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchMapping.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/jython/ast/MatchMapping.java @@ -5,20 +5,20 @@ import java.util.Arrays; public final class MatchMapping extends patternType { - public exprType[] keys; - public patternType[] values; + public patternType[] keyValues; + public NameTokType rest; - public MatchMapping(exprType[] keys, patternType[] values) { - this.keys = keys; - this.values = values; + public MatchMapping(patternType[] keyValues, NameTokType rest) { + this.keyValues = keyValues; + this.rest = rest; } @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + Arrays.hashCode(keys); - result = prime * result + Arrays.hashCode(values); + result = prime * result + Arrays.hashCode(keyValues); + result = prime * result + ((rest == null) ? 0 : rest.hashCode()); return result; } @@ -28,8 +28,9 @@ public boolean equals(Object obj) { if (obj == null) return false; if (getClass() != obj.getClass()) return false; MatchMapping other = (MatchMapping) obj; - if (!Arrays.equals(keys, other.keys)) return false; - if (!Arrays.equals(values, other.values)) return false; + if (!Arrays.equals(keyValues, other.keyValues)) return false; + if (rest == null) { if (other.rest != null) return false;} + else if (!rest.equals(other.rest)) return false; return true; } @Override @@ -38,26 +39,18 @@ public MatchMapping createCopy() { } @Override public MatchMapping createCopy(boolean copyComments) { - exprType[] new0; - if(this.keys != null){ - new0 = new exprType[this.keys.length]; - for(int i=0;i", @@ -28,5 +29,6 @@ public interface name_contextType { "GlobalName", "NonLocalName", "PatternName", + "PatternMappingRest", }; } diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/MakeAstValidForPrettyPrintingVisitor.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/MakeAstValidForPrettyPrintingVisitor.java index 1d6a4c56b7..f7384d7942 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/MakeAstValidForPrettyPrintingVisitor.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/MakeAstValidForPrettyPrintingVisitor.java @@ -46,7 +46,7 @@ import org.python.pydev.parser.jython.ast.Match; import org.python.pydev.parser.jython.ast.MatchAs; import org.python.pydev.parser.jython.ast.MatchClass; -import org.python.pydev.parser.jython.ast.MatchKeyword; +import org.python.pydev.parser.jython.ast.MatchKeyVal; import org.python.pydev.parser.jython.ast.MatchMapping; import org.python.pydev.parser.jython.ast.MatchOr; import org.python.pydev.parser.jython.ast.MatchSequence; @@ -218,7 +218,7 @@ public Object visitMatchOr(MatchOr node) throws Exception { } @Override - public Object visitMatchKeyword(MatchKeyword node) throws Exception { + public Object visitMatchKeyVal(MatchKeyVal node) throws Exception { fixNode(node); traverse(node); fixAfterNode(node); diff --git a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/PrettyPrinterVisitorV2.java b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/PrettyPrinterVisitorV2.java index d9e28b6e44..e3bfc16222 100644 --- a/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/PrettyPrinterVisitorV2.java +++ b/plugins/org.python.pydev.parser/src/org/python/pydev/parser/prettyprinterv2/PrettyPrinterVisitorV2.java @@ -44,7 +44,7 @@ import org.python.pydev.parser.jython.ast.Match; import org.python.pydev.parser.jython.ast.MatchAs; import org.python.pydev.parser.jython.ast.MatchClass; -import org.python.pydev.parser.jython.ast.MatchKeyword; +import org.python.pydev.parser.jython.ast.MatchKeyVal; import org.python.pydev.parser.jython.ast.MatchMapping; import org.python.pydev.parser.jython.ast.MatchOr; import org.python.pydev.parser.jython.ast.MatchSequence; @@ -1734,9 +1734,9 @@ public Object visitMatchClass(MatchClass node) throws Exception { } @Override - public Object visitMatchKeyword(MatchKeyword node) throws Exception { + public Object visitMatchKeyVal(MatchKeyVal node) throws Exception { beforeNode(node); - node.arg.accept(this); + node.key.accept(this); doc.addRequire("=", lastNode); node.value.accept(this); afterNode(node); @@ -1745,16 +1745,18 @@ public Object visitMatchKeyword(MatchKeyword node) throws Exception { @Override public Object visitMatchMapping(MatchMapping node) throws Exception { - int length = node.keys.length; + int length = node.keyValues.length; beforeNode(node); doc.addRequire("{", lastNode); for (int i = 0; i < length; i++) { if (i > 0) { doc.addRequire(",", lastNode); } - node.keys[i].accept(this); + MatchKeyVal keyValue = (MatchKeyVal) node.keyValues[i]; + + keyValue.key.accept(this); doc.addRequire(":", lastNode); - node.values[i].accept(this); + keyValue.value.accept(this); } doc.addRequire("}", lastNode); afterNode(node); diff --git a/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser310Test.java b/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser310Test.java index 2582caef0c..ee9da75b6c 100644 --- a/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser310Test.java +++ b/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser310Test.java @@ -487,4 +487,50 @@ public void testMatchCase() { }); } + public void testMatchLiteralAs() { + checkWithAllGrammars310Onwards((grammarVersion) -> { + String s = "" + + "def swallow_report(bird):\n" + + " match bird:\n" + + " case {\"family\": \"Hirundinidae\" as fam}:\n" + + " print(fam)\n" + + " case _:\n" + + " print(\"no match\")\n" + + "\n" + + "b1 = {\n" + + " \"family\": \"Hirundinidae\", \n" + + " \"status\": \"data deficient\"\n" + + " }\n" + + "\n" + + "swallow_report(b1)" + + ""; + + parseLegalDocStr(s); + return true; + }); + } + + public void testMatchOther() { + checkWithAllGrammars310Onwards((grammarVersion) -> { + String s = "" + + "def swallow_report(bird):\n" + + " match bird:\n" + + " case {\"family\": \"Hirundinidae\" as fam, **other}:\n" + + " print(fam)\n" + + " print(other)\n" + + " case _:\n" + + " print(\"no match\")\n" + + "\n" + + "b1 = {\n" + + " \"family\": \"Hirundinidae\", \n" + + " \"status\": \"data deficient\"\n" + + " }\n" + + "\n" + + "swallow_report(b1)" + + ""; + parseLegalDocStr(s); + return true; + }); + } + } diff --git a/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser311Test.java b/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser311Test.java index 6f28d03cda..d190499581 100644 --- a/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser311Test.java +++ b/plugins/org.python.pydev.parser/tests/org/python/pydev/parser/PyParser311Test.java @@ -49,13 +49,4 @@ public void testMatchExceptionGroups() { assertTrue(t.handlers[0].isExceptionGroup); } - public void testRegularMatchCase() { - String s = "" - + "match = {} \n" - + "match['x'] = 1 \n" - + "match.update({'x': 1})\n" - + ""; - - parseLegalDocStr(s); - } } diff --git a/plugins/org.python.pydev.refactoring/src/org/python/pydev/refactoring/ast/visitors/FindDuplicatesVisitor.java b/plugins/org.python.pydev.refactoring/src/org/python/pydev/refactoring/ast/visitors/FindDuplicatesVisitor.java index ac460c7dfd..c374ab2a01 100644 --- a/plugins/org.python.pydev.refactoring/src/org/python/pydev/refactoring/ast/visitors/FindDuplicatesVisitor.java +++ b/plugins/org.python.pydev.refactoring/src/org/python/pydev/refactoring/ast/visitors/FindDuplicatesVisitor.java @@ -50,7 +50,7 @@ import org.python.pydev.parser.jython.ast.Match; import org.python.pydev.parser.jython.ast.MatchAs; import org.python.pydev.parser.jython.ast.MatchClass; -import org.python.pydev.parser.jython.ast.MatchKeyword; +import org.python.pydev.parser.jython.ast.MatchKeyVal; import org.python.pydev.parser.jython.ast.MatchMapping; import org.python.pydev.parser.jython.ast.MatchOr; import org.python.pydev.parser.jython.ast.MatchSequence; @@ -839,7 +839,7 @@ public Object visitMatchClass(MatchClass node) throws Exception { } @Override - public Object visitMatchKeyword(MatchKeyword node) throws Exception { + public Object visitMatchKeyVal(MatchKeyVal node) throws Exception { boolean ret = unhandled_node(node); if (ret) { traverse(node);