From 4dfc94cbb37d7f6bc44eba8b62358b7a2289a446 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Tue, 26 Sep 2023 09:18:01 -0400 Subject: [PATCH 1/8] Fix type checking in ast.printer - Wrong types were used for MultilineStringNode and FormatMultilineStringNode - Simplify the `escape` method to avoid use of T.Cast --- mesonbuild/ast/printer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 06b05ba89c9f..e2191f9b7662 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -65,9 +65,7 @@ def visit_NumberNode(self, node: mparser.NumberNode) -> None: node.lineno = self.curr_line or node.lineno def escape(self, val: str) -> str: - return val.translate(str.maketrans(T.cast( - 'T.Dict[str, T.Union[str, int]]', - {'\'': '\\\'', '\\': '\\\\'}))) + return val.replace('\\', '\\\\').replace("'", "\'") def visit_StringNode(self, node: mparser.StringNode) -> None: assert isinstance(node.value, str) @@ -79,12 +77,12 @@ def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: self.append("f'" + self.escape(node.value) + "'", node) node.lineno = self.curr_line or node.lineno - def visit_MultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None: + def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: assert isinstance(node.value, str) self.append("'''" + node.value + "'''", node) node.lineno = self.curr_line or node.lineno - def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None: + def visit_FormatMultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None: assert isinstance(node.value, str) self.append("f'''" + node.value + "'''", node) node.lineno = self.curr_line or node.lineno From fe8dffdbc720e8f1084958f8665c8438c91d79cb Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Thu, 14 Sep 2023 15:04:43 -0400 Subject: [PATCH 2/8] fix colon wrongly named column in parser In #02ff955, I used the word `columns` instead of `colons`, but the meaning really was about the ':' symbol. --- mesonbuild/ast/printer.py | 10 +++++----- mesonbuild/mparser.py | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index e2191f9b7662..4fa120c5aa50 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -391,7 +391,7 @@ def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: varname.accept(self) if comma is not None: comma.accept(self) - node.column.accept(self) + node.colon.accept(self) node.items.accept(self) node.block.accept(self) node.endforeach.accept(self) @@ -427,7 +427,7 @@ def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: node.condition.accept(self) node.questionmark.accept(self) node.trueblock.accept(self) - node.column.accept(self) + node.colon.accept(self) node.falseblock.accept(self) if node.whitespaces: node.whitespaces.accept(self) @@ -443,10 +443,10 @@ def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: except StopIteration: pass - assert len(node.columns) == len(node.kwargs) - for (key, val), column in zip(node.kwargs.items(), node.columns): + assert len(node.colons) == len(node.kwargs) + for (key, val), colon in zip(node.kwargs.items(), node.colons): key.accept(self) - column.accept(self) + colon.accept(self) val.accept(self) try: comma = next(commas_iter) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index baccd2753e59..c1bceb294a76 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -338,14 +338,14 @@ class ArgumentNode(BaseNode): arguments: T.List[BaseNode] = field(hash=False) commas: T.List[SymbolNode] = field(hash=False) - columns: T.List[SymbolNode] = field(hash=False) + colons: T.List[SymbolNode] = field(hash=False) kwargs: T.Dict[BaseNode, BaseNode] = field(hash=False) def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) self.arguments = [] self.commas = [] - self.columns = [] + self.colons = [] self.kwargs = {} self.order_error = False @@ -556,17 +556,17 @@ class ForeachClauseNode(BaseNode): foreach_: SymbolNode = field(hash=False) varnames: T.List[IdNode] = field(hash=False) commas: T.List[SymbolNode] = field(hash=False) - column: SymbolNode = field(hash=False) + colon: SymbolNode = field(hash=False) items: BaseNode block: CodeBlockNode endforeach: SymbolNode = field(hash=False) - def __init__(self, foreach_: SymbolNode, varnames: T.List[IdNode], commas: T.List[SymbolNode], column: SymbolNode, items: BaseNode, block: CodeBlockNode, endforeach: SymbolNode): + def __init__(self, foreach_: SymbolNode, varnames: T.List[IdNode], commas: T.List[SymbolNode], colon: SymbolNode, items: BaseNode, block: CodeBlockNode, endforeach: SymbolNode): super().__init__(foreach_.lineno, foreach_.colno, foreach_.filename) self.foreach_ = foreach_ self.varnames = varnames self.commas = commas - self.column = column + self.colon = colon self.items = items self.block = block self.endforeach = endforeach @@ -629,15 +629,15 @@ class TernaryNode(BaseNode): condition: BaseNode questionmark: SymbolNode trueblock: BaseNode - column: SymbolNode + colon: SymbolNode falseblock: BaseNode - def __init__(self, condition: BaseNode, questionmark: SymbolNode, trueblock: BaseNode, column: SymbolNode, falseblock: BaseNode): + def __init__(self, condition: BaseNode, questionmark: SymbolNode, trueblock: BaseNode, colon: SymbolNode, falseblock: BaseNode): super().__init__(condition.lineno, condition.colno, condition.filename) self.condition = condition self.questionmark = questionmark self.trueblock = trueblock - self.column = column + self.colon = colon self.falseblock = falseblock @@ -780,10 +780,10 @@ def e1(self) -> BaseNode: self.in_ternary = True trueblock = self.e1() self.expect('colon') - column_node = self.create_node(SymbolNode, self.previous) + colon_node = self.create_node(SymbolNode, self.previous) falseblock = self.e1() self.in_ternary = False - return self.create_node(TernaryNode, left, qm_node, trueblock, column_node, falseblock) + return self.create_node(TernaryNode, left, qm_node, trueblock, colon_node, falseblock) return left def e2(self) -> BaseNode: @@ -946,7 +946,7 @@ def key_values(self) -> ArgumentNode: while not isinstance(s, EmptyNode): if self.accept('colon'): - a.columns.append(self.create_node(SymbolNode, self.previous)) + a.colons.append(self.create_node(SymbolNode, self.previous)) a.set_kwarg_no_check(s, self.statement()) if not self.accept('comma'): return a @@ -966,7 +966,7 @@ def args(self) -> ArgumentNode: a.commas.append(self.create_node(SymbolNode, self.previous)) a.append(s) elif self.accept('colon'): - a.columns.append(self.create_node(SymbolNode, self.previous)) + a.colons.append(self.create_node(SymbolNode, self.previous)) if not isinstance(s, IdNode): raise ParseException('Dictionary key must be a plain identifier.', self.getline(), s.lineno, s.colno) @@ -1021,11 +1021,11 @@ def foreachblock(self) -> ForeachClauseNode: varnames.append(self.create_node(IdNode, self.previous)) self.expect('colon') - column = self.create_node(SymbolNode, self.previous) + colon = self.create_node(SymbolNode, self.previous) items = self.statement() block = self.codeblock() endforeach = self.create_node(SymbolNode, self.current) - return self.create_node(ForeachClauseNode, foreach_, varnames, commas, column, items, block, endforeach) + return self.create_node(ForeachClauseNode, foreach_, varnames, commas, colon, items, block, endforeach) def ifblock(self) -> IfClauseNode: if_node = self.create_node(SymbolNode, self.previous) From a6a3cdfdb29cdca4c2401aa6a409fa04b292d11f Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Thu, 14 Sep 2023 15:05:44 -0400 Subject: [PATCH 3/8] Add ast visitor for whitespaces and symbols The `AstVisitor` intentionally ignores whitespaces and symbols, as they are not useful for tne interpreter. However, when formatting a build file, we need them. This commit introduces a `FullAstVisitor` that visits every Nodes, including whitespaces and symbols. --- mesonbuild/ast/visitor.py | 183 +++++++++++++++++++++++++++++++++++++- 1 file changed, 181 insertions(+), 2 deletions(-) diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 60aca61f9695..e75755ae3f44 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -6,6 +6,7 @@ from __future__ import annotations import typing as T +from itertools import zip_longest if T.TYPE_CHECKING: from .. import mparser @@ -32,10 +33,10 @@ def visit_StringNode(self, node: mparser.StringNode) -> None: def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: self.visit_default_func(node) - def visit_MultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None: + def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: self.visit_default_func(node) - def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None: + def visit_FormatMultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None: self.visit_default_func(node) def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: @@ -159,3 +160,181 @@ def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: self.visit_default_func(node) node.inner.accept(self) + +class FullAstVisitor(AstVisitor): + """Visit all nodes, including Symbol and Whitespaces""" + + def enter_node(self, node: mparser.BaseNode) -> None: + pass + + def exit_node(self, node: mparser.BaseNode) -> None: + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_default_func(self, node: mparser.BaseNode) -> None: + self.enter_node(node) + self.exit_node(node) + + def visit_UnaryOperatorNode(self, node: mparser.UnaryOperatorNode) -> None: + self.enter_node(node) + node.operator.accept(self) + node.value.accept(self) + self.exit_node(node) + + def visit_BinaryOperatorNode(self, node: mparser.BinaryOperatorNode) -> None: + self.enter_node(node) + node.left.accept(self) + node.operator.accept(self) + node.right.accept(self) + self.exit_node(node) + + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: + self.enter_node(node) + node.lbracket.accept(self) + node.args.accept(self) + node.rbracket.accept(self) + self.exit_node(node) + + def visit_DictNode(self, node: mparser.DictNode) -> None: + self.enter_node(node) + node.lcurl.accept(self) + node.args.accept(self) + node.rcurl.accept(self) + self.exit_node(node) + + def visit_OrNode(self, node: mparser.OrNode) -> None: + self.visit_BinaryOperatorNode(node) + + def visit_AndNode(self, node: mparser.AndNode) -> None: + self.visit_BinaryOperatorNode(node) + + def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: + self.visit_BinaryOperatorNode(node) + + def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: + self.visit_BinaryOperatorNode(node) + + def visit_NotNode(self, node: mparser.NotNode) -> None: + self.visit_UnaryOperatorNode(node) + + def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None: + self.enter_node(node) + if node.pre_whitespaces: + node.pre_whitespaces.accept(self) + for i in node.lines: + i.accept(self) + self.exit_node(node) + + def visit_IndexNode(self, node: mparser.IndexNode) -> None: + self.enter_node(node) + node.iobject.accept(self) + node.lbracket.accept(self) + node.index.accept(self) + node.rbracket.accept(self) + self.exit_node(node) + + def visit_MethodNode(self, node: mparser.MethodNode) -> None: + self.enter_node(node) + node.source_object.accept(self) + node.dot.accept(self) + node.name.accept(self) + node.lpar.accept(self) + node.args.accept(self) + node.rpar.accept(self) + self.exit_node(node) + + def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: + self.enter_node(node) + node.func_name.accept(self) + node.lpar.accept(self) + node.args.accept(self) + node.rpar.accept(self) + self.exit_node(node) + + def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: + self.enter_node(node) + node.var_name.accept(self) + node.operator.accept(self) + node.value.accept(self) + self.exit_node(node) + + def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: + self.visit_AssignmentNode(node) + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: + self.enter_node(node) + node.foreach_.accept(self) + for varname, comma in zip_longest(node.varnames, node.commas): + varname.accept(self) + if comma is not None: + comma.accept(self) + node.colon.accept(self) + node.items.accept(self) + node.block.accept(self) + node.endforeach.accept(self) + self.exit_node(node) + + def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: + self.enter_node(node) + for i in node.ifs: + i.accept(self) + node.elseblock.accept(self) + node.endif.accept(self) + self.exit_node(node) + + def visit_UMinusNode(self, node: mparser.UMinusNode) -> None: + self.visit_UnaryOperatorNode(node) + + def visit_IfNode(self, node: mparser.IfNode) -> None: + self.enter_node(node) + node.if_.accept(self) + node.condition.accept(self) + node.block.accept(self) + self.exit_node(node) + + def visit_ElseNode(self, node: mparser.ElseNode) -> None: + self.enter_node(node) + node.else_.accept(self) + node.block.accept(self) + self.exit_node(node) + + def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: + self.enter_node(node) + node.condition.accept(self) + node.questionmark.accept(self) + node.trueblock.accept(self) + node.colon.accept(self) + node.falseblock.accept(self) + self.exit_node(node) + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + self.enter_node(node) + commas_iter = iter(node.commas) + + for arg in node.arguments: + arg.accept(self) + try: + comma = next(commas_iter) + comma.accept(self) + except StopIteration: + pass + + assert len(node.colons) == len(node.kwargs) + for (key, val), colon in zip(node.kwargs.items(), node.colons): + key.accept(self) + colon.accept(self) + val.accept(self) + try: + comma = next(commas_iter) + comma.accept(self) + except StopIteration: + pass + + self.exit_node(node) + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + self.enter_node(node) + node.lpar.accept(self) + node.inner.accept(self) + node.rpar.accept(self) + self.exit_node(node) From 4e3948292e7ce52f76888e986b6562a4bfdb28bd Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 22 Sep 2023 15:42:34 -0400 Subject: [PATCH 4/8] visit full ast for AstConditionLevel Use the new `FullAstVisitor` for `AstConditionLevel`. This will allow proper formatting of `if / else / endif` and `foreach / endforeach` blocks. --- mesonbuild/ast/postprocess.py | 41 +++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/mesonbuild/ast/postprocess.py b/mesonbuild/ast/postprocess.py index e9f251345184..7acfc03a08a1 100644 --- a/mesonbuild/ast/postprocess.py +++ b/mesonbuild/ast/postprocess.py @@ -5,7 +5,7 @@ # or an interpreter-based tool from __future__ import annotations -from .visitor import AstVisitor +from .visitor import AstVisitor, FullAstVisitor import typing as T if T.TYPE_CHECKING: @@ -78,32 +78,41 @@ def visit_default_func(self, node: mparser.BaseNode) -> None: node.ast_id = name + '#' + str(self.counter[name]) self.counter[name] += 1 -class AstConditionLevel(AstVisitor): +class AstConditionLevel(FullAstVisitor): def __init__(self) -> None: self.condition_level = 0 - def visit_default_func(self, node: mparser.BaseNode) -> None: + def enter_node(self, node: mparser.BaseNode) -> None: node.condition_level = self.condition_level def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: - self.visit_default_func(node) - self.condition_level += 1 + self.enter_node(node) + node.foreach_.accept(self) + for varname in node.varnames: + varname.accept(self) + for comma in node.commas: + comma.accept(self) + node.colon.accept(self) node.items.accept(self) + self.condition_level += 1 node.block.accept(self) self.condition_level -= 1 - - def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: - self.visit_default_func(node) - for i in node.ifs: - i.accept(self) - if node.elseblock: - self.condition_level += 1 - node.elseblock.accept(self) - self.condition_level -= 1 + node.endforeach.accept(self) + self.exit_node(node) def visit_IfNode(self, node: mparser.IfNode) -> None: - self.visit_default_func(node) - self.condition_level += 1 + self.enter_node(node) + node.if_.accept(self) node.condition.accept(self) + self.condition_level += 1 + node.block.accept(self) + self.condition_level -= 1 + self.exit_node(node) + + def visit_ElseNode(self, node: mparser.ElseNode) -> None: + self.enter_node(node) + node.else_.accept(self) + self.condition_level += 1 node.block.accept(self) self.condition_level -= 1 + self.exit_node(node) From 8e3f2aa5f8e583a215bd0970c6903349e20ed094 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Thu, 14 Sep 2023 15:17:26 -0400 Subject: [PATCH 5/8] simplify RawPrinter using FullAstVisitor --- mesonbuild/ast/printer.py | 217 +++++--------------------------------- 1 file changed, 27 insertions(+), 190 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 4fa120c5aa50..736cf8c3037e 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -6,9 +6,8 @@ from __future__ import annotations from .. import mparser -from .visitor import AstVisitor +from .visitor import AstVisitor, FullAstVisitor -from itertools import zip_longest import re import typing as T @@ -239,223 +238,61 @@ def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: else: self.result = re.sub(r', $', '', self.result) -class RawPrinter(AstVisitor): +class RawPrinter(FullAstVisitor): def __init__(self) -> None: self.result = '' def visit_default_func(self, node: mparser.BaseNode) -> None: - # XXX: this seems like it could never actually be reached... - self.result += node.value # type: ignore[attr-defined] - if node.whitespaces: - node.whitespaces.accept(self) + self.enter_node(node) + assert hasattr(node, 'value') + self.result += node.value + self.exit_node(node) - def visit_unary_operator(self, node: mparser.UnaryOperatorNode) -> None: - node.operator.accept(self) - node.value.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_binary_operator(self, node: mparser.BinaryOperatorNode) -> None: - node.left.accept(self) - node.operator.accept(self) - node.right.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) + def visit_EmptyNode(self, node: mparser.EmptyNode) -> None: + self.enter_node(node) + self.exit_node(node) def visit_BooleanNode(self, node: mparser.BooleanNode) -> None: + self.enter_node(node) self.result += 'true' if node.value else 'false' - if node.whitespaces: - node.whitespaces.accept(self) + self.exit_node(node) def visit_NumberNode(self, node: mparser.NumberNode) -> None: + self.enter_node(node) self.result += node.raw_value - if node.whitespaces: - node.whitespaces.accept(self) + self.exit_node(node) def visit_StringNode(self, node: mparser.StringNode) -> None: + self.enter_node(node) self.result += f"'{node.raw_value}'" - if node.whitespaces: - node.whitespaces.accept(self) + self.exit_node(node) def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: + self.enter_node(node) self.result += f"'''{node.value}'''" - if node.whitespaces: - node.whitespaces.accept(self) + self.exit_node(node) def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: - self.result += 'f' - self.visit_StringNode(node) + self.enter_node(node) + self.result += f"f'{node.raw_value}'" + self.exit_node(node) def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None: - self.result += 'f' - self.visit_MultilineStringNode(node) + self.enter_node(node) + self.result += f"f'''{node.value}'''" + self.exit_node(node) def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: + self.enter_node(node) self.result += 'continue' - if node.whitespaces: - node.whitespaces.accept(self) + self.exit_node(node) def visit_BreakNode(self, node: mparser.BreakNode) -> None: + self.enter_node(node) self.result += 'break' - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: - node.lbracket.accept(self) - node.args.accept(self) - node.rbracket.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_DictNode(self, node: mparser.DictNode) -> None: - node.lcurl.accept(self) - node.args.accept(self) - node.rcurl.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: - node.lpar.accept(self) - node.inner.accept(self) - node.rpar.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_OrNode(self, node: mparser.OrNode) -> None: - self.visit_binary_operator(node) - - def visit_AndNode(self, node: mparser.AndNode) -> None: - self.visit_binary_operator(node) - - def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: - self.visit_binary_operator(node) - - def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: - self.visit_binary_operator(node) - - def visit_NotNode(self, node: mparser.NotNode) -> None: - self.visit_unary_operator(node) - - def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None: - if node.pre_whitespaces: - node.pre_whitespaces.accept(self) - for i in node.lines: - i.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_IndexNode(self, node: mparser.IndexNode) -> None: - node.iobject.accept(self) - node.lbracket.accept(self) - node.index.accept(self) - node.rbracket.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_MethodNode(self, node: mparser.MethodNode) -> None: - node.source_object.accept(self) - node.dot.accept(self) - node.name.accept(self) - node.lpar.accept(self) - node.args.accept(self) - node.rpar.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: - node.func_name.accept(self) - node.lpar.accept(self) - node.args.accept(self) - node.rpar.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: - node.var_name.accept(self) - node.operator.accept(self) - node.value.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: - node.var_name.accept(self) - node.operator.accept(self) - node.value.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: - node.foreach_.accept(self) - for varname, comma in zip_longest(node.varnames, node.commas): - varname.accept(self) - if comma is not None: - comma.accept(self) - node.colon.accept(self) - node.items.accept(self) - node.block.accept(self) - node.endforeach.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: - for i in node.ifs: - i.accept(self) - if not isinstance(node.elseblock, mparser.EmptyNode): - node.elseblock.accept(self) - node.endif.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_UMinusNode(self, node: mparser.UMinusNode) -> None: - self.visit_unary_operator(node) - - def visit_IfNode(self, node: mparser.IfNode) -> None: - node.if_.accept(self) - node.condition.accept(self) - node.block.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_ElseNode(self, node: mparser.ElseNode) -> None: - node.else_.accept(self) - node.block.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) + self.exit_node(node) - def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: - node.condition.accept(self) - node.questionmark.accept(self) - node.trueblock.accept(self) - node.colon.accept(self) - node.falseblock.accept(self) - if node.whitespaces: - node.whitespaces.accept(self) - - def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: - commas_iter = iter(node.commas) - - for arg in node.arguments: - arg.accept(self) - try: - comma = next(commas_iter) - comma.accept(self) - except StopIteration: - pass - - assert len(node.colons) == len(node.kwargs) - for (key, val), colon in zip(node.kwargs.items(), node.colons): - key.accept(self) - colon.accept(self) - val.accept(self) - try: - comma = next(commas_iter) - comma.accept(self) - except StopIteration: - pass - - if node.whitespaces: - node.whitespaces.accept(self) class AstJSONPrinter(AstVisitor): def __init__(self) -> None: From b9eec4ddfd6e590a62d6725f02cf16762bbb454d Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Thu, 14 Sep 2023 15:20:33 -0400 Subject: [PATCH 6/8] simplify astprinter by using operator value --- mesonbuild/ast/printer.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 736cf8c3037e..004a64953c3b 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -11,13 +11,6 @@ import re import typing as T -arithmic_map = { - 'add': '+', - 'sub': '-', - 'mod': '%', - 'mul': '*', - 'div': '/' -} class AstPrinter(AstVisitor): def __init__(self, indent: int = 2, arg_newline_cutoff: int = 5, update_ast_line_nos: bool = False): @@ -126,7 +119,7 @@ def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: node.left.accept(self) - self.append_padded(arithmic_map[node.operation], node) + self.append_padded(node.operator.value, node) node.lineno = self.curr_line or node.lineno node.right.accept(self) @@ -379,7 +372,7 @@ def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: self._accept('left', node.left) self._accept('right', node.right) - self.current['op'] = arithmic_map[node.operation] + self.current['op'] = node.operator.value self.setbase(node) def visit_NotNode(self, node: mparser.NotNode) -> None: From ab14d270f8a35a0deac984d2f5a2142cdc016c7d Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Thu, 28 Sep 2023 09:08:18 -0400 Subject: [PATCH 7/8] parser: revert to single StringNode type this will allow transforming string types in the formater --- mesonbuild/ast/interpreter.py | 6 +-- mesonbuild/ast/introspection.py | 10 ++-- mesonbuild/ast/printer.py | 46 +++++-------------- mesonbuild/ast/visitor.py | 9 ---- mesonbuild/coredata.py | 2 +- mesonbuild/interpreter/interpreter.py | 2 +- mesonbuild/interpreterbase/helpers.py | 4 +- mesonbuild/interpreterbase/interpreterbase.py | 17 +++---- mesonbuild/mintro.py | 4 +- mesonbuild/mparser.py | 34 +++++--------- mesonbuild/optinterpreter.py | 5 +- mesonbuild/rewriter.py | 28 +++++------ unittests/allplatformstests.py | 1 - 13 files changed, 62 insertions(+), 106 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 0887fa2ebab1..53ddc1052147 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -207,8 +207,8 @@ def func_subdir(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[str def method_call(self, node: BaseNode) -> bool: return True - def evaluate_fstring(self, node: mparser.FormatStringNode) -> str: - assert isinstance(node, mparser.FormatStringNode) + def evaluate_fstring(self, node: mparser.StringNode) -> str: + assert isinstance(node, mparser.StringNode) return node.value def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> TYPE_var: @@ -231,7 +231,7 @@ def evaluate_ternary(self, node: TernaryNode) -> None: def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs: def resolve_key(node: mparser.BaseNode) -> str: - if isinstance(node, mparser.BaseStringNode): + if isinstance(node, mparser.StringNode): return node.value return '__AST_UNKNOWN__' arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key) diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 17eb1d5b9669..5bf7e051a168 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -16,7 +16,7 @@ from ..compilers import detect_compiler_for from ..interpreterbase import InvalidArguments, SubProject from ..mesonlib import MachineChoice, OptionKey -from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, BaseStringNode +from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode from .interpreter import AstInterpreter if T.TYPE_CHECKING: @@ -118,7 +118,7 @@ def func_project(self, node: BaseNode, args: T.List[TYPE_var], kwargs: T.Dict[st if not self.is_subproject() and 'subproject_dir' in kwargs: spdirname = kwargs['subproject_dir'] - if isinstance(spdirname, BaseStringNode): + if isinstance(spdirname, StringNode): assert isinstance(spdirname.value, str) self.subproject_dir = spdirname.value if not self.is_subproject(): @@ -165,7 +165,7 @@ def _add_languages(self, raw_langs: T.List[TYPE_var], required: bool, for_machin for l in self.flatten_args(raw_langs): if isinstance(l, str): langs.append(l) - elif isinstance(l, BaseStringNode): + elif isinstance(l, StringNode): langs.append(l.value) for lang in sorted(langs, key=compilers.sort_clink): @@ -254,7 +254,7 @@ def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]: # Pop the first element if the function is a build target function if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS: arg_nodes.pop(0) - elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))] + elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, StringNode))] inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] if elementary_nodes: res += [curr] @@ -369,6 +369,6 @@ def extract_subproject_dir(self) -> T.Optional[str]: assert isinstance(kw, IdNode), 'for mypy' if kw.value == 'subproject_dir': # mypy does not understand "and isinstance" - if isinstance(val, BaseStringNode): + if isinstance(val, StringNode): return val.value return None diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 004a64953c3b..4ce3b3f1eec1 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -61,22 +61,13 @@ def escape(self, val: str) -> str: def visit_StringNode(self, node: mparser.StringNode) -> None: assert isinstance(node.value, str) - self.append("'" + self.escape(node.value) + "'", node) - node.lineno = self.curr_line or node.lineno - - def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: - assert isinstance(node.value, str) - self.append("f'" + self.escape(node.value) + "'", node) - node.lineno = self.curr_line or node.lineno - def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: - assert isinstance(node.value, str) - self.append("'''" + node.value + "'''", node) - node.lineno = self.curr_line or node.lineno - - def visit_FormatMultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None: - assert isinstance(node.value, str) - self.append("f'''" + node.value + "'''", node) + if node.is_fstring: + self.append('f', node) + if node.is_multiline: + self.append("'''" + node.value + "'''", node) + else: + self.append("'" + self.escape(node.value) + "'", node) node.lineno = self.curr_line or node.lineno def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: @@ -258,22 +249,12 @@ def visit_NumberNode(self, node: mparser.NumberNode) -> None: def visit_StringNode(self, node: mparser.StringNode) -> None: self.enter_node(node) - self.result += f"'{node.raw_value}'" - self.exit_node(node) - - def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: - self.enter_node(node) - self.result += f"'''{node.value}'''" - self.exit_node(node) - - def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: - self.enter_node(node) - self.result += f"f'{node.raw_value}'" - self.exit_node(node) - - def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None: - self.enter_node(node) - self.result += f"f'''{node.value}'''" + if node.is_fstring: + self.result += 'f' + if node.is_multiline: + self.result += f"'''{node.value}'''" + else: + self.result += f"'{node.raw_value}'" self.exit_node(node) def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: @@ -342,9 +323,6 @@ def visit_NumberNode(self, node: mparser.NumberNode) -> None: def visit_StringNode(self, node: mparser.StringNode) -> None: self.gen_ElementaryNode(node) - def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: - self.gen_ElementaryNode(node) - def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: self._accept('args', node.args) self.setbase(node) diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index e75755ae3f44..57a60bb44dd2 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -30,15 +30,6 @@ def visit_NumberNode(self, node: mparser.NumberNode) -> None: def visit_StringNode(self, node: mparser.StringNode) -> None: self.visit_default_func(node) - def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: - self.visit_default_func(node) - - def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: - self.visit_default_func(node) - - def visit_FormatMultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None: - self.visit_default_func(node) - def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: self.visit_default_func(node) diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index b12ec8203b27..eeeb8d1fedc1 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -1142,7 +1142,7 @@ def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[s return section def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]: - if isinstance(node, (mparser.BaseStringNode)): + if isinstance(node, (mparser.StringNode)): return node.value elif isinstance(node, mparser.BooleanNode): return node.value diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index f88f407de636..6e38c5722592 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -529,7 +529,7 @@ def handle_meson_version_from_ast(self) -> None: assert isinstance(kw, mparser.IdNode), 'for mypy' if kw.value == 'meson_version': # mypy does not understand "and isinstance" - if isinstance(val, mparser.BaseStringNode): + if isinstance(val, mparser.StringNode): self.handle_meson_version(val.value, val) def get_build_def_files(self) -> mesonlib.OrderedSet[str]: diff --git a/mesonbuild/interpreterbase/helpers.py b/mesonbuild/interpreterbase/helpers.py index 0ded85b0e604..3942f2c9f4c4 100644 --- a/mesonbuild/interpreterbase/helpers.py +++ b/mesonbuild/interpreterbase/helpers.py @@ -15,7 +15,7 @@ from .baseobjects import TYPE_var, TYPE_kwargs, SubProject def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']: - if isinstance(args, mparser.BaseStringNode): + if isinstance(args, mparser.StringNode): assert isinstance(args.value, str) return [args.value] if not isinstance(args, collections.abc.Sequence): @@ -25,7 +25,7 @@ def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var'] if isinstance(a, list): rest = flatten(a) result = result + rest - elif isinstance(a, mparser.BaseStringNode): + elif isinstance(a, mparser.StringNode): result.append(a.value) else: result.append(a) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 47dd46f328bd..ccc33497e7c9 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -198,11 +198,12 @@ def evaluate_statement(self, cur: mparser.BaseNode) -> T.Optional[InterpreterObj self.assignment(cur) elif isinstance(cur, mparser.MethodNode): return self.method_call(cur) - elif isinstance(cur, mparser.BaseStringNode): - if isinstance(cur, mparser.MultilineFormatStringNode): - return self.evaluate_multiline_fstring(cur) - elif isinstance(cur, mparser.FormatStringNode): - return self.evaluate_fstring(cur) + elif isinstance(cur, mparser.StringNode): + if cur.is_fstring: + if cur.is_multiline: + return self.evaluate_multiline_fstring(cur) + else: + return self.evaluate_fstring(cur) else: return self._holderify(cur.value) elif isinstance(cur, mparser.BooleanNode): @@ -256,7 +257,7 @@ def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> InterpreterObject: @FeatureNew('dict', '0.47.0') def evaluate_dictstatement(self, cur: mparser.DictNode) -> InterpreterObject: def resolve_key(key: mparser.BaseNode) -> str: - if not isinstance(key, mparser.BaseStringNode): + if not isinstance(key, mparser.StringNode): FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject) key_holder = self.evaluate_statement(key) if key_holder is None: @@ -424,11 +425,11 @@ def evaluate_ternary(self, node: mparser.TernaryNode) -> T.Optional[InterpreterO return self.evaluate_statement(node.falseblock) @FeatureNew('multiline format strings', '0.63.0') - def evaluate_multiline_fstring(self, node: mparser.MultilineFormatStringNode) -> InterpreterObject: + def evaluate_multiline_fstring(self, node: mparser.StringNode) -> InterpreterObject: return self.evaluate_fstring(node) @FeatureNew('format strings', '0.58.0') - def evaluate_fstring(self, node: T.Union[mparser.FormatStringNode, mparser.MultilineFormatStringNode]) -> InterpreterObject: + def evaluate_fstring(self, node: mparser.StringNode) -> InterpreterObject: def replace(match: T.Match[str]) -> str: var = str(match.group(1)) try: diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 81b15d27e8fb..bdbb59e3a18d 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -26,7 +26,7 @@ from . import environment from .interpreterbase import ObjectHolder from .mesonlib import OptionKey -from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode +from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode if T.TYPE_CHECKING: import argparse @@ -185,7 +185,7 @@ def nodes_to_paths(node_list: T.List[BaseNode]) -> T.List[Path]: elif isinstance(n, ArgumentNode): args = n.arguments for j in args: - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): assert isinstance(j.value, str) res += [Path(j.value)] elif isinstance(j, str): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index c1bceb294a76..0e26b9ed9fbc 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -298,31 +298,25 @@ def __init__(self, token: Token[str]): self.value = int(token.value, base=0) self.bytespan = token.bytespan -class BaseStringNode(ElementaryNode[str]): - pass - @dataclass(unsafe_hash=True) -class StringNode(BaseStringNode): +class StringNode(ElementaryNode[str]): raw_value: str = field(hash=False) + is_multiline: bool + is_fstring: bool def __init__(self, token: Token[str], escape: bool = True): super().__init__(token) - self.value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, token.value) if escape else token.value - self.raw_value = token.value -class FormatStringNode(StringNode): - pass - -@dataclass(unsafe_hash=True) -class MultilineStringNode(BaseStringNode): + self.is_multiline = 'multiline' in token.tid + self.is_fstring = 'fstring' in token.tid + self.raw_value = token.value - def __init__(self, token: Token[str]): - super().__init__(token) - self.value = token.value + if escape and not self.is_multiline: + self.value = self.escape() -class MultilineFormatStringNode(MultilineStringNode): - pass + def escape(self) -> str: + return ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, self.raw_value) class ContinueNode(ElementaryNode): pass @@ -930,14 +924,8 @@ def e9(self) -> BaseNode: return self.create_node(IdNode, t) if self.accept('number'): return self.create_node(NumberNode, t) - if self.accept('string'): + if self.accept_any(('string', 'fstring', 'multiline_string', 'multiline_fstring')): return self.create_node(StringNode, t) - if self.accept('fstring'): - return self.create_node(FormatStringNode, t) - if self.accept('multiline_string'): - return self.create_node(MultilineStringNode, t) - if self.accept('multiline_fstring'): - return self.create_node(MultilineFormatStringNode, t) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def key_values(self) -> ArgumentNode: diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 9da355a514e6..980dadd092f9 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -105,15 +105,14 @@ def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var': return arg if isinstance(arg, mparser.ParenthesizedNode): return self.reduce_single(arg.inner) - elif isinstance(arg, (mparser.BaseStringNode, mparser.BooleanNode, - mparser.NumberNode)): + elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, mparser.NumberNode)): return arg.value elif isinstance(arg, mparser.ArrayNode): return [self.reduce_single(curarg) for curarg in arg.args.arguments] elif isinstance(arg, mparser.DictNode): d = {} for k, v in arg.args.kwargs.items(): - if not isinstance(k, mparser.BaseStringNode): + if not isinstance(k, mparser.StringNode): raise OptionException('Dictionary keys must be a string literal') d[k.value] = self.reduce_single(v) return d diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index fd5413ac1c0f..0a40a711c4ea 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -13,7 +13,7 @@ from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode +from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, StringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, SymbolNode import json, os, re, sys import typing as T @@ -142,7 +142,7 @@ def __init__(self, node: T.Optional[BaseNode] = None): def new_node(cls, value=None): if value is None: value = '' - return StringNode(Token('', '', 0, 0, 0, None, str(value))) + return StringNode(Token('string', '', 0, 0, 0, None, str(value))) @classmethod def supported_nodes(cls): @@ -259,17 +259,17 @@ def __init__(self, node: T.Optional[BaseNode] = None): @classmethod def _new_element_node(cls, value): - return StringNode(Token('', '', 0, 0, 0, None, str(value))) + return StringNode(Token('string', '', 0, 0, 0, None, str(value))) @staticmethod def _check_is_equal(node, value) -> bool: - if isinstance(node, BaseStringNode): + if isinstance(node, StringNode): return node.value == value return False @staticmethod def _check_regex_matches(node, regex: str) -> bool: - if isinstance(node, BaseStringNode): + if isinstance(node, StringNode): return re.match(regex, node.value) is not None return False @@ -293,7 +293,7 @@ def _check_is_equal(node, value) -> bool: @staticmethod def _check_regex_matches(node, regex: str) -> bool: - if isinstance(node, BaseStringNode): + if isinstance(node, StringNode): return re.match(regex, node.value) is not None return False @@ -657,7 +657,7 @@ def arg_list_from_node(n): src_list = [] for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): src_list += [j.value] # Generate the new String nodes @@ -691,7 +691,7 @@ def arg_list_from_node(n): def find_node(src): for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): if j.value == src: return i, j return None, None @@ -750,7 +750,7 @@ def find_node(src): extra_files_list = [] for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): extra_files_list += [j.value] # Generate the new String nodes @@ -781,7 +781,7 @@ def find_node(src): def find_node(src): for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): if j.value == src: return i, j return None, None @@ -850,12 +850,12 @@ def find_node(src): src_list = [] for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): src_list += [j.value] extra_files_list = [] for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, BaseStringNode): + if isinstance(j, StringNode): extra_files_list += [j.value] test_data = { 'name': target['name'], @@ -870,8 +870,8 @@ def find_node(src): alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))]) - unknown = [x for x in i.arguments if not isinstance(x, BaseStringNode)] - sources = [x for x in i.arguments if isinstance(x, BaseStringNode)] + unknown = [x for x in i.arguments if not isinstance(x, StringNode)] + sources = [x for x in i.arguments if isinstance(x, StringNode)] sources = sorted(sources, key=lambda x: path_sorter(x.value)) i.arguments = unknown + sources diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 0e563758798c..60a3a8aea82e 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -3583,7 +3583,6 @@ def accept_kwargs(kwargs): 'IdNode': [('value', None, str)], 'NumberNode': [('value', None, int)], 'StringNode': [('value', None, str)], - 'FormatStringNode': [('value', None, str)], 'ContinueNode': [], 'BreakNode': [], 'ArgumentNode': [('positional', accept_node_list), ('kwargs', accept_kwargs)], From b7460142bd2bc1f386fd9a45b03c0f8af363bb85 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 15 Sep 2023 10:19:02 -0400 Subject: [PATCH 8/8] meson format command --- docs/markdown/Commands.md | 79 ++ docs/markdown/snippets/meson_format_cmd.md | 4 + mesonbuild/mesonmain.py | 4 +- mesonbuild/mformat.py | 979 ++++++++++++++++++ mesonbuild/mparser.py | 3 + run_format_tests.py | 1 + run_mypy.py | 1 + run_project_tests.py | 4 +- .../format/1 default/crazy_comments.meson | 47 + test cases/format/1 default/indentation.meson | 73 ++ test cases/format/1 default/meson.build | 14 + test cases/format/2 muon/crazy_comments.meson | 47 + test cases/format/2 muon/indentation.meson | 71 ++ test cases/format/2 muon/meson.build | 14 + test cases/format/2 muon/muon.ini | 15 + .../format/3 editorconfig/.editorconfig | 8 + .../3 editorconfig/crazy_comments.meson | 47 + .../format/3 editorconfig/indentation.meson | 73 ++ test cases/format/3 editorconfig/meson.build | 31 + test cases/format/3 editorconfig/options.ini | 1 + .../format/4 config/crazy_comments.meson | 47 + test cases/format/4 config/indentation.meson | 73 ++ test cases/format/4 config/meson.build | 19 + test cases/format/4 config/meson.format | 11 + .../format/5 transform/default.expected.meson | 69 ++ test cases/format/5 transform/default.ini | 15 + test cases/format/5 transform/file_compare.py | 7 + test cases/format/5 transform/genexpected.cmd | 7 + test cases/format/5 transform/meson.build | 29 + test cases/format/5 transform/meson.options | 1 + .../format/5 transform/muon.expected.meson | 69 ++ test cases/format/5 transform/muon.ini | 15 + .../format/5 transform/options.expected.meson | 48 + test cases/format/5 transform/options.ini | 15 + test cases/format/5 transform/source.meson | 37 + test cases/format/5 transform/test.json | 11 + unittests/allplatformstests.py | 1 + unittests/platformagnostictests.py | 43 + 38 files changed, 2031 insertions(+), 2 deletions(-) create mode 100644 docs/markdown/snippets/meson_format_cmd.md create mode 100644 mesonbuild/mformat.py create mode 100644 test cases/format/1 default/crazy_comments.meson create mode 100644 test cases/format/1 default/indentation.meson create mode 100644 test cases/format/1 default/meson.build create mode 100644 test cases/format/2 muon/crazy_comments.meson create mode 100644 test cases/format/2 muon/indentation.meson create mode 100644 test cases/format/2 muon/meson.build create mode 100644 test cases/format/2 muon/muon.ini create mode 100644 test cases/format/3 editorconfig/.editorconfig create mode 100644 test cases/format/3 editorconfig/crazy_comments.meson create mode 100644 test cases/format/3 editorconfig/indentation.meson create mode 100644 test cases/format/3 editorconfig/meson.build create mode 100644 test cases/format/3 editorconfig/options.ini create mode 100644 test cases/format/4 config/crazy_comments.meson create mode 100644 test cases/format/4 config/indentation.meson create mode 100644 test cases/format/4 config/meson.build create mode 100644 test cases/format/4 config/meson.format create mode 100644 test cases/format/5 transform/default.expected.meson create mode 100644 test cases/format/5 transform/default.ini create mode 100644 test cases/format/5 transform/file_compare.py create mode 100644 test cases/format/5 transform/genexpected.cmd create mode 100644 test cases/format/5 transform/meson.build create mode 100644 test cases/format/5 transform/meson.options create mode 100644 test cases/format/5 transform/muon.expected.meson create mode 100644 test cases/format/5 transform/muon.ini create mode 100644 test cases/format/5 transform/options.expected.meson create mode 100644 test cases/format/5 transform/options.ini create mode 100644 test cases/format/5 transform/source.meson create mode 100644 test cases/format/5 transform/test.json diff --git a/docs/markdown/Commands.md b/docs/markdown/Commands.md index 831cba85d489..542f1b269d40 100644 --- a/docs/markdown/Commands.md +++ b/docs/markdown/Commands.md @@ -396,3 +396,82 @@ format should be used. There are currently 3 formats supported: seems to be properly supported by vscode. {{ devenv_arguments.inc }} + + +### format + +*(since 1.5.0)* + +{{ format_usage.inc }} + +Format specified `meson.build` documents. For compatibility with `muon`, `fmt` +is an alias to `format`. + +{{ format_arguments.inc }} + +The configuration file is a `.ini` file. If a `meson.format` file exists +beside the provided build file to analyze, and no configuration file is +provided on the command line, the `meson.format` file is automatically used. + +If no build file is provided on the command line, the `meson.build` file in +current directory is analyzed. + +The following options are recognized: + +- max_line_length (int): When an array, a dict, a function or a method + would be longer that this, it is formatted one argument per line + (Default is 80). +- indent_by (str): Indentation to use (Default is four spaces `' '`). +- space_array (bool): Whether to add spaces between `[]` and array + arguments (default is false). +- kwargs_force_multiline (bool): If true, arguments are formatted one per + line as soon as there is a keyword argument (default is false). +- wide_colon (bool): If true, a space is placed before colon in dict + and in keyword arguments (default is false). +- no_single_comma_function (bool): If true, a comma is never appended + to function arguments if there is only one argument, even if + using multiline arguments (default is false). +- end_of_line ('cr', 'lf', 'crlf', 'native'): Line ending to use + (applied when using `--output` or `--inline` argument) (default + is 'native). +- indent_before_comments (str): Indentation to use before inline comments + (default is two spaces `' '`). +- simplify_string_literals (bool): When true, multiline strings are + converted to single line strings if they don't contain newlines. + Formatted strings are converted to normal strings if they don't + contain substitutions (default is true). +- insert_final_newline (bool): If true, force the `meson.build` file + to end with a newline character (default is true). +- tab_width (int): Width of tab stops, used to compute line length + when `indent_by` uses tab characters (default is 4). +- sort_files (bool): When true, arguments of `files()` function are + sorted alphabetically (default is false). +- group_arg_value (bool): When true, string argument with `--` prefix + followed by string argument without `--` prefix are grouped on the + same line, in multiline arguments (default is false). +- use_editor_config (bool): When true, also uses config from .editorconfig . + +The first six options are the same than for the `muon fmt` command. + +It is also possible to use a `.editorconfig` file, by providing +the `--editor-config` option on the command line, or with the +`use_editor_config` option in the config file. + +When `--recursive` option is specified, `meson.build` files from +`subdir` are also analyzed (must be used in conjunction with `--inplace` +or `--check-only` option). + + +#### Differences with `muon fmt` + +The `meson format` command should be compatible with the `muon fmt` command. +However, it has more features, and some differences: + +- By default, `meson format` put two spaces before inline comments, + while `muon fmt` only puts one. +- `muon fmt` can potentially mix crlf and lf end-of-lines, as it is not aware + of them. `meson format` will always be consistent in the output it produces. +- `muon fmt` only recognize the `indent_by` option from .editorconfig files. + `meson format` also recognizes `max_line_length`, `end_of_line`, + `insert_final_newline` and `tab_width` options. +- `meson format` has many additional format rules (see option list above). diff --git a/docs/markdown/snippets/meson_format_cmd.md b/docs/markdown/snippets/meson_format_cmd.md new file mode 100644 index 000000000000..390f15d581c7 --- /dev/null +++ b/docs/markdown/snippets/meson_format_cmd.md @@ -0,0 +1,4 @@ +## New meson format command + +This command is similar to `muon fmt` and allows to format a `meson.build` +document. diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index c01be49e59da..62ed8918cd2a 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -61,7 +61,7 @@ def errorhandler(e: Exception, command: str) -> int: class CommandLineParser: def __init__(self) -> None: # only import these once we do full argparse processing - from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects, munstable_coredata, mcompile, mdevenv + from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects, munstable_coredata, mcompile, mdevenv, mformat from .scripts import env2mfile from .wrap import wraptool import shutil @@ -100,6 +100,8 @@ def __init__(self) -> None: help_msg='Run commands in developer environment') self.add_command('env2mfile', env2mfile.add_arguments, env2mfile.run, help_msg='Convert current environment to a cross or native file') + self.add_command('format', mformat.add_arguments, mformat.run, aliases=['fmt'], + help_msg='Format meson source file') # Add new commands above this line to list them in help command self.add_command('help', self.add_help_arguments, self.run_help_command, help_msg='Print help of a subcommand') diff --git a/mesonbuild/mformat.py b/mesonbuild/mformat.py new file mode 100644 index 000000000000..49ece4f034a4 --- /dev/null +++ b/mesonbuild/mformat.py @@ -0,0 +1,979 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2024 The Meson development team + +from __future__ import annotations + +import argparse +import re +import typing as T +from configparser import ConfigParser, MissingSectionHeaderError +from copy import deepcopy +from dataclasses import dataclass, field, fields, asdict +from pathlib import Path + +from . import mparser +from .mesonlib import MesonException +from .ast.postprocess import AstConditionLevel +from .ast.printer import RawPrinter +from .ast.visitor import FullAstVisitor +from .environment import build_filename + +if T.TYPE_CHECKING: + from typing_extensions import Literal + + +class DefaultConfigParser(ConfigParser): + + def __init__(self, delimiters: T.Tuple[str, ...] = ('=', ':')): + super().__init__(delimiters=delimiters, interpolation=None) + + def read_default(self, filename: Path) -> None: + if not filename.exists(): + raise MesonException(f'Configuration file {filename} not found') + try: + super().read(filename, encoding='utf-8') + except MissingSectionHeaderError: + self.read_string(f'[{self.default_section}]\n' + filename.read_text(encoding='utf-8')) + + def getstr(self, section: str, key: str, fallback: T.Optional[str] = None) -> T.Optional[str]: + value: T.Optional[str] = self.get(section, key, fallback=fallback) + if value: + value = value.strip('"').strip("'") + return value + + +def match_path(filename: str, pattern: str) -> bool: + '''recursive glob match for editorconfig sections''' + index = 0 + num_ranges: T.List[T.Tuple[int, int]] = [] + + def curl_replace(m: re.Match) -> str: + nonlocal index + + if '\\.\\.' in m[1]: + index += 1 + low, high = m[1].split('\\.\\.') + num_ranges.append((int(low), int(high))) + return f'(?P-?[0-9]+)' + else: + return T.cast(str, m[1].replace(',', '|')) + + pattern_re = pattern.replace('.', '\\.') + pattern_re = re.sub(r'(? . + pattern_re = re.sub(r'(? ([^/]*) + pattern_re = re.sub(r'(? (.*) + pattern_re = re.sub(r'(? [^name] + pattern_re = re.sub(r'(? FormatterConfig: + defaults = {f.name: f.metadata['default'] for f in fields(cls)} + return cls(**defaults) + + def update(self, config: FormatterConfig) -> FormatterConfig: + """Returns copy of self updated with other config""" + new_config = deepcopy(self) + for key, value in asdict(config).items(): + if value is not None: + setattr(new_config, key, value) + return new_config + + def with_editorconfig(self, editorconfig: EditorConfig) -> FormatterConfig: + """Returns copy of self updated with editorconfig""" + config = deepcopy(self) + + if editorconfig.indent_style == 'space': + indent_size = editorconfig.indent_size or 4 + config.indent_by = indent_size * ' ' + elif editorconfig.indent_style == 'tab': + config.indent_by = '\t' + elif editorconfig.indent_size: + config.indent_by = editorconfig.indent_size * ' ' + + if editorconfig.max_line_length == 'off': + config.max_line_length = 0 + elif editorconfig.max_line_length: + config.max_line_length = int(editorconfig.max_line_length) + + if editorconfig.end_of_line: + config.end_of_line = editorconfig.end_of_line + if editorconfig.insert_final_newline: + config.insert_final_newline = editorconfig.insert_final_newline + if editorconfig.tab_width: + config.tab_width = editorconfig.tab_width + + return config + + @property + def newline(self) -> T.Optional[str]: + if self.end_of_line == 'crlf': + return '\r\n' + if self.end_of_line == 'lf': + return '\n' + if self.end_of_line == 'cr': + return '\r' + return None + + +class MultilineArgumentDetector(FullAstVisitor): + + def __init__(self, config: FormatterConfig): + self.config = config + self.is_multiline = False + + def enter_node(self, node: mparser.BaseNode) -> None: + if node.whitespaces and '#' in node.whitespaces.value: + self.is_multiline = True + + elif isinstance(node, mparser.StringNode) and node.is_multiline: + self.is_multiline = True + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + if node.is_multiline: + self.is_multiline = True + + if self.is_multiline: + return + + if self.config.kwargs_force_multiline and node.kwargs: + self.is_multiline = True + + super().visit_ArgumentNode(node) + + +class TrimWhitespaces(FullAstVisitor): + + def __init__(self, config: FormatterConfig): + self.config = config + + self.in_block_comments = False + self.in_arguments = 0 + self.indent_comments = '' + + def visit_default_func(self, node: mparser.BaseNode) -> None: + self.enter_node(node) + node.whitespaces.accept(self) + + def enter_node(self, node: mparser.BaseNode) -> None: + if isinstance(node, mparser.WhitespaceNode): + return + if not node.whitespaces: + # Ensure every node has a whitespace node + node.whitespaces = mparser.WhitespaceNode(mparser.Token('whitespace', node.filename, 0, 0, 0, (0, 0), '')) + node.whitespaces.condition_level = node.condition_level + + def exit_node(self, node: mparser.BaseNode) -> None: + pass + + def move_whitespaces(self, from_node: mparser.BaseNode, to_node: mparser.BaseNode) -> None: + to_node.whitespaces.value = from_node.whitespaces.value + to_node.whitespaces.value + from_node.whitespaces = None + to_node.whitespaces.accept(self) + + def add_space_after(self, node: mparser.BaseNode) -> None: + if not node.whitespaces.value: + node.whitespaces.value = ' ' + + def add_nl_after(self, node: mparser.BaseNode, force: bool = False) -> None: + if not node.whitespaces.value: + node.whitespaces.value = '\n' + elif force and not node.whitespaces.value.endswith('\n'): + node.whitespaces.value += '\n' + + def dedent(self, value: str) -> str: + if value.endswith(self.config.indent_by): + value = value[:-len(self.config.indent_by)] + return value + + def sort_arguments(self, node: mparser.ArgumentNode) -> None: + # TODO: natsort + def sort_key(arg: mparser.BaseNode) -> str: + if isinstance(arg, mparser.StringNode): + return arg.raw_value + return getattr(node, 'value', '') + + node.arguments.sort(key=sort_key) + + def visit_EmptyNode(self, node: mparser.EmptyNode) -> None: + self.enter_node(node) + self.in_block_comments = True + node.whitespaces.accept(self) + self.in_block_comments = False + + def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None: + lines = node.value.splitlines(keepends=True) + node.value = '' + in_block_comments = self.in_block_comments + with_comments = ['#' in line for line in lines] + [False] + for i, line in enumerate(lines): + has_nl = line.endswith('\n') + line = line.strip() + if line.startswith('#'): + if not in_block_comments: + node.value += self.config.indent_before_comments + else: + node.value += self.indent_comments + node.value += line + if has_nl and (line or with_comments[i+1] or not self.in_arguments): + node.value += '\n' + in_block_comments = True + if node.value.endswith('\n'): + node.value += self.indent_comments + + def visit_SymbolNode(self, node: mparser.SymbolNode) -> None: + super().visit_SymbolNode(node) + if node.value in "([{" and node.whitespaces.value == '\n': + node.whitespaces.value = '' + + def visit_StringNode(self, node: mparser.StringNode) -> None: + self.enter_node(node) + + if self.config.simplify_string_literals: + if node.is_multiline and '\n' not in node.value: + node.is_multiline = False + node.value = node.escape() + + if node.is_fstring and '@' not in node.value: + node.is_fstring = False + + self.exit_node(node) + + def visit_UnaryOperatorNode(self, node: mparser.UnaryOperatorNode) -> None: + super().visit_UnaryOperatorNode(node) + self.move_whitespaces(node.value, node) + + def visit_NotNode(self, node: mparser.NotNode) -> None: + super().visit_UnaryOperatorNode(node) + if not node.operator.whitespaces.value: + node.operator.whitespaces.value = ' ' + self.move_whitespaces(node.value, node) + + def visit_BinaryOperatorNode(self, node: mparser.BinaryOperatorNode) -> None: + super().visit_BinaryOperatorNode(node) + self.add_space_after(node.left) + self.add_space_after(node.operator) + self.move_whitespaces(node.right, node) + + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: + super().visit_ArrayNode(node) + self.move_whitespaces(node.rbracket, node) + + if node.lbracket.whitespaces.value: + node.args.is_multiline = True + if node.args.arguments and not node.args.is_multiline and self.config.space_array: + self.add_space_after(node.lbracket) + self.add_space_after(node.args) + + def visit_DictNode(self, node: mparser.DictNode) -> None: + super().visit_DictNode(node) + self.move_whitespaces(node.rcurl, node) + + if node.lcurl.whitespaces.value: + node.args.is_multiline = True + + def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None: + self.enter_node(node) + if node.pre_whitespaces: + self.in_block_comments = True + node.pre_whitespaces.accept(self) + self.in_block_comments = False + else: + node.pre_whitespaces = mparser.WhitespaceNode(mparser.Token('whitespace', node.filename, 0, 0, 0, (0, 0), '')) + + for i in node.lines: + i.accept(self) + self.exit_node(node) + + if node.lines: + self.move_whitespaces(node.lines[-1], node) + else: + node.whitespaces.accept(self) + + if node.condition_level == 0 and self.config.insert_final_newline: + self.add_nl_after(node, force=True) + + indent = node.condition_level * self.config.indent_by + if indent and node.lines: + node.pre_whitespaces.value += indent + for line in node.lines[:-1]: + line.whitespaces.value += indent + + def visit_IndexNode(self, node: mparser.IndexNode) -> None: + super().visit_IndexNode(node) + self.move_whitespaces(node.rbracket, node) + + def visit_MethodNode(self, node: mparser.MethodNode) -> None: + super().visit_MethodNode(node) + self.move_whitespaces(node.rpar, node) + + if node.lpar.whitespaces.value: + node.args.is_multiline = True + + def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: + if node.func_name.value == 'files': + if self.config.sort_files: + self.sort_arguments(node.args) + + if len(node.args.arguments) == 1 and not node.args.kwargs: + arg = node.args.arguments[0] + if isinstance(arg, mparser.ArrayNode): + if not arg.lbracket.whitespaces or not arg.lbracket.whitespaces.value.strip(): + # files([...]) -> files(...) + node.args = arg.args + + super().visit_FunctionNode(node) + self.move_whitespaces(node.rpar, node) + + if node.lpar.whitespaces.value: + node.args.is_multiline = True + + def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: + super().visit_AssignmentNode(node) + self.add_space_after(node.var_name) + self.add_space_after(node.operator) + self.move_whitespaces(node.value, node) + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: + super().visit_ForeachClauseNode(node) + self.add_space_after(node.foreach_) + self.add_space_after(node.varnames[-1]) + for comma in node.commas: + self.add_space_after(comma) + self.add_space_after(node.colon) + + node.block.whitespaces.value += node.condition_level * self.config.indent_by + + self.move_whitespaces(node.endforeach, node) + + def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: + super().visit_IfClauseNode(node) + self.move_whitespaces(node.endif, node) + + if isinstance(node.elseblock, mparser.ElseNode): + node.elseblock.whitespaces.value += node.condition_level * self.config.indent_by + else: + node.ifs[-1].whitespaces.value += node.condition_level * self.config.indent_by + + def visit_IfNode(self, node: mparser.IfNode) -> None: + super().visit_IfNode(node) + self.add_space_after(node.if_) + self.move_whitespaces(node.block, node) + + def visit_ElseNode(self, node: mparser.ElseNode) -> None: + super().visit_ElseNode(node) + self.move_whitespaces(node.block, node) + + def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: + super().visit_TernaryNode(node) + self.add_space_after(node.condition) + self.add_space_after(node.questionmark) + self.add_space_after(node.trueblock) + self.add_space_after(node.colon) + self.move_whitespaces(node.falseblock, node) + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + if not node.is_multiline: + ml_detector = MultilineArgumentDetector(self.config) + node.accept(ml_detector) + if ml_detector.is_multiline: + node.is_multiline = True + + self.in_arguments += 1 + super().visit_ArgumentNode(node) + self.in_arguments -= 1 + + if not node.arguments and not node.kwargs: + node.whitespaces.accept(self) + return + + last_node: mparser.BaseNode + has_trailing_comma = len(node.commas) == len(node.arguments) + len(node.kwargs) + if has_trailing_comma: + last_node = node.commas[-1] + elif node.kwargs: + for last_node in node.kwargs.values(): + pass + else: + last_node = node.arguments[-1] + + self.move_whitespaces(last_node, node) + + if not node.is_multiline and '#' not in node.whitespaces.value: + node.whitespaces.value = '' + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + self.enter_node(node) + + is_multiline = node.lpar.whitespaces and '#' in node.lpar.whitespaces.value + if is_multiline: + self.indent_comments += self.config.indent_by + + node.lpar.accept(self) + node.inner.accept(self) + + if is_multiline: + node.inner.whitespaces.value = self.dedent(node.inner.whitespaces.value) + self.indent_comments = self.dedent(self.indent_comments) + self.add_nl_after(node.inner) + + node.rpar.accept(self) + self.move_whitespaces(node.rpar, node) + + +class ArgumentFormatter(FullAstVisitor): + + def __init__(self, config: FormatterConfig): + self.config = config + self.level = 0 + self.indent_after = False + self.is_function_arguments = False + + def add_space_after(self, node: mparser.BaseNode) -> None: + if not node.whitespaces.value: + node.whitespaces.value = ' ' + + def add_nl_after(self, node: mparser.BaseNode, indent: int) -> None: + if not node.whitespaces.value or node.whitespaces.value == ' ': + node.whitespaces.value = '\n' + indent_by = (node.condition_level + indent) * self.config.indent_by + if indent_by: + node.whitespaces.value += indent_by + + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: + self.enter_node(node) + if node.args.is_multiline: + self.level += 1 + self.add_nl_after(node.lbracket, indent=self.level) + self.is_function_arguments = False + node.args.accept(self) + if node.args.is_multiline: + self.level -= 1 + self.exit_node(node) + + def visit_DictNode(self, node: mparser.DictNode) -> None: + self.enter_node(node) + if node.args.is_multiline: + self.level += 1 + self.add_nl_after(node.lcurl, indent=self.level) + self.is_function_arguments = False + node.args.accept(self) + if node.args.is_multiline: + self.level -= 1 + self.exit_node(node) + + def visit_MethodNode(self, node: mparser.MethodNode) -> None: + self.enter_node(node) + node.source_object.accept(self) + if node.args.is_multiline: + self.level += 1 + self.add_nl_after(node.lpar, indent=self.level) + self.is_function_arguments = True + node.args.accept(self) + if node.args.is_multiline: + self.level -= 1 + self.exit_node(node) + + def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: + self.enter_node(node) + if node.args.is_multiline: + self.level += 1 + self.add_nl_after(node.lpar, indent=self.level) + self.is_function_arguments = True + node.args.accept(self) + if node.args.is_multiline: + self.level -= 1 + self.exit_node(node) + + def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None: + lines = node.value.splitlines(keepends=True) + if lines: + indent = (node.condition_level + self.level) * self.config.indent_by + node.value = lines[0] + for line in lines[1:]: + if '#' in line and not line.startswith(indent): + node.value += indent + node.value += line + if self.indent_after and node.value.endswith(('\n', self.config.indent_by)): + node.value += indent + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + is_function_arguments = self.is_function_arguments # record it, because it may change when visiting children + super().visit_ArgumentNode(node) + + for colon in node.colons: + self.add_space_after(colon) + + if self.config.wide_colon: + for key in node.kwargs: + self.add_space_after(key) + + arguments_count = len(node.arguments) + len(node.kwargs) + has_trailing_comma = node.commas and len(node.commas) == arguments_count + if node.is_multiline: + need_comma = True + if arguments_count == 1 and is_function_arguments: + need_comma = not self.config.no_single_comma_function + + if need_comma and not has_trailing_comma: + comma = mparser.SymbolNode(mparser.Token('comma', node.filename, 0, 0, 0, (0, 0), ',')) + comma.condition_level = node.condition_level + node.commas.append(comma) + elif has_trailing_comma and not need_comma: + node.commas.pop(-1) + + arg_index = 0 + if self.config.group_arg_value: + for arg in node.arguments[:-1]: + group_args = False + if isinstance(arg, mparser.StringNode) and arg.value.startswith('--'): + next_arg = node.arguments[arg_index + 1] + if isinstance(next_arg, mparser.StringNode) and not next_arg.value.startswith('--'): + group_args = True + if group_args: + # keep '--arg', 'value' on same line + self.add_space_after(node.commas[arg_index]) + elif arg_index < len(node.commas): + self.add_nl_after(node.commas[arg_index], self.level) + arg_index += 1 + + for comma in node.commas[arg_index:-1]: + self.add_nl_after(comma, self.level) + self.add_nl_after(node, self.level - 1) + + else: + if has_trailing_comma and not (node.commas[-1].whitespaces and node.commas[-1].whitespaces.value): + node.commas.pop(-1) + + for comma in node.commas: + self.add_space_after(comma) + + self.exit_node(node) + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + self.enter_node(node) + is_multiline = '\n' in node.lpar.whitespaces.value + if is_multiline: + current_indent_after = self.indent_after + self.indent_after = True + node.lpar.accept(self) + node.inner.accept(self) + if is_multiline: + self.indent_after = current_indent_after + node.rpar.accept(self) + self.exit_node(node) + + +class ComputeLineLengths(FullAstVisitor): + + def __init__(self, config: FormatterConfig, level: int): + self.config = config + self.lengths: T.List[int] = [] + self.length = 0 + self.argument_stack: T.List[mparser.ArgumentNode] = [] + self.level = level + self.need_regenerate = False + + def visit_default_func(self, node: mparser.BaseNode) -> None: + self.enter_node(node) + assert hasattr(node, 'value') + self.length += len(str(node.value)) + self.exit_node(node) + + def len(self, line: str) -> int: + '''Compute line length, including tab stops''' + parts = line.split('\t') + line_length = len(parts[0]) + for p in parts[1:]: + tab_length = ((self.length + line_length) % self.config.tab_width) or self.config.tab_width + line_length += tab_length + len(p) + return line_length + + def count_multiline(self, value: str) -> None: + lines = value.splitlines(keepends=True) + for line in lines: + if line.endswith('\n'): + self.lengths.append(self.length + self.len(line) - 1) + self.length = 0 + else: + self.length += self.len(line) + + def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None: + self.count_multiline(node.value) + + def visit_EmptyNode(self, node: mparser.EmptyNode) -> None: + self.enter_node(node) + self.exit_node(node) + + def visit_NumberNode(self, node: mparser.NumberNode) -> None: + self.enter_node(node) + self.length += len(node.raw_value) + self.exit_node(node) + + def visit_StringNode(self, node: mparser.StringNode) -> None: + self.enter_node(node) + if node.is_fstring: + self.length += 1 + + if node.is_multiline: + self.length += 3 + self.count_multiline(node.value) + self.length += 3 + else: + self.length += self.len(node.raw_value) + 2 + + self.exit_node(node) + + def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: + self.enter_node(node) + self.length += len('continue') + self.exit_node(node) + + def visit_BreakNode(self, node: mparser.BreakNode) -> None: + self.enter_node(node) + self.length += len('break') + self.exit_node(node) + + def split_if_needed(self, node: mparser.ArgumentNode) -> None: + if not node.is_multiline and self.length > self.config.max_line_length: + arg = self.argument_stack[self.level] if len(self.argument_stack) > self.level else node + if not arg.is_multiline: + arg.is_multiline = True + self.need_regenerate = True + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + self.argument_stack.append(node) + super().visit_ArgumentNode(node) + self.split_if_needed(node) + self.argument_stack.pop(-1) + + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: + self.enter_node(node) + node.lbracket.accept(self) + node.args.accept(self) + node.rbracket.accept(self) + self.split_if_needed(node.args) # split if closing bracket is too far + self.exit_node(node) + + def visit_DictNode(self, node: mparser.DictNode) -> None: + self.enter_node(node) + node.lcurl.accept(self) + node.args.accept(self) + node.rcurl.accept(self) + self.split_if_needed(node.args) # split if closing bracket is too far + self.exit_node(node) + + +class SubdirFetcher(FullAstVisitor): + + def __init__(self, current_dir: Path): + self.current_dir = current_dir + self.subdirs: T.List[Path] = [] + + def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: + if node.func_name.value == 'subdir': + if node.args.arguments and isinstance(node.args.arguments[0], mparser.StringNode): + subdir = node.args.arguments[0].value + self.subdirs.append(self.current_dir / subdir) + super().visit_FunctionNode(node) + + +class Formatter: + + def __init__(self, configuration_file: T.Optional[Path], use_editor_config: bool, fetch_subdirs: bool): + self.fetch_subdirs = fetch_subdirs + self.use_editor_config = use_editor_config + self.config = self.load_configuration(configuration_file) + self.current_config = self.config + + self.current_dir = Path() + self.subdirs: T.List[Path] = [] + + def load_editor_config(self, source_file: Path) -> EditorConfig: + # See https://editorconfig.org/ + config = EditorConfig() + + for p in source_file.parents: + editorconfig_file = p / '.editorconfig' + if not editorconfig_file.exists(): + continue + + cp = DefaultConfigParser(delimiters=('=',)) + cp.read_default(editorconfig_file) + + sections = [section for section in cp.sections() if match_path(source_file.as_posix(), section)] + for f in fields(config): + if getattr(cp, f.name, None) is not None: + continue # value already set from higher file + + getter = f.metadata['getter'] + for section in sections: + value = getter(cp, section, f.name, fallback=None) + if value is not None: + setattr(config, f.name, value) + + if cp.getboolean(cp.default_section, 'root'): + break + + return config + + def load_configuration(self, configuration_file: T.Optional[Path]) -> FormatterConfig: + config = FormatterConfig() + if configuration_file: + cp = DefaultConfigParser() + cp.read_default(configuration_file) + + for f in fields(config): + getter = f.metadata['getter'] + value = getter(cp, cp.default_section, f.name, fallback=None) + if value is not None: + setattr(config, f.name, value) + + if config.use_editor_config: + self.use_editor_config = True + + return config + + def format(self, code: str, source_file: Path) -> str: + self.current_dir = source_file.parent + self.current_config = FormatterConfig.default() + if self.use_editor_config: + self.current_config = self.current_config.with_editorconfig(self.load_editor_config(source_file)) + self.current_config = self.current_config.update(self.config) + + ast = mparser.Parser(code, source_file.as_posix()).parse() + if self.fetch_subdirs: + subdir_fetcher = SubdirFetcher(self.current_dir) + ast.accept(subdir_fetcher) + self.subdirs = subdir_fetcher.subdirs + + ast.accept(AstConditionLevel()) + for level in range(5): + ast.accept(TrimWhitespaces(self.current_config)) + ast.accept(ArgumentFormatter(self.current_config)) + + cll = ComputeLineLengths(self.current_config, level) + ast.accept(cll) + if not cll.need_regenerate: + break + + printer = RawPrinter() + ast.accept(printer) + return printer.result + + +def add_arguments(parser: argparse.ArgumentParser) -> None: + inplace_group = parser.add_mutually_exclusive_group() + inplace_group.add_argument( + '-q', '--check-only', + action='store_true', + help='exit with 1 if files would be modified by meson format' + ) + inplace_group.add_argument( + '-i', '--inplace', + action='store_true', + help='format files in-place' + ) + parser.add_argument( + '-r', '--recursive', + action='store_true', + help='recurse subdirs (requires --check-only or --inplace option)', + ) + parser.add_argument( + '-c', '--configuration', + metavar='meson.format', + type=Path, + help='read configuration from meson.format' + ) + parser.add_argument( + '-e', '--editor-config', + action='store_true', + default=False, + help='try to read configuration from .editorconfig' + ) + parser.add_argument( + '-o', '--output', + type=Path, + help='output file (implies having exactly one input)' + ) + parser.add_argument( + 'sources', + nargs='*', + type=Path, + help='meson source files' + ) + +def run(options: argparse.Namespace) -> int: + if options.output and len(options.sources) != 1: + raise MesonException('--output argument implies having exactly one source file') + if options.recursive and not (options.inplace or options.check_only): + raise MesonException('--recursive argument requires either --inplace or --check-only option') + + sources: T.List[Path] = options.sources.copy() or [Path(build_filename)] + if not options.configuration: + default_config_path = sources[0].parent / 'meson.format' + if default_config_path.exists(): + options.configuration = default_config_path + formatter = Formatter(options.configuration, options.editor_config, options.recursive) + + while sources: + src_file = sources.pop(0) + if src_file.is_dir(): + src_file = src_file / build_filename + + try: + code = src_file.read_text(encoding='utf-8') + except IOError as e: + raise MesonException(f'Unable to read from {src_file}') from e + + formatted = formatter.format(code, src_file) + if options.recursive: + sources.extend(formatter.subdirs) + + if options.inplace: + try: + with src_file.open('w', encoding='utf-8', newline=formatter.current_config.newline) as sf: + sf.write(formatted) + except IOError as e: + raise MesonException(f'Unable to write to {src_file}') from e + elif options.check_only: + # TODO: add verbose output showing diffs + if code != formatted: + return 1 + elif options.output: + try: + with options.output.open('w', encoding='utf-8', newline=formatter.current_config.newline) as of: + of.write(formatted) + except IOError as e: + raise MesonException(f'Unable to write to {src_file}') from e + else: + print(formatted, end='') + + return 0 + +# TODO: remove empty newlines when more than N (2...) +# TODO: magic comment to prevent formatting +# TODO: handle meson.options ? +# TODO: split long lines on binary operators +# TODO: align series of assignements +# TODO: align comments +# TODO: move comments on long lines + +# Differences from muon format: +# - By default, uses two spaces before comment, and added option for that +# - Muon will mix CRLF and LF on Windows files... +# - Support for end_of_line char +# - Support for max_line_length, end_of_line, insert_final_newline, tab_width in .editorconfig +# - Option to simplify string literals +# - Option to recognize and parse meson.build in subdirs +# - Correctly compute line length when using tabs +# - By default, arguments in files() are sorted alphabetically +# - Option to group '--arg', 'value' on same line in multiline arguments diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 0e26b9ed9fbc..ec08ccfb2583 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -343,6 +343,9 @@ def __init__(self, token: Token[TV_TokenTypes]): self.kwargs = {} self.order_error = False + # Attributes for the visitors + self.is_multiline = False + def prepend(self, statement: BaseNode) -> None: if self.num_kwargs() > 0: self.order_error = True diff --git a/run_format_tests.py b/run_format_tests.py index 9be8549dfcfb..719b76b5ac21 100755 --- a/run_format_tests.py +++ b/run_format_tests.py @@ -52,6 +52,7 @@ def check_format() -> None: '.eggs', '_cache', # e.g. .mypy_cache 'venv', # virtualenvs have DOS line endings '120 rewrite', # we explicitly test for tab in meson.build file + '3 editorconfig', } for (root, _, filenames) in os.walk('.'): if any([x in root for x in skip_dirs]): diff --git a/run_mypy.py b/run_mypy.py index c57a75c12276..f1976226b753 100755 --- a/run_mypy.py +++ b/run_mypy.py @@ -45,6 +45,7 @@ 'mesonbuild/utils/universal.py', 'mesonbuild/mconf.py', 'mesonbuild/mdist.py', + 'mesonbuild/mformat.py', 'mesonbuild/minit.py', 'mesonbuild/minstall.py', 'mesonbuild/mintro.py', diff --git a/run_project_tests.py b/run_project_tests.py index b159b3d6e02a..ce88170d09ad 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -76,7 +76,8 @@ class ArgumentType(CompilerArgumentType): ALL_TESTS = ['cmake', 'common', 'native', 'warning-meson', 'failing-meson', 'failing-build', 'failing-test', 'keyval', 'platform-osx', 'platform-windows', 'platform-linux', 'java', 'C#', 'vala', 'cython', 'rust', 'd', 'objective c', 'objective c++', - 'fortran', 'swift', 'cuda', 'python3', 'python', 'fpga', 'frameworks', 'nasm', 'wasm', 'wayland' + 'fortran', 'swift', 'cuda', 'python3', 'python', 'fpga', 'frameworks', 'nasm', 'wasm', 'wayland', + 'format', ] @@ -1125,6 +1126,7 @@ def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandat TestCategory('nasm', 'nasm'), TestCategory('wasm', 'wasm', shutil.which('emcc') is None or backend is not Backend.ninja), TestCategory('wayland', 'wayland', should_skip_wayland()), + TestCategory('format', 'format'), ] categories = [t.category for t in all_tests] diff --git a/test cases/format/1 default/crazy_comments.meson b/test cases/format/1 default/crazy_comments.meson new file mode 100644 index 000000000000..f391ca28c263 --- /dev/null +++ b/test cases/format/1 default/crazy_comments.meson @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Stone Tickle +# SPDX-License-Identifier: GPL-3.0-only + +project('a') + +if ( + # comment + false # in a weird +) # place! # test +else +endif # test2 + +foreach a : ( + # test 7 + b # test 4 +) # test 6 # test 3 +endforeach +# test 5 + +a = [ + 1, + # inner + 2, # between comma + # between comma 2 +] # trailing + +( + # hello + a() +) +( + # comment 1 + # comment 2 + # comment 3 + a # comment 4 + # comment 5 + # comment 6 + = ( + # comment 7 + 1 # comment 8 + # comment 9 + + 2 # comment 10 + ) # comment 11 + # comment 12 +) # comment 13 + +# trailing comment diff --git a/test cases/format/1 default/indentation.meson b/test cases/format/1 default/indentation.meson new file mode 100644 index 000000000000..31a809abff51 --- /dev/null +++ b/test cases/format/1 default/indentation.meson @@ -0,0 +1,73 @@ +project( + 'indentation', + default_options: { + 'buildtype': 'release', + 'default_library': 'shared', + 'prefer_static': false, + 'unity': 'off', + }, + meson_version: '>= 1.5.0', + version: '1.2.3', +) + +a = [ + 1, + 2, + 3, + [ + 4, + 5, + 6, + [ + 7, + 8, + 9, + [ + 10, # 10 + 11, # 11 + 12, # 12 + ], + 13, + 14, + 15, + ], + ], +] +d = {} + +if meson.project_version().version_compare('>1.2') + if meson.version().version_compare('>1.0') + foreach i : a + e = { + 'a': 'a', + 'b': 'b', + 'c': 'c', + 'd': [ + 1, + 2, + 3, + { + 'e': 'e', + 'f': 'f', + 'g': 'g', + 'h': { + 'i': ( + # a + 1 + # b + + + # c + 2 + ), + 'j': [ + 1, # 1 + 2, # 2 + 3, # 3 + ], + }, + }, + ], + } + endforeach + endif +endif diff --git a/test cases/format/1 default/meson.build b/test cases/format/1 default/meson.build new file mode 100644 index 000000000000..5b5b1152af5e --- /dev/null +++ b/test cases/format/1 default/meson.build @@ -0,0 +1,14 @@ +# This file is for testing meson format with default options + +project('default format') + +meson_cmd = find_program('meson') +meson_files = { + 'self': files('meson.build'), + 'comments': files('crazy_comments.meson'), + 'indentation': files('indentation.meson'), +} + +foreach name, f : meson_files + test(name, meson_cmd, args: ['format', '--check-only', f]) +endforeach diff --git a/test cases/format/2 muon/crazy_comments.meson b/test cases/format/2 muon/crazy_comments.meson new file mode 100644 index 000000000000..5ebda7d63655 --- /dev/null +++ b/test cases/format/2 muon/crazy_comments.meson @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Stone Tickle +# SPDX-License-Identifier: GPL-3.0-only + +project('a') + +if ( + # comment + false # in a weird +) # place! # test +else +endif # test2 + +foreach a : ( + # test 7 + b # test 4 +) # test 6 # test 3 +endforeach +# test 5 + +a = [ + 1, + # inner + 2, # between comma + # between comma 2 +] # trailing + +( + # hello + a() +) +( + # comment 1 + # comment 2 + # comment 3 + a # comment 4 + # comment 5 + # comment 6 + = ( + # comment 7 + 1 # comment 8 + # comment 9 + + 2 # comment 10 + ) # comment 11 + # comment 12 +) # comment 13 + +# trailing comment diff --git a/test cases/format/2 muon/indentation.meson b/test cases/format/2 muon/indentation.meson new file mode 100644 index 000000000000..8f891d57cebf --- /dev/null +++ b/test cases/format/2 muon/indentation.meson @@ -0,0 +1,71 @@ +project( + 'indentation', + default_options: { + 'buildtype': 'release', + 'default_library': 'shared', + 'prefer_static': false, + 'unity': 'off', + }, + meson_version: '>= 1.5.0', + version: '1.2.3', +) + +a = [ + 1, + 2, + 3, + [ + 4, + 5, + 6, + [ + 7, + 8, + 9, + [ + 10, # 10 + 11, # 11 + 12, # 12 + ], + 13, + 14, + 15, + ], + ], +] +d = {} + +if meson.project_version().version_compare('>1.2') + if meson.version().version_compare('>1.0') + foreach i : a + e = { + 'a': 'a', + 'b': 'b', + 'c': 'c', + 'd': [ + 1, + 2, + 3, + { + 'e': 'e', + 'f': 'f', + 'g': 'g', + 'h': { + 'i': ( + # a + 1 # b + # c + + 2 + ), + 'j': [ + 1, # 1 + 2, # 2 + 3, # 3 + ], + }, + }, + ], + } + endforeach + endif +endif diff --git a/test cases/format/2 muon/meson.build b/test cases/format/2 muon/meson.build new file mode 100644 index 000000000000..165e38a5f4aa --- /dev/null +++ b/test cases/format/2 muon/meson.build @@ -0,0 +1,14 @@ +# This file is for testing meson format is compatible with muon format + +project('default format') + +meson_cmd = find_program('meson') +meson_files = { + 'self': files('meson.build'), + 'comments': files('crazy_comments.meson'), + 'indentation': files('indentation.meson'), +} + +foreach name, f : meson_files + test(name, meson_cmd, args: ['fmt', '-q', '-c', files('muon.ini'), f]) +endforeach diff --git a/test cases/format/2 muon/muon.ini b/test cases/format/2 muon/muon.ini new file mode 100644 index 000000000000..f35fa3a234ff --- /dev/null +++ b/test cases/format/2 muon/muon.ini @@ -0,0 +1,15 @@ +; This config should behave like muon default config + +; max_line_length = 80 +; indent_by = ' ' +; space_array = false +; kwargs_force_multiline = false +; wide_colon = false +; no_single_comma_function = false + +indent_before_comments = ' ' +end_of_line = lf +simplify_string_literals = false +; insert_final_newline = true +sort_files = false +; group_arg_value = false diff --git a/test cases/format/3 editorconfig/.editorconfig b/test cases/format/3 editorconfig/.editorconfig new file mode 100644 index 000000000000..5229226002c4 --- /dev/null +++ b/test cases/format/3 editorconfig/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] + +indent_style = tab +indent_size = 1 +tab_width = 4 +max_line_length = 60 diff --git a/test cases/format/3 editorconfig/crazy_comments.meson b/test cases/format/3 editorconfig/crazy_comments.meson new file mode 100644 index 000000000000..788ea1c88dff --- /dev/null +++ b/test cases/format/3 editorconfig/crazy_comments.meson @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Stone Tickle +# SPDX-License-Identifier: GPL-3.0-only + +project('a') + +if ( + # comment + false # in a weird +) # place! # test +else +endif # test2 + +foreach a : ( + # test 7 + b # test 4 +) # test 6 # test 3 +endforeach +# test 5 + +a = [ + 1, + # inner + 2, # between comma + # between comma 2 +] # trailing + +( + # hello + a() +) +( + # comment 1 + # comment 2 + # comment 3 + a # comment 4 + # comment 5 + # comment 6 + = ( + # comment 7 + 1 # comment 8 + # comment 9 + + 2 # comment 10 + ) # comment 11 + # comment 12 +) # comment 13 + +# trailing comment diff --git a/test cases/format/3 editorconfig/indentation.meson b/test cases/format/3 editorconfig/indentation.meson new file mode 100644 index 000000000000..2f348b08f6bc --- /dev/null +++ b/test cases/format/3 editorconfig/indentation.meson @@ -0,0 +1,73 @@ +project( + 'indentation', + default_options: { + 'buildtype': 'release', + 'default_library': 'shared', + 'prefer_static': false, + 'unity': 'off', + }, + meson_version: '>= 1.5.0', + version: '1.2.3', +) + +a = [ + 1, + 2, + 3, + [ + 4, + 5, + 6, + [ + 7, + 8, + 9, + [ + 10, # 10 + 11, # 11 + 12, # 12 + ], + 13, + 14, + 15, + ], + ], +] +d = {} + +if meson.project_version().version_compare('>1.2') + if meson.version().version_compare('>1.0') + foreach i : a + e = { + 'a': 'a', + 'b': 'b', + 'c': 'c', + 'd': [ + 1, + 2, + 3, + { + 'e': 'e', + 'f': 'f', + 'g': 'g', + 'h': { + 'i': ( + # a + 1 + # b + + + # c + 2 + ), + 'j': [ + 1, # 1 + 2, # 2 + 3, # 3 + ], + }, + }, + ], + } + endforeach + endif +endif diff --git a/test cases/format/3 editorconfig/meson.build b/test cases/format/3 editorconfig/meson.build new file mode 100644 index 000000000000..b32974cb9e8e --- /dev/null +++ b/test cases/format/3 editorconfig/meson.build @@ -0,0 +1,31 @@ +# This file is for testing meson format with editor config + +project('default format') + +meson_cmd = find_program('meson') +meson_files = { + 'self': files('meson.build'), + 'comments': files('crazy_comments.meson'), + 'indentation': files('indentation.meson'), +} + +foreach name, f : meson_files + test( + name, + meson_cmd, + args: ['format', '-e', '--check-only', f], + ) + + # Test that .editorconfig can also be loaded from options file + test( + name + '-fromconfig', + meson_cmd, + args: [ + 'format', + '-c', + files('options.ini'), + '--check-only', + f, + ], + ) +endforeach diff --git a/test cases/format/3 editorconfig/options.ini b/test cases/format/3 editorconfig/options.ini new file mode 100644 index 000000000000..d9f9f338bae8 --- /dev/null +++ b/test cases/format/3 editorconfig/options.ini @@ -0,0 +1 @@ +use_editor_config = true diff --git a/test cases/format/4 config/crazy_comments.meson b/test cases/format/4 config/crazy_comments.meson new file mode 100644 index 000000000000..557d5d409f65 --- /dev/null +++ b/test cases/format/4 config/crazy_comments.meson @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Stone Tickle +# SPDX-License-Identifier: GPL-3.0-only + +project('a') + +if ( + # comment + false # in a weird +) # place! # test +else +endif # test2 + +foreach a : ( + # test 7 + b # test 4 +) # test 6 # test 3 +endforeach +# test 5 + +a = [ + 1, + # inner + 2, # between comma + # between comma 2 +] # trailing + +( + # hello + a() +) +( + # comment 1 + # comment 2 + # comment 3 + a # comment 4 + # comment 5 + # comment 6 + = ( + # comment 7 + 1 # comment 8 + # comment 9 + + 2 # comment 10 + ) # comment 11 + # comment 12 +) # comment 13 + +# trailing comment diff --git a/test cases/format/4 config/indentation.meson b/test cases/format/4 config/indentation.meson new file mode 100644 index 000000000000..816b5f3026a1 --- /dev/null +++ b/test cases/format/4 config/indentation.meson @@ -0,0 +1,73 @@ +project( + 'indentation', + default_options : { + 'buildtype' : 'release', + 'default_library' : 'shared', + 'prefer_static' : false, + 'unity' : 'off', + }, + meson_version : '>= 1.5.0', + version : '1.2.3', +) + +a = [ + 1, + 2, + 3, + [ + 4, + 5, + 6, + [ + 7, + 8, + 9, + [ + 10, # 10 + 11, # 11 + 12, # 12 + ], + 13, + 14, + 15, + ], + ], +] +d = {} + +if meson.project_version().version_compare('>1.2') + if meson.version().version_compare('>1.0') + foreach i : a + e = { + 'a' : 'a', + 'b' : 'b', + 'c' : 'c', + 'd' : [ + 1, + 2, + 3, + { + 'e' : 'e', + 'f' : 'f', + 'g' : 'g', + 'h' : { + 'i' : ( + # a + 1 + # b + + + # c + 2 + ), + 'j' : [ + 1, # 1 + 2, # 2 + 3, # 3 + ], + }, + }, + ], + } + endforeach + endif +endif diff --git a/test cases/format/4 config/meson.build b/test cases/format/4 config/meson.build new file mode 100644 index 000000000000..7b49145dc115 --- /dev/null +++ b/test cases/format/4 config/meson.build @@ -0,0 +1,19 @@ +# This file is for testing meson format with custom options. +# It ensures 'meson.format' file is automatically loaded. + +project('default format') + +meson_cmd = find_program('meson') +meson_files = { + 'self' : files('meson.build'), + 'comments' : files('crazy_comments.meson'), + 'indentation' : files('indentation.meson'), +} + +foreach name, f : meson_files + test( + name, + meson_cmd, + args : [ 'format', '--check-only', f ], + ) +endforeach diff --git a/test cases/format/4 config/meson.format b/test cases/format/4 config/meson.format new file mode 100644 index 000000000000..91f9143ec76b --- /dev/null +++ b/test cases/format/4 config/meson.format @@ -0,0 +1,11 @@ +; Different options for config + +max_line_length = 120 +indent_by = ' ' +space_array = true +kwargs_force_multiline = true +wide_colon = true +no_single_comma_function = true + +indent_before_comments = ' ' +; end_of_line = 'native' diff --git a/test cases/format/5 transform/default.expected.meson b/test cases/format/5 transform/default.expected.meson new file mode 100644 index 000000000000..4201053e1f18 --- /dev/null +++ b/test cases/format/5 transform/default.expected.meson @@ -0,0 +1,69 @@ +project('a') # should be on one line + + +# List should be removed, and should be on one line +options_ini = 'options.ini' +f = files(options_ini, 'expected.meson', 'source.meson') + +# This array should fit on one line +a1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + +# This array is too long and should be splitted +a2 = [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, +] + +# space array +a3 = [1, 2, 3] + +# no single comma function +fct = files( + 'meson.build', # force multiline +) + +# wide colon +d = {'a': 1, 'b': 2, 'c': 3} + +# string conversion +'This is not a multiline' +'This is not a fstring' + +# group arg value +arguments = [ + 'a', + '--opt_a', + 'opt_a_value', + 'b', + 'c', + '--opt_d', + '--opt_e', + 'opt_e_value', + '--opt_f', + '--opt_g', + 'opt_g_value', + 'other_value', + 'again', + '--x', +] + +# no final endline diff --git a/test cases/format/5 transform/default.ini b/test cases/format/5 transform/default.ini new file mode 100644 index 000000000000..a0ff816fedf8 --- /dev/null +++ b/test cases/format/5 transform/default.ini @@ -0,0 +1,15 @@ +; Use default values for config + +; max_line_length = 80 +; indent_by = ' ' +; space_array = false +; kwargs_force_multiline = false +; wide_colon = false +; no_single_comma_function = false + +; indent_before_comments = ' ' +; end_of_line = 'native' +; simplify_string_literals = true +; insert_final_newline = true +; sort_files = true +; group_arg_value = false diff --git a/test cases/format/5 transform/file_compare.py b/test cases/format/5 transform/file_compare.py new file mode 100644 index 000000000000..7b0d1b85668f --- /dev/null +++ b/test cases/format/5 transform/file_compare.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys + +with open(sys.argv[1], 'r', encoding='utf-8') as f, open(sys.argv[2], 'r', encoding='utf-8') as g: + if f.read() != g.read(): + sys.exit('contents are not equal') diff --git a/test cases/format/5 transform/genexpected.cmd b/test cases/format/5 transform/genexpected.cmd new file mode 100644 index 000000000000..de3699ddebc7 --- /dev/null +++ b/test cases/format/5 transform/genexpected.cmd @@ -0,0 +1,7 @@ +@echo off +REM This script generates the expected files +REM Please double-check the contents of those files before commiting them!!! + +python ../../../meson.py format -o default.expected.meson source.meson +python ../../../meson.py format -c muon.ini -o muon.expected.meson source.meson +python ../../../meson.py format -c options.ini -o options.expected.meson source.meson diff --git a/test cases/format/5 transform/meson.build b/test cases/format/5 transform/meson.build new file mode 100644 index 000000000000..d15fd1866517 --- /dev/null +++ b/test cases/format/5 transform/meson.build @@ -0,0 +1,29 @@ +project('format') + +fs = import('fs') + +meson_cmd = find_program('meson') +file_compare = find_program(files('file_compare.py')) +config = get_option('fmt_config') + +source = files('source.meson') +config_file = files(config + '.ini') +expected = files(config + '.expected.meson') + +transform = custom_target( + input: [config_file, source], + output: 'transformed.meson', + command: [ + meson_cmd, + 'format', + '--output', '@OUTPUT@', + '--configuration', '@INPUT@', + ], +) + + +test( + 'transform', + file_compare, + args: [transform, expected], +) diff --git a/test cases/format/5 transform/meson.options b/test cases/format/5 transform/meson.options new file mode 100644 index 000000000000..16927831a880 --- /dev/null +++ b/test cases/format/5 transform/meson.options @@ -0,0 +1 @@ +option('fmt_config', type: 'string', value: 'default') diff --git a/test cases/format/5 transform/muon.expected.meson b/test cases/format/5 transform/muon.expected.meson new file mode 100644 index 000000000000..871ce27d722d --- /dev/null +++ b/test cases/format/5 transform/muon.expected.meson @@ -0,0 +1,69 @@ +project('a') # should be on one line + + +# List should be removed, and should be on one line +options_ini = 'options.ini' +f = files('expected.meson', 'source.meson', options_ini) + +# This array should fit on one line +a1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + +# This array is too long and should be splitted +a2 = [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, +] + +# space array +a3 = [1, 2, 3] + +# no single comma function +fct = files( + 'meson.build', # force multiline +) + +# wide colon +d = {'a': 1, 'b': 2, 'c': 3} + +# string conversion +'''This is not a multiline''' +f'This is not a fstring' + +# group arg value +arguments = [ + 'a', + '--opt_a', + 'opt_a_value', + 'b', + 'c', + '--opt_d', + '--opt_e', + 'opt_e_value', + '--opt_f', + '--opt_g', + 'opt_g_value', + 'other_value', + 'again', + '--x', +] + +# no final endline diff --git a/test cases/format/5 transform/muon.ini b/test cases/format/5 transform/muon.ini new file mode 100644 index 000000000000..9bf765948c81 --- /dev/null +++ b/test cases/format/5 transform/muon.ini @@ -0,0 +1,15 @@ +; This config should behave like muon default config + +; max_line_length = 80 +; indent_by = ' ' +; space_array = false +; kwargs_force_multiline = false +; wide_colon = false +; no_single_comma_function = false + +indent_before_comments = ' ' +end_of_line = lf +simplify_string_literals = false +; insert_final_newline = true +sort_files = false +; group_arg_value = false \ No newline at end of file diff --git a/test cases/format/5 transform/options.expected.meson b/test cases/format/5 transform/options.expected.meson new file mode 100644 index 000000000000..f7f45658d5c3 --- /dev/null +++ b/test cases/format/5 transform/options.expected.meson @@ -0,0 +1,48 @@ +project('a') # should be on one line + + +# List should be removed, and should be on one line +options_ini = 'options.ini' +f = files(options_ini, 'expected.meson', 'source.meson') + +# This array should fit on one line +a1 = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ] + +# This array is too long and should be splitted +a2 = [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22 ] + +# space array +a3 = [ 1, 2, 3 ] + +# no single comma function +fct = files( + 'meson.build' # force multiline +) + +# wide colon +d = { + 'a' : 1, + 'b' : 2, + 'c' : 3, +} + +# string conversion +'This is not a multiline' +'This is not a fstring' + +# group arg value +arguments = [ + 'a', + '--opt_a', 'opt_a_value', + 'b', + 'c', + '--opt_d', + '--opt_e', 'opt_e_value', + '--opt_f', + '--opt_g', 'opt_g_value', + 'other_value', + 'again', + '--x', +] + +# no final endline \ No newline at end of file diff --git a/test cases/format/5 transform/options.ini b/test cases/format/5 transform/options.ini new file mode 100644 index 000000000000..823400b8c0cd --- /dev/null +++ b/test cases/format/5 transform/options.ini @@ -0,0 +1,15 @@ +; Different options for config + +max_line_length = 120 +indent_by = ' ' +space_array = true +kwargs_force_multiline = true +wide_colon = true +no_single_comma_function = true + +indent_before_comments = ' ' +; end_of_line = 'native' +; simplify_string_literals = true +insert_final_newline = false +; sort_files = true +group_arg_value = true diff --git a/test cases/format/5 transform/source.meson b/test cases/format/5 transform/source.meson new file mode 100644 index 000000000000..7274d4802699 --- /dev/null +++ b/test cases/format/5 transform/source.meson @@ -0,0 +1,37 @@ +project( + 'a' +) # should be on one line + + +# List should be removed, and should be on one line +options_ini = 'options.ini' +f = files( + [ + 'expected.meson', 'source.meson', options_ini]) + +# This array should fit on one line +a1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + +# This array is too long and should be splitted +a2 = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] + +# space array +a3 = [ 1, 2, 3 ] + +# no single comma function +fct = files( + 'meson.build', # force multiline +) + +# wide colon +d = {'a': 1, 'b': 2, 'c': 3} + +# string conversion +'''This is not a multiline''' +f'This is not a fstring' + +# group arg value +arguments = ['a', '--opt_a', 'opt_a_value', 'b', 'c', '--opt_d', '--opt_e', 'opt_e_value', +'--opt_f', '--opt_g', 'opt_g_value', 'other_value', 'again', '--x'] + +# no final endline \ No newline at end of file diff --git a/test cases/format/5 transform/test.json b/test cases/format/5 transform/test.json new file mode 100644 index 000000000000..fe05a52b3926 --- /dev/null +++ b/test cases/format/5 transform/test.json @@ -0,0 +1,11 @@ +{ + "matrix": { + "options": { + "fmt_config": [ + { "val": "default"}, + { "val": "muon"}, + { "val": "options"} + ] + } + } +} diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 60a3a8aea82e..c334d9e5461e 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -3945,6 +3945,7 @@ def test_commands_documented(self): cmndstr = cmndline.split('{')[1] self.assertIn('}', cmndstr) help_commands = set(cmndstr.split('}')[0].split(',')) + help_commands.remove('fmt') # Remove the alias self.assertTrue(len(help_commands) > 0, 'Must detect some command names.') self.assertEqual(md_commands | {'help'}, help_commands, f'Doc file: `{doc_path}`') diff --git a/unittests/platformagnostictests.py b/unittests/platformagnostictests.py index ba3f5013c55e..ffc4b47ad2d3 100644 --- a/unittests/platformagnostictests.py +++ b/unittests/platformagnostictests.py @@ -16,6 +16,7 @@ from .baseplatformtests import BasePlatformTests from .helpers import is_ci from mesonbuild.mesonlib import EnvironmentVariables, ExecutableSerialisation, MesonException, is_linux, python_command +from mesonbuild.mformat import match_path from mesonbuild.optinterpreter import OptionInterpreter, OptionException from run_tests import Backend @@ -291,6 +292,48 @@ def test_cmake_openssl_not_found_bug(self): out = self.init(testdir, allow_fail=True) self.assertNotIn('Unhandled python exception', out) + def test_editorconfig_match_path(self): + '''match_path function used to parse editorconfig in meson format''' + cases = [ + ('a.txt', '*.txt', True), + ('a.txt', '?.txt', True), + ('a.txt', 'a.t?t', True), + ('a.txt', '*.build', False), + + ('/a.txt', '*.txt', True), + ('/a.txt', '/*.txt', True), + ('a.txt', '/*.txt', False), + + ('a/b/c.txt', 'a/b/*.txt', True), + ('a/b/c.txt', 'a/*/*.txt', True), + ('a/b/c.txt', '*/*.txt', True), + ('a/b/c.txt', 'b/*.txt', True), + ('a/b/c.txt', 'a/*.txt', False), + + ('a/b/c/d.txt', 'a/**/*.txt', True), + ('a/b/c/d.txt', 'a/*', False), + ('a/b/c/d.txt', 'a/**', True), + + ('a.txt', '[abc].txt', True), + ('a.txt', '[!xyz].txt', True), + ('a.txt', '[xyz].txt', False), + ('a.txt', '[!abc].txt', False), + + ('a.txt', '{a,b,c}.txt', True), + ('a.txt', '*.{txt,tex,cpp}', True), + ('a.hpp', '*.{txt,tex,cpp}', False), + + ('a1.txt', 'a{0..9}.txt', True), + ('a001.txt', 'a{0..9}.txt', True), + ('a-1.txt', 'a{-10..10}.txt', True), + ('a99.txt', 'a{0..9}.txt', False), + ('a099.txt', 'a{0..9}.txt', False), + ('a-1.txt', 'a{0..10}.txt', False), + ] + + for filename, pattern, expected in cases: + self.assertTrue(match_path(filename, pattern) is expected, f'{filename} -> {pattern}') + def test_error_configuring_subdir(self): testdir = os.path.join(self.common_test_dir, '152 index customtarget') out = self.init(os.path.join(testdir, 'subdir'), allow_fail=True)