Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: Add unstable formatter #10971

Closed
wants to merge 58 commits into from
Closed
Changes from 7 commits
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
481c81b
Add first implementation for attaching comments to nodes
JCWasmx86 Dec 28, 2022
24fb269
Linter fixes
JCWasmx86 Dec 28, 2022
9b4ec2c
Fix some remaining bugs.
JCWasmx86 Dec 28, 2022
90e933e
Fix last issues, found during building wrapdb
JCWasmx86 Dec 28, 2022
d83d1e8
Remove prints
JCWasmx86 Dec 28, 2022
a03d6fa
Change type of comment based on attempt number
JCWasmx86 Dec 28, 2022
7d9f4bd
mfmt: Initial commit
JCWasmx86 Sep 27, 2022
0ab1a86
mfmt: Implement parsing of files
JCWasmx86 Sep 27, 2022
fead869
Added rough draft of formatter
JCWasmx86 Sep 29, 2022
f152388
Added plumbing for adding some comments back
JCWasmx86 Oct 19, 2022
0036f92
Add fmt_unstable to documentation
JCWasmx86 Oct 19, 2022
5d78e09
Make flake8 happy
JCWasmx86 Oct 19, 2022
474adf6
Added docs for fmt_unstable
JCWasmx86 Oct 20, 2022
cfde2d1
More formatting changes
JCWasmx86 Oct 20, 2022
307e7e5
Add initial tests
JCWasmx86 Oct 23, 2022
f3ffeec
Add inplace editing
JCWasmx86 Oct 25, 2022
9a83943
Add config
JCWasmx86 Oct 25, 2022
0560f29
Add support for space_array
JCWasmx86 Oct 25, 2022
37c7eae
Added support for wide_colon
JCWasmx86 Oct 25, 2022
2aee9d4
Improve docs.
JCWasmx86 Oct 25, 2022
4ba14b5
Fix tests
JCWasmx86 Oct 25, 2022
a3cf11e
Add ParenthesizedNode
JCWasmx86 Oct 26, 2022
3b499f3
Add length estimate
JCWasmx86 Oct 26, 2022
99c8433
Improve output
JCWasmx86 Oct 27, 2022
a5335d1
Improved idempotency
JCWasmx86 Oct 31, 2022
1e9a7cd
Update docs
JCWasmx86 Oct 31, 2022
e35d342
Fixed invalid syntax when formatting systemd
JCWasmx86 Nov 2, 2022
951e757
Add more testcases
JCWasmx86 Nov 5, 2022
ef065dd
Add code to test for foreach-statements with inline-lists
JCWasmx86 Nov 12, 2022
8292930
Add idempotency tests
JCWasmx86 Nov 12, 2022
70aaeea
Remove unused import
JCWasmx86 Nov 12, 2022
e05b4a5
Add check that the reformatted code still parses
JCWasmx86 Nov 15, 2022
ab161fd
Remove note about eating comments
JCWasmx86 Nov 15, 2022
4957feb
Add more comment readding fixes
JCWasmx86 Nov 15, 2022
8407e17
mesa: Reduce number of lost comments (To ~12)
JCWasmx86 Nov 15, 2022
11200b8
Add more comments
JCWasmx86 Nov 15, 2022
2c2c465
Minor fixes
JCWasmx86 Nov 15, 2022
ea09583
Add more testcases
JCWasmx86 Nov 15, 2022
90fb9a3
Add support for recursing.
JCWasmx86 Nov 15, 2022
9029bed
Add support for -q flag
JCWasmx86 Nov 15, 2022
6b6cab7
Add verbose option.
JCWasmx86 Nov 15, 2022
2a0fcb6
Fix linter
JCWasmx86 Nov 15, 2022
eb785db
Allow invalid files during recursing
JCWasmx86 Nov 15, 2022
82ccd55
Add comments in dict literals.
JCWasmx86 Nov 15, 2022
615f8d2
Add more stuff
JCWasmx86 Nov 15, 2022
d9cd7fe
Minor bugfixes
JCWasmx86 Nov 15, 2022
f975df2
Add support for things at the end of file
JCWasmx86 Nov 15, 2022
11f9ab5
Add reminder for broken gstreamer
JCWasmx86 Nov 15, 2022
bafaeed
Add tests for GStreamer dicts
JCWasmx86 Nov 15, 2022
c7e28b2
Fix LGTM warning
JCWasmx86 Nov 15, 2022
2c4216d
Add more stuff for GStreamer
JCWasmx86 Nov 16, 2022
d303f19
Partial fixes for GLib
JCWasmx86 Nov 16, 2022
6bc73e9
Print to stderr
JCWasmx86 Nov 16, 2022
c7f12c0
Add stats script for easily showing the amount of removed comments
JCWasmx86 Nov 16, 2022
7cf756e
stats.sh: Add confirmation prompt and tests until when it converges t…
JCWasmx86 Nov 16, 2022
23da1d7
Update script to add total stats
JCWasmx86 Nov 16, 2022
d4c0776
Merge branch 'comments'
JCWasmx86 Dec 28, 2022
ca2a515
Add formatter2
JCWasmx86 Dec 28, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
286 changes: 244 additions & 42 deletions mesonbuild/mparser.py
Original file line number Diff line number Diff line change
@@ -104,7 +104,6 @@ def __eq__(self, other: object) -> bool:
return self.tid == other.tid
return NotImplemented

# TODO: What type annotations?
@dataclass(eq=False)
class Comment:
line_start: int
@@ -125,6 +124,7 @@ def __init__(self, code: str):
'endif', 'and', 'or', 'not', 'foreach', 'endforeach',
'in', 'continue', 'break'}
self.future_keywords = {'return'}
self.comments = [] # type: T.List[Comment]
self.token_specification = [
# Need to be sorted longest to shortest.
('ignore', re.compile(r'[ \t]')),
@@ -192,9 +192,6 @@ def lex(self, filename: str) -> T.Generator[Token, None, None]:
match_text = mo.group()
if tid == 'ignore':
break
elif tid == 'comment':
self.comments.append(Comment(curline_start, curline, col, bytespan, match_text))
break
elif tid == 'lparen':
par_count += 1
elif tid == 'rparen':
@@ -248,6 +245,9 @@ def lex(self, filename: str) -> T.Generator[Token, None, None]:
line_start = loc
if par_count > 0 or bracket_count > 0 or curl_count > 0:
break
elif tid == 'comment':
self.comments.append(Comment(curline_start, curline, col, bytespan, match_text))
break
elif tid == 'id':
if match_text in self.keywords:
tid = match_text
@@ -268,6 +268,10 @@ class BaseNode:
filename: str
end_lineno: T.Optional[int] = None
end_colno: T.Optional[int] = None
bytespan: T.Optional[T.Tuple[int, int]] = None
pre_comments: T.Optional[T.List['Comment']] = None
comments: T.Optional[T.List['Comment']] = None
post_comments: T.Optional[T.List['Comment']] = None

def __post_init__(self) -> None:
if self.end_lineno is None:
@@ -279,6 +283,9 @@ def __post_init__(self) -> None:
self.level = 0 # type: int
self.ast_id = '' # type: str
self.condition_level = 0 # type: int
self.pre_comments = []
self.comments = []
self.post_comments = []

def accept(self, visitor: 'AstVisitor') -> None:
fname = 'visit_{}'.format(type(self).__name__)
@@ -532,6 +539,18 @@ def __init__(self, code: str, filename: str):
self.current = Token('eof', '', 0, 0, 0, (0, 0), None) # type: Token
self.getsym()
self.in_ternary = False
self.stack = [] # type: T.List[Token]
self.nodes = set() # type: T.Set[BaseNode]

def begin(self) -> None:
self.stack.append(self.current)

def end(self, node: BaseNode) -> None:
assert len(self.stack) > 0
token = self.stack.pop()
# print("%s: [%s:%s] -> [%s:%s] (%s %s)"%(type(node), token.lineno, token.colno, self.current.lineno, self.current.colno, token.bytespan, self.current.bytespan))
node.bytespan = (token.bytespan[0], self.current.bytespan[0])
self.nodes.add(node)

def comments(self) -> T.List[Comment]:
return self.lexer.comments
@@ -571,26 +590,88 @@ def block_expect(self, s: str, block_start: Token) -> bool:
def parse(self) -> CodeBlockNode:
block = self.codeblock()
self.expect('eof')
for s in self.stack:
print(s)
assert len(self.stack) == 0
for c in self.lexer.comments:
self.attach_comment(c)
return block

def attach_comment(self, comment: Comment) -> None:
if len(self.nodes) == 0:
return
smallest_distance = 100000000000
node = None
for n in self.nodes:
# Comment is before node
if n.bytespan[0] > comment.bytespan[0]:
continue
# Comment is after node
if n.bytespan[1] < comment.bytespan[0]:
continue
distance = n.bytespan[1] - n.bytespan[0]
if smallest_distance > distance:
smallest_distance = distance
node = n
if node is not None:
node.comments.append(comment)
return
smallest_distance = 100000000000
node = None
# Now we try to attach the comment to nodes that are before it
for n in self.nodes:
# Comment is before node
if n.bytespan[0] > comment.bytespan[0]:
continue
distance = n.bytespan[1] - comment.bytespan[0]
if smallest_distance > distance:
smallest_distance = distance
node = n
if node is not None:
node.pre_comments.append(comment)
return
smallest_distance = 100000000000
node = None
# Now we try to attach the comment to nodes that are after it
for n in self.nodes:
# Comment is after node
if comment.bytespan[1] < n.bytespan[0]:
continue
distance = comment.bytespan[1] - n.bytespan[0]
if smallest_distance > distance:
smallest_distance = distance
node = n
if node is not None:
node.post_comments.append(comment)
return
assert node is not None

def statement(self) -> BaseNode:
return self.e1()
self.begin()
e = self.e1()
self.end(e)
return e

def e1(self) -> BaseNode:
self.begin()
left = self.e2()
if self.accept('plusassign'):
value = self.e1()
if not isinstance(left, IdNode):
raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno)
assert isinstance(left.value, str)
return PlusAssignmentNode(left.filename, left.lineno, left.colno, left.value, value)
e = PlusAssignmentNode(left.filename, left.lineno, left.colno, left.value, value)
self.end(e)
return e
elif self.accept('assign'):
value = self.e1()
if not isinstance(left, IdNode):
raise ParseException('Assignment target must be an id.',
self.getline(), left.lineno, left.colno)
assert isinstance(left.value, str)
return AssignmentNode(left.filename, left.lineno, left.colno, left.value, value)
f = AssignmentNode(left.filename, left.lineno, left.colno, left.value, value)
self.end(f)
return f
elif self.accept('questionmark'):
if self.in_ternary:
raise ParseException('Nested ternary operators are not allowed.',
@@ -600,40 +681,67 @@ def e1(self) -> BaseNode:
self.expect('colon')
falseblock = self.e1()
self.in_ternary = False
return TernaryNode(left, trueblock, falseblock)
g = TernaryNode(left, trueblock, falseblock)
self.end(g)
return g
self.end(left)
return left

def e2(self) -> BaseNode:
self.begin()
left = self.e3()
started = False
while self.accept('or'):
started = True
if isinstance(left, EmptyNode):
raise ParseException('Invalid or clause.',
self.getline(), left.lineno, left.colno)
left = OrNode(left, self.e3())
x1 = OrNode(left, self.e3())
self.end(x1)
left = x1
started = False
self.begin()
if not started:
self.end(left)
return left

def e3(self) -> BaseNode:
self.begin()
left = self.e4()
started = False
while self.accept('and'):
if isinstance(left, EmptyNode):
raise ParseException('Invalid and clause.',
self.getline(), left.lineno, left.colno)
left = AndNode(left, self.e4())
x1 = AndNode(left, self.e4())
self.end(x1)
left = x1
started = False
self.begin()
if not started:
self.end(left)
return left

def e4(self) -> BaseNode:
self.begin()
left = self.e5()
for nodename, operator_type in comparison_map.items():
if self.accept(nodename):
return ComparisonNode(operator_type, left, self.e5())
x = ComparisonNode(operator_type, left, self.e5())
self.end(x)
return x
if self.accept('not') and self.accept('in'):
return ComparisonNode('notin', left, self.e5())
x = ComparisonNode('notin', left, self.e5())
self.end(x)
return x
self.end(left)
return left

def e5(self) -> BaseNode:
return self.e5addsub()

def e5addsub(self) -> BaseNode:
self.begin()
op_map = {
'plus': 'add',
'dash': 'sub',
@@ -642,12 +750,17 @@ def e5addsub(self) -> BaseNode:
while True:
op = self.accept_any(tuple(op_map.keys()))
if op:
left = ArithmeticNode(op_map[op], left, self.e5muldiv())
x1 = ArithmeticNode(op_map[op], left, self.e5muldiv())
self.end(x1)
left = x1
self.begin()
else:
break
self.end(left)
return left

def e5muldiv(self) -> BaseNode:
self.begin()
op_map = {
'percent': 'mod',
'star': 'mul',
@@ -657,19 +770,31 @@ def e5muldiv(self) -> BaseNode:
while True:
op = self.accept_any(tuple(op_map.keys()))
if op:
left = ArithmeticNode(op_map[op], left, self.e6())
x1 = ArithmeticNode(op_map[op], left, self.e6())
self.end(x1)
left = x1
self.begin()
else:
break
self.end(left)
return left

def e6(self) -> BaseNode:
self.begin()
if self.accept('not'):
return NotNode(self.current, self.e7())
x = NotNode(self.current, self.e7())
self.end(x)
return x
if self.accept('dash'):
return UMinusNode(self.current, self.e7())
return self.e7()
x1 = UMinusNode(self.current, self.e7())
self.end(x1)
return x1
x2 = self.e7()
self.end(x2)
return x2

def e7(self) -> BaseNode:
self.begin()
left = self.e8()
block_start = self.current
if self.accept('lparen'):
@@ -679,56 +804,93 @@ def e7(self) -> BaseNode:
raise ParseException('Function call must be applied to plain id',
self.getline(), left.lineno, left.colno)
assert isinstance(left.value, str)
left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left.value, args)
x1 = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left.value, args)
self.end(left)
left = x1
self.begin()
go_again = True
while go_again:
go_again = False
if self.accept('dot'):
go_again = True
left = self.method_call(left)
x2 = self.method_call(left)
self.end(left)
left = x2
self.begin()
if self.accept('lbracket'):
go_again = True
left = self.index_call(left)
x3 = self.index_call(left)
self.end(left)
left = x3
self.begin()
self.end(left)
return left

def e8(self) -> BaseNode:
self.begin()
block_start = self.current
if self.accept('lparen'):
e = self.statement()
self.block_expect('rparen', block_start)
return ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno)
e1 = ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno)
self.end(e1)
return e1
elif self.accept('lbracket'):
args = self.args()
self.block_expect('rbracket', block_start)
return ArrayNode(args, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno)
x1 = ArrayNode(args, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno)
self.end(x1)
return x1
elif self.accept('lcurl'):
key_values = self.key_values()
self.block_expect('rcurl', block_start)
return DictNode(key_values, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno)
x2 = DictNode(key_values, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno)
self.end(x2)
return x2
else:
return self.e9()
x3 = self.e9()
self.end(x3)
return x3

def e9(self) -> BaseNode:
self.begin()
t = self.current
if self.accept('true'):
t.value = True
return BooleanNode(t)
x1 = BooleanNode(t)
self.end(x1)
return x1
if self.accept('false'):
t.value = False
return BooleanNode(t)
x2 = BooleanNode(t)
self.end(x2)
return x2
if self.accept('id'):
return IdNode(t)
x3 = IdNode(t)
self.end(x3)
return x3
if self.accept('number'):
return NumberNode(t)
x4 = NumberNode(t)
self.end(x4)
return x4
if self.accept('string'):
return StringNode(t)
x5 = StringNode(t)
self.end(x5)
return x5
if self.accept('fstring'):
return FormatStringNode(t)
x6 = FormatStringNode(t)
self.end(x6)
return x6
if self.accept('multiline_fstring'):
return MultilineFormatStringNode(t)
return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
x7 = MultilineFormatStringNode(t)
self.end(x7)
return x7
x8 = EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
self.end(x8)
return x8

def key_values(self) -> ArgumentNode:
self.begin()
s = self.statement() # type: BaseNode
a = ArgumentNode(self.current)

@@ -737,15 +899,18 @@ def key_values(self) -> ArgumentNode:
a.set_kwarg_no_check(s, self.statement())
potential = self.current
if not self.accept('comma'):
self.end(a)
return a
a.commas.append(potential)
else:
raise ParseException('Only key:value pairs are valid in dict construction.',
self.getline(), s.lineno, s.colno)
s = self.statement()
self.end(a)
return a

def args(self) -> ArgumentNode:
self.begin()
s = self.statement() # type: BaseNode
a = ArgumentNode(self.current)

@@ -761,15 +926,19 @@ def args(self) -> ArgumentNode:
a.set_kwarg(s, self.statement())
potential = self.current
if not self.accept('comma'):
self.end(a)
return a
a.commas.append(potential)
else:
a.append(s)
self.end(a)
return a
s = self.statement()
self.end(a)
return a

def method_call(self, source_object: BaseNode) -> MethodNode:
self.begin()
methodname = self.e9()
if not isinstance(methodname, IdNode):
raise ParseException('Method name must be plain id',
@@ -780,15 +949,22 @@ def method_call(self, source_object: BaseNode) -> MethodNode:
self.expect('rparen')
method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname.value, args)
if self.accept('dot'):
return self.method_call(method)
x1 = self.method_call(method)
self.end(x1)
return x1
self.end(method)
return method

def index_call(self, source_object: BaseNode) -> IndexNode:
self.begin()
index_statement = self.statement()
self.expect('rbracket')
return IndexNode(source_object, index_statement)
x1 = IndexNode(source_object, index_statement)
self.end(x1)
return x1

def foreachblock(self) -> ForeachClauseNode:
self.begin()
t = self.current
self.expect('id')
assert isinstance(t.value, str)
@@ -804,55 +980,81 @@ def foreachblock(self) -> ForeachClauseNode:
self.expect('colon')
items = self.statement()
block = self.codeblock()
return ForeachClauseNode(varname, varnames, items, block)
x1 = ForeachClauseNode(varname, varnames, items, block)
self.end(x1)
return x1

def ifblock(self) -> IfClauseNode:
self.begin()
condition = self.statement()
clause = IfClauseNode(condition)
self.expect('eol')
block = self.codeblock()
clause.ifs.append(IfNode(clause, condition, block))
self.begin()
i = IfNode(clause, condition, block)
self.end(i)
clause.ifs.append(i)
self.elseifblock(clause)
clause.elseblock = self.elseblock()
self.end(clause)
return clause

def elseifblock(self, clause: IfClauseNode) -> None:
while self.accept('elif'):
self.begin()
s = self.statement()
self.expect('eol')
b = self.codeblock()
clause.ifs.append(IfNode(s, s, b))
x1 = IfNode(s, s, b)
self.end(x1)
clause.ifs.append(x1)

def elseblock(self) -> T.Union[CodeBlockNode, EmptyNode]:
if self.accept('else'):
self.expect('eol')
return self.codeblock()
return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
self.begin()
x1 = EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
self.end(x1)
return x1

def line(self) -> BaseNode:
block_start = self.current
self.begin()
if self.current == 'eol':
return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
x1 = EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
self.end(x1)
return x1
if self.accept('if'):
ifblock = self.ifblock()
self.block_expect('endif', block_start)
self.end(ifblock)
return ifblock
if self.accept('foreach'):
forblock = self.foreachblock()
self.block_expect('endforeach', block_start)
self.end(forblock)
return forblock
if self.accept('continue'):
return ContinueNode(self.current)
x2 = ContinueNode(self.current)
self.end(x2)
return x2
if self.accept('break'):
return BreakNode(self.current)
return self.statement()
x3 = BreakNode(self.current)
self.end(x3)
return x3
s = self.statement()
self.end(s)
return s

def codeblock(self) -> CodeBlockNode:
self.begin()
block = CodeBlockNode(self.current)
cond = True
while cond:
curline = self.line()
if not isinstance(curline, EmptyNode):
block.lines.append(curline)
cond = self.accept('eol')
self.end(block)
return block