MacroPy crashes on complex and bytes

[Technologicat]

I get a crash during macro expansion when the source AST being transformed contains complex or bytes literals. The transformed AST contains the bare complex or bytes object (and hence is an invalid AST), instead of having that object inside the appropriate type of AST node.

This affects at least macroyp3 version 1.1.0b2 (latest) on PyPI, the HEAD version of MacroPy from this repo, and the HEAD version of MacroPy from my fork (which contains my PRs from 1-2 years ago).

Tested on Python 3.6. Occurs on both CPython 3.6.9 and PyPy3 7.3.0.

Test case:

# ----------------------------------------
# let.py

from macropy.core.macros import Macros
from macropy.core.quotes import macros, q, ast_literal

from ast import arg

macros = Macros()  # noqa: F811

@macros.expr
def let(tree, args, **kw):  # args; ast.Tuple: (k1, v1), (k2, v2), ..., (kn, vn)
    names = [k.id for k, _ in (a.elts for a in args)]
    if len(set(names)) < len(names):
        assert False, "binding names must be unique in the same let"
    values = [v for _, v in (a.elts for a in args)]
    lam = q[lambda: ast_literal[tree]]
    lam.args.args = [arg(arg=x) for x in names]  # inject args
    return q[ast_literal[lam](ast_literal[values])]

# ----------------------------------------
# main.py

from let import macros, let  # noqa: F401

let((x, 21))[2 * x]  # runs fine  # noqa: F821, the `let` defines `x`.
let((x, 1 + 2j))[2 * x]  # complex number, crash (invalid AST)  # noqa: F821
let((x, b"wiki"))[2 * x]  # bytestring, crash (invalid AST)  # noqa: F821

# ----------------------------------------
# wrapper.py

import macropy.activate  # noqa: F401
import main  # noqa: F401

Run with python3 wrapper.py.

Output:

jje@arcturus:~/Documents/koodit/macropy/complexcrash/$ python3 wrapper.py
Traceback (most recent call last):
  File "wrapper.py", line 2, in <module>
    import main  # noqa: F401
  File "<frozen importlib._bootstrap>", line 971, in _find_and_load
  File "<frozen importlib._bootstrap>", line 951, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 894, in _find_spec
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/import_hooks.py", line 147, in find_spec
    code, tree = self.expand_macros(source, origin, spec)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/import_hooks.py", line 111, in expand_macros
    tree, source_code, modules).expand_macros()
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 630, in expand_macros
    tree = super().expand_macros(tree)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 436, in expand_macros
    return self.walk_tree(tree)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 586, in walk_tree
    self.walk_children(tree)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 550, in walk_children
    new_value = self.walk_tree(old_value)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 586, in walk_tree
    self.walk_children(tree)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 555, in walk_children
    new_t = self.walk_tree(t)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 586, in walk_tree
    self.walk_children(tree)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 550, in walk_children
    new_value = self.walk_tree(old_value)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 577, in walk_tree
    new_tree = self.walk_tree(expand_gen.send(new_tree))
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 467, in macro_expand
    new_tree = yield expand_single_gen.send(new_tree)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/macros.py", line 535, in macro_expand_single
    tuple(self.file_vars.items()))
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/cleanup.py", line 62, in fill_line_numbers
    fill_line_numbers(sub, tree.lineno, tree.col_offset)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/cleanup.py", line 56, in fill_line_numbers
    fill_line_numbers(sub, lineno, col_offset)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/cleanup.py", line 62, in fill_line_numbers
    fill_line_numbers(sub, tree.lineno, tree.col_offset)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/cleanup.py", line 62, in fill_line_numbers
    fill_line_numbers(sub, tree.lineno, tree.col_offset)
  File "/home/jje/.local/lib/python3.6/site-packages/macropy3-1.1.0b2-py3.6.egg/macropy/core/cleanup.py", line 67, in fill_line_numbers
    "after expansion".format(tree, type(tree)))
TypeError: Invalid AST node '2j',  type: '<class 'complex'>' after expansion

[Technologicat]

Hmm, the check in https://github.com/azazel75/macropy/blob/master/macropy/core/cleanup.py#L63...

Shouldn't we have bytes and complex there, too?