From 6d3d72cb3d8d971d3451862f021761c786793619 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Fri, 19 Feb 2021 18:01:24 -0500 Subject: [PATCH] Upgrade mypy and linting fixes --- manticore/core/smtlib/expression.py | 2 +- manticore/core/state.py | 8 +-- manticore/native/cpu/aarch64.py | 4 +- manticore/native/cpu/x86.py | 2 +- manticore/native/state.py | 7 +-- manticore/platforms/platform.py | 7 +-- mypy.ini | 41 +--------------- setup.py | 2 +- tests/auto_generators/flags.py | 4 +- tests/auto_generators/make_dump.py | 76 +++++++++++++++-------------- tests/auto_generators/make_tests.py | 3 +- tests/wasm/json2mc.py | 35 ++++++++----- 12 files changed, 84 insertions(+), 107 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index c1cddba276..f65001e4df 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1168,7 +1168,7 @@ def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size self._written: Optional[Set[Any]] = None # Cache of the known indexes - self._concrete_cache: Dict[Any, Any] = None + self._concrete_cache: Optional[Dict[Any, Any]] = None self._length = array.length self._default = array.default diff --git a/manticore/core/state.py b/manticore/core/state.py index efc1c0d4da..b408940896 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,6 +1,6 @@ import copy import logging -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import Any, Dict, List, Optional, TypeVar, TYPE_CHECKING from .smtlib import Bool, ConstraintSet, Expression, issymbolic, BitVecConstant, MutableArray from ..utils.event import Eventful @@ -172,8 +172,8 @@ class StateBase(Eventful): """ Representation of a unique program state/path. - :param ConstraintSet constraints: Initial constraints - :param Platform platform: Initial operating system state + :param constraints: Initial constraints + :param platform: Initial operating system state :ivar dict context: Local context for arbitrary data storage """ @@ -183,7 +183,7 @@ def __init__( self, *, constraints: ConstraintSet, - platform: "Platform", + platform: Platform, manticore: Optional["ManticoreBase"] = None, **kwargs, ): diff --git a/manticore/native/cpu/aarch64.py b/manticore/native/cpu/aarch64.py index 2957c5fb3b..ce861b753d 100644 --- a/manticore/native/cpu/aarch64.py +++ b/manticore/native/cpu/aarch64.py @@ -40,7 +40,7 @@ class Aarch64InvalidInstruction(CpuException): # See "C1.2.4 Condition code". -Condspec = collections.namedtuple("CondSpec", "inverse func") +Condspec = collections.namedtuple("Condspec", "inverse func") COND_MAP = { cs.arm64.ARM64_CC_EQ: Condspec(cs.arm64.ARM64_CC_NE, lambda n, z, c, v: z == 1), cs.arm64.ARM64_CC_NE: Condspec(cs.arm64.ARM64_CC_EQ, lambda n, z, c, v: z == 0), @@ -75,7 +75,7 @@ class Aarch64InvalidInstruction(CpuException): class Aarch64RegisterFile(RegisterFile): - Regspec = collections.namedtuple("RegSpec", "parent size") + Regspec = collections.namedtuple("Regspec", "parent size") # Register table. _table = {} diff --git a/manticore/native/cpu/x86.py b/manticore/native/cpu/x86.py index 5076bc15aa..730514df46 100644 --- a/manticore/native/cpu/x86.py +++ b/manticore/native/cpu/x86.py @@ -144,7 +144,7 @@ def new_method(cpu, *args, **kw_args): class AMD64RegFile(RegisterFile): - Regspec = collections.namedtuple("RegSpec", "register_id ty offset size reset") + Regspec = collections.namedtuple("Regspec", "register_id ty offset size reset") _flags = {"CF": 0, "PF": 2, "AF": 4, "ZF": 6, "SF": 7, "IF": 9, "DF": 10, "OF": 11} _table = { "CS": Regspec("CS", int, 0, 16, False), diff --git a/manticore/native/state.py b/manticore/native/state.py index bb747920c5..9927922341 100644 --- a/manticore/native/state.py +++ b/manticore/native/state.py @@ -11,9 +11,10 @@ from ..platforms import linux_syscalls if TYPE_CHECKING: - from ..platforms.platform import Platform + from ..platforms.linux import Linux + from ..platforms.decree import Decree -HookCallback = Callable[[StateBase], None] +HookCallback = Callable[["State"], None] logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ class CheckpointData(NamedTuple): class State(StateBase): - def __init__(self, *, constraints: ConstraintSet, platform: "Platform", **kwargs): + def __init__(self, *, constraints: ConstraintSet, platform: Union[Linux, Decree], **kwargs): super().__init__(constraints=constraints, platform=platform, **kwargs) self._hooks: Dict[Optional[int], Set[HookCallback]] = {} self._after_hooks: Dict[Optional[int], Set[HookCallback]] = {} diff --git a/manticore/platforms/platform.py b/manticore/platforms/platform.py index 43a4e5aeec..e92194db91 100644 --- a/manticore/platforms/platform.py +++ b/manticore/platforms/platform.py @@ -1,11 +1,10 @@ import logging -from typing import Optional - from functools import wraps -from typing import Any, Callable, TypeVar +from typing import Any, Callable, TypeVar, Optional from ..utils.event import Eventful from ..core.state import StateBase +from ..native.cpu.abstractcpu import Cpu logger = logging.getLogger(__name__) @@ -50,6 +49,8 @@ class Platform(Eventful): Base class for all platforms e.g. operating systems or virtual machines. """ + current: Any + _published_events = {"solve"} def __init__(self, *, state: Optional[StateBase] = None, **kwargs): diff --git a/mypy.ini b/mypy.ini index c15877c4e6..8597e3660f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,7 @@ [mypy] python_version = 3.6 files = manticore, tests, examples/**/*.py +ignore_missing_imports = True # TODO: LOTS OF ERRORS # check_untyped_defs = True @@ -11,45 +12,5 @@ check_untyped_defs = True [mypy-manticore.ethereum.parsetab] ignore_errors = True -# 3rd-party libraries with no typing information -[mypy-capstone.*] -ignore_missing_imports = True - -[mypy-crytic_compile.*] -ignore_missing_imports = True - -[mypy-elftools.*] -ignore_missing_imports = True - -[mypy-sha3.*] -ignore_missing_imports = True - -[mypy-pyevmasm.*] -ignore_missing_imports = True - -[mypy-unicorn.*] -ignore_missing_imports = True - -[mypy-keystone.*] -ignore_missing_imports = True - -[mypy-ply.*] -ignore_missing_imports = True - -[mypy-rlp.*] -ignore_missing_imports = True - -[mypy-setuptools.*] -ignore_missing_imports = True - -[mypy-toposort.*] -ignore_missing_imports = True - -[mypy-prettytable.*] -ignore_missing_imports = True - -[mypy-wasm.*] -ignore_missing_imports = True - [mypy-manticore.core.state_pb2] ignore_errors = True diff --git a/setup.py b/setup.py index 0316785172..ef8968c849 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def rtd_dependent_deps(): # (we need to know how to import a given native dependency so we can check if native dependencies are installed) native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc2"] -lint_deps = ["black==20.8b1", "mypy==0.790"] +lint_deps = ["black==20.8b1", "mypy==0.812"] auto_test_deps = ["py-evm"] diff --git a/tests/auto_generators/flags.py b/tests/auto_generators/flags.py index 62e8f7eebd..99f026ef2e 100644 --- a/tests/auto_generators/flags.py +++ b/tests/auto_generators/flags.py @@ -1,4 +1,6 @@ -flags = { +from typing import Dict, List + +flags: Dict[str, Dict[str, List[str]]] = { "AAA": { "undefined": ["OF", "SF", "ZF", "PF"], "defined": ["AF", "CF"], diff --git a/tests/auto_generators/make_dump.py b/tests/auto_generators/make_dump.py index 259342ad3d..76e3dd4dd0 100644 --- a/tests/auto_generators/make_dump.py +++ b/tests/auto_generators/make_dump.py @@ -5,8 +5,10 @@ import sys import time import subprocess -from capstone import * -from capstone.x86 import * +from typing import Any, Dict +from capstone import Cs +from capstone.x86 import CS_ARCH_X86, CS_MODE_32, CS_MODE_64, X86_OP_MEM, X86_OP_REG, X86_OP_IMM +import capstone.x86 as csr from flags import flags flags_maks = { @@ -228,7 +230,7 @@ def read_operand(o): groups = map(instruction.group_name, instruction.groups) PC = {"i386": "EIP", "amd64": "RIP"}[arch] - registers = {PC: gdb.getR(PC)} + registers: Dict[Any, Any] = {PC: gdb.getR(PC)} memory = {} # save the encoded instruction @@ -246,11 +248,11 @@ def read_operand(o): if instruction.insn_name().upper() in ["PUSHF", "PUSHFD"]: registers["EFLAGS"] = gdb.getR("EFLAGS") - if instruction.insn_name().upper() in ["XLAT", "XLATB"]: - registers["AL"] = gdb.getR("AL") - registers[B] = gdb.getR(B) - address = registers[B] + registers["AL"] - memory[address] = chr(gdb.getByte(address)) + # if instruction.insn_name().upper() in ["XLAT", "XLATB"]: + # registers["AL"] = gdb.getR("AL") + # registers[B] = gdb.getR(B) + # address = registers[B] + registers["AL"] + # memory[address] = chr(gdb.getByte(address)) if instruction.insn_name().upper() in ["BTC", "BTR", "BTS", "BT"]: if instruction.operands[0].type == X86_OP_MEM: @@ -310,34 +312,34 @@ def read_operand(o): # registers[reg_name] = gdb.getR(reg_name) reg_sizes = { - X86_REG_AH: X86_REG_AX, - X86_REG_AL: X86_REG_AX, - X86_REG_AX: X86_REG_EAX, - X86_REG_EAX: X86_REG_RAX, - X86_REG_RAX: X86_REG_INVALID, - X86_REG_BH: X86_REG_BX, - X86_REG_BL: X86_REG_BX, - X86_REG_BX: X86_REG_EBX, - X86_REG_EBX: X86_REG_RBX, - X86_REG_RBX: X86_REG_INVALID, - X86_REG_CH: X86_REG_CX, - X86_REG_CL: X86_REG_CX, - X86_REG_CX: X86_REG_ECX, - X86_REG_ECX: X86_REG_RCX, - X86_REG_RCX: X86_REG_INVALID, - X86_REG_DH: X86_REG_DX, - X86_REG_DL: X86_REG_DX, - X86_REG_DX: X86_REG_EDX, - X86_REG_EDX: X86_REG_RDX, - X86_REG_RDX: X86_REG_INVALID, - X86_REG_DIL: X86_REG_EDI, - X86_REG_DI: X86_REG_EDI, - X86_REG_EDI: X86_REG_RDI, - X86_REG_RDI: X86_REG_INVALID, - X86_REG_SIL: X86_REG_ESI, - X86_REG_SI: X86_REG_ESI, - X86_REG_ESI: X86_REG_RSI, - X86_REG_RSI: X86_REG_INVALID, + csr.X86_REG_AH: csr.X86_REG_AX, + csr.X86_REG_AL: csr.X86_REG_AX, + csr.X86_REG_AX: csr.X86_REG_EAX, + csr.X86_REG_EAX: csr.X86_REG_RAX, + csr.X86_REG_RAX: csr.X86_REG_INVALID, + csr.X86_REG_BH: csr.X86_REG_BX, + csr.X86_REG_BL: csr.X86_REG_BX, + csr.X86_REG_BX: csr.X86_REG_EBX, + csr.X86_REG_EBX: csr.X86_REG_RBX, + csr.X86_REG_RBX: csr.X86_REG_INVALID, + csr.X86_REG_CH: csr.X86_REG_CX, + csr.X86_REG_CL: csr.X86_REG_CX, + csr.X86_REG_CX: csr.X86_REG_ECX, + csr.X86_REG_ECX: csr.X86_REG_RCX, + csr.X86_REG_RCX: csr.X86_REG_INVALID, + csr.X86_REG_DH: csr.X86_REG_DX, + csr.X86_REG_DL: csr.X86_REG_DX, + csr.X86_REG_DX: csr.X86_REG_EDX, + csr.X86_REG_EDX: csr.X86_REG_RDX, + csr.X86_REG_RDX: csr.X86_REG_INVALID, + csr.X86_REG_DIL: csr.X86_REG_EDI, + csr.X86_REG_DI: csr.X86_REG_EDI, + csr.X86_REG_EDI: csr.X86_REG_RDI, + csr.X86_REG_RDI: csr.X86_REG_INVALID, + csr.X86_REG_SIL: csr.X86_REG_ESI, + csr.X86_REG_SI: csr.X86_REG_ESI, + csr.X86_REG_ESI: csr.X86_REG_RSI, + csr.X86_REG_RSI: csr.X86_REG_INVALID, } # There is a capstone branch that should fix all these annoyances... soon # https://github.com/aquynh/capstone/tree/next @@ -387,7 +389,7 @@ def read_operand(o): registers[reg_name] = gdb.getR(reg_name) address += o.mem.scale * registers[reg_name] address = address & ({"i386": 0xFFFFFFFF, "amd64": 0xFFFFFFFFFFFFFFFF}[arch]) - for i in xrange(address, address + o.size): + for i in range(address, address + o.size): memory[i] = chr(gdb.getByte(i)) # gather PRE info diff --git a/tests/auto_generators/make_tests.py b/tests/auto_generators/make_tests.py index 318b35a277..b56af1ae73 100644 --- a/tests/auto_generators/make_tests.py +++ b/tests/auto_generators/make_tests.py @@ -1,5 +1,6 @@ from __future__ import print_function import sys +from typing import Any, Dict import random tests = [] @@ -15,7 +16,7 @@ random.shuffle(tests) -op_count = {} +op_count: Dict[str, int] = {} test_dic = {} for test in tests: try: diff --git a/tests/wasm/json2mc.py b/tests/wasm/json2mc.py index 3611cc1c4e..0294cd7468 100644 --- a/tests/wasm/json2mc.py +++ b/tests/wasm/json2mc.py @@ -3,6 +3,7 @@ from jinja2 import Environment, FileSystemLoader from base64 import b64encode from copy import copy +from typing import Any, List parser = argparse.ArgumentParser("Generate Manticore tests from the WASM Spec") parser.add_argument("filename", type=argparse.FileType("r"), help="JSON file output from wast2json") @@ -60,7 +61,7 @@ def escape_null(in_str: str): template = env.get_template("test_template.jinja2") -modules = [] +modules: List[Any] = [] registered_modules = {} imports = [] current_module = None @@ -100,14 +101,16 @@ def escape_null(in_str: str): mod_name=d["action"].get("module", None), ) elif d["action"]["type"] == "get": - modules[current_module].add_test( - d["action"]["field"], - d["line"], - [], - convert_types(d["expected"]), - "assert_global", - d["action"].get("module", None), - ) + if current_module: + modules[current_module].add_test( + d["action"]["field"], + d["line"], + [], + convert_types(d["expected"]), + "assert_global", + d["action"].get("module", None), + ) + current_module = None else: raise NotImplementedError("assert_return with action type: " + d["action"]["type"]) elif d["type"] == "assert_return_arithmetic_nan": @@ -155,10 +158,16 @@ def escape_null(in_str: str): if maybe_name: # This is an alias for another registered module imports.append({"type": "alias", "alias": d["as"], "orig": maybe_name}) else: # This is an alias for the current module - imports.append( - {"type": "import", "name": d["as"], "filename": modules[current_module].filename} - ) - modules[current_module].registered_name = d["as"] + if current_module: + imports.append( + { + "type": "import", + "name": d["as"], + "filename": modules[current_module].filename, + } + ) + modules[current_module].registered_name = d["as"] + current_module = None if current_module: modules[current_module].imports = copy(imports)