Upgrade mypy and linting fixes

manticore/core/smtlib/expression.py

@@ -1168,7 +1168,7 @@ def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs):
         assert index.size == array.index_size
         assert value.size == array.value_size
         self._written: Optional[Set[Any]] = None  # Cache of the known indexes
-        self._concrete_cache: Dict[Any, Any] = None
+        self._concrete_cache: Optional[Dict[Any, Any]] = None
         self._length = array.length
         self._default = array.default
 

manticore/core/state.py

@@ -1,6 +1,6 @@
 import copy
 import logging
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Dict, List, Optional, TypeVar, TYPE_CHECKING
 
 from .smtlib import Bool, ConstraintSet, Expression, issymbolic, BitVecConstant, MutableArray
 from ..utils.event import Eventful
@@ -172,8 +172,8 @@ class StateBase(Eventful):
     """
     Representation of a unique program state/path.
 
-    :param ConstraintSet constraints: Initial constraints
-    :param Platform platform: Initial operating system state
+    :param constraints: Initial constraints
+    :param platform: Initial operating system state
     :ivar dict context: Local context for arbitrary data storage
     """
 
@@ -183,7 +183,7 @@ def __init__(
         self,
         *,
         constraints: ConstraintSet,
-        platform: "Platform",
+        platform: Platform,
         manticore: Optional["ManticoreBase"] = None,
         **kwargs,
     ):

manticore/native/cpu/aarch64.py

@@ -40,7 +40,7 @@ class Aarch64InvalidInstruction(CpuException):
 
 
 # See "C1.2.4 Condition code".
-Condspec = collections.namedtuple("CondSpec", "inverse func")
+Condspec = collections.namedtuple("Condspec", "inverse func")
 COND_MAP = {
     cs.arm64.ARM64_CC_EQ: Condspec(cs.arm64.ARM64_CC_NE, lambda n, z, c, v: z == 1),
     cs.arm64.ARM64_CC_NE: Condspec(cs.arm64.ARM64_CC_EQ, lambda n, z, c, v: z == 0),
@@ -75,7 +75,7 @@ class Aarch64InvalidInstruction(CpuException):
 
 
 class Aarch64RegisterFile(RegisterFile):
-    Regspec = collections.namedtuple("RegSpec", "parent size")
+    Regspec = collections.namedtuple("Regspec", "parent size")
 
     # Register table.
     _table = {}

manticore/native/cpu/x86.py

@@ -144,7 +144,7 @@ def new_method(cpu, *args, **kw_args):
 
 
 class AMD64RegFile(RegisterFile):
-    Regspec = collections.namedtuple("RegSpec", "register_id ty offset size reset")
+    Regspec = collections.namedtuple("Regspec", "register_id ty offset size reset")
     _flags = {"CF": 0, "PF": 2, "AF": 4, "ZF": 6, "SF": 7, "IF": 9, "DF": 10, "OF": 11}
     _table = {
         "CS": Regspec("CS", int, 0, 16, False),

manticore/native/state.py

@@ -11,9 +11,10 @@
 from ..platforms import linux_syscalls
 
 if TYPE_CHECKING:
-    from ..platforms.platform import Platform
+    from ..platforms.linux import Linux
+    from ..platforms.decree import Decree
 
-HookCallback = Callable[[StateBase], None]
+HookCallback = Callable[["State"], None]
 logger = logging.getLogger(__name__)
 
 
@@ -23,7 +24,7 @@ class CheckpointData(NamedTuple):
 
 
 class State(StateBase):
-    def __init__(self, *, constraints: ConstraintSet, platform: "Platform", **kwargs):
+    def __init__(self, *, constraints: ConstraintSet, platform: Union[Linux, Decree], **kwargs):
         super().__init__(constraints=constraints, platform=platform, **kwargs)
         self._hooks: Dict[Optional[int], Set[HookCallback]] = {}
         self._after_hooks: Dict[Optional[int], Set[HookCallback]] = {}

manticore/platforms/platform.py

@@ -1,11 +1,10 @@
 import logging
-from typing import Optional
-
 from functools import wraps
-from typing import Any, Callable, TypeVar
+from typing import Any, Callable, TypeVar, Optional
 
 from ..utils.event import Eventful
 from ..core.state import StateBase
+from ..native.cpu.abstractcpu import Cpu
 
 
 logger = logging.getLogger(__name__)
@@ -50,6 +49,8 @@ class Platform(Eventful):
     Base class for all platforms e.g. operating systems or virtual machines.
     """
 
+    current: Any
+
     _published_events = {"solve"}
 
     def __init__(self, *, state: Optional[StateBase] = None, **kwargs):

mypy.ini

@@ -1,6 +1,7 @@
 [mypy]
 python_version = 3.6
 files = manticore, tests, examples/**/*.py
+ignore_missing_imports = True
 # TODO: LOTS OF ERRORS
 # check_untyped_defs = True
 
@@ -11,45 +12,5 @@ check_untyped_defs = True
 [mypy-manticore.ethereum.parsetab]
 ignore_errors = True
 
-# 3rd-party libraries with no typing information
-[mypy-capstone.*]
-ignore_missing_imports = True
-
-[mypy-crytic_compile.*]
-ignore_missing_imports = True
-
-[mypy-elftools.*]
-ignore_missing_imports = True
-
-[mypy-sha3.*]
-ignore_missing_imports = True
-
-[mypy-pyevmasm.*]
-ignore_missing_imports = True
-
-[mypy-unicorn.*]
-ignore_missing_imports = True
-
-[mypy-keystone.*]
-ignore_missing_imports = True
-
-[mypy-ply.*]
-ignore_missing_imports = True
-
-[mypy-rlp.*]
-ignore_missing_imports = True
-
-[mypy-setuptools.*]
-ignore_missing_imports = True
-
-[mypy-toposort.*]
-ignore_missing_imports = True
-
-[mypy-prettytable.*]
-ignore_missing_imports = True
-
-[mypy-wasm.*]
-ignore_missing_imports = True
-
 [mypy-manticore.core.state_pb2]
 ignore_errors = True

setup.py

@@ -18,7 +18,7 @@ def rtd_dependent_deps():
 # (we need to know how to import a given native dependency so we can check if native dependencies are installed)
 native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc2"]
 
-lint_deps = ["black==20.8b1", "mypy==0.790"]
+lint_deps = ["black==20.8b1", "mypy==0.812"]
 
 auto_test_deps = ["py-evm"]
 

tests/auto_generators/flags.py

@@ -1,4 +1,6 @@
-flags = {
+from typing import Dict, List
+
+flags: Dict[str, Dict[str, List[str]]] = {
     "AAA": {
         "undefined": ["OF", "SF", "ZF", "PF"],
         "defined": ["AF", "CF"],

tests/auto_generators/make_dump.py

@@ -5,8 +5,10 @@
 import sys
 import time
 import subprocess
-from capstone import *
-from capstone.x86 import *
+from typing import Any, Dict
+from capstone import Cs
+from capstone.x86 import CS_ARCH_X86, CS_MODE_32, CS_MODE_64, X86_OP_MEM, X86_OP_REG, X86_OP_IMM
+import capstone.x86 as csr
 from flags import flags
 
 flags_maks = {
@@ -228,7 +230,7 @@ def read_operand(o):
         groups = map(instruction.group_name, instruction.groups)
 
         PC = {"i386": "EIP", "amd64": "RIP"}[arch]
-        registers = {PC: gdb.getR(PC)}
+        registers: Dict[Any, Any] = {PC: gdb.getR(PC)}
         memory = {}
 
         # save the encoded instruction
@@ -246,11 +248,11 @@ def read_operand(o):
         if instruction.insn_name().upper() in ["PUSHF", "PUSHFD"]:
             registers["EFLAGS"] = gdb.getR("EFLAGS")
 
-        if instruction.insn_name().upper() in ["XLAT", "XLATB"]:
-            registers["AL"] = gdb.getR("AL")
-            registers[B] = gdb.getR(B)
-            address = registers[B] + registers["AL"]
-            memory[address] = chr(gdb.getByte(address))
+        # if instruction.insn_name().upper() in ["XLAT", "XLATB"]:
+        #     registers["AL"] = gdb.getR("AL")
+        #     registers[B] = gdb.getR(B)
+        #     address = registers[B] + registers["AL"]
+        #     memory[address] = chr(gdb.getByte(address))
 
         if instruction.insn_name().upper() in ["BTC", "BTR", "BTS", "BT"]:
             if instruction.operands[0].type == X86_OP_MEM:
@@ -310,34 +312,34 @@ def read_operand(o):
         #            registers[reg_name] = gdb.getR(reg_name)
 
         reg_sizes = {
-            X86_REG_AH: X86_REG_AX,
-            X86_REG_AL: X86_REG_AX,
-            X86_REG_AX: X86_REG_EAX,
-            X86_REG_EAX: X86_REG_RAX,
-            X86_REG_RAX: X86_REG_INVALID,
-            X86_REG_BH: X86_REG_BX,
-            X86_REG_BL: X86_REG_BX,
-            X86_REG_BX: X86_REG_EBX,
-            X86_REG_EBX: X86_REG_RBX,
-            X86_REG_RBX: X86_REG_INVALID,
-            X86_REG_CH: X86_REG_CX,
-            X86_REG_CL: X86_REG_CX,
-            X86_REG_CX: X86_REG_ECX,
-            X86_REG_ECX: X86_REG_RCX,
-            X86_REG_RCX: X86_REG_INVALID,
-            X86_REG_DH: X86_REG_DX,
-            X86_REG_DL: X86_REG_DX,
-            X86_REG_DX: X86_REG_EDX,
-            X86_REG_EDX: X86_REG_RDX,
-            X86_REG_RDX: X86_REG_INVALID,
-            X86_REG_DIL: X86_REG_EDI,
-            X86_REG_DI: X86_REG_EDI,
-            X86_REG_EDI: X86_REG_RDI,
-            X86_REG_RDI: X86_REG_INVALID,
-            X86_REG_SIL: X86_REG_ESI,
-            X86_REG_SI: X86_REG_ESI,
-            X86_REG_ESI: X86_REG_RSI,
-            X86_REG_RSI: X86_REG_INVALID,
+            csr.X86_REG_AH: csr.X86_REG_AX,
+            csr.X86_REG_AL: csr.X86_REG_AX,
+            csr.X86_REG_AX: csr.X86_REG_EAX,
+            csr.X86_REG_EAX: csr.X86_REG_RAX,
+            csr.X86_REG_RAX: csr.X86_REG_INVALID,
+            csr.X86_REG_BH: csr.X86_REG_BX,
+            csr.X86_REG_BL: csr.X86_REG_BX,
+            csr.X86_REG_BX: csr.X86_REG_EBX,
+            csr.X86_REG_EBX: csr.X86_REG_RBX,
+            csr.X86_REG_RBX: csr.X86_REG_INVALID,
+            csr.X86_REG_CH: csr.X86_REG_CX,
+            csr.X86_REG_CL: csr.X86_REG_CX,
+            csr.X86_REG_CX: csr.X86_REG_ECX,
+            csr.X86_REG_ECX: csr.X86_REG_RCX,
+            csr.X86_REG_RCX: csr.X86_REG_INVALID,
+            csr.X86_REG_DH: csr.X86_REG_DX,
+            csr.X86_REG_DL: csr.X86_REG_DX,
+            csr.X86_REG_DX: csr.X86_REG_EDX,
+            csr.X86_REG_EDX: csr.X86_REG_RDX,
+            csr.X86_REG_RDX: csr.X86_REG_INVALID,
+            csr.X86_REG_DIL: csr.X86_REG_EDI,
+            csr.X86_REG_DI: csr.X86_REG_EDI,
+            csr.X86_REG_EDI: csr.X86_REG_RDI,
+            csr.X86_REG_RDI: csr.X86_REG_INVALID,
+            csr.X86_REG_SIL: csr.X86_REG_ESI,
+            csr.X86_REG_SI: csr.X86_REG_ESI,
+            csr.X86_REG_ESI: csr.X86_REG_RSI,
+            csr.X86_REG_RSI: csr.X86_REG_INVALID,
         }
         # There is a capstone branch that should fix all these annoyances... soon
         # https://github.com/aquynh/capstone/tree/next
@@ -387,7 +389,7 @@ def read_operand(o):
                     registers[reg_name] = gdb.getR(reg_name)
                     address += o.mem.scale * registers[reg_name]
                 address = address & ({"i386": 0xFFFFFFFF, "amd64": 0xFFFFFFFFFFFFFFFF}[arch])
-                for i in xrange(address, address + o.size):
+                for i in range(address, address + o.size):
                     memory[i] = chr(gdb.getByte(i))
 
         # gather PRE info

tests/auto_generators/make_tests.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 import sys
+from typing import Any, Dict
 import random
 
 tests = []
@@ -15,7 +16,7 @@
 
 random.shuffle(tests)
 
-op_count = {}
+op_count: Dict[str, int] = {}
 test_dic = {}
 for test in tests:
     try:

tests/wasm/json2mc.py

@@ -3,6 +3,7 @@
 from jinja2 import Environment, FileSystemLoader
 from base64 import b64encode
 from copy import copy
+from typing import Any, List
 
 parser = argparse.ArgumentParser("Generate Manticore tests from the WASM Spec")
 parser.add_argument("filename", type=argparse.FileType("r"), help="JSON file output from wast2json")
@@ -60,7 +61,7 @@ def escape_null(in_str: str):
 template = env.get_template("test_template.jinja2")
 
 
-modules = []
+modules: List[Any] = []
 registered_modules = {}
 imports = []
 current_module = None
@@ -100,14 +101,16 @@ def escape_null(in_str: str):
                     mod_name=d["action"].get("module", None),
                 )
         elif d["action"]["type"] == "get":
-            modules[current_module].add_test(
-                d["action"]["field"],
-                d["line"],
-                [],
-                convert_types(d["expected"]),
-                "assert_global",
-                d["action"].get("module", None),
-            )
+            if current_module:
+                modules[current_module].add_test(
+                    d["action"]["field"],
+                    d["line"],
+                    [],
+                    convert_types(d["expected"]),
+                    "assert_global",
+                    d["action"].get("module", None),
+                )
+            current_module = None
         else:
             raise NotImplementedError("assert_return with action type: " + d["action"]["type"])
     elif d["type"] == "assert_return_arithmetic_nan":
@@ -155,10 +158,16 @@ def escape_null(in_str: str):
         if maybe_name:  # This is an alias for another registered module
             imports.append({"type": "alias", "alias": d["as"], "orig": maybe_name})
         else:  # This is an alias for the current module
-            imports.append(
-                {"type": "import", "name": d["as"], "filename": modules[current_module].filename}
-            )
-            modules[current_module].registered_name = d["as"]
+            if current_module:
+                imports.append(
+                    {
+                        "type": "import",
+                        "name": d["as"],
+                        "filename": modules[current_module].filename,
+                    }
+                )
+                modules[current_module].registered_name = d["as"]
+            current_module = None
 
     if current_module:
         modules[current_module].imports = copy(imports)