From 1d24ef9d505f04323255e17d1ca7d3d7a687f4a3 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 17 Jan 2020 16:54:52 -0300 Subject: [PATCH 001/126] Quick fix to serialize big concrete Arrays --- manticore/core/smtlib/expression.py | 57 +++++++++++++++++++++-------- manticore/platforms/evm.py | 12 ++++-- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 39ec1b198..9e85948eb 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -905,7 +905,7 @@ class ArraySlice(Array): def __init__( self, array: Union["Array", "ArrayProxy"], offset: int, size: int, *args, **kwargs ): - if not isinstance(array, Array): + if not isinstance(array, (Array, ArrayProxy)): raise ValueError("Array expected") if isinstance(array, ArrayProxy): array = array._array @@ -950,14 +950,13 @@ def store(self, index, value): ) -class ArrayProxy(Array): +class ArrayProxy(): def __init__(self, array: Array, default: Optional[int] = None): self._default = default self._concrete_cache: Dict[int, int] = {} self._written = None if isinstance(array, ArrayProxy): # copy constructor - super().__init__(array.index_bits, array.index_max, array.value_bits) self._array: Array = array._array self._name: str = array._name if default is None: @@ -966,12 +965,10 @@ def __init__(self, array: Array, default: Optional[int] = None): self._written = set(array.written) elif isinstance(array, ArrayVariable): # fresh array proxy - super().__init__(array.index_bits, array.index_max, array.value_bits) self._array = array self._name = array.name else: # arrayproxy for a prepopulated array - super().__init__(array.index_bits, array.index_max, array.value_bits) self._name = array.underlying_variable.name self._array = array @@ -1010,11 +1007,14 @@ def taint(self): def select(self, index): return self.get(index) + def __len__(self): + return self._array.index_max + def store(self, index, value): if not isinstance(index, Expression): - index = self.cast_index(index) + index = self._array.cast_index(index) if not isinstance(value, Expression): - value = self.cast_value(value) + value = self._array.cast_value(value) from .visitors import simplify index = simplify(index) @@ -1031,8 +1031,8 @@ def store(self, index, value): def __getitem__(self, index): if isinstance(index, slice): - start, stop = self._fix_index(index) - size = self._get_size(index) + start, stop = self._array._fix_index(index) + size = self._array._get_size(index) array_proxy_slice = ArrayProxy(ArraySlice(self, start, size), default=self._default) array_proxy_slice._concrete_cache = {} for k, v in self._concrete_cache.items(): @@ -1050,8 +1050,8 @@ def __getitem__(self, index): def __setitem__(self, index, value): if isinstance(index, slice): - start, stop = self._fix_index(index) - size = self._get_size(index) + start, stop = self._array._fix_index(index) + size = self._array._get_size(index) assert len(value) == size for i in range(size): self.store(start + i, value[i]) @@ -1059,20 +1059,30 @@ def __setitem__(self, index, value): self.store(index, value) def __getstate__(self): + if self.index_max is not None and self.index_max == len(self._concrete_cache): + array = (self._array.index_bits, self._array.index_max, self._array.value_bits) + else: + array = self._array + state = {} state["_default"] = self._default - state["_array"] = self._array + state["_array"] = array state["name"] = self.name state["_concrete_cache"] = self._concrete_cache state["_written"] = self._written return state def __setstate__(self, state): - self._default = state["_default"] self._array = state["_array"] + self._default = state["_default"] self._name = state["name"] self._concrete_cache = state["_concrete_cache"] self._written = state["_written"] + if isinstance(self._array, tuple): + index_bits, index_max, value_bits = self._array + self._array=ArrayVariable(index_bits, index_max, value_bits, name=self._name) + for x,y in self._concrete_cache.items(): + self._array = self._array.store(x, y) def __copy__(self): return ArrayProxy(self) @@ -1113,7 +1123,7 @@ def is_known(self, index): def get(self, index, default=None): if default is None: default = self._default - index = self.cast_index(index) + index = self._array.cast_index(index) if self.index_max is not None: from .visitors import simplify @@ -1129,9 +1139,26 @@ def get(self, index, default=None): return value is_known = self.is_known(index) - default = self.cast_value(default) + default = self._array.cast_value(default) return BitVecITE(self._array.value_bits, is_known, value, default) + def write_BE(self, address, value, size): + address = self._array.cast_index(address) + value = BitVec(size * self.value_bits).cast(value) + array = self + for offset in range(size): + array = array.store( + address + offset, + BitVecExtract(value, (size - 1 - offset) * self.value_bits, self.value_bits), + ) + return array + + def read_BE(self, address, size): + bytes = [] + for offset in range(size): + bytes.append(self.get(address + offset, 0)) + return BitVecConcat(size * self.value_bits, *bytes) + class ArraySelect(BitVec, Operation): def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 3ea508701..479754eec 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -343,7 +343,7 @@ def set_result(self, result, return_data=None): if result not in {None, "TXERROR", "REVERT", "RETURN", "THROW", "STOP", "SELFDESTRUCT"}: raise EVMException("Invalid transaction result") if result in {"RETURN", "REVERT"}: - if not isinstance(return_data, (bytes, bytearray, Array)): + if not isinstance(return_data, (bytes, bytearray, Array, ArrayProxy)): raise EVMException( "Invalid transaction return_data type:", type(return_data).__name__ ) @@ -436,7 +436,7 @@ def __init__(self, result, data=None): raise EVMException("Invalid end transaction result") if result is None and data is not None: raise EVMException("Invalid end transaction result") - if not isinstance(data, (type(None), Array, bytes)): + if not isinstance(data, (type(None), Array, ArrayProxy, bytes)): raise EVMException("Invalid end transaction data type") self.result = result self.data = data @@ -1736,7 +1736,11 @@ def CODECOPY(self, mem_offset, code_offset, size): self._consume(copyfee) if issymbolic(size): - max_size = Z3Solver().max(self.constraints, size) + size = simplify(size) + if isinstance(size, Constant): + max_size = size.value + else: + max_size = Z3Solver().max(self.constraints, size) else: max_size = size @@ -2667,7 +2671,7 @@ def get_code(self, address): return self._world_state[address]["code"] def set_code(self, address, data): - assert data is not None and isinstance(data, (bytes, Array)) + assert data is not None and isinstance(data, (bytes, Array, ArrayProxy)) if self._world_state[address]["code"]: raise EVMException("Code already set") self._world_state[address]["code"] = data From d76d8b1cc6222488b296f00a808d4da2ea4a3607 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 20 Jan 2020 10:25:33 -0300 Subject: [PATCH 002/126] blacken --- manticore/core/smtlib/expression.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 9e85948eb..0b4f52e0b 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -950,7 +950,7 @@ def store(self, index, value): ) -class ArrayProxy(): +class ArrayProxy: def __init__(self, array: Array, default: Optional[int] = None): self._default = default self._concrete_cache: Dict[int, int] = {} @@ -1080,8 +1080,8 @@ def __setstate__(self, state): self._written = state["_written"] if isinstance(self._array, tuple): index_bits, index_max, value_bits = self._array - self._array=ArrayVariable(index_bits, index_max, value_bits, name=self._name) - for x,y in self._concrete_cache.items(): + self._array = ArrayVariable(index_bits, index_max, value_bits, name=self._name) + for x, y in self._concrete_cache.items(): self._array = self._array.store(x, y) def __copy__(self): From f4ba5462c73b14c4a5de29d32ecfa857c6e09c9f Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 23 Jan 2020 12:40:03 -0300 Subject: [PATCH 003/126] smtlib expressions with kwargs and slots --- manticore/core/smtlib/constraints.py | 2 +- manticore/core/smtlib/expression.py | 304 ++++++++++++++++++--------- manticore/core/smtlib/expressions.py | 76 +++++++ manticore/core/smtlib/solver.py | 4 +- manticore/ethereum/manticore.py | 2 +- tests/other/test_smtlibv2.py | 19 +- 6 files changed, 295 insertions(+), 112 deletions(-) create mode 100644 manticore/core/smtlib/expressions.py diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index a96e6ec49..07489d512 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -372,7 +372,7 @@ def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False) name = self._make_unique_name(name) if not avoid_collisions and name in self._declarations: raise ValueError(f"Name {name} already used") - var = BitVecVariable(size, name, taint=taint) + var = BitVecVariable(size=size, name=name, taint=taint) return self._declare(var) def new_array( diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 0b4f52e0b..fe0a28ebd 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -4,7 +4,7 @@ import re import copy -from typing import Union, Optional, Dict +from typing import Union, Optional, Dict, Tuple class ExpressionException(SmtlibError): @@ -15,12 +15,11 @@ class ExpressionException(SmtlibError): pass -class Expression: +class Expression(object): """ Abstract taintable Expression. """ - + __slots__ = () + xslots = ('_taint',) def __init__(self, taint: Union[tuple, frozenset] = ()): - if self.__class__ is Expression: - raise TypeError super().__init__() self._taint = frozenset(taint) @@ -45,7 +44,7 @@ def issymbolic(value) -> bool: :return: whether `value` is symbolic :rtype: bool """ - return isinstance(value, Expression) + return isinstance(value, (Expression, ArrayProxy)) def istainted(arg, taint=None): @@ -111,11 +110,10 @@ def taint_with(arg, *taints, value_bits=256, index_bits=256): class Variable(Expression): - def __init__(self, name: str, *args, **kwargs): - if self.__class__ is Variable: - raise TypeError - assert " " not in name - super().__init__(*args, **kwargs) + __slots__ = () + xslots = ('_name',) + def __init__(self, name: str, **kwargs): + super().__init__(**kwargs) self._name = name @property @@ -137,10 +135,10 @@ def __repr__(self): class Constant(Expression): - def __init__(self, value: Union[bool, int], *args, **kwargs): - if self.__class__ is Constant: - raise TypeError - super().__init__(*args, **kwargs) + __slots__ = () + xslots = ('_value',) + def __init__(self, value: Union[bool, int], **kwargs): + super().__init__(**kwargs) self._value = value @property @@ -149,18 +147,18 @@ def value(self): class Operation(Expression): - def __init__(self, *operands, **kwargs): - if self.__class__ is Operation: - raise TypeError + __slots__ = () + xslots = ('_operands',) + def __init__(self, operands:Tuple[Expression], taint=None, **kwargs): # assert len(operands) > 0 # assert all(isinstance(x, Expression) for x in operands) self._operands = operands # If taint was not forced by a keyword argument, calculate default - if "taint" not in kwargs: - kwargs["taint"] = reduce(lambda x, y: x.union(y.taint), operands, frozenset()) + if taint is None: + taint = reduce(lambda x, y: x.union(y.taint), operands, frozenset()) - super().__init__(**kwargs) + super().__init__(taint=taint, **kwargs) @property def operands(self): @@ -170,9 +168,7 @@ def operands(self): ############################################################################### # Booleans class Bool(Expression): - def __init__(self, *operands, **kwargs): - super().__init__(*operands, **kwargs) - + __slots__ = () def cast(self, value: Union[int, bool], **kwargs) -> Union["BoolConstant", "Bool"]: if isinstance(value, Bool): return value @@ -216,60 +212,65 @@ def __bool__(self): class BoolVariable(Bool, Variable): - def __init__(self, name, *args, **kwargs): - super().__init__(name, *args, **kwargs) - + __slots__ = Bool.xslots + Variable.xslots @property def declaration(self): return f"(declare-fun {self.name} () Bool)" class BoolConstant(Bool, Constant): - def __init__(self, value: bool, *args, **kwargs): - super().__init__(value, *args, **kwargs) + __slots__ = Bool.xslots + Constant.xslots + def __init__(self, value: bool, **kwargs): + super().__init__(value=value, **kwargs) def __bool__(self): return self.value class BoolOperation(Operation, Bool): + __slots__ = () + xslots = Operation.xslots + Bool.xslots def __init__(self, *operands, **kwargs): - super().__init__(*operands, **kwargs) + super().__init__(operands=operands, **kwargs) class BoolNot(BoolOperation): - def __init__(self, value, **kwargs): - super().__init__(value, **kwargs) - + __slots__ = BoolOperation.xslots + pass class BoolAnd(BoolOperation): - def __init__(self, a, b, **kwargs): - super().__init__(a, b, **kwargs) - + __slots__ = BoolOperation.xslots + pass class BoolOr(BoolOperation): - def __init__(self, a: "Bool", b: "Bool", **kwargs): - super().__init__(a, b, **kwargs) - + __slots__ = BoolOperation.xslots + pass class BoolXor(BoolOperation): + __slots__ = BoolOperation.xslots def __init__(self, a, b, **kwargs): super().__init__(a, b, **kwargs) class BoolITE(BoolOperation): + __slots__ = BoolOperation.xslots def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): super().__init__(cond, true, false, **kwargs) class BitVec(Expression): + __slots__ = () + xslots = Expression.xslots + ('_size',) """ This adds a bitsize to the Expression class """ - def __init__(self, size, *operands, **kwargs): - super().__init__(*operands, **kwargs) - self.size = size + def __init__(self, size, **kwargs): + super().__init__(**kwargs) + self._size = size @property + def size(self): + return self._size + @property def mask(self): return (1 << self.size) - 1 @@ -466,8 +467,9 @@ def Bool(self): class BitVecVariable(BitVec, Variable): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + __slots__ = BitVec.xslots + Variable.xslots + def __init__(self, **kwargs): + super().__init__( **kwargs) @property def declaration(self): @@ -475,8 +477,9 @@ def declaration(self): class BitVecConstant(BitVec, Constant): - def __init__(self, size: int, value: int, *args, **kwargs): - super().__init__(size, value, *args, **kwargs) + __slots__ = BitVec.xslots + Constant.xslots + def __init__(self, size: int, value: int, **kwargs): + super().__init__(size=size, value=value, **kwargs) def __bool__(self): return self.value != 0 @@ -491,108 +494,128 @@ def __hash__(self): class BitVecOperation(BitVec, Operation): - def __init__(self, size, *operands, **kwargs): - super().__init__(size, *operands, **kwargs) + xslots = BitVec.xslots + Operation.xslots + __slots__ = () class BitVecAdd(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecSub(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecMul(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecDiv(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecUnsignedDiv(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecMod(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = () + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecRem(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecUnsignedRem(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecShiftLeft(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecShiftRight(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecArithmeticShiftLeft(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecArithmeticShiftRight(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecAnd(BitVecOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecOr(BitVecOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a: BitVec, b: BitVec, *args, **kwargs): assert a.size == b.size - super().__init__(a.size, a, b, *args, **kwargs) + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecXor(BitVecOperation): - def __init__(self, a, b, *args, **kwargs): - super().__init__(a.size, a, b, *args, **kwargs) + __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): + super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecNot(BitVecOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, **kwargs): - super().__init__(a.size, a, **kwargs) + super().__init__(size=a.size, operands=(a,), **kwargs) class BitVecNeg(BitVecOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, *args, **kwargs): super().__init__(a.size, a, *args, **kwargs) # Comparing two bitvectors results in a Bool class LessThan(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): super().__init__(a, b, *args, **kwargs) class LessOrEqual(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): super().__init__(a, b, *args, **kwargs) class BoolEqual(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): if isinstance(a, BitVec) or isinstance(b, BitVec): assert a.size == b.size @@ -600,36 +623,42 @@ def __init__(self, a, b, *args, **kwargs): class GreaterThan(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) class GreaterOrEqual(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) class UnsignedLessThan(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): super().__init__(a, b, *args, **kwargs) assert a.size == b.size class UnsignedLessOrEqual(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) class UnsignedGreaterThan(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) class UnsignedGreaterOrEqual(BoolOperation): + __slots__ = BitVecOperation.xslots def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super(UnsignedGreaterOrEqual, self).__init__(a, b, *args, **kwargs) @@ -638,16 +667,18 @@ def __init__(self, a, b, *args, **kwargs): ############################################################################### # Array BV32 -> BV8 or BV64 -> BV8 class Array(Expression): + __slots__ = () + xslots = Expression.xslots + ('_index_bits', '_index_max', '_value_bits') def __init__( - self, index_bits: int, index_max: Optional[int], value_bits: int, *operands, **kwargs + self, index_bits: int, index_max: Optional[int], value_bits: int, **kwargs ): assert index_bits in (32, 64, 256) assert value_bits in (8, 16, 32, 64, 256) assert index_max is None or index_max >= 0 and index_max < 2 ** index_bits - self._index_bits = index_bits + self._index_bits= index_bits self._index_max = index_max self._value_bits = value_bits - super().__init__(*operands, **kwargs) + super().__init__(**kwargs) assert type(self) is not Array, "Abstract class" def _get_size(self, index): @@ -791,7 +822,7 @@ def read_LE(self, address, size): def write_BE(self, address, value, size): address = self.cast_index(address) - value = BitVec(size * self.value_bits).cast(value) + value = BitVecConstant(size=size * self.value_bits, value=0).cast(value) array = self for offset in range(size): array = array.store( @@ -863,8 +894,9 @@ def __radd__(self, other): class ArrayVariable(Array, Variable): - def __init__(self, index_bits, index_max, value_bits, name, *operands, **kwargs): - super().__init__(index_bits, index_max, value_bits, name, **kwargs) + __slots__ = Array.xslots + Variable.xslots + def __init__(self, index_bits, index_max, value_bits, name, **kwargs): + super().__init__(index_bits=index_bits, index_max=index_max, value_bits=value_bits, name=name, **kwargs) @property def declaration(self): @@ -872,17 +904,15 @@ def declaration(self): class ArrayOperation(Array, Operation): - def __init__(self, array: Array, *operands, **kwargs): - super().__init__( - array.index_bits, array.index_max, array.value_bits, array, *operands, **kwargs - ) - + __slots__ = () + xslots = Array.xslots + Operation.xslots class ArrayStore(ArrayOperation): + __slots__ = ArrayOperation.xslots def __init__(self, array: "Array", index: "BitVec", value: "BitVec", *args, **kwargs): assert index.size == array.index_bits assert value.size == array.value_bits - super().__init__(array, index, value, *args, **kwargs) + super().__init__(index_bits=array.index_bits, value_bits=array.value_bits, index_max=array.index_max,operands=(array, index, value), **kwargs) @property def array(self): @@ -1144,7 +1174,7 @@ def get(self, index, default=None): def write_BE(self, address, value, size): address = self._array.cast_index(address) - value = BitVec(size * self.value_bits).cast(value) + value = BitVecConstant(size=size * self.value_bits, value=0).cast(value) array = self for offset in range(size): array = array.store( @@ -1159,11 +1189,88 @@ def read_BE(self, address, size): bytes.append(self.get(address + offset, 0)) return BitVecConcat(size * self.value_bits, *bytes) + def write(self, offset, buf): + if not isinstance(buf, (Array, bytearray)): + raise TypeError("Array or bytearray expected got {:s}".format(type(buf))) + arr = self + for i, val in enumerate(buf): + arr = arr.store(offset + i, val) + return arr + + def read(self, offset, size): + return ArraySlice(self, offset, size) + + def __eq__(self, other): + # FIXME taint + def compare_buffers(a, b): + if len(a) != len(b): + return BoolConstant(False) + cond = BoolConstant(True) + for i in range(len(a)): + cond = BoolAnd(cond.cast(a[i] == b[i]), cond) + if cond is BoolConstant(False): + return BoolConstant(False) + return cond + return compare_buffers(self, other) + + def __ne__(self, other): + return BoolNot(self == other) + + def __add__(self, other): + if not isinstance(other, (Array, ArrayProxy, bytes, bytearray)): + raise TypeError("can't concat Array to {}".format(type(other))) + if isinstance(other, Array): + if self.index_bits != other.index_bits or self.value_bits != other.value_bits: + raise ValueError("Array sizes do not match for concatenation") + + from .visitors import simplify + + # FIXME This should be related to a constrainSet + new_arr = ArrayProxy( + ArrayVariable( + self.index_bits, + self.index_max + len(other), + self.value_bits, + "concatenation{}".format(uuid.uuid1()), + ) + ) + for index in range(self.index_max): + new_arr[index] = simplify(self[index]) + for index in range(len(other)): + new_arr[index + self.index_max] = simplify(other[index]) + return new_arr + + def __radd__(self, other): + if not isinstance(other, (Array, bytearray, bytes)): + raise TypeError("can't concat Array to {}".format(type(other))) + if isinstance(other, Array): + if self.index_bits != other.index_bits or self.value_bits != other.value_bits: + raise ValueError("Array sizes do not match for concatenation") + + from .visitors import simplify + + # FIXME This should be related to a constrainSet + new_arr = ArrayProxy( + ArrayVariable( + self.index_bits, + self.index_max + len(other), + self.value_bits, + "concatenation{}".format(uuid.uuid1()), + ) + ) + for index in range(len(other)): + new_arr[index] = simplify(other[index]) + _concrete_cache = new_arr._concrete_cache + for index in range(self.index_max): + new_arr[index + len(other)] = simplify(self[index]) + new_arr._concrete_cache.update(_concrete_cache) + return new_arr class ArraySelect(BitVec, Operation): + __slots__ = BitVec.xslots + Operation.xslots def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): assert index.size == array.index_bits - super().__init__(array.value_bits, array, index, *args, **kwargs) + super().__init__(size=array.value_bits, operands=(array, index), **kwargs) @property def array(self): @@ -1187,14 +1294,16 @@ def __init__(self, operand: "BitVec", size_dest: int, *args, **kwargs): class BitVecZeroExtend(BitVecOperation): def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): assert size_dest >= operand.size - super().__init__(size_dest, operand, *args, **kwargs) - self.extend = size_dest - operand.size - + super().__init__(size=size_dest, operands=(operand,), **kwargs) + self._extend = size_dest - operand.size + @property + def extend(self): + return self._extend class BitVecExtract(BitVecOperation): def __init__(self, operand: "BitVec", offset: int, size: int, *args, **kwargs): assert offset >= 0 and offset + size <= operand.size - super().__init__(size, operand, *args, **kwargs) + super().__init__(size=size, operands=(operand,), **kwargs) self._begining = offset self._end = offset + size - 1 @@ -1215,7 +1324,7 @@ class BitVecConcat(BitVecOperation): def __init__(self, size_dest: int, *operands, **kwargs): assert all(isinstance(x, BitVec) for x in operands) assert size_dest == sum(x.size for x in operands) - super().__init__(size_dest, *operands, **kwargs) + super().__init__(size=size_dest, operands=operands, **kwargs) class BitVecITE(BitVecOperation): @@ -1225,9 +1334,8 @@ def __init__( condition: Union["Bool", bool], true_value: "BitVec", false_value: "BitVec", - *args, **kwargs, ): assert true_value.size == size assert false_value.size == size - super().__init__(size, condition, true_value, false_value, *args, **kwargs) + super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) diff --git a/manticore/core/smtlib/expressions.py b/manticore/core/smtlib/expressions.py new file mode 100644 index 000000000..71ec78246 --- /dev/null +++ b/manticore/core/smtlib/expressions.py @@ -0,0 +1,76 @@ +from typing import Union, Optional, Dict + +class ExpressionException(Exception): + """ + Expression exception + """ + pass + + +class Expression(object): + """ Abstract Unmutable Taintable Expression. """ + __slots__ = ("_taint",) + def __init__(self, taint: Union[tuple, frozenset] = ()): + #if self.__class__ is Expression: + # raise TypeError + super().__init__() + self._taint = frozenset(taint) + #print (dir(self)) + def __repr__(self): + return "<{:s} at {:x}{:s}>".format(type(self).__name__, id(self), self.taint and "-T" or "") + + @property + def is_tainted(self): + return len(self._taint) != 0 + + @property + def taint(self): + return self._taint + + +class Variable(Expression): + __slots__ = Expression.__slots__ + ("_name",) + def __init__(self, name: str, **kwargs): + #if self.__class__ is Variable: + # raise TypeError # Abstract class + super().__init__(**kwargs) + self._name = name + + @property + def declaration(self): + pass + + @property + def name(self): + return self._name + + def __copy__(self, memo): + raise ExpressionException("Copying of Variables is not allowed.") + + def __deepcopy__(self, memo): + raise ExpressionException("Copying of Variables is not allowed.") + + def __repr__(self): + return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) + + +class Constant(Expression): + __slots__ = Expression.__slots__ + ("_value", ) + def __init__(self, value: Union[bool, int], **kwargs): + #if self.__class__ is Constant: + # raise TypeError + super().__init__(**kwargs) + self._value = value + + @property + def value(self): + return self._value + + + +if __name__ == "__main__": + import sys + x = Expression() + y = Variable(name="Aa") + z = Constant(value=1) + import pdb; pdb.set_trace() \ No newline at end of file diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index cb47d52d8..c1b8dfedb 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -550,12 +550,12 @@ def get_value(self, constraints, *expressions): if not issymbolic(expression): values.append(expression) continue - assert isinstance(expression, (Bool, BitVec, Array)) + assert isinstance(expression, (Bool, BitVec, Array, ArrayProxy)) if isinstance(expression, Bool): var = temp_cs.new_bool() elif isinstance(expression, BitVec): var = temp_cs.new_bitvec(expression.size) - elif isinstance(expression, Array): + elif isinstance(expression, (Array, ArrayProxy)): var = [] result = [] for i in range(expression.index_max): diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 5827d9dbb..5ec420013 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -853,7 +853,7 @@ def _transaction(self, sort, caller, value=0, address=None, data=None, gaslimit= data = bytearray(b"") if isinstance(data, (str, bytes)): data = bytearray(data) - if not isinstance(data, (bytearray, Array)): + if not isinstance(data, (bytearray, Array, ArrayProxy)): raise TypeError("code bad type") # Check types diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 87903cb19..ebe174bd3 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -47,15 +47,15 @@ def testBasicAST_001(self): """ Can't build abstract classes """ a = BitVecConstant(32, 100) - self.assertRaises(TypeError, Expression, ()) - self.assertRaises(TypeError, Constant, 123) - self.assertRaises(TypeError, Variable, "NAME") - self.assertRaises(TypeError, Operation, a) + self.assertRaises(Exception, Expression, ()) + self.assertRaises(Exception, Constant, 123) + self.assertRaises(Exception, Variable, "NAME") + self.assertRaises(Exception, Operation, a) def testBasicOperation(self): """ Add """ - a = BitVecConstant(32, 100) - b = BitVecVariable(32, "VAR") + a = BitVecConstant(size=32, value=100) + b = BitVecVariable(size=32, name="VAR") c = a + b self.assertIsInstance(c, BitVecAdd) self.assertIsInstance(c, Operation) @@ -281,7 +281,6 @@ def testBasicArrayConcatSlice(self): array = cs.new_array(32, index_max=12) array = array.write(0, hw) - self.assertTrue(self.solver.must_be_true(cs, array == hw)) self.assertTrue(self.solver.must_be_true(cs, array.read(0, 12) == hw)) @@ -577,9 +576,9 @@ def test_constant_folding_udiv(self): def testBasicReplace(self): """ Add """ - a = BitVecConstant(32, 100) - b1 = BitVecVariable(32, "VAR1") - b2 = BitVecVariable(32, "VAR2") + a = BitVecConstant(size=32, value=100) + b1 = BitVecVariable(size=32, name="VAR1") + b2 = BitVecVariable(size=32, name="VAR2") c = a + b1 From 5c187bd313a8b8c2572918e34e0a476079dfd289 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 28 Jan 2020 17:44:31 -0300 Subject: [PATCH 004/126] remove debug script: --- manticore/core/smtlib/expressions.py | 76 ---------------------------- 1 file changed, 76 deletions(-) delete mode 100644 manticore/core/smtlib/expressions.py diff --git a/manticore/core/smtlib/expressions.py b/manticore/core/smtlib/expressions.py deleted file mode 100644 index 71ec78246..000000000 --- a/manticore/core/smtlib/expressions.py +++ /dev/null @@ -1,76 +0,0 @@ -from typing import Union, Optional, Dict - -class ExpressionException(Exception): - """ - Expression exception - """ - pass - - -class Expression(object): - """ Abstract Unmutable Taintable Expression. """ - __slots__ = ("_taint",) - def __init__(self, taint: Union[tuple, frozenset] = ()): - #if self.__class__ is Expression: - # raise TypeError - super().__init__() - self._taint = frozenset(taint) - #print (dir(self)) - def __repr__(self): - return "<{:s} at {:x}{:s}>".format(type(self).__name__, id(self), self.taint and "-T" or "") - - @property - def is_tainted(self): - return len(self._taint) != 0 - - @property - def taint(self): - return self._taint - - -class Variable(Expression): - __slots__ = Expression.__slots__ + ("_name",) - def __init__(self, name: str, **kwargs): - #if self.__class__ is Variable: - # raise TypeError # Abstract class - super().__init__(**kwargs) - self._name = name - - @property - def declaration(self): - pass - - @property - def name(self): - return self._name - - def __copy__(self, memo): - raise ExpressionException("Copying of Variables is not allowed.") - - def __deepcopy__(self, memo): - raise ExpressionException("Copying of Variables is not allowed.") - - def __repr__(self): - return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) - - -class Constant(Expression): - __slots__ = Expression.__slots__ + ("_value", ) - def __init__(self, value: Union[bool, int], **kwargs): - #if self.__class__ is Constant: - # raise TypeError - super().__init__(**kwargs) - self._value = value - - @property - def value(self): - return self._value - - - -if __name__ == "__main__": - import sys - x = Expression() - y = Variable(name="Aa") - z = Constant(value=1) - import pdb; pdb.set_trace() \ No newline at end of file From 346788640aa5b27bbdff236d87adeb0c90f30903 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 28 Jan 2020 17:45:03 -0300 Subject: [PATCH 005/126] blkn --- manticore/core/smtlib/expression.py | 88 ++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index fe0a28ebd..a65bf337e 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -17,8 +17,10 @@ class ExpressionException(SmtlibError): class Expression(object): """ Abstract taintable Expression. """ + __slots__ = () - xslots = ('_taint',) + xslots = ("_taint",) + def __init__(self, taint: Union[tuple, frozenset] = ()): super().__init__() self._taint = frozenset(taint) @@ -111,7 +113,8 @@ def taint_with(arg, *taints, value_bits=256, index_bits=256): class Variable(Expression): __slots__ = () - xslots = ('_name',) + xslots = ("_name",) + def __init__(self, name: str, **kwargs): super().__init__(**kwargs) self._name = name @@ -136,7 +139,8 @@ def __repr__(self): class Constant(Expression): __slots__ = () - xslots = ('_value',) + xslots = ("_value",) + def __init__(self, value: Union[bool, int], **kwargs): super().__init__(**kwargs) self._value = value @@ -148,8 +152,9 @@ def value(self): class Operation(Expression): __slots__ = () - xslots = ('_operands',) - def __init__(self, operands:Tuple[Expression], taint=None, **kwargs): + xslots = ("_operands",) + + def __init__(self, operands: Tuple[Expression], taint=None, **kwargs): # assert len(operands) > 0 # assert all(isinstance(x, Expression) for x in operands) self._operands = operands @@ -169,6 +174,7 @@ def operands(self): # Booleans class Bool(Expression): __slots__ = () + def cast(self, value: Union[int, bool], **kwargs) -> Union["BoolConstant", "Bool"]: if isinstance(value, Bool): return value @@ -213,6 +219,7 @@ def __bool__(self): class BoolVariable(Bool, Variable): __slots__ = Bool.xslots + Variable.xslots + @property def declaration(self): return f"(declare-fun {self.name} () Bool)" @@ -220,7 +227,8 @@ def declaration(self): class BoolConstant(Bool, Constant): __slots__ = Bool.xslots + Constant.xslots - def __init__(self, value: bool, **kwargs): + + def __init__(self, value: bool, **kwargs): super().__init__(value=value, **kwargs) def __bool__(self): @@ -230,6 +238,7 @@ def __bool__(self): class BoolOperation(Operation, Bool): __slots__ = () xslots = Operation.xslots + Bool.xslots + def __init__(self, *operands, **kwargs): super().__init__(operands=operands, **kwargs) @@ -238,29 +247,34 @@ class BoolNot(BoolOperation): __slots__ = BoolOperation.xslots pass + class BoolAnd(BoolOperation): __slots__ = BoolOperation.xslots pass + class BoolOr(BoolOperation): __slots__ = BoolOperation.xslots pass + class BoolXor(BoolOperation): __slots__ = BoolOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(a, b, **kwargs) class BoolITE(BoolOperation): __slots__ = BoolOperation.xslots + def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): super().__init__(cond, true, false, **kwargs) class BitVec(Expression): __slots__ = () - xslots = Expression.xslots + ('_size',) + xslots = Expression.xslots + ("_size",) """ This adds a bitsize to the Expression class """ def __init__(self, size, **kwargs): @@ -270,6 +284,7 @@ def __init__(self, size, **kwargs): @property def size(self): return self._size + @property def mask(self): return (1 << self.size) - 1 @@ -468,8 +483,9 @@ def Bool(self): class BitVecVariable(BitVec, Variable): __slots__ = BitVec.xslots + Variable.xslots + def __init__(self, **kwargs): - super().__init__( **kwargs) + super().__init__(**kwargs) @property def declaration(self): @@ -478,6 +494,7 @@ def declaration(self): class BitVecConstant(BitVec, Constant): __slots__ = BitVec.xslots + Constant.xslots + def __init__(self, size: int, value: int, **kwargs): super().__init__(size=size, value=value, **kwargs) @@ -500,84 +517,98 @@ class BitVecOperation(BitVec, Operation): class BitVecAdd(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecSub(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecMul(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecDiv(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecUnsignedDiv(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecMod(BitVecOperation): __slots__ = () + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecRem(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecUnsignedRem(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecShiftLeft(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecShiftRight(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecArithmeticShiftLeft(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecArithmeticShiftRight(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecAnd(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecOr(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a: BitVec, b: BitVec, *args, **kwargs): assert a.size == b.size super().__init__(size=a.size, operands=(a, b), **kwargs) @@ -585,18 +616,21 @@ def __init__(self, a: BitVec, b: BitVec, *args, **kwargs): class BitVecXor(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecNot(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, **kwargs): super().__init__(size=a.size, operands=(a,), **kwargs) class BitVecNeg(BitVecOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, *args, **kwargs): super().__init__(a.size, a, *args, **kwargs) @@ -604,18 +638,21 @@ def __init__(self, a, *args, **kwargs): # Comparing two bitvectors results in a Bool class LessThan(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): super().__init__(a, b, *args, **kwargs) class LessOrEqual(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): super().__init__(a, b, *args, **kwargs) class BoolEqual(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): if isinstance(a, BitVec) or isinstance(b, BitVec): assert a.size == b.size @@ -624,6 +661,7 @@ def __init__(self, a, b, *args, **kwargs): class GreaterThan(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) @@ -631,6 +669,7 @@ def __init__(self, a, b, *args, **kwargs): class GreaterOrEqual(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) @@ -638,6 +677,7 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedLessThan(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): super().__init__(a, b, *args, **kwargs) assert a.size == b.size @@ -645,6 +685,7 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedLessOrEqual(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) @@ -652,6 +693,7 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedGreaterThan(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super().__init__(a, b, *args, **kwargs) @@ -659,6 +701,7 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedGreaterOrEqual(BoolOperation): __slots__ = BitVecOperation.xslots + def __init__(self, a, b, *args, **kwargs): assert a.size == b.size super(UnsignedGreaterOrEqual, self).__init__(a, b, *args, **kwargs) @@ -668,14 +711,13 @@ def __init__(self, a, b, *args, **kwargs): # Array BV32 -> BV8 or BV64 -> BV8 class Array(Expression): __slots__ = () - xslots = Expression.xslots + ('_index_bits', '_index_max', '_value_bits') - def __init__( - self, index_bits: int, index_max: Optional[int], value_bits: int, **kwargs - ): + xslots = Expression.xslots + ("_index_bits", "_index_max", "_value_bits") + + def __init__(self, index_bits: int, index_max: Optional[int], value_bits: int, **kwargs): assert index_bits in (32, 64, 256) assert value_bits in (8, 16, 32, 64, 256) assert index_max is None or index_max >= 0 and index_max < 2 ** index_bits - self._index_bits= index_bits + self._index_bits = index_bits self._index_max = index_max self._value_bits = value_bits super().__init__(**kwargs) @@ -895,8 +937,11 @@ def __radd__(self, other): class ArrayVariable(Array, Variable): __slots__ = Array.xslots + Variable.xslots + def __init__(self, index_bits, index_max, value_bits, name, **kwargs): - super().__init__(index_bits=index_bits, index_max=index_max, value_bits=value_bits, name=name, **kwargs) + super().__init__( + index_bits=index_bits, index_max=index_max, value_bits=value_bits, name=name, **kwargs + ) @property def declaration(self): @@ -907,12 +952,20 @@ class ArrayOperation(Array, Operation): __slots__ = () xslots = Array.xslots + Operation.xslots + class ArrayStore(ArrayOperation): __slots__ = ArrayOperation.xslots + def __init__(self, array: "Array", index: "BitVec", value: "BitVec", *args, **kwargs): assert index.size == array.index_bits assert value.size == array.value_bits - super().__init__(index_bits=array.index_bits, value_bits=array.value_bits, index_max=array.index_max,operands=(array, index, value), **kwargs) + super().__init__( + index_bits=array.index_bits, + value_bits=array.value_bits, + index_max=array.index_max, + operands=(array, index, value), + **kwargs, + ) @property def array(self): @@ -1211,6 +1264,7 @@ def compare_buffers(a, b): if cond is BoolConstant(False): return BoolConstant(False) return cond + return compare_buffers(self, other) def __ne__(self, other): @@ -1266,8 +1320,10 @@ def __radd__(self, other): new_arr._concrete_cache.update(_concrete_cache) return new_arr + class ArraySelect(BitVec, Operation): __slots__ = BitVec.xslots + Operation.xslots + def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): assert index.size == array.index_bits super().__init__(size=array.value_bits, operands=(array, index), **kwargs) @@ -1296,10 +1352,12 @@ def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): assert size_dest >= operand.size super().__init__(size=size_dest, operands=(operand,), **kwargs) self._extend = size_dest - operand.size + @property def extend(self): return self._extend + class BitVecExtract(BitVecOperation): def __init__(self, operand: "BitVec", offset: int, size: int, *args, **kwargs): assert offset >= 0 and offset + size <= operand.size From 0cd1bc50582e581791368bf85f50dd85911829c1 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 13 May 2020 18:48:55 -0300 Subject: [PATCH 006/126] Add ONE failing test --- tests/ethereum/data/ErrRelated.pkl.gz | Bin 0 -> 17638 bytes tests/ethereum/test_regressions.py | 35 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/ethereum/data/ErrRelated.pkl.gz diff --git a/tests/ethereum/data/ErrRelated.pkl.gz b/tests/ethereum/data/ErrRelated.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..eb1036567ca2ce788a1c79c0ea6973bfe1b2240d GIT binary patch literal 17638 zcmYIvbyO72`!(I&ow{`IE(_9KOSq)uN;gUfh%DV50t*TP(kY-K2$E7uw}9jlqQru* zi0seL_x7I z-}r{s*Hb>c)i@&<0$IbQ^;Ywk494|s6M{Vgo;`b3AVc`RFX2am?=lIe5&Qrgx&L|} zeGzejI(gCc3^Bdic@@z!wU0(;wPc*7a{Cw>K$D;$3SOzI4@s&4fMx7XWuAF~1iuLA zepzS4+CnzRmxwwWo0maoPPSas*RR#z>59j*^TuvV9{nnv6puDmnnX^%3*&J&aaT~v zB%HM|p4zcH6#uaQL7cWSgrs8C)O%iY^^XscL&1Eu*ee~WG`o% z(Tu|ebEhzO)=zba}C_AFot4exg*{U61l|1`VLsHWI%jtBxF$0 zb7P*>t_%I&VY?V9e{BzvH;vVu{D>XP0iAE~Ne){w=(6H@3NlSQy);(S`RcSzC2yt#A z#O{6NN2W!-z`~5t+gb2#mJZh}WV&e7)lB~hegnDVVxCm$*!;U+`Xc0nIb2VEVll>& zpCjcmJSOQSWaVw_zHiW2J-ui#PqgGmHv1P%K1yaCxTB#C;fWH3Y+{EQDdJl zGRnt{HYRa0ev9K{;h57`Miam78Z9LgB^I`FSQ}_hrVE~hUOC#eG8>wdLu|EbB*Y<- z_ar5O0{1p7<&_{x^DL~aTwE-VKKN;cLW+go=jcbAyZrpAJTq=7JA|rwv;tIl{zo_8YCW&0{-R+O*m1_25aY>&#NaZu3u3yYCBr?}x`0&jaj-y}Mp_zrPJG8f;!O zl7tjkS+nKZ|2p^^vqDn+UfX{+5bVh;50MK&qx{|vjlF;R8{B0*FHvQ4^-`ju(WmQl zwU71ZJd5v@U1!y6v1*CcJ(+J36TeT}uDskYNr}*bft-qUYM?gjKi<)jB)u(NljmL~C$FOdp%(I2Bs1uLKBXapa6Jsj-)scP8 z8~Sn8YsZFSe^L^we;WHbDhw=@E|7RWA5ys6?RIbe_@&eimdu|z0nn-)?aHibPyLS90 zUwL~gGo<=A8kfAo`@=sn{U(kua=5ei<4UvTZ%$eStGph|PZI2o>IUB}V0pxGj$4Kv z_alAkq`rEaC2TnLZ|%B(3H1jR|KDpU2kRBXjpO`%<1mR|dGZl6j=0BxtO!ONiuG=Q zwdXG@r7~imU5|}p^L%uB2j<%#Mv0q|S>GE++)^@pPj&XK#g|l7yTxzCV<;TWEU@QJ z&akE#rO*Sby(8RSd$X?B#6?D^RpUc>EBoo``LxPRekv{?A*sIMZD&VN+2lm*UKqI? zZ~v{LT|*a~OTUcuGs**BzN|G#R{RF^zBTu@54m?-1V&N7jZENb* zKIW6W*GSd|>IW;zC64(}x;+^4X_kH96sFsM9M{q|No87O|E2w*n^pb_^>`Pf8*Q9& zu$yr{JE5D@*m0xm_r7zLxE0o|E=9S5v13v<#8`W+(Sw&?8Qi!u5+^4bWq10x)QXJr zi=y@j$ph*7wO$qJy3&xyCClbopy|Zj;|?_n%jGb26Xw!$!htno5wzop^yq*%!5}x4 zd5pOVc`$Drllfnb2N#x{6jP#zmt9Z10C8j?pu(XS$-$?1b1F1%=K6Q z`FdO@>6XJQ8za|Wgn_EAJf=mYt+#Y=9-UAvcNVK6-l(uVSyl5QUO4t^9G1M_B-o8S zKa6T(CofFHH8NqZlo8EJW^y!h+!eWtcx117rix9@>|h+q5@I`-OE8l7iQ&ByEEPwn z$2uN1d>?H1KGdKp{iC+zlB(@)-rZ*795*7gx$o3dHc=3RHIR0U#p=l72r&M%ZI*58 ziDVdF>F-`f(U`M^0GkL>2O*xBT(%R?e>5lq@*Q3##uCb}X$4h(AwUFPY=t zVYlC|2|sevEgV=nZ7p3Qrwe|XqS$FF^r5+?O0>mN4D3zU_!j<4V&VR2gzpEhn5u^UZ4N6c3DK9=HnBph$%S=Up&w z+OcF&nV-K!5PYegMHjXxNctFWmP}Qm_MKk5SuW8EN?DCaz7VTGdG5Mj$ZpM}#m7qI zR!R!Mtau*}*3k@j1#eNz@IZ@Yy8#79!(#t0pPH7})PizX*EGVvd0-^YPcCHP^Jto} zHPFghF8B&RbaNg394|5pOdF39gkuCJvHGh@BtMcLW8}&mYf#O42)WhUj9c2zM#3Fo zDY8>WV1z;OK3?~!ABG{yY8Ktm5+>+#*ROyuEJl0@uV22tP8Lq(Z9UUI+IMd`)Ac*k zEc!GLs4o@HMNN6+u1`7V5=P;vHiH%LNA~txM}JnY?4Hf}oZln|o@2v&&SNdUNDWl+ zm21pbNzV%C6qk>r9=dAOK%eA^z~708_!{7wu69A$@x}~_pY~UEx76Ylr_5F{Sm!er zizd}QELN*rNCPiqFHMgV!HSW>uyGfd?dm)}-$)l!8jpXrs;PyX84PbBF96d!Ex>#P z=ohNN)LCHQDo|s?T8dH27g=Tm%Z28(dy=RHo`5Crl3s!(;-O;jIS#OICDgWMRRsK# zKDZ3bogE^(8lDhl*Gh&jM5SJ*r$5 z*J+Pi71#bW;nBv0rTA<_?bXKMYFXrCX;=)xYq<1lqlvnlb&G9T5%vN1xwW;g5G+~_ zxp9{NoOdIgeWg=uKl1E{YaOguS-5SNSMzAnO4@XAW)`ldpw_@QQlscApjc=Y?T#0D=7$l(>(wv*9*^SfM~nEp z)-P7=3vXKPR=dbq?bf(i^}vu|9m9)6=h6Oo*sm@a%hl`a9nGd*D^5oQ{wd~a`TCDH zm<53X)b~P!ihI8T9#|{f%J~q`?Oh0gJG0Qg<(GHWUt3ARidJ_ui*E09iV5y`ievDY z&i!C)Q3A77H7z0lpOk^BxONe8OOc@}V7cskmH6yao#OKmu>)68v*HwC*r*H4X4M?O ztnOK{8eW}#@lk(OLQ5@q@uS%)uyrs~F`ri7Jb=j4z5*(IF8$d^+lw(nmL5?zL{ygB{-1P{af#WNlvRx!nG10GbWzHG>rR2QIW3j=yYDn+H-z9R$Ic`0 zXXABn+9Rw#K;x#k`@(JW^G}P8db!e=2NC!CE!h3x+r1pSAi((}E^DE`OO!b3euUty zygm+9_)h!mdx{6=<@b!B#2JLpxqUclb{-)=b}ZAeR)2Fs==?9&h&pu|E`EK@OMq=j zjvyjc$-2{x$0hdT1g{a|e^vYPOqg-bNf|I4qn@hV4I<-~^0S-B5rV z*mD~}vbB=t8*^Psi0w{}z$cY7z!~=A-ktUm;Ob{_990pG*NE5y#mnC4i>etM%lWr( zRL(rkHV->Ff1YuKU`=~Ih*Rk&W#^CYZ@kcm@G^;@zs9*kn7?iuc&|O5pX5+69Jwj8{T2A8SNq=riM35EO)_i2)f(Yw+OPTd5c?3 z`gk*3Kqb8nE__7r5~*E_c#-Gnv*FyBi#W(}3-s=HhNa$tP>obT8(`yaWP z_?-pGYn-GT*>^AE5RO|o$=g5rxVyr~u-l(Wk82TJ4-;opd9Zk&-QY(jT17;#@#s1n zpEN@<&M8j+KW0rx!jh6F0>0e;3yVY4otnhrOb>B9RaCIsL_rSqzq$h4t`{b{cRdbm zbeFv8vd%^lU+eC6zOb3x3h@QG`-l%B1V(a=c1I!7+BYW5cXM&LpzEouJ2@HGm)okG zGh9citcOY!xQJcK3nXzZP9x98#jDf$(ve{J;dU?8)_Gcb{B<*4*xTf83Q|kMf9H*8 zJar_+UCts1t9;CF<@r~Q{(j(F_qzC=5Ygq(hIjcrTJ`}{;#uuoSKqSc?Ule|AQrtv`Ugu5s<-_J-ciw(AJ%gV_ql;jl^by7ooEkcU^f9w2P z|6y$W%BG_VHv#qc`|N-9^O5Df2bV*5OmmuJyp%PGiH4Ds0VcE!?7EZwmu0R@v`Z47 zmEvx}9)rK>FkkW6mSRoBAKrzUX84;LN0BmwYoSec-|U$j|Lfc8C6y(r>~=YnSQwsx z$i4|Nu*3Tf_+{?Md-x@$dO-G&+SZK@;~u925J9*_*(hBP3hYf$?D>%F9f$If?oa7s z1I_N%MoJSg`|+vvBPY}&x|1iA^HZQ+)l<9|(BcwRx^W`*_f#yH{aBqy5d?k|vGRWy zQbc?GBzpp(Wy1SXI@na$9m9xY4yGW+(32m*uI9RTyBGEH8OjkqRQm-~N&#_S9JbQp z$}EGh@G#Fkw<5J3_}`w8rcJJc(V4H_3F7e>zyrN*F*2{G3;08BJu63dGBNO<3oD5%x@?*5m8 zBkxDMx8p4nPX}I#4c>~qX*C$@<=8~lFs9}qB=VNn^Okm^sY?)od*8LN@`5;)$zEp3 zZ*jNZ@PB4uPKZ8@gK&PE&o4mAySL9}Zc;Pc3-2KghOOW#@w%MtuK0%^68QZM6laPl4 ze6juCc6^1u%LsoTBD{j&UX0r3B%CeFV)#y!Tzf7so^I;XF+5 zu&h6oZ4Wc1H+J~B2#z$0x5D_}tu-;d^D(`PA?+U~nz)Q zBQdlIY5K*(J1RW6NB;`}qP+={J@HUJvHd9>Y#Z#3Yos(6vmayV$B*Dq3){O5Dcfkm zzKHsBEU`mmonncpE)UmUB)@IN`qak7ZTWJW94a%I#>4l3wXODGh9@077QW zq}LL^fZ^$Kj+$X-f&_JUS`Q3Z;8-6Os4!r#*Fk!BY3phba=1w~(?eesL+r1F!&;fR z-$yYx2D2V{tab-u7NM{5aZY=^n_>*yjRUVN+HrU5t<`OHJs<9dfD0|b!DTDg_U>Om zZx_As8+^Csif)_a1QtPB4Z2g+zqRUCXBxbuJS`qH*RGD*FGTj&u4aZ=bS;{7P^wsT z9a(f3A}qRaW*x<97F~>%9UlnIe=5^kJa^Vx*x-hBd?2&<`Qo9)^ET^+4H>JBOltF= zZHyLw@bni{v|$~Y3>H5z<`#cUZ5C8)tU4T}4mKN19$>zg`Ksw^YbuIzY}I%)xUwDi z^lt$j$j&s9SxFoq1_7t^Eb1jygPtVDfdS^|c0Oan?RJ7{5X84|?j_`B(hgCzGK9Nu zE*tVHX^y=_ax=))SbjT5$e1=DJZ3Ic;dAm_BIHHiA*nF}O~p5ObTzO0Vdf3hGXP16n2v3q9KfZMMRS>Xe0hf^KBzScSVR?pMXXG1di6y3l#X|UZu;0}U#7q-2G{77o!DWTr< zahU{c`v4|c0^Y~8r7L_(Zi|IH=vyJ4^g|!>PdaQL6S}KHNDBkAAc;wVY$bx5T&|Nc z+gw7E>k5IrC=- zK`aB9(4q8SbxM{u2l0g@1GnjgRGu(I<sIOacvFU;*Bp4`5FC_QmT~sb}iOUuiPrAu1DR z_cM5(-1d><31um$50V3nRRYdqVNnXi$*?3ye4hw`u^F04(D=!A9$xi5$lF2~65^T! zV<~yMndfdyylsBZ_S=kcp0A(w@)QV4a@99@ad zCeY28aXXGqh^a(6pucE`&zR^5GG>md1U+GvJQo9z=<_5t_C~k!8{2NT6MiK(evS^} zGuGV>BB+*w*cZ;_L)Me#xJ!68FI4~liomox`|a8lMo84}F< zq{0z#{X+IuxXHySwcZgw$LPr1nG-d{!9w0G;opmSzj}nHJAK4b3+98V9Z+n8u&w-K zP=C5}5Osa-kExY&wpcMe`cuS;%@%@`Sw?1;5YIBaLF9RJH z`ti($%2p1)$h#HGYt7m z6RNO`=t*nK8%ll4@I`UUPteQu?b{@O-jGKf9p)(Qg+wNf^ZqO-XSqi!y+j5Z6av_O zboRTbf#-YK>!^7~4Q2Xzp@eV-d*#N|@P{=*xeK|R^RTRNgPQ8cm-D73J8oC)rY1u5 z)$%p&e3xmaQ-eoEZ(a}GTv-`f*0|>`v~kV{W`!Hq?9V0$(5|dWk2nwY)$(qHMXv^% zKWIA4RAsGwm!H`;R?u+kv^S+rP{x+Tx7uBDk4!(t6&pc&{AL7ozpLIy@woBqfF0HTJ$^g&W|If&S#a&m{53Yq z1UR#Tu|=#m=xHO(5(if`VBXsX`$|p!^Lrhx;|9U{B~Z{{pycU;pVZ`vrZE|M)5+fssR|_h9M6njN zpq*h6WyEWXYY(kVmG5=yP2zk7@c}k*jV$Bb5;6_%^AcXdJrtg`6S#@32enfi)hl2) zVox72IFN2#Zu&dj=@23q&*Wo!wU0f-LRJ7l?cCEeOBo>q-(B-f5Xc?BBM;in?lH#JwUr*Gf| zVQ_6JO0kxK?GH;nIE8T4++T%AwkUXZ$3xf z6SdN5zLO}v#Ai{05D1?%P*{3RetzmJR_(bTmK2%eSF#I&BxM@5s?6S?{bW^SyV@>lh%e$f}w zm>e3#Fd2)hFm09QElk3TjsE>t7T0j`>+ZxU(#X*lz}9OmmQwgxJdIgnNOd_$SIrsf`1 zBKyx%Lsg#Fr&p9&I({oHQFi;+vW0I|{N5iX@R^8)iuM*P?M2E!bzJB~(leG*k^6u0 z{;g}OxNgBRUW^S?$DChLMyH(C-uq(?KC9JGvDkt^>HV1M2kH`T)0T)H8R~XaBk!&Q zyZDP2Wn>x+t1g`$l@5TSLNtd^63&8^-r}4f8HL}PBf?BNa5mb4uiSv=I@6)BGr zz2(NJX6k8$3g=Ov%Vj6YWmu&O?-8Z-3;ualk^YY8vL3c0!@QC4oVO%fuJ^jR`o#{k z%eccTdl-)D*J_$A4|LBt!mc_H7D5%R~++K**B3jyiS*~rmDMWtru)Zx-OR$B$v*Wkz7YYgL4>twvwDF z^6KcMH4#)#U*|VYvNufTE)ZMW@A4Itg|4jYV<~d%^7oqAcbju>sYkvKPD_Lx zyoVpqP_GY7D^soSJ6}d`-_8Hc0`S7mTN$uA>x`9QbypSk3pJ?PQulMoDQLT6)^9_P z-t8&}%|Gb^Yxrc^pNCSnUL9jpMBBEimq^dCWThs0I2#_^<9BE)4qU?72^)?-69;^P z)86To;X$Pi<1i`5DxV6T@31+nx)@2VF*(h~3P}KRUjsddD!1=<|JdBq-dpW1KkK4t z#UH}2^zzrJS$%+q@FVW7%sVsU+R1e$fB-I1A|x5rAG9ugckI!p6BS7`K=1X8?7CzV zu!zVMr3k@_#S0PubB|?TS?h)$7pT_=3YHm#b3b};t%V=TB5(HY@f)9~t}D><-Ll4~ zkL&I1Z2YcwpX}tas)W$|@_oFe$q@+;^_OPPciD~6WE$I^o6p4(wjn(c_+=MoJYs7nT|f=FAsj0F;ENzy&>t!F_ixp^fgj>m*hEe@Q3!!XPJ#e&ST1Nku8md zlO6qc(i~s3seor}T0F5^GDNwWVNVP7;|DyS5&?8mm^1ogD&i@=;&J?Vv<2DRxfJh_ ziSq)OCNhq?G57$S`$V25!l38SinBwu(mw9VxD#=C-7TN%i7f#%(t1KL729GRH5#wf zIa1j_1neZrH5{TSfcNfc#EaoI1~mpSpDu*Q<3En~uIcY^X`=!EXx${~>`w<^!86xx zY5rRjANrpa7gKx~a3^JLof~}2YIB@0#AEbpZ}bI#`B(KQVWE<+WhmJnZWd7`f`c0G z_^(mCOJe_yOi7*j^}S+Pi$Xf_)Cluy(CD*_;VI`^0;^E^o`MSs=T7rbb``6SN%a7h zrIBq#aX+VXfXRkz?I{?#(YI^g+W&m{q{noNP>t4JEQ08^)d9}m+An0j1$!4pQoL_{ z8gI8{m>bdTaSIy$9uV)p^=A@gcFm*0`!OjKz*0K0EhpaNbS`ERDOY>?0DAXj*QB-I zarxwfDICZd={bUa2oKtn@u_L4v4$IQzSwF>kL>-b17_irIPlCx^nOLFlxplauzgq& z5KE)}4td{e#m1^~=98^GDBr55#zGFyCd*9@lTAjwh}ZX&We4Pbi=%)!_2#{=p@88V z{CouG*z~-lFic{us-T$5g_11048|1gQpgqWj%)1L$h=M$o1pp5LXhF@s3%A&Dn^Ampl3+zF>YZzhn^;S7TG>{SE(Ph_E}an# zvlv&x-vmnL3_Rd$N59z@-5<=B^kQ6^%o+EKpR}-N5(|wm8v{q>lZSRQ>n(xf57spS3rt;>X|2_AV+v=au((T#5ScB;t$MC+OVV z(1V48@WlW3i5%EJ@vnu-J@0tmJKTyjy09{m$*ECq97$ zgs;srPtAcnb3s^kyU3TBr(D04ahX!Pl*{`9yn}B&-hVW*wd-TCda?)VIS&WwhQ>B^ zCyV)y>V|S+*0VECCCi`R2zF=QMnn3C&*@bv1tYY+DGh^7q&H!gWTo0W&y5RuzwAk2 z$sFO|mvE4N`Q?2H-oY-9_n{`X5not%pX>?soCgARuj89;Q^a~lb+5TGf9dw`tB~U4 zT7NfJ^wwmnV0i>%R8Q}06@RLJR7}u*wkpZe-8vjI=LaYa5l;Cdr*2PMfams-VJey=XDmd48=puE)0aa18iSwo&_&?V+?&_>+y*v$;+7EO(|@_Zk6i zXMn`+v-fe-P zH>^fNYW4h$gzASd$O>BD*TDnx@}!dXS7G!0qtsE_?B2Un5xlUnkDY(( z4COyW`IR+f)V7;2O{3#m!r-PUHI9$9UtKU&jBVG5_c_Ynmw}5IS)1WZs^7LgV^FTT3kUXhNhB^G(ts0i4RRv_$X z7Jbnh+-9O~`~ET-j|G_P|~?ze>5_wZ#fmoKWbF+SH< zqE>>yw{f?4mLmf^3-wH`d39ISuDhYWmZyFWO-@5s1y!ORz%i3$L1z92Rzk|_HSZpX zK0k#XCX+u;E@p6i1`BsSQHe-<@oMcoukSbBZ!~J()ciPiUA{SaamKuV6L)aQDnpgD zrAAet!XHhod!A5HvK9dJg^+M&EUd#LBO zjG2P}7p<=7_x?=ZhpfNCW-%c?< z43dw9><}OAbWV@PI%wsmnsvtUPQ5J*C<;}wujGDd+s^yVaynS-$+~FTJEZ>=hfuY( zq3@6B_dLFwAAKuweT!>+Bpr8kU+d3v zZW*sd5r^84!>WyJ`409D)v5r>!f)oK;O#1Htnx0+i>#WK{ zie%q<120n|%Isen{8{3kS7r##tDZZWwUR{73K0IiFKu>VEyf__W8(4o_taN?d!${INL8 zwHDn-_``nbIDa8F>p#o7|4d6&t5W$=mHux(p`5_v$Fl1VF4fug>zga#$ho$+>9>i1 z6nHwv%1}!A{a8WgNBPa8|Gsxz=M8$(9-WQXF!!d_QSQw36SYh8vjApU6gRR8mklsB z4Q3Q#0#^0E?hjLUb}j7QA9nkDWtjE~Y!ZXYeLUuJlsDQq`J%D9Y%H(F$ThfouyQ!g zvtihu(DRUMb|(1ni9JbLjv4p;+}sM6sxk4JiIg!PFV_$uu|QK-V{>%QiKQPNCdpQC zBkwhjtYJES62Xl1o4-*t zw0kLnl4@^P2|@SXm4yq(mnh}+2MKFW+vfR)$xN88)&22Rr!pmwudMv^Oho8P68gMA z-POkYW!t%`j>3v%^r~oiV@||zesryXUvtQATtiCts%qAPj2Shv&D?0K*NkbNRrhCw!|9*z);ns4T<=N*buD z$IvxC@V7Wx2gCaFS91NMW0vMC9QnScE0(X`y~@r1S^JJqRIo9_UxvGJ^+TcY?1#e1 zZJRFb??1b0xSq#7a{xK8HE_^ZR_IN1)>nYBqPH`Lvkm4T$TMzHLZaJeXT zK-Io!wL`bxj~c&+zIk8kQJnDAL-7MD;GHj^1EulJr+$ZYQvp)h38{o5dYy*g69daA#dogxxKLpGgfNX~}NE$7`+_M^TMn{d!ohmF3dcEC_#{bh{QEij}5g8>WPc6Qvz!lux<;`&;^dokw0 zC&3=upJq#?gBO5j_MT7zt3&|*Cx%WPPX@tvA4NID9$Hzw`Y*EBs-o&z;cQ*%xp5$U zv#bfbTrOe}>)ae*tzIq%64;=h+7y%nP%fvc0|?Fi}X2qi)kK?ob-0 zUEtEL#=p9eU4E#;>~1VT)mv$LPsFpEF07t|x3VWZo%bn1#wekNlTqS z5DI5vfBA_Is|~fWO`meAc*<|;s_HXDA&-3<|_A zOy&w@^X%83<3h4DcqTWI>CaJ^d0Whpc-nrh;y(ZVn9b2*4u3$41ITSNM^5|-1+q7t z&IhgQ&AkFDmmQ8G9S)8pJa71p>b-9Ge>N{ssi~M!?NoS?zzh3GP;`q_+I`Z2r~sxk zCzRf7ngGhfW4Ogl{L`>L@+=8ZgG|;FIt1lro6rx0)+1}#+>q*WuRwC9(sOoll*DZ| za*4n|eK}erw!tQ?8jePcPI7)VRd`Q}%&W$+1#&f#iG0KanL1w8zyH9bk zK?p@M>l5e3=nby$POV%?@hGla`RA5$Xk)}m%A{4-t)#<$vjLymw9yj(L4in3PkFs* za?o_1G@5~NAl;c@qggK9AdQ6*t#E^8h4kArwqebMk|F7RDkP?%Gb$z(jWiVgIDr(a z>@+}nPusbpsDsw?fFJ1AEkVz+HWs{p@%Kt4j5d4mI29(6pad6{bov+rQ7|SYXH?*9 zHy_H-!=V)o`m~{k{E)O!i@efPJpl1$BNzsDVW3i(3D#J&n0L>c&8|)~DMjQvG^=N1 zE}9l7v&O~2Du0x6$nrKhUif`+d2%eUso0p2HY`G(D%Wb^UO)w$;nN`~d5sG>DBSX~ z$g4p(AUM!tZS7#{s>_q3n6t694M6@W#{lV)lv9sn(t{m=YO|_Y26oLrp3+8-VgcOV zvlV=?od7`!*;as@=esO`7;wcpm)l<99@JQ?(;Vto8Fqo}9>FBTpVlL9*@g$o&Ro)2 zDY{W072#9YbY9O}eMUSepEpHWeOaYQTun=zNKAW;1B@!{iHwn?1&m-I59ndgS_PyS zEA$>KFXQqa%dH~t6xDzwgYsuAv}et=&7lkCx{oxQYm>WQvH_~=YSZCIY`|a}7%lN7 z3Pf!>odrtQn=1rD%RI-BZU+=Oz`i66bExpF^k)E9SfGp%jl~?nKs(Swz;lyd3eEw(q7OC$*=L8$d4Cc-;ZGQC@(O=k=s*JXU%XIPcMOE8 zGF`Z%@Mfv*q4+#V+T6%KB^V&#NeT5xdOh7EP(@ZI%YX|EPw-9s5^wV9)QDg!T&og@3n@OO1cESp=jdRrDK)wE^Vu8oM; z(55X7IIiDDfXr{OiEmoSaicEpsB`0~aoC+nA#ORBbC|EB(ku;bpg`m5dnf zh82CJedP)@if8dydr3vVjdup$0>e8S8;;`NO&pRgvOv*O1u6-aK!9v{r`29Msl@C0ukmUa>zu*XIVT7qJu$a|w=lh9iw z2hHfIqKSzIInG={XCrhkG#75QZVa2T>9?6|fY`d^E#rZYAAf*k*13_^zpnY0*3B77 ziJ$zNh5kYUdb+CgH;ug+K0N>?PvZ z6{hI7W;8~kD?Ra{hKpKd_4$>a={4zfo~iAx9Oih!NM(#fEf|yGiX6aK)^qH{_7fm` zv*~ouy8c`&Q2EW_5OU+-NYL{};HcW`Mv%K~iAqaFk1F@!0@bjzCKd8c1A9Wu78*$} z%r=TNt7L~FWv0_yQIMeY0w5zD)ExBhO|j_x&!_#5v^QQ=@WMk96xHH{R-bqv%71I0TJnH`5K=vjU09Oo+z1fxH*XoEkgbxG5If z+z36vIc|)=Vw*N%d)xacu+}#V6iwxWOXNS3b}Brl7OBkU)~w!^3HoCe@Wl?pOF{GQ zKRM1tlL2*Y0;%l3-bayZ30a^-DyuJ%awAhI@SHkiARD?{Jun)?Y8EhR7syU*WgVCg zT+$2V{$EbfWSfHqmp`aOOU^zIN4?mk9hBBLM`==FM#{n_kV3+tUQJ;MG5ZZ+>d1o1 zFk+PVVz>H|tv-fk=D7t1|4{ccn(=(l6nU%P09spbp0c$g|dMf=>WZp0W&V<%lWE_|SkPU6t_X+e67j9WK0yPN- zv?&08fuYWAo*WN=`qsHj_M`WpPqjqMp$3(;7s&aMxg_|S1@sYB?qr$%6p~7K#G`36 zE@q-(R0G*iIZ6TzS=?2#GS<(fH1o5_CG~Xp4B$P#eat3ru@enfqUMlV46XR`)ov%1 zKy~pt7Z3%Eo@R1_ss}OW0ORSI%|P1OEOXu|f>QhmPb@(9WsF2(tAO5PWn~SWRa_Kf zZU1R;P89<%-v&l(Z*d9Qdfd5k&i~aQ{fGDRWu2;-qZ8W&=*WV=_R0+y5J%Z z0A-v7##ga>$4oPeduq_d*CC}Vx5A-;yAp%aEEaPN16pjrM(a5aV&4gnui114=%@Z% zG%$NQ$OS4C6#NosPZw+fTF(ku@($-a5lPT*@Ct_)ZjeBE7caEcRRf_2CMqYC+ANg- z%E=?4#ZCBAfsfyAGr4rxJi@;*SRUrQir2-c8dq4$<0x2RBc0LD`QAs;bp(3mFSpP4J+En^~x1O zl*8h&=2E17Al?k3?!Q8sa1?x3cToDw0%b-OI8mlBiDVRpcs41-$ILY-Xd>Gx6-ZHG zi*{;D+WLXyGp-ha#D^B2(H!SL9v6nzp-B=tJ*U?mqkNgxWR}*n`;-^ggiwUDN^xj< zZ=Q-D$(3O#2Ss&Z85i*m7zV7EjFFuMhGU>x#$)3hnww&y*juG+Q_x!_8!hMAqlU=` zeV@LxpS@MPfGyl=+}Ji_W0exP2L{bRk|7*`D*NVS*JmD&zny6|{ZnIB;z=u(Rou;lqGdm2~(uB zRzW-sva0R@-&(^s>|O6cZM62xp<$KG7s!+mv1GWNInH$vW6+F5>sEp~2d7`+*f9)yBm^U*_4 z5Xz&6(dgkcdUK540&80Gnh09ciq^Eonl@O|me;hSHSK9l2ds(2nvT4t6Rqh?Yr0@f zSFGvAYr4~#9<-(>*7U-f-n^y{t?5f^`e99ftQo*-2GW{Av}Q2Y48fYAyk;1!8BS|P zV9iLZ8O3Wx)0#1~W-Qi>!-+^6qw5}sWfRc z>0++HT!pztbDidfn42)SU~bdgp}8yO9?X5144O=u2Vx$=Jc4;l^MvN9m}fB0VP4R@ zqVM7+V-Sn9($2 zXvT^e2V)O2o@N5gL@^F9lVB#(OrdcUGZn@OW*Ut%&2%v?Ff(9g(#)cnEoKhPT$p(@ z^Jx}{SqQTTW-*N`%@Q$7VV1!xr*WfMA!a4aDwx$YYiQPrafew4vz}%H%|xr$>n z$Hkn0ISF%$=Co?;G+lwbv7Gb zqyPHiYO8T$9L!yu>>O;h9sL!aB2m;n1Oe*BX7?coWK2GhWSSuExB4VsS}O#rV?0!@ z%VXmq*r8DFP?)+yu8!eq;NM60BXUiO#H1)ridHm*atsrnSL9DZjut7g{EaT;j2@RO zdOV^hFnXdyPh#|B&26Ohm(QQ+FEaflxkXZPWxkBesmz>4pZ@F1;3t3KgxBWh000+E BKM4Q; literal 0 HcmV?d00001 diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index 67c6830fd..0b09e60b7 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -353,6 +353,41 @@ def test_mulmod(self): # 0x8000000000000000000000000000000000000000000000000000000082000011 self.assertEqual(Z3Solver.instance().get_all_values(constraints, result), [2423129]) + def test_related_to(self): + import gzip + from manticore import config + from manticore.core.smtlib.visitors import translate_to_smtlib + from manticore.core.smtlib import ConstraintSet, Z3Solver, Operators + import pickle, sys + filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) + + constraints, constraint = pickle.loads(gzip.open(filename,'rb').read()) + + consts = config.get_group("smt") + consts.related_constraints = False + + Z3Solver.instance().can_be_true.cache_clear() + ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) + self.assertEqual(ground_truth, True) + + consts.related_constraints = True + Z3Solver.instance().can_be_true.cache_clear() + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, constraint)) + + #Replace + new_constraint = Operators.UGE( Operators.SEXTEND(BitVecConstant(256,0x1a),256,512) * BitVecConstant(512,1), 0 ) + + self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) + + consts.related_constraints = False + Z3Solver.instance().can_be_true.cache_clear() + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) + + consts.related_constraints = True + Z3Solver.instance().can_be_true.cache_clear() + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) + + if __name__ == "__main__": unittest.main() From 609a37ec9f6c21779dc7721a7df304518fb4af5c Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 13 May 2020 18:54:52 -0300 Subject: [PATCH 007/126] Fix test --- manticore/core/smtlib/constraints.py | 2 +- tests/ethereum/test_regressions.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 36aff2f5f..494f00d17 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -146,7 +146,7 @@ def __get_related(self, related_to=None): break variables = get_variables(constraint) - if related_variables & variables: + if related_variables & variables:# or not variables or not related_variables: remaining_constraints.remove(constraint) related_constraints.add(constraint) related_variables |= variables diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index 0b09e60b7..cf6d2c9b9 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -357,7 +357,7 @@ def test_related_to(self): import gzip from manticore import config from manticore.core.smtlib.visitors import translate_to_smtlib - from manticore.core.smtlib import ConstraintSet, Z3Solver, Operators + from manticore.core.smtlib import ConstraintSet, Z3Solver, Operators, BitVecConstant import pickle, sys filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) @@ -368,15 +368,14 @@ def test_related_to(self): Z3Solver.instance().can_be_true.cache_clear() ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) - self.assertEqual(ground_truth, True) + self.assertEqual(ground_truth, False) consts.related_constraints = True Z3Solver.instance().can_be_true.cache_clear() self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, constraint)) #Replace - new_constraint = Operators.UGE( Operators.SEXTEND(BitVecConstant(256,0x1a),256,512) * BitVecConstant(512,1), 0 ) - + new_constraint = Operators.UGE( Operators.SEXTEND(BitVecConstant(256,0x1a),256,512) * BitVecConstant(512,1), 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000 ) self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) consts.related_constraints = False From b29f3acc742b1d3b40c1485ff8b83842319f8a17 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 13 May 2020 18:56:04 -0300 Subject: [PATCH 008/126] Try fix get-related --- manticore/core/smtlib/constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 494f00d17..d34ff67df 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -146,7 +146,7 @@ def __get_related(self, related_to=None): break variables = get_variables(constraint) - if related_variables & variables:# or not variables or not related_variables: + if related_variables & variables or not variables: remaining_constraints.remove(constraint) related_constraints.add(constraint) related_variables |= variables From 3bbb483d6cadc1c114535358e9306ca72bdd4c1b Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 26 May 2020 19:32:58 -0300 Subject: [PATCH 009/126] Playing with mypy --- manticore/core/smtlib/expression.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index d3a840621..f99ff071f 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -18,10 +18,11 @@ class ExpressionException(SmtlibError): class Expression(object): """ Abstract taintable Expression. """ - __slots__ = () - xslots = ("_taint",) + __slots__:Tuple[str, ...] = () + xslots:Tuple[str, ...] = ("_taint",) - def __init__(self, taint: Union[tuple, frozenset] = ()): + def __init__(self, taint: Union[tuple, frozenset] = (), **kwargs): + assert not kwargs super().__init__() self._taint = frozenset(taint) @@ -112,7 +113,7 @@ def taint_with(arg, *taints, value_bits=256, index_bits=256): class Variable(Expression): __slots__ = () - xslots = ("_name",) + xslots: Tuple[str, ...] = ("_name",) def __init__(self, name: str, **kwargs): super().__init__(**kwargs) @@ -138,7 +139,7 @@ def __repr__(self): class Constant(Expression): __slots__ = () - xslots = ("_value",) + xslots: Tuple[str, ...] = ("_value",) def __init__(self, value: Union[bool, int], **kwargs): super().__init__(**kwargs) @@ -151,9 +152,9 @@ def value(self): class Operation(Expression): __slots__ = () - xslots = ("_operands",) + xslots:Tuple[str, ...] = ("_operands",) - def __init__(self, operands: Tuple[Expression], taint=None, **kwargs): + def __init__(self, operands: Tuple[Expression, ...], taint=None, **kwargs): # assert len(operands) > 0 # assert all(isinstance(x, Expression) for x in operands) self._operands = operands @@ -172,7 +173,7 @@ def operands(self): ############################################################################### # Booleans class Bool(Expression): - __slots__:Tuple[str] = tuple() + __slots__: Tuple[str, ...] = tuple() def cast(self, value: Union["Bool", int, bool], **kwargs) -> Union["BoolConstant", "Bool"]: if isinstance(value, Bool): @@ -269,7 +270,7 @@ def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): class BitVec(Expression): __slots__ = () - xslots = Expression.xslots + ("size",) + xslots: Tuple[str, ...] = Expression.xslots + ("size",) """ This adds a bitsize to the Expression class """ def __init__(self, size=None, **kwargs): @@ -518,9 +519,6 @@ def value(self): return self._value -class BitVecOperation(BitVec): - __slots__ = ["_operands"] - class BitVecOperation(BitVec, Operation): xslots = BitVec.xslots + Operation.xslots __slots__ = () From 154326ce697708131632f67515a20fe53cf0852d Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 26 May 2020 20:06:08 -0300 Subject: [PATCH 010/126] CC --- manticore/core/smtlib/expression.py | 30 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index f99ff071f..6e2dbd90e 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -18,8 +18,8 @@ class ExpressionException(SmtlibError): class Expression(object): """ Abstract taintable Expression. """ - __slots__:Tuple[str, ...] = () - xslots:Tuple[str, ...] = ("_taint",) + __slots__: Tuple[str, ...] = () + xslots: Tuple[str, ...] = ("_taint",) def __init__(self, taint: Union[tuple, frozenset] = (), **kwargs): assert not kwargs @@ -111,6 +111,7 @@ def taint_with(arg, *taints, value_bits=256, index_bits=256): return arg + class Variable(Expression): __slots__ = () xslots: Tuple[str, ...] = ("_name",) @@ -152,7 +153,7 @@ def value(self): class Operation(Expression): __slots__ = () - xslots:Tuple[str, ...] = ("_operands",) + xslots: Tuple[str, ...] = ("_operands",) def __init__(self, operands: Tuple[Expression, ...], taint=None, **kwargs): # assert len(operands) > 0 @@ -224,14 +225,17 @@ class BoolVariable(Bool, Variable): def declaration(self): return f"(declare-fun {self.name} () Bool)" + class BoolConstant(Bool, Constant): __slots__ = Bool.xslots + Constant.xslots + def __init__(self, value, **kwargs): super().__init__(value=value, **kwargs) def __bool__(self): return self._value + class BoolOperation(Operation, Bool): __slots__ = () xslots = Operation.xslots + Bool.xslots @@ -239,24 +243,28 @@ class BoolOperation(Operation, Bool): class BoolNot(BoolOperation): __slots__ = BoolOperation.xslots + def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolAnd(BoolOperation): __slots__ = BoolOperation.xslots + def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolOr(BoolOperation): __slots__ = BoolOperation.xslots + def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolXor(BoolOperation): __slots__ = BoolOperation.xslots + def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) @@ -665,7 +673,7 @@ class BoolEqual(BoolOperation): def __init__(self, a, b, *args, **kwargs): if isinstance(a, BitVec) or isinstance(b, BitVec): assert a.size == b.size - super().__init__(operands=(a,b), **kwargs) + super().__init__(operands=(a, b), **kwargs) class GreaterThan(BoolOperation): @@ -686,7 +694,7 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedLessThan(BoolOperation): __slots__ = BitVecOperation.xslots - def __init__(self, a, b, **kwargs): + def __init__(self, a, b, **kwargs): super().__init__(operands=(a, b), **kwargs) @@ -992,7 +1000,7 @@ def value(self): class ArraySlice(Array): - __slots__ = Array.xslots+("_array", "_slice_offset", "_slice_size") + __slots__ = Array.xslots + ("_array", "_slice_offset", "_slice_size") def __init__( self, array: Union["Array", "ArrayProxy"], offset: int, size: int, *args, **kwargs @@ -1358,12 +1366,13 @@ class BitVecSignExtend(BitVecOperation): def __init__(self, operand: "BitVec", size_dest: int, *args, **kwargs): assert size_dest >= operand.size - super().__init__(size=size_dest, operands=(operand,), **kwargs) + super().__init__(size=size_dest, operands=(operand,), **kwargs) self.extend = size_dest - operand.size class BitVecZeroExtend(BitVecOperation): __slots__ = BitVecOperation.xslots + ("extend",) + def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): assert size_dest >= operand.size super().__init__(size=size_dest, operands=(operand,), **kwargs) @@ -1372,6 +1381,7 @@ def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): class BitVecExtract(BitVecOperation): __slots__ = BitVecOperation.xslots + ("_begining", "_end") + def __init__(self, operand: "BitVec", offset: int, size: int, *args, **kwargs): assert offset >= 0 and offset + size <= operand.size super().__init__(size=size, operands=(operand,), **kwargs) @@ -1416,6 +1426,6 @@ def __init__( super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) -#Constant = (BitVecConstant, BoolConstant) -#Variable = (BitVecVariable, BoolVariable, ArrayVariable) -#Operation = (BitVecOperation, BoolOperation, ArrayOperation, ArraySelect) +# Constant = (BitVecConstant, BoolConstant) +# Variable = (BitVecVariable, BoolVariable, ArrayVariable) +# Operation = (BitVecOperation, BoolOperation, ArrayOperation, ArraySelect) From 4192b6e42f6d45e9d58cfab0d9c62ba4a9d88904 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 27 May 2020 00:25:22 -0300 Subject: [PATCH 011/126] Test fix --- manticore/core/smtlib/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 6e2dbd90e..9219a58d5 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -281,7 +281,7 @@ class BitVec(Expression): xslots: Tuple[str, ...] = Expression.xslots + ("size",) """ This adds a bitsize to the Expression class """ - def __init__(self, size=None, **kwargs): + def __init__(self, size:int, **kwargs): super().__init__(**kwargs) self.size = size From 7885c84b6429c1a14e77f07566738709c0af34c2 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 27 May 2020 18:08:05 -0300 Subject: [PATCH 012/126] Fixing ArrayProxy usages --- manticore/core/smtlib/expression.py | 24 ++++++++++-------------- manticore/core/smtlib/visitors.py | 9 +++++++++ manticore/ethereum/abi.py | 27 +++++++++++++-------------- tests/ethereum/test_general.py | 6 +++--- tests/native/test_linux.py | 2 +- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 9219a58d5..41d2d5f96 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -900,7 +900,7 @@ def write_LE(self, address, value, size): return array def __add__(self, other): - if not isinstance(other, (Array, bytearray)): + if not isinstance(other, (Array, bytes)): raise TypeError("can't concat Array to {}".format(type(other))) if isinstance(other, Array): if self.index_bits != other.index_bits or self.value_bits != other.value_bits: @@ -909,22 +909,21 @@ def __add__(self, other): from .visitors import simplify # FIXME This should be related to a constrainSet - new_arr = ArrayProxy( - ArrayVariable( + new_arr = ArrayVariable( self.index_bits, self.index_max + len(other), self.value_bits, "concatenation{}".format(uuid.uuid1()), ) - ) + for index in range(self.index_max): - new_arr[index] = simplify(self[index]) + new_arr = new_arr.store(index, simplify(self[index])) for index in range(len(other)): - new_arr[index + self.index_max] = simplify(other[index]) + new_arr = new_arr.store(index + self.index_max, simplify(other[index])) return new_arr def __radd__(self, other): - if not isinstance(other, (Array, bytearray, bytes)): + if not isinstance(other, (Array, bytes)): raise TypeError("can't concat Array to {}".format(type(other))) if isinstance(other, Array): if self.index_bits != other.index_bits or self.value_bits != other.value_bits: @@ -933,20 +932,17 @@ def __radd__(self, other): from .visitors import simplify # FIXME This should be related to a constrainSet - new_arr = ArrayProxy( - ArrayVariable( + new_arr = ArrayVariable( self.index_bits, self.index_max + len(other), self.value_bits, "concatenation{}".format(uuid.uuid1()), ) - ) + for index in range(len(other)): - new_arr[index] = simplify(other[index]) - _concrete_cache = new_arr._concrete_cache + new_arr = new_arr.store(index, simplify(other[index])) for index in range(self.index_max): - new_arr[index + len(other)] = simplify(self[index]) - new_arr._concrete_cache.update(_concrete_cache) + new_arr = new_arr.store(index + len(other), simplify(self[index])) return new_arr diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index be126fff4..29a21e821 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -783,6 +783,8 @@ def to_constant(expression): Iff the expression can be simplified to a Constant get the actual concrete value. This discards/ignore any taint """ + if isinstance(expression , ArrayProxy): + expression = expression.array value = simplify(expression) if isinstance(value, Expression) and value.taint: raise ValueError("Can not simplify tainted values to constant") @@ -933,6 +935,8 @@ def result(self): def translate_to_smtlib(expression, **kwargs): + if isinstance(expression, ArrayProxy): + expression = expression.array translator = TranslatorSmtlib(**kwargs) translator.visit(expression) return translator.result @@ -960,6 +964,9 @@ def _visit_variable(self, expression): def replace(expression, bindings): if not bindings: return expression + if isinstance(expression, ArrayProxy): + expression = expression.array + visitor = Replace(bindings) visitor.visit(expression, use_fixed_point=True) result_expression = visitor.result @@ -991,6 +998,8 @@ def simplify_array_select(array_exp): def get_variables(expression): + if isinstance(expression, ArrayProxy): + expression = expression.array visitor = GetDeclarations() visitor.visit(expression) return visitor.result diff --git a/manticore/ethereum/abi.py b/manticore/ethereum/abi.py index dfbcab45a..107e087e1 100644 --- a/manticore/ethereum/abi.py +++ b/manticore/ethereum/abi.py @@ -11,7 +11,6 @@ Operators, BitVec, ArrayVariable, - ArrayProxy, to_constant, issymbolic, ) @@ -110,8 +109,8 @@ def _serialize(ty, value, dyn_offset=None): if dyn_offset is None: dyn_offset = ABI._type_size(ty) - result = bytearray() - dyn_result = bytearray() + result = bytes() + dyn_result = bytes() if ty[0] == "int": result += ABI._serialize_int(value, size=ty[1] // 8, padding=32 - ty[1] // 8) @@ -158,8 +157,8 @@ def _serialize_bytes(value): @staticmethod def _serialize_tuple(types, value, dyn_offset=None): - result = bytearray() - dyn_result = bytearray() + result = bytes() + dyn_result = bytes() if len(types) != len(value): raise ValueError( f"The number of values to serialize is {'less' if len(value) < len(types) else 'greater'} than the number of types" @@ -206,10 +205,10 @@ def function_selector(method_name_and_signature): def deserialize(type_spec, data): try: if isinstance(data, str): - data = bytearray(data.encode()) + data = bytes(data.encode()) elif isinstance(data, bytes): - data = bytearray(data) - assert isinstance(data, (bytearray, Array)) + data = bytes(data) + assert isinstance(data, (bytes, Array)) m = re.match(r"(?P[a-zA-Z_0-9]+)(?P\(.*\))", type_spec) if m and m.group("name"): @@ -229,7 +228,7 @@ def deserialize(type_spec, data): @staticmethod def _deserialize(ty, buf: typing.Union[bytearray, bytes, Array], offset=0): - assert isinstance(buf, (bytearray, bytes, Array)) + assert isinstance(buf, ( bytes, Array)) result = None if ty[0] == "int": result = ABI._deserialize_int(buf[offset : offset + 32], nbytes=ty[1] // 8) @@ -320,7 +319,7 @@ def _serialize_int(value: typing.Union[int, BitVec], size=32, padding=0): index_bits=256, index_max=32, value_bits=8, name="temp{}".format(uuid.uuid1()) ) value = Operators.SEXTEND(value, value.size, size * 8) - return ArrayProxy(buf.write_BE(padding, value, size)) + return buf.write_BE(padding, value, size) else: buf_arr = bytearray() for _ in range(padding): @@ -359,7 +358,7 @@ def _readBE(data, nbytes, padding=False, offset=0): @staticmethod def _deserialize_uint( - data: typing.Union[bytearray, bytes, Array], nbytes=32, padding=0, offset=0 + data: typing.Union[bytes, Array], nbytes=32, padding=0, offset=0 ): """ Read a `nbytes` bytes long big endian unsigned integer from `data` starting at `offset` @@ -368,13 +367,13 @@ def _deserialize_uint( :param nbytes: number of bytes to read starting from least significant byte :rtype: int or Expression """ - assert isinstance(data, (bytearray, bytes, Array)) + assert isinstance(data, (bytes, Array)) value = ABI._readBE(data, nbytes, padding=True, offset=offset) value = Operators.ZEXTEND(value, (nbytes + padding) * 8) return value @staticmethod - def _deserialize_int(data: typing.Union[bytearray, bytes, Array], nbytes=32, padding=0): + def _deserialize_int(data: typing.Union[bytes, Array], nbytes=32, padding=0): """ Read a `nbytes` bytes long big endian signed integer from `data` starting at `offset` @@ -382,7 +381,7 @@ def _deserialize_int(data: typing.Union[bytearray, bytes, Array], nbytes=32, pad :param nbytes: number of bytes to read starting from least significant byte :rtype: int or Expression """ - assert isinstance(data, (bytearray, bytes, Array)) + assert isinstance(data, (bytes, Array)) value = ABI._readBE(data, nbytes, padding=True) value = Operators.SEXTEND(value, nbytes * 8, (nbytes + padding) * 8) if not issymbolic(value): diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index d0903cb79..21f29f9e8 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -13,7 +13,7 @@ from manticore.core.plugin import Plugin from manticore.core.smtlib import ConstraintSet, operators from manticore.core.smtlib import Z3Solver -from manticore.core.smtlib.expression import BitVec +from manticore.core.smtlib.expression import BitVecVariable from manticore.core.smtlib.visitors import to_constant from manticore.core.state import TerminateState from manticore.ethereum import ( @@ -125,7 +125,7 @@ def test_dyn_address(self): def test_dyn_bytes(self): d = [ b"AAAA", # function hash - self._pack_int_to_32(32), # offset to data start + self._pack_int_to_32(32), # offset to data start1350 self._pack_int_to_32(30), # data start; # of elements b"Z" * 30, b"\x00" * 2, @@ -1347,7 +1347,7 @@ def will_evm_execute_instruction_callback(self, state, i, *args, **kwargs): class EthHelpersTest(unittest.TestCase): def setUp(self): - self.bv = BitVec(256) + self.bv = BitVecVariable(size=256, name='bv') def test_concretizer(self): policy = "SOME_NONSTANDARD_POLICY" diff --git a/tests/native/test_linux.py b/tests/native/test_linux.py index a12580192..b015aa597 100644 --- a/tests/native/test_linux.py +++ b/tests/native/test_linux.py @@ -285,7 +285,7 @@ def test_armv7_syscall_openat_concrete(self) -> None: def test_armv7_syscall_openat_symbolic(self) -> None: platform, temp_dir = self._armv7_create_openat_state() try: - platform.current.R0 = BitVecVariable(32, "fd") + platform.current.R0 = BitVecVariable(size=32, name="fd") with self.assertRaises(ConcretizeRegister) as cm: platform.syscall() From df96058c0f8785d21f859274b4c694e48aad8daa Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 28 May 2020 11:35:12 -0300 Subject: [PATCH 013/126] Remove bytearray from expression and tests --- manticore/core/smtlib/expression.py | 26 +++++++++++++------------- manticore/ethereum/abi.py | 15 +++------------ tests/other/test_smtlibv2.py | 10 +++++----- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 41d2d5f96..f9b1afe39 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -281,7 +281,7 @@ class BitVec(Expression): xslots: Tuple[str, ...] = Expression.xslots + ("size",) """ This adds a bitsize to the Expression class """ - def __init__(self, size:int, **kwargs): + def __init__(self, size: int, **kwargs): super().__init__(**kwargs) self.size = size @@ -910,11 +910,11 @@ def __add__(self, other): # FIXME This should be related to a constrainSet new_arr = ArrayVariable( - self.index_bits, - self.index_max + len(other), - self.value_bits, - "concatenation{}".format(uuid.uuid1()), - ) + self.index_bits, + self.index_max + len(other), + self.value_bits, + "concatenation{}".format(uuid.uuid1()), + ) for index in range(self.index_max): new_arr = new_arr.store(index, simplify(self[index])) @@ -933,11 +933,11 @@ def __radd__(self, other): # FIXME This should be related to a constrainSet new_arr = ArrayVariable( - self.index_bits, - self.index_max + len(other), - self.value_bits, - "concatenation{}".format(uuid.uuid1()), - ) + self.index_bits, + self.index_max + len(other), + self.value_bits, + "concatenation{}".format(uuid.uuid1()), + ) for index in range(len(other)): new_arr = new_arr.store(index, simplify(other[index])) @@ -1256,8 +1256,8 @@ def read_BE(self, address, size): return BitVecConcat(size * self.value_bits, *bytes) def write(self, offset, buf): - if not isinstance(buf, (Array, bytearray)): - raise TypeError("Array or bytearray expected got {:s}".format(type(buf))) + if not isinstance(buf, (Array, bytes)): + raise TypeError("Array or bytes expected got {:s}".format(type(buf))) arr = self for i, val in enumerate(buf): arr = arr.store(offset + i, val) diff --git a/manticore/ethereum/abi.py b/manticore/ethereum/abi.py index 107e087e1..88a6a481f 100644 --- a/manticore/ethereum/abi.py +++ b/manticore/ethereum/abi.py @@ -6,14 +6,7 @@ import sha3 from . import abitypes -from ..core.smtlib import ( - Array, - Operators, - BitVec, - ArrayVariable, - to_constant, - issymbolic, -) +from ..core.smtlib import Array, Operators, BitVec, ArrayVariable, to_constant, issymbolic from ..exceptions import EthereumError logger = logging.getLogger(__name__) @@ -228,7 +221,7 @@ def deserialize(type_spec, data): @staticmethod def _deserialize(ty, buf: typing.Union[bytearray, bytes, Array], offset=0): - assert isinstance(buf, ( bytes, Array)) + assert isinstance(buf, (bytes, Array)) result = None if ty[0] == "int": result = ABI._deserialize_int(buf[offset : offset + 32], nbytes=ty[1] // 8) @@ -357,9 +350,7 @@ def _readBE(data, nbytes, padding=False, offset=0): return Operators.CONCAT(nbytes * 8, *values) @staticmethod - def _deserialize_uint( - data: typing.Union[bytes, Array], nbytes=32, padding=0, offset=0 - ): + def _deserialize_uint(data: typing.Union[bytes, Array], nbytes=32, padding=0, offset=0): """ Read a `nbytes` bytes long big endian unsigned integer from `data` starting at `offset` diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 48f768042..6a85731cc 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -275,7 +275,7 @@ def testBasicArraySymbIdx2(self): self.assertFalse(self.solver.check(cs)) def testBasicArrayConcatSlice(self): - hw = bytearray(b"Hello world!") + hw = b"Hello world!" cs = ConstraintSet() # make array of 32->8 bits array = cs.new_array(32, index_max=12) @@ -287,18 +287,18 @@ def testBasicArrayConcatSlice(self): self.assertTrue(self.solver.must_be_true(cs, array.read(6, 6) == hw[6:12])) - self.assertTrue(self.solver.must_be_true(cs, bytearray(b"Hello ") + array.read(6, 6) == hw)) + self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 6) == hw)) self.assertTrue( self.solver.must_be_true( - cs, bytearray(b"Hello ") + array.read(6, 5) + bytearray(b"!") == hw + cs, b"Hello " + array.read(6, 5) + b"!" == hw ) ) self.assertTrue( self.solver.must_be_true( cs, - array.read(0, 1) + bytearray(b"ello ") + array.read(6, 5) + bytearray(b"!") == hw, + array.read(0, 1) + b"ello " + array.read(6, 5) + b"!" == hw, ) ) @@ -312,7 +312,7 @@ def testBasicArrayConcatSlice(self): self.assertTrue(len(results) == 5) def testBasicArraySlice(self): - hw = bytearray(b"Hello world!") + hw = b"Hello world!" cs = ConstraintSet() # make array of 32->8 bits array = cs.new_array(32, index_max=12) From cc24e0d42c3f90d682d5bda20bc943f37d9e1f68 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 1 Jun 2020 11:36:02 -0300 Subject: [PATCH 014/126] Fix uses of unhasheable arrayproxy --- manticore/core/smtlib/expression.py | 13 +++++++++++++ manticore/core/smtlib/solver.py | 2 +- manticore/ethereum/manticore.py | 5 ++++- manticore/platforms/evm.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index f9b1afe39..eb311bd3f 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -996,6 +996,9 @@ def value(self): class ArraySlice(Array): + ''' Provides a projection of an underlying array. + Lets you slice an array without copying it + ''' __slots__ = Array.xslots + ("_array", "_slice_offset", "_slice_size") def __init__( @@ -1047,6 +1050,16 @@ def store(self, index, value): class ArrayProxy: + ''' + Arrayproxy is a layer on top of an array that provides mutability and some + simple optimizations for concrete indexes. + + It is not hasheable. + Think: + bytearray <-> ArrayProxy ::: not hasheable, mutable + bytes <-> Array (ArraySlice, ArrayVariable, ArrayStore) ::: hasheable, notmutable + + ''' def __init__(self, array: Array, default: Optional[int] = None): self._default = default self._concrete_cache: Dict[int, int] = {} diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 70fe8e542..64674c5b4 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -162,7 +162,7 @@ def __init__(self): # http://smtlib.cs.uiowa.edu/logics-all.shtml#QF_AUFBV # Closed quantifier-free formulas over the theory of bitvectors and bitvector arrays extended with # free sort and function symbols. - "(set-logic QF_AUFBV)", + "(set-logic QF_ABV)", # The declarations and definitions will be scoped "(set-option :global-decls false)", # sam.moelius: Option "tactic.solve_eqs.context_solve" was turned on by this commit in z3: diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index cdb8c8bb5..52f275603 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -1180,6 +1180,8 @@ def _on_unsound_symbolication(self, state, func, data, result): name = func.__name__ value = func(data) # If func returns None then result should considered unknown/symbolic + if isinstance(data, ArrayProxy): + data = data.array # Value is known. Let's add it to our concrete database if value is not None: with self.locked_context("ethereum", dict) as ethereum_context: @@ -1207,7 +1209,7 @@ def _on_unsound_symbolication(self, state, func, data, result): data_var = state.new_symbolic_buffer(len(data)) # FIXME: generalize to bitvec state.constrain(data_var == data) - data = data_var + data = data_var.array # symbolic_pairs list of symbolic applications of func in sate symbolic_pairs = state.context.get(f"symbolic_func_sym_{name}", []) @@ -1230,6 +1232,7 @@ def _on_unsound_symbolication(self, state, func, data, result): else: constraint = y != value state.constrain(constraint) + print (type(data), type(value)) symbolic_pairs.append((data, value)) state.context[f"symbolic_func_sym_{name}"] = symbolic_pairs diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 037b6bac7..efd7a19b0 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -1335,7 +1335,7 @@ def read_buffer(self, offset, size): if size == 0: return b"" self._allocate(offset, size) - return self.memory[offset : offset + size] + return self.memory[offset : offset + size].array def write_buffer(self, offset, data): self._allocate(offset, len(data)) From 114610408ceedd5286401f2411def9cb97f73d9d Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 1 Jun 2020 13:33:09 -0300 Subject: [PATCH 015/126] blkn --- manticore/core/smtlib/visitors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 29a21e821..7c220ddac 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -783,7 +783,7 @@ def to_constant(expression): Iff the expression can be simplified to a Constant get the actual concrete value. This discards/ignore any taint """ - if isinstance(expression , ArrayProxy): + if isinstance(expression, ArrayProxy): expression = expression.array value = simplify(expression) if isinstance(value, Expression) and value.taint: From 79aa5ccb2a544c4e6d5f60a5b4aa98964981d11a Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 2 Jun 2020 17:29:26 -0300 Subject: [PATCH 016/126] Move regression to other --- tests/ethereum/test_regressions.py | 33 ---------------- .../data/ErrRelated.pkl.gz | Bin tests/other/test_smtlibv2.py | 36 ++++++++++++++++++ 3 files changed, 36 insertions(+), 33 deletions(-) rename tests/{ethereum => other}/data/ErrRelated.pkl.gz (100%) diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index cf6d2c9b9..a7cbeb4fd 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -353,39 +353,6 @@ def test_mulmod(self): # 0x8000000000000000000000000000000000000000000000000000000082000011 self.assertEqual(Z3Solver.instance().get_all_values(constraints, result), [2423129]) - def test_related_to(self): - import gzip - from manticore import config - from manticore.core.smtlib.visitors import translate_to_smtlib - from manticore.core.smtlib import ConstraintSet, Z3Solver, Operators, BitVecConstant - import pickle, sys - filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) - - constraints, constraint = pickle.loads(gzip.open(filename,'rb').read()) - - consts = config.get_group("smt") - consts.related_constraints = False - - Z3Solver.instance().can_be_true.cache_clear() - ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) - self.assertEqual(ground_truth, False) - - consts.related_constraints = True - Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, constraint)) - - #Replace - new_constraint = Operators.UGE( Operators.SEXTEND(BitVecConstant(256,0x1a),256,512) * BitVecConstant(512,1), 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000 ) - self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) - - consts.related_constraints = False - Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) - - consts.related_constraints = True - Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) - if __name__ == "__main__": diff --git a/tests/ethereum/data/ErrRelated.pkl.gz b/tests/other/data/ErrRelated.pkl.gz similarity index 100% rename from tests/ethereum/data/ErrRelated.pkl.gz rename to tests/other/data/ErrRelated.pkl.gz diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index c9b230fdf..0b767c75c 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1,4 +1,5 @@ import unittest +import os from manticore.core.smtlib import ( ConstraintSet, @@ -11,15 +12,50 @@ arithmetic_simplify, constant_folder, replace, + BitVecConstant, ) from manticore.core.smtlib.solver import Z3Solver from manticore.core.smtlib.expression import * from manticore.utils.helpers import pickle_dumps +from manticore import config # logging.basicConfig(filename = "test.log", # format = "%(asctime)s: %(name)s:%(levelname)s: %(message)s", # level = logging.DEBUG) +DIRPATH = os.path.dirname(__file__) + +class RegressionTest(unittest.TestCase): + def test_related_to(self): + import gzip + import pickle, sys + filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) + + constraints, constraint = pickle.loads(gzip.open(filename,'rb').read()) + + consts = config.get_group("smt") + consts.related_constraints = False + + Z3Solver.instance().can_be_true.cache_clear() + ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) + self.assertEqual(ground_truth, False) + + consts.related_constraints = True + Z3Solver.instance().can_be_true.cache_clear() + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, constraint)) + + #Replace + new_constraint = Operators.UGE( Operators.SEXTEND(BitVecConstant(256,0x1a),256,512) * BitVecConstant(512,1), 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000 ) + self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) + + consts.related_constraints = False + Z3Solver.instance().can_be_true.cache_clear() + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) + + consts.related_constraints = True + Z3Solver.instance().can_be_true.cache_clear() + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) + class ExpressionTest(unittest.TestCase): _multiprocess_can_split_ = True From 7339eb96799f7f71d596c6a5f3b1945738e697db Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 2 Jun 2020 18:06:57 -0300 Subject: [PATCH 017/126] blkn --- tests/other/test_smtlibv2.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 0b767c75c..ca97d6524 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -25,13 +25,15 @@ DIRPATH = os.path.dirname(__file__) + class RegressionTest(unittest.TestCase): def test_related_to(self): import gzip import pickle, sys + filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) - constraints, constraint = pickle.loads(gzip.open(filename,'rb').read()) + constraints, constraint = pickle.loads(gzip.open(filename, "rb").read()) consts = config.get_group("smt") consts.related_constraints = False @@ -44,14 +46,17 @@ def test_related_to(self): Z3Solver.instance().can_be_true.cache_clear() self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, constraint)) - #Replace - new_constraint = Operators.UGE( Operators.SEXTEND(BitVecConstant(256,0x1a),256,512) * BitVecConstant(512,1), 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000 ) + # Replace + new_constraint = Operators.UGE( + Operators.SEXTEND(BitVecConstant(256, 0x1A), 256, 512) * BitVecConstant(512, 1), + 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000, + ) self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) consts.related_constraints = False Z3Solver.instance().can_be_true.cache_clear() self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) - + consts.related_constraints = True Z3Solver.instance().can_be_true.cache_clear() self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) From c121c381402b55d741f6c6a9cae1fe4366745a1a Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 2 Jun 2020 18:49:56 -0300 Subject: [PATCH 018/126] blkn --- tests/ethereum/test_regressions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index a7cbeb4fd..67c6830fd 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -354,6 +354,5 @@ def test_mulmod(self): self.assertEqual(Z3Solver.instance().get_all_values(constraints, result), [2423129]) - if __name__ == "__main__": unittest.main() From b40de82cd14ec258d7915e39202d2de97fa2a997 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 11:11:28 -0300 Subject: [PATCH 019/126] Various expression.py refactors --- manticore/core/smtlib/constraints.py | 8 +- manticore/core/smtlib/expression.py | 483 ++++++++---------- manticore/core/smtlib/solver.py | 8 +- manticore/core/smtlib/visitors.py | 11 +- manticore/ethereum/manticore.py | 2 +- manticore/platforms/evm.py | 7 +- tests/ethereum/test_general.py | 4 +- .../ethereum_vm/VMTests_concrete/__init__.py | 2 +- tests/other/test_smtlibv2.py | 30 +- 9 files changed, 271 insertions(+), 284 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index cf25998e8..acc2c3219 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -417,5 +417,9 @@ def new_array( name = self._make_unique_name(name) if not avoid_collisions and name in self._declarations: raise ValueError(f"Name {name} already used") - var = self._declare(ArrayVariable(index_bits, index_max, value_bits, name, taint=taint)) - return ArrayProxy(var, default=default) + var = self._declare( + ArrayVariable( + index_bits, index_max, value_bits, name=name, taint=taint, default=default + ) + ) + return ArrayProxy(var) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index eb311bd3f..1e3033ad6 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -19,7 +19,7 @@ class Expression(object): """ Abstract taintable Expression. """ __slots__: Tuple[str, ...] = () - xslots: Tuple[str, ...] = ("_taint",) + __xslots__: Tuple[str, ...] = ("_taint",) def __init__(self, taint: Union[tuple, frozenset] = (), **kwargs): assert not kwargs @@ -153,7 +153,7 @@ def value(self): class Operation(Expression): __slots__ = () - xslots: Tuple[str, ...] = ("_operands",) + __xslots__: Tuple[str, ...] = ("_operands",) def __init__(self, operands: Tuple[Expression, ...], taint=None, **kwargs): # assert len(operands) > 0 @@ -219,7 +219,7 @@ def __bool__(self): class BoolVariable(Bool, Variable): - __slots__ = Bool.xslots + Variable.xslots + __slots__ = Bool.__xslots__ + Variable.xslots @property def declaration(self): @@ -227,7 +227,7 @@ def declaration(self): class BoolConstant(Bool, Constant): - __slots__ = Bool.xslots + Constant.xslots + __slots__ = Bool.__xslots__ + Constant.xslots def __init__(self, value, **kwargs): super().__init__(value=value, **kwargs) @@ -238,7 +238,7 @@ def __bool__(self): class BoolOperation(Operation, Bool): __slots__ = () - xslots = Operation.xslots + Bool.xslots + xslots = Operation.__xslots__ + Bool.__xslots__ class BoolNot(BoolOperation): @@ -278,7 +278,7 @@ def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): class BitVec(Expression): __slots__ = () - xslots: Tuple[str, ...] = Expression.xslots + ("size",) + xslots: Tuple[str, ...] = Expression.__xslots__ + ("size",) """ This adds a bitsize to the Expression class """ def __init__(self, size: int, **kwargs): @@ -528,7 +528,7 @@ def value(self): class BitVecOperation(BitVec, Operation): - xslots = BitVec.xslots + Operation.xslots + xslots = BitVec.xslots + Operation.__xslots__ __slots__ = () @@ -724,22 +724,53 @@ def __init__(self, a, b, **kwargs): ############################################################################### # Array BV32 -> BV8 or BV64 -> BV8 class Array(Expression): - __slots__ = ["_index_bits", "_index_max", "_value_bits"] + __slots__ = [ + "_index_bits", + "_index_max", + "_value_bits", + "_default", + "_written", + "_concrete_cache", + ] def __init__( - self, index_bits: int, index_max: Optional[int], value_bits: int, *operands, **kwargs + self, + index_bits: int, + index_max: Optional[int], + value_bits: int, + default: Optional[int] = None, + **kwargs, ): + """ + This is a mapping from BV to BV. Normally used to represent a memory. + + :param index_bits: Number of bits in the addressing side + :param index_max: Max address allowed + :param value_bits: Number of bits in tha value side + :param default: Reading from an uninitialized index will return default + if provided. If not the behaivor mimics thtat from smtlib, + the returned value is a free symbol. + :param kwargs: Used in other parent classes + """ assert index_bits in (32, 64, 256) assert value_bits in (8, 16, 32, 64, 256) assert index_max is None or index_max >= 0 and index_max < 2 ** index_bits self._index_bits = index_bits self._index_max = index_max self._value_bits = value_bits + self._default = default + self._written = None # Cache of the known indexs + self._concrete_cache: Dict[int, int] = {} # Cache of concrete indexes super().__init__(**kwargs) assert type(self) is not Array, "Abstract class" - def _get_size(self, index): - start, stop = self._fix_index(index) + def _fix_slice(self, index: slice): + """Used to calculate the size of slices""" + stop, start = index.stop, index.start + if start is None: + start = 0 + if stop is None: + stop = len(self) size = stop - start if isinstance(size, BitVec): from .visitors import simplify @@ -748,24 +779,14 @@ def _get_size(self, index): else: size = BitVecConstant(self.index_bits, size) assert isinstance(size, BitVecConstant) - return size.value - - def _fix_index(self, index): - """ - :param slice index: - """ - stop, start = index.stop, index.start - if start is None: - start = 0 - if stop is None: - stop = len(self) - return start, stop + return start, stop, size.value def cast(self, possible_array): - if isinstance(possible_array, bytearray): + """ Builds an Array from a bytes or bytearray""" + if isinstance(possible_array, (bytearray, bytes)): # FIXME This should be related to a constrainSet arr = ArrayVariable( - self.index_bits, len(possible_array), 8, "cast{}".format(uuid.uuid1()) + self.index_bits, len(possible_array), 8, name="cast{}".format(uuid.uuid1()) ) for pos, byte in enumerate(possible_array): arr = arr.store(pos, byte) @@ -779,9 +800,7 @@ def cast_index(self, index: Union[int, "BitVec"]) -> Union["BitVecConstant", "Bi assert index.size == self.index_bits return index - def cast_value( - self, value: Union["BitVec", str, bytes, int] - ) -> Union["BitVecConstant", "BitVec"]: + def cast_value(self, value: Union["BitVec", str, bytes, int]) -> "BitVec": if isinstance(value, BitVec): assert value.size == self.value_bits return value @@ -808,34 +827,122 @@ def value_bits(self): def index_max(self): return self._index_max - def select(self, index): + @property + def default(self): + return self._default + + def get(self, index, default=None): + """ Gets an element from the Array. + If the element was not previously the default is used. + """ index = self.cast_index(index) - return ArraySelect(self, index) + + # Emulate list[-1] + if self.index_max is not None: + from .visitors import simplify + + index = simplify( + BitVecITE(self.index_bits, index < 0, self.index_max + index + 1, index) + ) + + if isinstance(index, Constant) and index.value in self._concrete_cache: + return self._concrete_cache[index.value] + value = ArraySelect(self, index) + + # No default. Returns free symbol + if default is None: + default = self._default + if default is None: + return value + + # If a default is provided calculate check if the index is known + is_known = self.is_known(index) + default = self.cast_value(default) + if isinstance(is_known, Constant): + if is_known.value: + return value + else: + return default + return BitVecITE(self.value_bits, is_known, value, default) + + def select(self, index): + return self.get(index) def store(self, index, value): - return ArrayStore(self, self.cast_index(index), self.cast_value(value)) + from .visitors import simplify + + index = simplify(self.cast_index(index)) + value = self.cast_value(value) + new_array = ArrayStore(self, index, value) + if isinstance(index, Constant): + new_array._concrete_cache = copy.copy(self._concrete_cache) + new_array._concrete_cache[index.value] = value + else: + # delete all cache as we do not know what this may overwrite. + new_array._concrete_cache = {} + if self.default is not None: + # potentially generate and update .written set + new_array.written.add(index) + return new_array + + @property + def written(self): + # Calculate only first time + if self._written is None: + written = set() + # take out Proxy sleve + array = self + offset = 0 + while not isinstance(array, ArrayVariable): + if array._written is not None: + written = written.union((x - offset for x in array.written)) + break + if isinstance(array, ArraySlice): + # if it is a proxy over a slice take out the slice too + offset += array.slice_offset + array = array.array + else: + # The index written to underlaying Array are displaced when sliced + written.add(array.index - offset) + array = array.array + self._written = written + return self._written + + def is_known(self, index): + if isinstance(index, Constant) and index.value in self._concrete_cache: + return BoolConstant(True) + + is_known_index = BoolConstant(False) + written = self.written + for known_index in written: + if isinstance(index, Constant) and isinstance(known_index, Constant): + if known_index.value == index.value: + return BoolConstant(True) + is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) + return is_known_index def write(self, offset, buf): - if not isinstance(buf, (Array, bytearray)): + """ Creates a new Array instance by writing buf at offset """ + if not isinstance(buf, (Array, bytes)): raise TypeError("Array or bytearray expected got {:s}".format(type(buf))) arr = self for i, val in enumerate(buf): arr = arr.store(offset + i, val) return arr - def read(self, offset, size): - return ArraySlice(self, offset, size) + def read(self, offset, size, default: Optional[int] = None): + default = self._default if default is None else default + return ArraySlice(self, offset=offset, size=size, default=default) def __getitem__(self, index): if isinstance(index, slice): - start, stop = self._fix_index(index) - size = self._get_size(index) - return ArraySlice(self, start, size) + start, stop, size = self._fix_slice(index) + return self.read(start, size, self.default) else: if self.index_max is not None: if not isinstance(index, Expression) and index >= self.index_max: raise IndexError - return self.select(self.cast_index(index)) + return self.get(index, self._default) def __eq__(self, other): # FIXME taint @@ -867,14 +974,14 @@ def underlying_variable(self): def read_BE(self, address, size): bytes = [] for offset in range(size): - bytes.append(self.get(address + offset, 0)) + bytes.append(self.get(address + offset, self._default)) return BitVecConcat(size * self.value_bits, *bytes) def read_LE(self, address, size): address = self.cast_index(address) bytes = [] for offset in range(size): - bytes.append(self.get(address + offset, 0)) + bytes.append(self.get(address + offset, self._default)) return BitVecConcat(size * self.value_bits, *reversed(bytes)) def write_BE(self, address, value, size): @@ -913,7 +1020,8 @@ def __add__(self, other): self.index_bits, self.index_max + len(other), self.value_bits, - "concatenation{}".format(uuid.uuid1()), + default=self._default, + name="concatenation{}".format(uuid.uuid1()), ) for index in range(self.index_max): @@ -936,7 +1044,8 @@ def __radd__(self, other): self.index_bits, self.index_max + len(other), self.value_bits, - "concatenation{}".format(uuid.uuid1()), + default=self._default, + name="concatenation{}".format(uuid.uuid1()), ) for index in range(len(other)): @@ -947,12 +1056,7 @@ def __radd__(self, other): class ArrayVariable(Array, Variable): - __slots__ = Array.xslots + Variable.xslots - - def __init__(self, index_bits, index_max, value_bits, name, **kwargs): - super().__init__( - index_bits=index_bits, index_max=index_max, value_bits=value_bits, name=name, **kwargs - ) + __slots__ = Array.__xslots__ + Variable.xslots @property def declaration(self): @@ -961,19 +1065,20 @@ def declaration(self): class ArrayOperation(Array, Operation): __slots__ = () - xslots = Array.xslots + Operation.xslots + __xslots__ = Array.__xslots__ + Operation.__xslots__ class ArrayStore(ArrayOperation): - __slots__ = ArrayOperation.xslots + __slots__ = ArrayOperation.__xslots__ - def __init__(self, array: "Array", index: "BitVec", value: "BitVec", *args, **kwargs): + def __init__(self, array: "Array", index: "BitVec", value: "BitVec", **kwargs): assert index.size == array.index_bits assert value.size == array.value_bits super().__init__( index_bits=array.index_bits, value_bits=array.value_bits, index_max=array.index_max, + default=array.default, operands=(array, index, value), **kwargs, ) @@ -995,62 +1100,68 @@ def value(self): return self.operands[2] -class ArraySlice(Array): - ''' Provides a projection of an underlying array. - Lets you slice an array without copying it - ''' - __slots__ = Array.xslots + ("_array", "_slice_offset", "_slice_size") +class ArraySlice(ArrayOperation): + """ Provides a projection of an underlying array. + Lets you slice an array without copying it. + (It needs to be simplified out before translating it smtlib) + """ + + __slots__ = ArrayOperation.__xslots__ + ("_slice_offset", "_slice_size") def __init__( - self, array: Union["Array", "ArrayProxy"], offset: int, size: int, *args, **kwargs + self, + array: "Array", + offset: int, + size: int, + default: Optional[int] = None, + **kwargs, ): - if not isinstance(array, (Array, ArrayProxy)): + assert size + if not isinstance(array, Array): raise ValueError("Array expected") - if isinstance(array, ArrayProxy): - array = array._array - super().__init__(array.index_bits, array.index_max, array.value_bits, *args, **kwargs) + default = default if default is not None else array.default + + super().__init__( + index_bits=array.index_bits, + value_bits=array.value_bits, + index_max=size, + operands=(array,), + default=default, + **kwargs, + ) - self._array = array self._slice_offset = offset self._slice_size = size @property def underlying_variable(self): - return self._array.underlying_variable + return self.array.underlying_variable @property - def operands(self): - return self._array.operands - - @property - def index_bits(self): - return self._array.index_bits + def array(self): + return self.operands[0] @property - def index_max(self): - return self._slice_size + def slice_offset(self): + return self._slice_offset @property def value_bits(self): - return self._array.value_bits + return self.array.value_bits - @property - def taint(self): - return self._array.taint - - def select(self, index): - return self._array.select(index + self._slice_offset) + def get(self, index, default): + return self.array.get(index + self._slice_offset, default) def store(self, index, value): return ArraySlice( - self._array.store(index + self._slice_offset, value), + self.array.store(index + self._slice_offset, value), self._slice_offset, self._slice_size, ) class ArrayProxy: - ''' + """ Arrayproxy is a layer on top of an array that provides mutability and some simple optimizations for concrete indexes. @@ -1059,27 +1170,12 @@ class ArrayProxy: bytearray <-> ArrayProxy ::: not hasheable, mutable bytes <-> Array (ArraySlice, ArrayVariable, ArrayStore) ::: hasheable, notmutable - ''' - def __init__(self, array: Array, default: Optional[int] = None): - self._default = default - self._concrete_cache: Dict[int, int] = {} - self._written = None - if isinstance(array, ArrayProxy): - # copy constructor - self._array: Array = array._array - self._name: str = array._name - if default is None: - self._default = array._default - self._concrete_cache = dict(array._concrete_cache) - self._written = set(array.written) - elif isinstance(array, ArrayVariable): - # fresh array proxy - self._array = array - self._name = array.name - else: - # arrayproxy for a prepopulated array - self._name = array.underlying_variable.name - self._array = array + """ + def __hash__(self): + return hash(self.array) + + def __init__(self, array: Array): + self._array = array @property def underlying_variable(self): @@ -1089,10 +1185,6 @@ def underlying_variable(self): def array(self): return self._array - @property - def name(self): - return self._name - @property def operands(self): return self._array.operands @@ -1113,185 +1205,56 @@ def value_bits(self): def taint(self): return self._array.taint - def select(self, index): - return self.get(index) - def __len__(self): - return self._array.index_max + return len(self._array) - def store(self, index, value): - if not isinstance(index, Expression): - index = self._array.cast_index(index) - if not isinstance(value, Expression): - value = self._array.cast_value(value) - from .visitors import simplify + def select(self, index): + return self._array.select(index) - index = simplify(index) - if isinstance(index, Constant): - self._concrete_cache[index.value] = value - else: - # delete all cache as we do not know what this may overwrite. - self._concrete_cache = {} + def store(self, index, value): + return self._array.store(index, value) - # potentially generate and update .written set - self.written.add(index) - self._array = self._array.store(index, value) - return self + @property + def written(self): + return self._array.written def __getitem__(self, index): + result = self._array[index] if isinstance(index, slice): - start, stop = self._array._fix_index(index) - size = self._array._get_size(index) - array_proxy_slice = ArrayProxy(ArraySlice(self, start, size), default=self._default) - array_proxy_slice._concrete_cache = {} - for k, v in self._concrete_cache.items(): - if k >= start and k < start + size: - array_proxy_slice._concrete_cache[k - start] = v - - for i in self.written: - array_proxy_slice.written.add(i - start) - return array_proxy_slice - else: - if self.index_max is not None: - if not isinstance(index, Expression) and index >= self.index_max: - raise IndexError - return self.get(index, self._default) + return ArrayProxy(result) + return result def __setitem__(self, index, value): if isinstance(index, slice): - start, stop = self._array._fix_index(index) - size = self._array._get_size(index) + start, stop, size = self._array._fix_slice(index) assert len(value) == size for i in range(size): - self.store(start + i, value[i]) + self._array = self._array.store(start + i, value[i]) else: - self.store(index, value) - - def __getstate__(self): - if self.index_max is not None and self.index_max == len(self._concrete_cache): - array = (self._array.index_bits, self._array.index_max, self._array.value_bits) - else: - array = self._array - - state = {} - state["_default"] = self._default - state["_array"] = array - state["name"] = self.name - state["_concrete_cache"] = self._concrete_cache - state["_written"] = self._written - return state - - def __setstate__(self, state): - self._array = state["_array"] - self._default = state["_default"] - self._name = state["name"] - self._concrete_cache = state["_concrete_cache"] - self._written = state["_written"] - if isinstance(self._array, tuple): - index_bits, index_max, value_bits = self._array - self._array = ArrayVariable(index_bits, index_max, value_bits, name=self._name) - for x, y in self._concrete_cache.items(): - self._array = self._array.store(x, y) + self._array = self._array.store(index, value) def __copy__(self): - return ArrayProxy(self) - - @property - def written(self): - # Calculate only first time - if self._written is None: - written = set() - # take out Proxy sleve - array = self._array - offset = 0 - while isinstance(array, ArraySlice): - # if it is a proxy over a slice take out the slice too - offset += array._slice_offset - array = array._array - while not isinstance(array, ArrayVariable): - # The index written to underlaying Array are displaced when sliced - written.add(array.index - offset) - array = array.array - assert isinstance(array, ArrayVariable) - self._written = written - return self._written - - def is_known(self, index): - if isinstance(index, Constant) and index.value in self._concrete_cache: - return BoolConstant(True) - - is_known_index = BoolConstant(False) - written = self.written - for known_index in written: - if isinstance(index, Constant) and isinstance(known_index, Constant): - if known_index.value == index.value: - return BoolConstant(True) - is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) - return is_known_index + return ArrayProxy(self.array) def get(self, index, default=None): - if default is None: - default = self._default - index = self._array.cast_index(index) - - if self.index_max is not None: - from .visitors import simplify - - index = simplify( - BitVecITE(self.index_bits, index < 0, self.index_max + index + 1, index) - ) - if isinstance(index, Constant) and index.value in self._concrete_cache: - return self._concrete_cache[index.value] - - value = self._array.select(index) - if default is None: - return value - - is_known = self.is_known(index) - default = self._array.cast_value(default) - return BitVecITE(self._array.value_bits, is_known, value, default) + return self._array.get(index, default) def write_BE(self, address, value, size): - address = self._array.cast_index(address) - value = BitVecConstant(size=size * self.value_bits, value=0).cast(value) - array = self - for offset in range(size): - array = array.store( - address + offset, - BitVecExtract(value, (size - 1 - offset) * self.value_bits, self.value_bits), - ) - return array + self._array = self._array.write_BE(address, value, size) def read_BE(self, address, size): - bytes = [] - for offset in range(size): - bytes.append(self.get(address + offset, 0)) - return BitVecConcat(size * self.value_bits, *bytes) + return self._array.read_BE(address, size) def write(self, offset, buf): - if not isinstance(buf, (Array, bytes)): - raise TypeError("Array or bytes expected got {:s}".format(type(buf))) - arr = self for i, val in enumerate(buf): - arr = arr.store(offset + i, val) - return arr + self[offset + i] = val + return self def read(self, offset, size): - return ArraySlice(self, offset, size) + return ArrayProxy(self._array[offset:offset+size]) def __eq__(self, other): - # FIXME taint - def compare_buffers(a, b): - if len(a) != len(b): - return BoolConstant(False) - cond = BoolConstant(True) - for i in range(len(a)): - cond = BoolAnd(cond.cast(a[i] == b[i]), cond) - if cond is BoolConstant(False): - return BoolConstant(False) - return cond - - return compare_buffers(self, other) + return other == self.array def __ne__(self, other): return BoolNot(self == other) @@ -1311,7 +1274,7 @@ def __add__(self, other): self.index_bits, self.index_max + len(other), self.value_bits, - "concatenation{}".format(uuid.uuid1()), + name="concatenation{}".format(uuid.uuid1()), ) ) for index in range(self.index_max): @@ -1335,20 +1298,18 @@ def __radd__(self, other): self.index_bits, self.index_max + len(other), self.value_bits, - "concatenation{}".format(uuid.uuid1()), + name="concatenation{}".format(uuid.uuid1()), ) ) for index in range(len(other)): new_arr[index] = simplify(other[index]) - _concrete_cache = new_arr._concrete_cache for index in range(self.index_max): new_arr[index + len(other)] = simplify(self[index]) - new_arr._concrete_cache.update(_concrete_cache) return new_arr class ArraySelect(BitVec, Operation): - __slots__ = BitVec.xslots + Operation.xslots + __slots__ = BitVec.xslots + Operation.__xslots__ def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): assert index.size == array.index_bits diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 64674c5b4..188f77f47 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -443,10 +443,10 @@ def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool] @lru_cache(maxsize=32) def get_all_values(self, constraints, expression, maxcnt=None, silent=False): """Returns a list with all the possible values for the symbol x""" - if not isinstance(expression, Expression): + if not issymbolic(expression): return [expression] - assert isinstance(constraints, ConstraintSet) - assert isinstance(expression, Expression) + #assert isinstance(constraints, ConstraintSet) + #assert isinstance(expression, Expression) expression = simplify(expression) if maxcnt is None: maxcnt = consts.maxsolutions @@ -456,7 +456,7 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): var = temp_cs.new_bool() elif isinstance(expression, BitVec): var = temp_cs.new_bitvec(expression.size) - elif isinstance(expression, Array): + elif isinstance(expression, (Array, ArrayProxy)): var = temp_cs.new_array( index_max=expression.index_max, value_bits=expression.value_bits, diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 7c220ddac..17daff3bd 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -730,6 +730,7 @@ def visit_ArraySelect(self, expression, *operands): """ ArraySelect (ArrayStore((ArrayStore(x0,v0) ...),xn, vn), x0) -> v0 """ + return self._visit_operation(expression, *operands) arr, index = operands if isinstance(arr, ArrayVariable): return self._visit_operation(expression, *operands) @@ -893,18 +894,13 @@ def visit_BitVecConstant(self, expression): def visit_BoolConstant(self, expression): return expression.value and "true" or "false" - def _visit_variable(self, expression): + def visit_Variable(self, expression): return expression.name - visit_ArrayVariable = _visit_variable - visit_BitVecVariable = _visit_variable - visit_BoolVariable = _visit_variable - def visit_ArraySelect(self, expression, *operands): array_smt, index_smt = operands if isinstance(expression.array, ArrayStore): array_smt = self._add_binding(expression.array, array_smt) - return "(select %s %s)" % (array_smt, index_smt) def _visit_operation(self, expression, *operands): @@ -938,6 +934,9 @@ def translate_to_smtlib(expression, **kwargs): if isinstance(expression, ArrayProxy): expression = expression.array translator = TranslatorSmtlib(**kwargs) + if isinstance(expression, ArrayProxy): + print (expression) + import pdb; pdb.set_trace() translator.visit(expression) return translator.result diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 52f275603..d03ca8a4d 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -1694,7 +1694,7 @@ def finalizer(state_id): if self.fix_unsound_symbolication(st): last_tx = st.platform.last_transaction # Do not generate killed state if only_alive_states is True - if only_alive_states and last_tx.result in {"REVERT", "THROW", "TXERROR"}: + if not last_tx or only_alive_states and last_tx.result in {"REVERT", "THROW", "TXERROR"}: return logger.debug("Generating testcase for state_id %d", state_id) message = last_tx.result if last_tx else "NO STATE RESULT (?)" diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index efd7a19b0..843fbdabd 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -1335,7 +1335,7 @@ def read_buffer(self, offset, size): if size == 0: return b"" self._allocate(offset, size) - return self.memory[offset : offset + size].array + return self.memory[offset : offset + size] def write_buffer(self, offset, data): self._allocate(offset, len(data)) @@ -1343,7 +1343,10 @@ def write_buffer(self, offset, data): self._store(offset + i, Operators.ORD(c)) def _load(self, offset, size=1): - value = self.memory.read_BE(offset, size) + if size == 1: + value = self.memory[offset] + else: + value = self.memory.read_BE(offset, size) try: value = simplify(value) if not value.taint: diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index 1f5680cf3..4da8d9731 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -446,7 +446,7 @@ def setUp(self): def tearDown(self): workspace = self.mevm.workspace del self.mevm - shutil.rmtree(workspace) + #shutil.rmtree(workspace) def test_solidity_create_contract_no_args(self): source_code = "contract A { constructor() {} }" @@ -1014,7 +1014,7 @@ def did_evm_execute_instruction_callback(self, state, instruction, arguments, re filename = os.path.join(THIS_DIR, "contracts/int_overflow.sol") mevm.multi_tx_analysis(filename, tx_limit=1) - mevm.finalize() + mevm.finalize(only_alive_states=True) worksp = mevm.workspace listdir = os.listdir(worksp) diff --git a/tests/ethereum_vm/VMTests_concrete/__init__.py b/tests/ethereum_vm/VMTests_concrete/__init__.py index 3755e783b..6c742a293 100644 --- a/tests/ethereum_vm/VMTests_concrete/__init__.py +++ b/tests/ethereum_vm/VMTests_concrete/__init__.py @@ -1 +1 @@ -# DO NOT DELETE +# DO NOT DELETE \ No newline at end of file diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 6a85731cc..9203fb42b 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -228,9 +228,6 @@ def testBasicArrayStore(self): # 1001 position of array can be 'B' self.assertTrue(self.solver.can_be_true(cs, array.select(1001) == ord("B"))) - # name is correctly proxied - self.assertEqual(array.name, name) - with cs as temp_cs: # but if it is 'B' ... temp_cs.add(array.select(1001) == ord("B")) @@ -274,6 +271,30 @@ def testBasicArraySymbIdx2(self): # It should not be another solution for index self.assertFalse(self.solver.check(cs)) + def testBasicArrayDefault(self): + cs = ConstraintSet() + array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + key = cs.new_bitvec(32, name="key") + self.assertTrue(self.solver.must_be_true(cs, array[key] == 0)) + + + def testBasicArrayDefault2(self): + cs = ConstraintSet() + array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + index1 = cs.new_bitvec(32) + index2 = cs.new_bitvec(32) + value = cs.new_bitvec(32) + array[index2] = value + cs.add(index1 != index2) + cs.add(value != 0) + self.assertTrue(self.solver.must_be_true(cs, array[index1] == 0)) + + def testBasicArrayIndexConcrete(self): + cs = ConstraintSet() + array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + array[0] = 100 + self.assertTrue(array[0] == 100) + def testBasicArrayConcatSlice(self): hw = b"Hello world!" cs = ConstraintSet() @@ -284,7 +305,6 @@ def testBasicArrayConcatSlice(self): self.assertTrue(self.solver.must_be_true(cs, array == hw)) self.assertTrue(self.solver.must_be_true(cs, array.read(0, 12) == hw)) - self.assertTrue(self.solver.must_be_true(cs, array.read(6, 6) == hw[6:12])) self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 6) == hw)) @@ -335,7 +355,7 @@ def testBasicArraySlice(self): def testBasicArrayProxySymbIdx(self): cs = ConstraintSet() - array = ArrayProxy(cs.new_array(index_bits=32, value_bits=32, name="array"), default=0) + array = ArrayProxy(cs.new_array(index_bits=32, value_bits=32, name="array", default=0)) key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") From b5d6b9cfa1624598c936f8e90a3009e8961aff1f Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 11:58:11 -0300 Subject: [PATCH 020/126] blkn --- manticore/core/smtlib/expression.py | 10 +++------- manticore/core/smtlib/visitors.py | 6 ++++-- manticore/ethereum/manticore.py | 8 ++++++-- tests/ethereum/test_general.py | 4 ++-- tests/other/test_smtlibv2.py | 11 +++-------- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 1e3033ad6..a04135cc5 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1109,12 +1109,7 @@ class ArraySlice(ArrayOperation): __slots__ = ArrayOperation.__xslots__ + ("_slice_offset", "_slice_size") def __init__( - self, - array: "Array", - offset: int, - size: int, - default: Optional[int] = None, - **kwargs, + self, array: "Array", offset: int, size: int, default: Optional[int] = None, **kwargs ): assert size if not isinstance(array, Array): @@ -1171,6 +1166,7 @@ class ArrayProxy: bytes <-> Array (ArraySlice, ArrayVariable, ArrayStore) ::: hasheable, notmutable """ + def __hash__(self): return hash(self.array) @@ -1251,7 +1247,7 @@ def write(self, offset, buf): return self def read(self, offset, size): - return ArrayProxy(self._array[offset:offset+size]) + return ArrayProxy(self._array[offset : offset + size]) def __eq__(self, other): return other == self.array diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 17daff3bd..5fa8e23cd 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -935,8 +935,10 @@ def translate_to_smtlib(expression, **kwargs): expression = expression.array translator = TranslatorSmtlib(**kwargs) if isinstance(expression, ArrayProxy): - print (expression) - import pdb; pdb.set_trace() + print(expression) + import pdb + + pdb.set_trace() translator.visit(expression) return translator.result diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index d03ca8a4d..344e20654 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -1232,7 +1232,7 @@ def _on_unsound_symbolication(self, state, func, data, result): else: constraint = y != value state.constrain(constraint) - print (type(data), type(value)) + print(type(data), type(value)) symbolic_pairs.append((data, value)) state.context[f"symbolic_func_sym_{name}"] = symbolic_pairs @@ -1694,7 +1694,11 @@ def finalizer(state_id): if self.fix_unsound_symbolication(st): last_tx = st.platform.last_transaction # Do not generate killed state if only_alive_states is True - if not last_tx or only_alive_states and last_tx.result in {"REVERT", "THROW", "TXERROR"}: + if ( + not last_tx + or only_alive_states + and last_tx.result in {"REVERT", "THROW", "TXERROR"} + ): return logger.debug("Generating testcase for state_id %d", state_id) message = last_tx.result if last_tx else "NO STATE RESULT (?)" diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index 4da8d9731..611972c1d 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -446,7 +446,7 @@ def setUp(self): def tearDown(self): workspace = self.mevm.workspace del self.mevm - #shutil.rmtree(workspace) + # shutil.rmtree(workspace) def test_solidity_create_contract_no_args(self): source_code = "contract A { constructor() {} }" @@ -1347,7 +1347,7 @@ def will_evm_execute_instruction_callback(self, state, i, *args, **kwargs): class EthHelpersTest(unittest.TestCase): def setUp(self): - self.bv = BitVecVariable(size=256, name='bv') + self.bv = BitVecVariable(size=256, name="bv") def test_concretizer(self): policy = "SOME_NONSTANDARD_POLICY" diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 9203fb42b..ac08446e5 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -277,7 +277,6 @@ def testBasicArrayDefault(self): key = cs.new_bitvec(32, name="key") self.assertTrue(self.solver.must_be_true(cs, array[key] == 0)) - def testBasicArrayDefault2(self): cs = ConstraintSet() array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) @@ -309,16 +308,11 @@ def testBasicArrayConcatSlice(self): self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 6) == hw)) - self.assertTrue( - self.solver.must_be_true( - cs, b"Hello " + array.read(6, 5) + b"!" == hw - ) - ) + self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 5) + b"!" == hw)) self.assertTrue( self.solver.must_be_true( - cs, - array.read(0, 1) + b"ello " + array.read(6, 5) + b"!" == hw, + cs, array.read(0, 1) + b"ello " + array.read(6, 5) + b"!" == hw ) ) @@ -1023,5 +1017,6 @@ def test_API(self): self.assertTrue(hasattr(cls, attr), f"{cls.__name__} is missing attribute {attr}") ''' + if __name__ == "__main__": unittest.main() From 198e0d825d5e783d70d5818bc77b4cd5efe1d0e7 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 13:14:38 -0300 Subject: [PATCH 021/126] fix tests --- manticore/core/smtlib/solver.py | 4 ++-- manticore/core/workspace.py | 2 +- tests/ethereum_vm/VMTests_concrete/__init__.py | 2 +- tests/native/test_armv7cpu.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 188f77f47..4cca30af1 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -445,8 +445,8 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): """Returns a list with all the possible values for the symbol x""" if not issymbolic(expression): return [expression] - #assert isinstance(constraints, ConstraintSet) - #assert isinstance(expression, Expression) + # assert isinstance(constraints, ConstraintSet) + # assert isinstance(expression, Expression) expression = simplify(expression) if maxcnt is None: maxcnt = consts.maxsolutions diff --git a/manticore/core/workspace.py b/manticore/core/workspace.py index f77093e61..69ff91e31 100644 --- a/manticore/core/workspace.py +++ b/manticore/core/workspace.py @@ -659,4 +659,4 @@ def save_input_symbols(testcase, state: StateBase): with testcase.open_stream("input") as f: for symbol in state.input_symbols: buf = Z3Solver.instance().get_value(state.constraints, symbol) - f.write(f"{symbol.name}: {buf!r}\n") + f.write(f"{symbol.underlaying_name}: {buf!r}\n") diff --git a/tests/ethereum_vm/VMTests_concrete/__init__.py b/tests/ethereum_vm/VMTests_concrete/__init__.py index 6c742a293..3755e783b 100644 --- a/tests/ethereum_vm/VMTests_concrete/__init__.py +++ b/tests/ethereum_vm/VMTests_concrete/__init__.py @@ -1 +1 @@ -# DO NOT DELETE \ No newline at end of file +# DO NOT DELETE diff --git a/tests/native/test_armv7cpu.py b/tests/native/test_armv7cpu.py index 144c96d95..42920dbb1 100644 --- a/tests/native/test_armv7cpu.py +++ b/tests/native/test_armv7cpu.py @@ -1568,7 +1568,7 @@ def test_uqsub8_concrete_saturated(self): @itest_custom("uqsub8 r3, r1, r2") @itest_setregs("R2=0x01010101") def test_uqsub8_sym(self): - op1 = BitVecVariable(32, "op1") + op1 = BitVecVariable(size=32, name= "op1") self.cpu.memory.constraints.add(op1 >= 0x04030201) self.cpu.memory.constraints.add(op1 < 0x04030204) self.cpu.R1 = op1 @@ -2416,7 +2416,7 @@ def test_sxth(self): @itest_custom("blx r1") def test_blx_reg_sym(self): - dest = BitVecVariable(32, "dest") + dest = BitVecVariable(size=size=32, name= name= "dest") self.cpu.memory.constraints.add(dest >= 0x1000) self.cpu.memory.constraints.add(dest <= 0x1001) self.cpu.R1 = dest @@ -2462,7 +2462,7 @@ def test_symbolic_conditional(self): self._setupCpu(asm, mode=CS_MODE_THUMB) # code starts at 0x1004 # Set R0 as a symbolic value - self.cpu.R0 = BitVecVariable(32, "val") + self.cpu.R0 = BitVecVariable(size=32, name= "val") self.cpu.execute() # tst r0, r0 self.cpu.execute() # beq label From d82b398b28d9d4863c64bd77cd2cce2d68cbc6e6 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 13:18:46 -0300 Subject: [PATCH 022/126] CC --- tests/native/test_armv7cpu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/native/test_armv7cpu.py b/tests/native/test_armv7cpu.py index 42920dbb1..6c923f57b 100644 --- a/tests/native/test_armv7cpu.py +++ b/tests/native/test_armv7cpu.py @@ -1568,7 +1568,7 @@ def test_uqsub8_concrete_saturated(self): @itest_custom("uqsub8 r3, r1, r2") @itest_setregs("R2=0x01010101") def test_uqsub8_sym(self): - op1 = BitVecVariable(size=32, name= "op1") + op1 = BitVecVariable(size=32, name="op1") self.cpu.memory.constraints.add(op1 >= 0x04030201) self.cpu.memory.constraints.add(op1 < 0x04030204) self.cpu.R1 = op1 @@ -2416,7 +2416,7 @@ def test_sxth(self): @itest_custom("blx r1") def test_blx_reg_sym(self): - dest = BitVecVariable(size=size=32, name= name= "dest") + dest = BitVecVariable(size=32, name="dest") self.cpu.memory.constraints.add(dest >= 0x1000) self.cpu.memory.constraints.add(dest <= 0x1001) self.cpu.R1 = dest @@ -2462,7 +2462,7 @@ def test_symbolic_conditional(self): self._setupCpu(asm, mode=CS_MODE_THUMB) # code starts at 0x1004 # Set R0 as a symbolic value - self.cpu.R0 = BitVecVariable(size=32, name= "val") + self.cpu.R0 = BitVecVariable(size=32, name="val") self.cpu.execute() # tst r0, r0 self.cpu.execute() # beq label From 487f2bb56024b7545b640f61c63ef48b5155a4c7 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 15:34:12 -0300 Subject: [PATCH 023/126] underlying_variable --- manticore/core/smtlib/expression.py | 101 ++++++++++++++-------------- manticore/core/workspace.py | 2 +- tests/native/test_register.py | 4 +- tests/native/test_state.py | 10 +-- 4 files changed, 57 insertions(+), 60 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index a04135cc5..1a8f2dda2 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -103,18 +103,15 @@ def taint_with(arg, *taints, value_bits=256, index_bits=256): raise ValueError("type not supported") else: - if isinstance(arg, BitVecVariable): - arg = arg + BitVecConstant(value_bits, 0, taint=tainted_fset) - else: - arg = copy.copy(arg) - arg._taint |= tainted_fset + arg = copy.copy(arg) + arg._taint |= tainted_fset return arg class Variable(Expression): __slots__ = () - xslots: Tuple[str, ...] = ("_name",) + __xslots__: Tuple[str, ...] = ("_name",) def __init__(self, name: str, **kwargs): super().__init__(**kwargs) @@ -140,7 +137,7 @@ def __repr__(self): class Constant(Expression): __slots__ = () - xslots: Tuple[str, ...] = ("_value",) + __xslots__: Tuple[str, ...] = ("_value",) def __init__(self, value: Union[bool, int], **kwargs): super().__init__(**kwargs) @@ -219,7 +216,7 @@ def __bool__(self): class BoolVariable(Bool, Variable): - __slots__ = Bool.__xslots__ + Variable.xslots + __slots__ = Bool.__xslots__ + Variable.__xslots__ @property def declaration(self): @@ -227,7 +224,7 @@ def declaration(self): class BoolConstant(Bool, Constant): - __slots__ = Bool.__xslots__ + Constant.xslots + __slots__ = Bool.__xslots__ + Constant.__xslots__ def __init__(self, value, **kwargs): super().__init__(value=value, **kwargs) @@ -238,39 +235,39 @@ def __bool__(self): class BoolOperation(Operation, Bool): __slots__ = () - xslots = Operation.__xslots__ + Bool.__xslots__ + __xslots__ = Operation.__xslots__ + Bool.__xslots__ class BoolNot(BoolOperation): - __slots__ = BoolOperation.xslots + __slots__ = BoolOperation.__xslots__ def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolAnd(BoolOperation): - __slots__ = BoolOperation.xslots + __slots__ = BoolOperation.__xslots__ def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolOr(BoolOperation): - __slots__ = BoolOperation.xslots + __slots__ = BoolOperation.__xslots__ def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolXor(BoolOperation): - __slots__ = BoolOperation.xslots + __slots__ = BoolOperation.__xslots__ def __init__(self, *args, **kwargs): super().__init__(operands=args, **kwargs) class BoolITE(BoolOperation): - __slots__ = BoolOperation.xslots + __slots__ = BoolOperation.__xslots__ def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): super().__init__(operands=(cond, true, false), **kwargs) @@ -278,7 +275,7 @@ def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): class BitVec(Expression): __slots__ = () - xslots: Tuple[str, ...] = Expression.__xslots__ + ("size",) + __xslots__: Tuple[str, ...] = Expression.__xslots__ + ("size",) """ This adds a bitsize to the Expression class """ def __init__(self, size: int, **kwargs): @@ -482,7 +479,7 @@ def Bool(self): class BitVecVariable(BitVec, Variable): - __slots__ = BitVec.xslots + Variable.xslots + __slots__ = BitVec.__xslots__ + Variable.__xslots__ def __init__(self, **kwargs): super().__init__(**kwargs) @@ -506,7 +503,7 @@ def declaration(self): class BitVecConstant(BitVec, Constant): - __slots__ = BitVec.xslots + Constant.xslots + __slots__ = BitVec.__xslots__ + Constant.__xslots__ def __init__(self, size: int, value: int, **kwargs): super().__init__(size=size, value=value, **kwargs) @@ -528,103 +525,103 @@ def value(self): class BitVecOperation(BitVec, Operation): - xslots = BitVec.xslots + Operation.__xslots__ + __xslots__ = BitVec.__xslots__ + Operation.__xslots__ __slots__ = () class BitVecAdd(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecSub(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecMul(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecDiv(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecUnsignedDiv(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecMod(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecRem(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecUnsignedRem(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecShiftLeft(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecShiftRight(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecArithmeticShiftLeft(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecArithmeticShiftRight(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecAnd(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecOr(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a: BitVec, b: BitVec, *args, **kwargs): assert a.size == b.size @@ -632,21 +629,21 @@ def __init__(self, a: BitVec, b: BitVec, *args, **kwargs): class BitVecXor(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(size=a.size, operands=(a, b), **kwargs) class BitVecNot(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, **kwargs): super().__init__(size=a.size, operands=(a,), **kwargs) class BitVecNeg(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, **kwargs): super().__init__(size=a.size, operands=(a,), **kwargs) @@ -654,21 +651,21 @@ def __init__(self, a, **kwargs): # Comparing two bitvectors results in a Bool class LessThan(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): super().__init__(operands=(a, b), **kwargs) class LessOrEqual(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): super().__init__(operands=(a, b), **kwargs) class BoolEqual(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): if isinstance(a, BitVec) or isinstance(b, BitVec): @@ -677,14 +674,14 @@ def __init__(self, a, b, *args, **kwargs): class GreaterThan(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): super().__init__(operands=(a, b), **kwargs) class GreaterOrEqual(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): assert a.size == b.size @@ -692,14 +689,14 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedLessThan(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super().__init__(operands=(a, b), **kwargs) class UnsignedLessOrEqual(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): assert a.size == b.size @@ -707,7 +704,7 @@ def __init__(self, a, b, **kwargs): class UnsignedGreaterThan(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, *args, **kwargs): assert a.size == b.size @@ -715,7 +712,7 @@ def __init__(self, a, b, *args, **kwargs): class UnsignedGreaterOrEqual(BoolOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, a, b, **kwargs): super(UnsignedGreaterOrEqual, self).__init__(operands=(a, b), **kwargs) @@ -1056,7 +1053,7 @@ def __radd__(self, other): class ArrayVariable(Array, Variable): - __slots__ = Array.__xslots__ + Variable.xslots + __slots__ = Array.__xslots__ + Variable.__xslots__ @property def declaration(self): @@ -1305,7 +1302,7 @@ def __radd__(self, other): class ArraySelect(BitVec, Operation): - __slots__ = BitVec.xslots + Operation.__xslots__ + __slots__ = BitVec.__xslots__ + Operation.__xslots__ def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): assert index.size == array.index_bits @@ -1328,7 +1325,7 @@ def __repr__(self): class BitVecSignExtend(BitVecOperation): - __slots__ = BitVecOperation.xslots + ("extend",) + __slots__ = BitVecOperation.__xslots__ + ("extend",) def __init__(self, operand: "BitVec", size_dest: int, *args, **kwargs): assert size_dest >= operand.size @@ -1337,7 +1334,7 @@ def __init__(self, operand: "BitVec", size_dest: int, *args, **kwargs): class BitVecZeroExtend(BitVecOperation): - __slots__ = BitVecOperation.xslots + ("extend",) + __slots__ = BitVecOperation.__xslots__ + ("extend",) def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): assert size_dest >= operand.size @@ -1346,7 +1343,7 @@ def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): class BitVecExtract(BitVecOperation): - __slots__ = BitVecOperation.xslots + ("_begining", "_end") + __slots__ = BitVecOperation.__xslots__ + ("_begining", "_end") def __init__(self, operand: "BitVec", offset: int, size: int, *args, **kwargs): assert offset >= 0 and offset + size <= operand.size @@ -1368,7 +1365,7 @@ def end(self): class BitVecConcat(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__(self, size_dest: int, *operands, **kwargs): assert all(isinstance(x, BitVec) for x in operands) @@ -1377,7 +1374,7 @@ def __init__(self, size_dest: int, *operands, **kwargs): class BitVecITE(BitVecOperation): - __slots__ = BitVecOperation.xslots + __slots__ = BitVecOperation.__xslots__ def __init__( self, diff --git a/manticore/core/workspace.py b/manticore/core/workspace.py index 69ff91e31..75b4cb432 100644 --- a/manticore/core/workspace.py +++ b/manticore/core/workspace.py @@ -659,4 +659,4 @@ def save_input_symbols(testcase, state: StateBase): with testcase.open_stream("input") as f: for symbol in state.input_symbols: buf = Z3Solver.instance().get_value(state.constraints, symbol) - f.write(f"{symbol.underlaying_name}: {buf!r}\n") + f.write(f"{symbol.underlying_variable.name}: {buf!r}\n") diff --git a/tests/native/test_register.py b/tests/native/test_register.py index e099fc292..0663e811b 100644 --- a/tests/native/test_register.py +++ b/tests/native/test_register.py @@ -1,6 +1,6 @@ import unittest -from manticore.core.smtlib import Bool, BitVecConstant +from manticore.core.smtlib import Bool, BoolVariable, BitVecConstant from manticore.native.cpu.register import Register @@ -47,7 +47,7 @@ def test_bool_write_nonflag(self): def test_Bool(self): r = Register(32) - b = Bool() + b = BoolVariable(name="B") r.write(b) self.assertIs(r.read(), b) diff --git a/tests/native/test_state.py b/tests/native/test_state.py index ac7df6e09..3b06f2286 100644 --- a/tests/native/test_state.py +++ b/tests/native/test_state.py @@ -76,27 +76,27 @@ def setUp(self): def test_solve_one(self): val = 42 - expr = BitVecVariable(32, "tmp") + expr = BitVecVariable(size=32, name= "tmp") self.state.constrain(expr == val) solved = self.state.solve_one(expr) self.assertEqual(solved, val) def test_solve_n(self): - expr = BitVecVariable(32, "tmp") + expr = BitVecVariable(size=32, name= "tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) solved = sorted(self.state.solve_n(expr, 2)) self.assertEqual(solved, [5, 6]) def test_solve_n2(self): - expr = BitVecVariable(32, "tmp") + expr = BitVecVariable(size=32, name= "tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 100) solved = self.state.solve_n(expr, 5) self.assertEqual(len(solved), 5) def test_solve_min_max(self): - expr = BitVecVariable(32, "tmp") + expr = BitVecVariable(size=32, name= "tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) self.assertEqual(self.state.solve_min(expr), 5) @@ -104,7 +104,7 @@ def test_solve_min_max(self): self.assertEqual(self.state.solve_minmax(expr), (5, 6)) def test_policy_one(self): - expr = BitVecVariable(32, "tmp") + expr = BitVecVariable(size=32, name= "tmp") self.state.constrain(expr > 0) self.state.constrain(expr < 100) solved = self.state.concretize(expr, "ONE") From ec250a00116071f7e20c1071f8e8262ad8dce436 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 17:10:41 -0300 Subject: [PATCH 024/126] CC --- tests/native/test_state.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/native/test_state.py b/tests/native/test_state.py index 3b06f2286..daf8035b1 100644 --- a/tests/native/test_state.py +++ b/tests/native/test_state.py @@ -76,27 +76,27 @@ def setUp(self): def test_solve_one(self): val = 42 - expr = BitVecVariable(size=32, name= "tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr == val) solved = self.state.solve_one(expr) self.assertEqual(solved, val) def test_solve_n(self): - expr = BitVecVariable(size=32, name= "tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) solved = sorted(self.state.solve_n(expr, 2)) self.assertEqual(solved, [5, 6]) def test_solve_n2(self): - expr = BitVecVariable(size=32, name= "tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 100) solved = self.state.solve_n(expr, 5) self.assertEqual(len(solved), 5) def test_solve_min_max(self): - expr = BitVecVariable(size=32, name= "tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) self.assertEqual(self.state.solve_min(expr), 5) @@ -104,7 +104,7 @@ def test_solve_min_max(self): self.assertEqual(self.state.solve_minmax(expr), (5, 6)) def test_policy_one(self): - expr = BitVecVariable(size=32, name= "tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 0) self.state.constrain(expr < 100) solved = self.state.concretize(expr, "ONE") From f922e6fd40f3265a90e6e7a07d0b7031f3c60fec Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 17:49:18 -0300 Subject: [PATCH 025/126] Add .name --- manticore/core/smtlib/expression.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 1a8f2dda2..b73b22a87 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1174,6 +1174,10 @@ def __init__(self, array: Array): def underlying_variable(self): return self._array.underlying_variable + @property + def name(self): + return self._array.name + @property def array(self): return self._array @@ -1386,9 +1390,4 @@ def __init__( ): assert true_value.size == size assert false_value.size == size - super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) - - -# Constant = (BitVecConstant, BoolConstant) -# Variable = (BitVecVariable, BoolVariable, ArrayVariable) -# Operation = (BitVecOperation, BoolOperation, ArrayOperation, ArraySelect) + super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) \ No newline at end of file From 3476a11543d6b0947b963ee8e26c6d9b3700309d Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 18:51:55 -0300 Subject: [PATCH 026/126] blkn --- manticore/core/smtlib/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index b73b22a87..17aa35716 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1390,4 +1390,4 @@ def __init__( ): assert true_value.size == size assert false_value.size == size - super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) \ No newline at end of file + super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) From 866bc0ba1ca83c4e0ef3e4328c2eb0040dc528f1 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 3 Jun 2020 20:02:45 -0300 Subject: [PATCH 027/126] fix array name --- manticore/core/workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/workspace.py b/manticore/core/workspace.py index 75b4cb432..f77093e61 100644 --- a/manticore/core/workspace.py +++ b/manticore/core/workspace.py @@ -659,4 +659,4 @@ def save_input_symbols(testcase, state: StateBase): with testcase.open_stream("input") as f: for symbol in state.input_symbols: buf = Z3Solver.instance().get_value(state.constraints, symbol) - f.write(f"{symbol.underlying_variable.name}: {buf!r}\n") + f.write(f"{symbol.name}: {buf!r}\n") From 3ff6b95e31cf07a56414f39dc2231de8fe1c7438 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 00:36:18 -0300 Subject: [PATCH 028/126] Move get related --- manticore/core/smtlib/constraints.py | 100 +++++++++++++++------------ manticore/core/smtlib/solver.py | 14 ++-- manticore/core/smtlib/visitors.py | 14 +++- tests/other/test_smtlibv2.py | 31 ++++++++- 4 files changed, 105 insertions(+), 54 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 711e5e91a..a9b652e8d 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -1,9 +1,10 @@ import itertools import sys - +from typing import Optional from ...utils.helpers import PickleSerializer from ...exceptions import SmtlibError from .expression import ( + Expression, BitVecVariable, BoolVariable, ArrayVariable, @@ -115,7 +116,7 @@ def _get_sid(self) -> int: self._sid += 1 return self._sid - def __get_related(self, related_to=None): + def related_to(self, related_to: Optional[Expression]=None): # sam.moelius: There is a flaw in how __get_related works: when called on certain # unsatisfiable sets, it can return a satisfiable one. The flaw arises when: # * self consists of a single constraint C @@ -127,49 +128,59 @@ def __get_related(self, related_to=None): # set. Thus, __get_related was called on an unsatisfiable set, {C}, but it returned a # satisfiable one, {}. # In light of the above, the core __get_related logic is currently disabled. - # if related_to is not None: - # feliam: This assumes the previous constraints are already SAT (normal SE forking) - if consts.related_constraints and related_to is not None: - number_of_constraints = len(self.constraints) - remaining_constraints = set(self.constraints) - related_variables = get_variables(related_to) - related_constraints = set() - - added = True - while added: - added = False - logger.debug("Related variables %r", [x.name for x in related_variables]) - for constraint in list(remaining_constraints): - if isinstance(constraint, BoolConstant): - if constraint.value: - continue - else: - related_constraints = {constraint} - break - - variables = get_variables(constraint) - if related_variables & variables or not (variables): - remaining_constraints.remove(constraint) - related_constraints.add(constraint) - related_variables |= variables - added = True - - logger.debug( - "Reduced %d constraints!!", number_of_constraints - len(related_constraints) - ) - else: - related_variables = set() - for constraint in self.constraints: - related_variables |= get_variables(constraint) - related_constraints = set(self.constraints) - return related_variables, related_constraints - - def to_string(self, related_to=None, replace_constants=False): - related_variables, related_constraints = self.__get_related(related_to) + """ + Slices this ConstraintSet keeping only the related constraints. + Two constraints are independient if they can be expressed full using a + disjoint set of variables. + Todo: Research. constraints refering differen not overlapping parts of the same array + should be considered independient. + :param related_to: An expression + :return: + """ + if related_to is None: + return self + number_of_constraints = len(self.constraints) + remaining_constraints = set(self.constraints) + related_variables = get_variables(related_to) + related_constraints = set() + + added = True + while added: + added = False + logger.debug("Related variables %r", [x.name for x in related_variables]) + for constraint in list(remaining_constraints): + if isinstance(constraint, BoolConstant): + if constraint.value: + continue + else: + related_constraints = {constraint} + break + + variables = get_variables(constraint) + if related_variables & variables or not (variables): + remaining_constraints.remove(constraint) + related_constraints.add(constraint) + related_variables |= variables + added = True + + logger.debug( + "Reduced %d constraints!!", number_of_constraints - len(related_constraints) + ) + + # related_variables, related_constraints + cs = ConstraintSet() + for var in related_variables: + cs._declare(var) + for constraint in related_constraints: + cs.add(constraint) + return cs + + def to_string(self, replace_constants=True): + variables, constraints = self.get_declared_variables(), self.constraints if replace_constants: constant_bindings = {} - for expression in related_constraints: + for expression in constraints: if ( isinstance(expression, BoolEqual) and isinstance(expression.operands[0], Variable) @@ -179,7 +190,7 @@ def to_string(self, related_to=None, replace_constants=False): tmp = set() result = "" - for var in related_variables: + for var in variables: # FIXME # band aid hack around the fact that we are double declaring stuff :( :( if var.declaration in tmp: @@ -187,8 +198,9 @@ def to_string(self, related_to=None, replace_constants=False): continue tmp.add(var.declaration) result += var.declaration + "\n" + translator = TranslatorSmtlib(use_bindings=True) - for constraint in related_constraints: + for constraint in constraints: if replace_constants: constraint = simplify(replace(constraint, constant_bindings)) # if no variables then it is a constant diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index f389a186f..eea4654e1 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -435,7 +435,7 @@ def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool] with constraints as temp_cs: temp_cs.add(expression) - self._reset(temp_cs.to_string(related_to=expression)) + self._reset(temp_cs.to_string()) return self._is_sat() # get-all-values min max minmax @@ -454,7 +454,7 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): maxcnt = 2 silent = True - with constraints as temp_cs: + with constraints.related_to(expression) as temp_cs: if isinstance(expression, Bool): var = temp_cs.new_bool() elif isinstance(expression, BitVec): @@ -471,7 +471,7 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): ) temp_cs.add(var == expression) - self._reset(temp_cs.to_string(related_to=var)) + self._reset(temp_cs.related_to(var).to_string()) result = [] start = time.time() while self._is_sat(): @@ -490,11 +490,11 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): if time.time() - start > consts.timeout: if silent: logger.info("Timeout searching for all solutions") - return result + return result/home/felipe/Projects/manticore/tests/other/test_smtlibv2.py raise SolverError("Timeout") # Sometimes adding a new contraint after a check-sat eats all the mem temp_cs.add(var != value) - self._reset(temp_cs.to_string(related_to=var)) + self._reset(temp_cs.to_string()) # self._assert(var != value) return list(result) @@ -516,8 +516,8 @@ def optimize(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10 X = temp_cs.new_bitvec(x.size) temp_cs.add(X == x) aux = temp_cs.new_bitvec(X.size, name="optimized_") - self._reset(temp_cs.to_string(related_to=X)) - self._send(aux.declaration) + self._reset(temp_cs.to_string()) + #self._send(aux.declaration) start = time.time() if consts.optimize and getattr(self, f"support_{goal}", False): diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index cbb256e11..d48a3c24e 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -285,7 +285,6 @@ def __init__(self, **kw): BitVecAdd: operator.__add__, BitVecSub: operator.__sub__, BitVecMul: operator.__mul__, - BitVecDiv: operator.__floordiv__, BitVecShiftLeft: operator.__lshift__, BitVecShiftRight: operator.__rshift__, BitVecAnd: operator.__and__, @@ -308,6 +307,19 @@ def __init__(self, **kw): UnsignedGreaterOrEqual: lambda x, y: (x & UNSIGN_MASK) >= (y & UNSIGN_MASK), } + def visit_BitVecDiv(self, expression, *operands): + if all(isinstance(o, Constant) for o in operands): + signmask=operands[0].signmask + mask=operands[0].mask + numeral = operands[0].value + dividend = operands[1].value + if numeral & signmask: + numeral = -(mask - numeral -1) + if dividend & signmask: + dividend = -(mask - dividend -1) + result = int(numeral / dividend) + return BitVecConstant(expression.size, result, taint=expression.taint) + def visit_BitVecConcat(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): result = 0 diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 2a93d8648..959b9b0a4 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -905,9 +905,8 @@ def test_SDIV(self): cs.add(b == 0x86) # -122 cs.add(c == 0x11) # 17 cs.add(a == Operators.SDIV(b, c)) - cs.add(d == b / c) + cs.add(d == (b // c)) cs.add(a == d) - self.assertTrue(solver.check(cs)) self.assertEqual(solver.get_value(cs, a), -7 & 0xFF) @@ -986,6 +985,34 @@ def test_check_solver_undefined(self): ) self.assertTrue(self.solver._solver_version() > Version(major=4, minor=4, patch=1)) + def testRelated(self): + cs = ConstraintSet() + aa1 = cs.new_bool(name="AA1") + aa2 = cs.new_bool(name="AA2") + bb1 = cs.new_bool(name="BB1") + bb2 = cs.new_bool(name="BB2") + cs.add(Operators.OR(aa1, aa2)) + cs.add(Operators.OR(bb1, bb2)) + self.assertTrue(self.solver.check(cs)) + #No BB variables related to AA + self.assertNotIn("BB", cs.related_to(aa1).to_string()) + self.assertNotIn("BB", cs.related_to(aa2).to_string()) + self.assertNotIn("BB", cs.related_to(aa1 == aa2).to_string()) + self.assertNotIn("BB", cs.related_to(aa1 == False).to_string()) + #No AA variables related to BB + self.assertNotIn("AA", cs.related_to(bb1).to_string()) + self.assertNotIn("AA", cs.related_to(bb2).to_string()) + self.assertNotIn("AA", cs.related_to(bb1 == bb2).to_string()) + self.assertNotIn("AA", cs.related_to(bb1 == False).to_string()) + + #Nothing is related to tautologies? + self.assertEqual('', cs.related_to(simplify(bb1 == bb1)).to_string()) + + #But if the tautollogy can not get simplified we have to ask the solver + #and send in all the other stuff + self.assertNotIn("AA", cs.related_to(bb1 == bb1).to_string()) + + def test_API(self): """ As we've split up the Constant, Variable, and Operation classes to avoid using multiple inheritance, From 00e819978656aa15c8b22b2b8b91c06f071071c1 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 14:18:58 -0300 Subject: [PATCH 029/126] CC --- tests/other/test_smtlibv2.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 613541d87..12a883903 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1035,25 +1035,24 @@ def testRelated(self): cs.add(Operators.OR(aa1, aa2)) cs.add(Operators.OR(bb1, bb2)) self.assertTrue(self.solver.check(cs)) - #No BB variables related to AA + # No BB variables related to AA self.assertNotIn("BB", cs.related_to(aa1).to_string()) self.assertNotIn("BB", cs.related_to(aa2).to_string()) self.assertNotIn("BB", cs.related_to(aa1 == aa2).to_string()) self.assertNotIn("BB", cs.related_to(aa1 == False).to_string()) - #No AA variables related to BB + # No AA variables related to BB self.assertNotIn("AA", cs.related_to(bb1).to_string()) self.assertNotIn("AA", cs.related_to(bb2).to_string()) self.assertNotIn("AA", cs.related_to(bb1 == bb2).to_string()) self.assertNotIn("AA", cs.related_to(bb1 == False).to_string()) - #Nothing is related to tautologies? - self.assertEqual('', cs.related_to(simplify(bb1 == bb1)).to_string()) + # Nothing is related to tautologies? + self.assertEqual("", cs.related_to(simplify(bb1 == bb1)).to_string()) - #But if the tautollogy can not get simplified we have to ask the solver - #and send in all the other stuff + # But if the tautollogy can not get simplified we have to ask the solver + # and send in all the other stuff self.assertNotIn("AA", cs.related_to(bb1 == bb1).to_string()) - def test_API(self): """ As we've split up the Constant, Variable, and Operation classes to avoid using multiple inheritance, From 6b7f6717eeec30875e68fa27a8711648f25db94d Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 16:08:05 -0300 Subject: [PATCH 030/126] fix concolic --- examples/script/concolic.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/script/concolic.py b/examples/script/concolic.py index ab75e682a..26885821e 100755 --- a/examples/script/concolic.py +++ b/examples/script/concolic.py @@ -22,6 +22,8 @@ from manticore.core.plugin import ExtendedTracer, Follower, Plugin from manticore.core.smtlib.constraints import ConstraintSet from manticore.core.smtlib.solver import Z3Solver +from manticore.core.smtlib.visitors import GetDeclarations + from manticore.utils import config import copy @@ -136,8 +138,18 @@ def perm(lst, func): def constraints_to_constraintset(constupl): + # originally those constraints belonged to a different ConstraintSet + # This is a hack x = ConstraintSet() - x._constraints = list(constupl) + + declarations = GetDeclarations() + for a in constupl: + declarations.visit(a) + x.add(a) + for d in declarations.result: + x._declare(d) + + return x From 6732eb2b35f3217508aa46952781fb78cff59db6 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 16:17:31 -0300 Subject: [PATCH 031/126] lint --- examples/script/concolic.py | 1 - manticore/core/smtlib/constraints.py | 6 ++---- manticore/core/smtlib/solver.py | 13 +++++++++++-- manticore/core/smtlib/visitors.py | 8 ++++---- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/examples/script/concolic.py b/examples/script/concolic.py index 26885821e..ce3a90292 100755 --- a/examples/script/concolic.py +++ b/examples/script/concolic.py @@ -149,7 +149,6 @@ def constraints_to_constraintset(constupl): for d in declarations.result: x._declare(d) - return x diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index a9b652e8d..78143d400 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -116,7 +116,7 @@ def _get_sid(self) -> int: self._sid += 1 return self._sid - def related_to(self, related_to: Optional[Expression]=None): + def related_to(self, related_to: Optional[Expression] = None): # sam.moelius: There is a flaw in how __get_related works: when called on certain # unsatisfiable sets, it can return a satisfiable one. The flaw arises when: # * self consists of a single constraint C @@ -163,9 +163,7 @@ def related_to(self, related_to: Optional[Expression]=None): related_variables |= variables added = True - logger.debug( - "Reduced %d constraints!!", number_of_constraints - len(related_constraints) - ) + logger.debug("Reduced %d constraints!!", number_of_constraints - len(related_constraints)) # related_variables, related_constraints cs = ConstraintSet() diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index eea4654e1..9c2c4e4cd 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -490,7 +490,16 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): if time.time() - start > consts.timeout: if silent: logger.info("Timeout searching for all solutions") - return result/home/felipe/Projects/manticore/tests/other/test_smtlibv2.py + return ( + result + / home + / felipe + / Projects + / manticore + / tests + / other + / test_smtlibv2.py + ) raise SolverError("Timeout") # Sometimes adding a new contraint after a check-sat eats all the mem temp_cs.add(var != value) @@ -517,7 +526,7 @@ def optimize(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10 temp_cs.add(X == x) aux = temp_cs.new_bitvec(X.size, name="optimized_") self._reset(temp_cs.to_string()) - #self._send(aux.declaration) + # self._send(aux.declaration) start = time.time() if consts.optimize and getattr(self, f"support_{goal}", False): diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index d48a3c24e..6f9d2aecd 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -309,14 +309,14 @@ def __init__(self, **kw): def visit_BitVecDiv(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): - signmask=operands[0].signmask - mask=operands[0].mask + signmask = operands[0].signmask + mask = operands[0].mask numeral = operands[0].value dividend = operands[1].value if numeral & signmask: - numeral = -(mask - numeral -1) + numeral = -(mask - numeral - 1) if dividend & signmask: - dividend = -(mask - dividend -1) + dividend = -(mask - dividend - 1) result = int(numeral / dividend) return BitVecConstant(expression.size, result, taint=expression.taint) From 4adeac2feca4ef733ea56aa10732151a8541ff07 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 17:01:15 -0300 Subject: [PATCH 032/126] DivByZero default to zero --- manticore/core/smtlib/visitors.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 6f9d2aecd..77ecf9d71 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -300,7 +300,7 @@ def __init__(self, **kw): BoolAnd: operator.__and__, BoolOr: operator.__or__, BoolNot: operator.__not__, - BitVecUnsignedDiv: lambda x, y: (x & UNSIGN_MASK) // (y & UNSIGN_MASK), + BitVecUnsignedDiv: lambda x, y: 0 if (y & UNSIGN_MASK) == 0 else (x & UNSIGN_MASK) // (y & UNSIGN_MASK), UnsignedLessThan: lambda x, y: (x & UNSIGN_MASK) < (y & UNSIGN_MASK), UnsignedLessOrEqual: lambda x, y: (x & UNSIGN_MASK) <= (y & UNSIGN_MASK), UnsignedGreaterThan: lambda x, y: (x & UNSIGN_MASK) > (y & UNSIGN_MASK), @@ -317,7 +317,10 @@ def visit_BitVecDiv(self, expression, *operands): numeral = -(mask - numeral - 1) if dividend & signmask: dividend = -(mask - dividend - 1) - result = int(numeral / dividend) + if dividend == 0: + result = 0 + else: + result = int(numeral / dividend) return BitVecConstant(expression.size, result, taint=expression.taint) def visit_BitVecConcat(self, expression, *operands): From 7565b4a9b0c2c1c054455a84de87dcb572e39bfb Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 17:09:03 -0300 Subject: [PATCH 033/126] blkn --- manticore/core/smtlib/visitors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 77ecf9d71..45b59662c 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -300,7 +300,9 @@ def __init__(self, **kw): BoolAnd: operator.__and__, BoolOr: operator.__or__, BoolNot: operator.__not__, - BitVecUnsignedDiv: lambda x, y: 0 if (y & UNSIGN_MASK) == 0 else (x & UNSIGN_MASK) // (y & UNSIGN_MASK), + BitVecUnsignedDiv: lambda x, y: 0 + if (y & UNSIGN_MASK) == 0 + else (x & UNSIGN_MASK) // (y & UNSIGN_MASK), UnsignedLessThan: lambda x, y: (x & UNSIGN_MASK) < (y & UNSIGN_MASK), UnsignedLessOrEqual: lambda x, y: (x & UNSIGN_MASK) <= (y & UNSIGN_MASK), UnsignedGreaterThan: lambda x, y: (x & UNSIGN_MASK) > (y & UNSIGN_MASK), From 1680397e6cd3c2734fce713437f0c66e808d4d50 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 17:55:13 -0300 Subject: [PATCH 034/126] Update manticore/core/smtlib/solver.py Co-authored-by: Eric Kilmer --- manticore/core/smtlib/solver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 9c2c4e4cd..dad96c67e 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -526,7 +526,6 @@ def optimize(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10 temp_cs.add(X == x) aux = temp_cs.new_bitvec(X.size, name="optimized_") self._reset(temp_cs.to_string()) - # self._send(aux.declaration) start = time.time() if consts.optimize and getattr(self, f"support_{goal}", False): From cbe55b9acc1f0b3107fc838ede3524d7d4caf60f Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 17:56:05 -0300 Subject: [PATCH 035/126] Update manticore/core/smtlib/constraints.py Co-authored-by: Eric Kilmer --- manticore/core/smtlib/constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 78143d400..2e162cdf9 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -116,7 +116,7 @@ def _get_sid(self) -> int: self._sid += 1 return self._sid - def related_to(self, related_to: Optional[Expression] = None): + def related_to(self, related_to: Optional[Expression] = None) -> ConstraintSet: # sam.moelius: There is a flaw in how __get_related works: when called on certain # unsatisfiable sets, it can return a satisfiable one. The flaw arises when: # * self consists of a single constraint C From 315ef64acddf5e763e0bf161d28cc356cc657134 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 17:56:25 -0300 Subject: [PATCH 036/126] Update manticore/core/smtlib/constraints.py Co-authored-by: Eric Kilmer --- manticore/core/smtlib/constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 2e162cdf9..0e1a08c72 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -173,7 +173,7 @@ def related_to(self, related_to: Optional[Expression] = None) -> ConstraintSet: cs.add(constraint) return cs - def to_string(self, replace_constants=True): + def to_string(self, replace_constants: bool =True) -> str: variables, constraints = self.get_declared_variables(), self.constraints if replace_constants: From 6155a4df3c89cfc689f244a969be82f3cc56befc Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 18:00:40 -0300 Subject: [PATCH 037/126] remove odd string --- manticore/core/smtlib/solver.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 9c2c4e4cd..afbb566a7 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -490,16 +490,7 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): if time.time() - start > consts.timeout: if silent: logger.info("Timeout searching for all solutions") - return ( - result - / home - / felipe - / Projects - / manticore - / tests - / other - / test_smtlibv2.py - ) + return list(result) raise SolverError("Timeout") # Sometimes adding a new contraint after a check-sat eats all the mem temp_cs.add(var != value) From 1491cbd64f3360c83611136a79a57766b53c20aa Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 18:12:14 -0300 Subject: [PATCH 038/126] lint --- manticore/core/smtlib/constraints.py | 2 +- manticore/core/smtlib/solver.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 0e1a08c72..7507c83d6 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -173,7 +173,7 @@ def related_to(self, related_to: Optional[Expression] = None) -> ConstraintSet: cs.add(constraint) return cs - def to_string(self, replace_constants: bool =True) -> str: + def to_string(self, replace_constants: bool = True) -> str: variables, constraints = self.get_declared_variables(), self.constraints if replace_constants: diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 6c0218647..ab4736a48 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -495,7 +495,6 @@ def get_all_values(self, constraints, expression, maxcnt=None, silent=False): # Sometimes adding a new contraint after a check-sat eats all the mem temp_cs.add(var != value) self._reset(temp_cs.to_string()) - # self._assert(var != value) return list(result) def optimize(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): From 264eb041c7d7cf27bc86de22958a691a5a69e308 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 18:13:53 -0300 Subject: [PATCH 039/126] mypy lint --- manticore/core/smtlib/constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 7507c83d6..259d3c72f 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -116,7 +116,7 @@ def _get_sid(self) -> int: self._sid += 1 return self._sid - def related_to(self, related_to: Optional[Expression] = None) -> ConstraintSet: + def related_to(self, related_to: Optional[Expression] = None) -> "ConstraintSet": # sam.moelius: There is a flaw in how __get_related works: when called on certain # unsatisfiable sets, it can return a satisfiable one. The flaw arises when: # * self consists of a single constraint C From e0130a473b7f0167d2ce727a5f24bc04d5fc95e4 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 18:15:32 -0300 Subject: [PATCH 040/126] Update manticore/core/smtlib/visitors.py Co-authored-by: Eric Kilmer --- manticore/core/smtlib/visitors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 45b59662c..fc289950c 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -309,7 +309,7 @@ def __init__(self, **kw): UnsignedGreaterOrEqual: lambda x, y: (x & UNSIGN_MASK) >= (y & UNSIGN_MASK), } - def visit_BitVecDiv(self, expression, *operands): + def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask mask = operands[0].mask From d0e9abf673fab434ea7ba46b4d07c995100174c5 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 5 Jun 2020 18:27:55 -0300 Subject: [PATCH 041/126] lint --- manticore/core/smtlib/visitors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index fc289950c..e44dc5a3a 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -324,6 +324,7 @@ def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: else: result = int(numeral / dividend) return BitVecConstant(expression.size, result, taint=expression.taint) + return None def visit_BitVecConcat(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): From 9807b3745ce965ba55b74073b8fe6dcdbd9617eb Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 8 Jun 2020 13:43:50 -0300 Subject: [PATCH 042/126] Add Docs --- tests/other/test_smtlibv2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 12a883903..5706b79c1 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -33,6 +33,7 @@ def test_related_to(self): filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) + # A constraint set and a contraint caught in the act of making related_to fail constraints, constraint = pickle.loads(gzip.open(filename, "rb").read()) consts = config.get_group("smt") From bf3a2b933a83400920e6afaf04d09267f3987339 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Tue, 9 Jun 2020 18:40:40 -0700 Subject: [PATCH 043/126] Replace modulo with masks --- manticore/wasm/executor.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/manticore/wasm/executor.py b/manticore/wasm/executor.py index 962863ed2..dd6eeede7 100644 --- a/manticore/wasm/executor.py +++ b/manticore/wasm/executor.py @@ -28,6 +28,9 @@ import operator import math +MASK_64 = (1 << 64) - 1 +MASK_32 = (1 << 32) - 1 + class Executor(Eventful): """ @@ -450,14 +453,13 @@ def int_store(self, store, stack, imm: MemoryImm, ty: type, n=None): ) # TODO - Implement a symbolic memory model ea = i + imm.offset N = n if n else (32 if ty is I32 else 64) + mask = (1 << N) - 1 if ea not in mem: raise OutOfBoundsMemoryTrap(ea) if (ea + (N // 8)) - 1 not in mem: raise OutOfBoundsMemoryTrap(ea + (N // 8)) if n: - b = [ - Operators.CHR(Operators.EXTRACT(c % 2 ** N, offset, 8)) for offset in range(0, N, 8) - ] + b = [Operators.CHR(Operators.EXTRACT(c & mask, offset, 8)) for offset in range(0, N, 8)] else: b = [Operators.CHR(Operators.EXTRACT(c, offset, 8)) for offset in range(0, N, 8)] @@ -742,6 +744,17 @@ def i32_clz(self, store, stack): res = Operators.ITEBV(32, flag, res, 32) stack.push(I32.cast(res)) + # value = src.read() + # flag = Operators.EXTRACT(value, 0, 1) == 1 + # res = 0 + # for pos in range(1, src.size): + # res = Operators.ITEBV(dest.size, flag, res, pos) + # flag = Operators.OR(flag, Operators.EXTRACT(value, pos, 1) == 1) + # + # cpu.CF = res == src.size + # cpu.ZF = res == 0 + # dest.write(res) + def i32_ctz(self, store, stack): # Copied from x86 TZCNT stack.has_type_on_top(I32, 1) c1 = stack.pop() @@ -768,19 +781,19 @@ def i32_add(self, store, stack): stack.has_type_on_top(I32, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I32.cast((c2 + c1) % 2 ** 32)) + stack.push(I32.cast((c2 + c1) & MASK_32)) def i32_sub(self, store, stack): stack.has_type_on_top(I32, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I32.cast((c1 - c2 + 2 ** 32) % 2 ** 32)) + stack.push(I32.cast((c1 - c2 + 2 ** 32) & MASK_32)) def i32_mul(self, store, stack): stack.has_type_on_top(I32, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I32.cast((c2 * c1) % 2 ** 32)) + stack.push(I32.cast((c2 * c1) & MASK_32)) def i32_div_s(self, store, stack): stack.has_type_on_top(I32, 2) @@ -850,7 +863,7 @@ def i32_shl(self, store, stack): stack.has_type_on_top(I32, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I32.cast((c1 << (c2 % 32)) % 2 ** 32)) + stack.push(I32.cast((c1 << (c2 % 32)) & MASK_32)) def i32_shr_s(self, store, stack): stack.has_type_on_top(I32, 2) @@ -925,19 +938,19 @@ def i64_add(self, store, stack): stack.has_type_on_top(I64, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I64.cast((c2 + c1) % 2 ** 64)) + stack.push(I64.cast((c2 + c1) & MASK_64)) def i64_sub(self, store, stack): stack.has_type_on_top(I64, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I64.cast((c1 - c2 + 2 ** 64) % 2 ** 64)) + stack.push(I64.cast((c1 - c2 + 2 ** 64) & MASK_64)) def i64_mul(self, store, stack): stack.has_type_on_top(I64, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I64.cast((c2 * c1) % 2 ** 64)) + stack.push(I64.cast((c2 * c1) & MASK_64)) def i64_div_s(self, store, stack): stack.has_type_on_top(I64, 2) @@ -1014,7 +1027,7 @@ def i64_shl(self, store, stack): stack.has_type_on_top(I64, 2) c2 = stack.pop() c1 = stack.pop() - stack.push(I64.cast((c1 << (c2 % 64)) % 2 ** 64)) + stack.push(I64.cast((c1 << (c2 % 64)) & MASK_64)) def i64_shr_s(self, store, stack): stack.has_type_on_top(I64, 2) @@ -1054,7 +1067,7 @@ def i64_rotr(self, store, stack): def i32_wrap_i64(self, store, stack): stack.has_type_on_top(I64, 1) c1: I64 = stack.pop() - c1 %= 2 ** 32 + c1 &= MASK_32 c1 = Operators.EXTRACT(c1, 0, 32) stack.push(I32.cast(c1)) From 3eb96f86d491b50b0f1ec50adf783327d3258ada Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Tue, 9 Jun 2020 18:40:58 -0700 Subject: [PATCH 044/126] Blacken --- tests/other/test_smtlibv2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index d2f0a0717..f7a77d437 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1044,7 +1044,6 @@ def test_NOT(self): self.assertTrue(solver.must_be_true(cs, Operators.NOT(False))) self.assertTrue(solver.must_be_true(cs, Operators.NOT(a == b))) - def testRelated(self): cs = ConstraintSet() aa1 = cs.new_bool(name="AA1") From 5e54189e2f49b145bb57662654661f66eb27a473 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 9 Jun 2020 22:59:31 -0300 Subject: [PATCH 045/126] blkn --- tests/other/test_smtlibv2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index d2f0a0717..f7a77d437 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1044,7 +1044,6 @@ def test_NOT(self): self.assertTrue(solver.must_be_true(cs, Operators.NOT(False))) self.assertTrue(solver.must_be_true(cs, Operators.NOT(a == b))) - def testRelated(self): cs = ConstraintSet() aa1 = cs.new_bool(name="AA1") From 536ec71d55a6c9209526791d93d348b74ae73ea5 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 9 Jun 2020 23:29:17 -0300 Subject: [PATCH 046/126] fix mypy --- manticore/core/smtlib/solver.py | 104 ++++++++++++++++---------------- 1 file changed, 51 insertions(+), 53 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 077fdd0d8..3c08a8885 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -432,64 +432,62 @@ def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, ma last_value: Optional[Union[int, bool, bytes]] = None start = time.time() - with constraints as temp_cs: - X = temp_cs.new_bitvec(x.size) - temp_cs.add(X == x) - aux = temp_cs.new_bitvec(X.size, name="optimized_") - self._reset(temp_cs.to_string(related_to=X)) - - # Find one value and use it as currently known min/Max - if not self._is_sat(): - raise SolverException("UNSAT") - last_value = self._getvalue(X) - self._assert(operation(X, last_value)) - - # This uses a binary search to find a suitable range for aux - # Use known solution as min or max depending on the goal - if goal == "maximize": - m, M = last_value, (1 << x.size) - 1 + temp_cs = constraints.related_to(x) + self._reset(temp_cs.to_string()) + + # Find one value and use it as currently known min/Max + if not self._is_sat(): + raise SolverException("UNSAT") + last_value = self._getvalue(x) + self._assert(operation(x, last_value)) + + # This uses a binary search to find a suitable range for aux + # Use known solution as min or max depending on the goal + if goal == "maximize": + m, M = last_value, (1 << x.size) - 1 + else: + m, M = 0, last_value + + # Iteratively divide the range + L = None + while L not in (M, m): + L = (m + M) // 2 + self._assert(operation(x, L)) + sat = self._is_sat() + + # depending on the goal move one of the extremes + if goal == "maximize" and sat or goal == "minimize" and not sat: + m = L else: - m, M = 0, last_value - - # Iteratively divide the range - L = None - while L not in (M, m): - L = (m + M) // 2 - self._assert(operation(X, L)) - sat = self._is_sat() - - # depending on the goal move one of the extremes - if goal == "maximize" and sat or goal == "minimize" and not sat: - m = L - else: - M = L + M = L - if time.time() - start > consts.timeout: - raise SolverError("Timeout") + if time.time() - start > consts.timeout: + raise SolverError("Timeout") - # reset to before the dichotomic search - self._reset(temp_cs.to_string(related_to=X)) + # reset to before the dichotomic search + temp_cs = constraints.related_to(x) + self._reset(temp_cs.to_string()) - # At this point we know aux is inside [m,M] - # Lets constrain it to that range - self._assert(Operators.UGE(X, m)) - self._assert(Operators.ULE(X, M)) + # At this point we know aux is inside [m,M] + # Lets constrain it to that range + self._assert(Operators.UGE(x, m)) + self._assert(Operators.ULE(x, M)) - # And now check all remaining possible extremes - last_value = None - i = 0 - while self._is_sat(): - last_value = self._getvalue(X) - self._assert(operation(X, last_value)) - self._assert(X != last_value) - i = i + 1 - if i > max_iter: - raise SolverError("Optimizing error, maximum number of iterations was reached") - if time.time() - start > consts.timeout: - raise SolverError("Timeout") - if last_value is not None: - return last_value - raise SolverError("Optimizing error, unsat or unknown core") + # And now check all remaining possible extremes + last_value = None + i = 0 + while self._is_sat(): + last_value = self._getvalue(X) + self._assert(operation(x, last_value)) + self._assert(X != last_value) + i = i + 1 + if i > max_iter: + raise SolverError("Optimizing error, maximum number of iterations was reached") + if time.time() - start > consts.timeout: + raise SolverError("Timeout") + if last_value is not None: + return last_value + raise SolverError("Optimizing error, unsat or unknown core") @lru_cache(maxsize=32) def get_all_values( From a83e8c29b3cb344dee7443f5cfbf74d2a1021429 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 9 Jun 2020 23:34:01 -0300 Subject: [PATCH 047/126] fix mypy --- manticore/core/smtlib/solver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 3c08a8885..0b546fb09 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -477,9 +477,9 @@ def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, ma last_value = None i = 0 while self._is_sat(): - last_value = self._getvalue(X) + last_value = self._getvalue(x) self._assert(operation(x, last_value)) - self._assert(X != last_value) + self._assert(x != last_value) i = i + 1 if i > max_iter: raise SolverError("Optimizing error, maximum number of iterations was reached") From 37144d949d74876f05e3bf671fa52e2acf607e5c Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Wed, 10 Jun 2020 11:48:10 -0700 Subject: [PATCH 048/126] Add tests for signed LT behavior --- tests/other/test_smtlibv2.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index f7a77d437..ce6ecb377 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1093,6 +1093,39 @@ def test_API(self): for attr in attrs: self.assertTrue(hasattr(cls, attr), f"{cls.__name__} is missing attribute {attr}") + def test_signed_unsigned_LT_simple(self): + cs = ConstraintSet() + a = cs.new_bitvec(32) + b = cs.new_bitvec(32) + + cs.add(a == 0x1) + cs.add(b == 0x80000000) + + lt = b < a + ult = b.ult(a) + + self.assertFalse(self.solver.can_be_true(cs, ult)) + self.assertTrue(self.solver.must_be_true(cs, lt)) + + def test_signed_unsigned_LT_complex(self): + mask = (1 << 32) - 1 + + cs = ConstraintSet() + _a = cs.new_bitvec(32) + _b = cs.new_bitvec(32) + + cs.add(_a == 0x1) + cs.add(_b == (0x80000000 - 1)) + + a = _a & mask + b = (_b + 1) & mask + + lt = b < a + ult = b.ult(a) + + self.assertFalse(self.solver.can_be_true(cs, ult)) + self.assertTrue(self.solver.must_be_true(cs, lt)) + class ExpressionTestYices(ExpressionTest): def setUp(self): From 1b8754d655af27a4e6bd29e691c57f7d99cf3bac Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 16:53:56 -0300 Subject: [PATCH 049/126] New test --- manticore/core/smtlib/solver.py | 3 ++- tests/other/test_smtlibv2.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 0b546fb09..0d411ba58 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -214,6 +214,7 @@ def __readline_and_count(self): buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error # Error messages may contain an unbalanced parenthesis situation + print (buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -229,7 +230,7 @@ def send(self, cmd: str) -> None: """ if self._debug: logger.debug(">%s", cmd) - print(">", cmd) + print(">", cmd) self._proc.stdout.flush() # type: ignore self._proc.stdin.write(f"{cmd}\n") # type: ignore diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index f7a77d437..394768811 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1093,6 +1093,19 @@ def test_API(self): for attr in attrs: self.assertTrue(hasattr(cls, attr), f"{cls.__name__} is missing attribute {attr}") + def test_signed_unsigned_LT_simple(self): + cs = ConstraintSet() + a = cs.new_bitvec(32) + b = cs.new_bitvec(32) + cs.add(a == 0x1) + cs.add(b == 0x80000000) + lt = b < a + ult = b.ult(a) + print ("lt", translate_to_smtlib(lt)) + print ("ult",translate_to_smtlib(ult)) + self.assertFalse(self.solver.can_be_true(cs, ult)) + self.assertTrue(self.solver.must_be_true(cs, lt)) + class ExpressionTestYices(ExpressionTest): def setUp(self): From 787e51fb801c88a75aac8ea8f8750e1039648771 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 17:42:14 -0300 Subject: [PATCH 050/126] Fix constant folding --- manticore/core/smtlib/expression.py | 7 ++++ manticore/core/smtlib/solver.py | 2 -- manticore/core/smtlib/visitors.py | 51 +++++++++++++++++++++-------- tests/other/test_smtlibv2.py | 2 -- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 7e964ad8a..6cfa26a23 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -504,6 +504,13 @@ def __hash__(self): def value(self): return self._value + @property + def signed_value(self): + if self._value & self.signmask: + return self._value - (1< None: """ if self._debug: logger.debug(">%s", cmd) - print(">", cmd) self._proc.stdout.flush() # type: ignore self._proc.stdin.write(f"{cmd}\n") # type: ignore diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index e44dc5a3a..170734f73 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -7,7 +7,6 @@ import operator logger = logging.getLogger(__name__) -UNSIGN_MASK = (1 << 256) - 1 class Visitor: @@ -292,23 +291,49 @@ def __init__(self, **kw): BitVecXor: operator.__xor__, BitVecNot: operator.__not__, BitVecNeg: operator.__invert__, - LessThan: operator.__lt__, - LessOrEqual: operator.__le__, - BoolEqual: operator.__eq__, - GreaterThan: operator.__gt__, - GreaterOrEqual: operator.__ge__, BoolAnd: operator.__and__, BoolOr: operator.__or__, BoolNot: operator.__not__, - BitVecUnsignedDiv: lambda x, y: 0 - if (y & UNSIGN_MASK) == 0 - else (x & UNSIGN_MASK) // (y & UNSIGN_MASK), - UnsignedLessThan: lambda x, y: (x & UNSIGN_MASK) < (y & UNSIGN_MASK), - UnsignedLessOrEqual: lambda x, y: (x & UNSIGN_MASK) <= (y & UNSIGN_MASK), - UnsignedGreaterThan: lambda x, y: (x & UNSIGN_MASK) > (y & UNSIGN_MASK), - UnsignedGreaterOrEqual: lambda x, y: (x & UNSIGN_MASK) >= (y & UNSIGN_MASK), + UnsignedLessThan: operator.__lt__, + UnsignedLessOrEqual: operator.__le__, + UnsignedGreaterThan: operator.__gt__, + UnsignedGreaterOrEqual: operator.__le__, } + def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConstant]: + if all(isinstance(o, Constant) for o in operands): + a = operands[0].value + b = operands[1].value + if a == 0: + ret = 0 + else: + ret = int(x / y) + return BitVecConstant(ret, taint=expression.taint) + + def visit_LessThan(self, expression, *operands) -> Optional[BoolConstant]: + if all(isinstance(o, Constant) for o in operands): + a = operands[0].signed_value + b = operands[1].signed_value + return BoolConstant(a < b, taint=expression.taint) + + def visit_LessOrEqual(self, expression, *operands) -> Optional[BoolConstant]: + if all(isinstance(o, Constant) for o in operands): + a = operands[0].signed_value + b = operands[1].signed_value + return BoolConstant(a <= b, taint=expression.taint) + + def visit_GreaterThan(self, expression, *operands) -> Optional[BoolConstant]: + if all(isinstance(o, Constant) for o in operands): + a = operands[0].signed_value + b = operands[1].signed_value + return BoolConstant(a > b, taint=expression.taint) + + def visit_GreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: + if all(isinstance(o, Constant) for o in operands): + a = operands[0].signed_value + b = operands[1].signed_value + return BoolConstant(a >= b, taint=expression.taint) + def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 394768811..934d587ab 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1101,8 +1101,6 @@ def test_signed_unsigned_LT_simple(self): cs.add(b == 0x80000000) lt = b < a ult = b.ult(a) - print ("lt", translate_to_smtlib(lt)) - print ("ult",translate_to_smtlib(ult)) self.assertFalse(self.solver.can_be_true(cs, ult)) self.assertTrue(self.solver.must_be_true(cs, lt)) From dfa7a84ac205cc88d1b4dea0a484d5d41bf2c293 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 17:48:44 -0300 Subject: [PATCH 051/126] lint --- manticore/core/smtlib/expression.py | 2 +- manticore/core/smtlib/visitors.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 6cfa26a23..6f1649f6b 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -507,7 +507,7 @@ def value(self): @property def signed_value(self): if self._value & self.signmask: - return self._value - (1< Optional[BitVecConst ret = 0 else: ret = int(x / y) - return BitVecConstant(ret, taint=expression.taint) + return BitVecConstant(ret, taint=expression.taint) def visit_LessThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): From b841038482a47007549623304da51032a9b845af Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 17:58:08 -0300 Subject: [PATCH 052/126] lint --- manticore/core/smtlib/visitors.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 8be6448da..93d871943 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -307,7 +307,7 @@ def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConst if a == 0: ret = 0 else: - ret = int(x / y) + ret = int(a / b) return BitVecConstant(ret, taint=expression.taint) def visit_LessThan(self, expression, *operands) -> Optional[BoolConstant]: @@ -315,24 +315,28 @@ def visit_LessThan(self, expression, *operands) -> Optional[BoolConstant]: a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a < b, taint=expression.taint) + return None def visit_LessOrEqual(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a <= b, taint=expression.taint) + return None def visit_GreaterThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a > b, taint=expression.taint) + return None def visit_GreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a >= b, taint=expression.taint) + return None def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: if all(isinstance(o, Constant) for o in operands): From 735a96350de598eacde8a500a31fe8540c1c0536 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 18:47:56 -0300 Subject: [PATCH 053/126] lint --- manticore/core/smtlib/visitors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 93d871943..e85809460 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -308,7 +308,8 @@ def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConst ret = 0 else: ret = int(a / b) - return BitVecConstant(ret, taint=expression.taint) + return BitVecConstant(expression.size, ret, taint=expression.taint) + return None def visit_LessThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): From b54a4480869bb80684ac73d14803ec2c012554ea Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 19:01:28 -0300 Subject: [PATCH 054/126] Unittesting power --- manticore/core/smtlib/visitors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index e85809460..29cb04603 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -297,7 +297,7 @@ def __init__(self, **kw): UnsignedLessThan: operator.__lt__, UnsignedLessOrEqual: operator.__le__, UnsignedGreaterThan: operator.__gt__, - UnsignedGreaterOrEqual: operator.__le__, + UnsignedGreaterOrEqual: operator.__ge__, } def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConstant]: From 02563cdbd6147fc6e39a9de5eb504750fb3d2b00 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 10 Jun 2020 19:06:30 -0300 Subject: [PATCH 055/126] Permisive read_buffer --- manticore/platforms/evm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 0b85987f4..435c6d83e 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -1381,6 +1381,8 @@ def setstate(state, value): def read_buffer(self, offset, size): if issymbolic(size) and not isinstance(size, Constant): raise EVMException("Symbolic size not supported") + if isinstance(size, Constant): + size = size.value if size == 0: return b"" self._allocate(offset, size) From 3fab981e353272ce16e7b6f9b5c626277d189d07 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Wed, 10 Jun 2020 16:59:02 -0700 Subject: [PATCH 056/126] Preserve precision in constant folding --- manticore/core/smtlib/visitors.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 29cb04603..b2f3de8c3 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -5,6 +5,8 @@ import copy import logging import operator +import math +from decimal import Decimal logger = logging.getLogger(__name__) @@ -307,7 +309,7 @@ def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConst if a == 0: ret = 0 else: - ret = int(a / b) + ret = math.trunc(Decimal(a) / Decimal(b)) return BitVecConstant(expression.size, ret, taint=expression.taint) return None @@ -340,6 +342,7 @@ def visit_GreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: return None def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: + print("HOO BABY VISITING A BITVECDIV") if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask mask = operands[0].mask @@ -352,7 +355,7 @@ def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: if dividend == 0: result = 0 else: - result = int(numeral / dividend) + result = math.trunc(Decimal(numeral) / Decimal(dividend)) return BitVecConstant(expression.size, result, taint=expression.taint) return None From f7fb8bb5bfcdc7f4d2d7700ef1c4c9b32979b797 Mon Sep 17 00:00:00 2001 From: Eric Hennenfent Date: Wed, 10 Jun 2020 17:14:08 -0700 Subject: [PATCH 057/126] Strip left-in print debugging --- manticore/core/smtlib/visitors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index b2f3de8c3..c361c49c2 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -342,7 +342,6 @@ def visit_GreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: return None def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: - print("HOO BABY VISITING A BITVECDIV") if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask mask = operands[0].mask From 2f5d4d22fa0c8698774a191ac3dc44d71cd6f162 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 11 Jun 2020 18:27:06 -0300 Subject: [PATCH 058/126] fix wasm --- manticore/core/smtlib/constraints.py | 24 +++++++++++++++++------- manticore/core/smtlib/visitors.py | 9 +++++++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 259d3c72f..3e86443e6 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -1,5 +1,6 @@ import itertools import sys +import copy from typing import Optional from ...utils.helpers import PickleSerializer from ...exceptions import SmtlibError @@ -17,7 +18,14 @@ Variable, Constant, ) -from .visitors import GetDeclarations, TranslatorSmtlib, get_variables, simplify, replace +from .visitors import ( + GetDeclarations, + TranslatorSmtlib, + get_variables, + simplify, + replace, + pretty_print, +) from ...utils import config import logging @@ -116,7 +124,7 @@ def _get_sid(self) -> int: self._sid += 1 return self._sid - def related_to(self, related_to: Optional[Expression] = None) -> "ConstraintSet": + def related_to(self, *related_to) -> "ConstraintSet": # sam.moelius: There is a flaw in how __get_related works: when called on certain # unsatisfiable sets, it can return a satisfiable one. The flaw arises when: # * self consists of a single constraint C @@ -137,11 +145,14 @@ def related_to(self, related_to: Optional[Expression] = None) -> "ConstraintSet" :param related_to: An expression :return: """ - if related_to is None: - return self + + if not related_to: + return copy.copy(self) number_of_constraints = len(self.constraints) remaining_constraints = set(self.constraints) - related_variables = get_variables(related_to) + related_variables = set() + for expression in related_to: + related_variables |= get_variables(expression) related_constraints = set() added = True @@ -164,7 +175,6 @@ def related_to(self, related_to: Optional[Expression] = None) -> "ConstraintSet" added = True logger.debug("Reduced %d constraints!!", number_of_constraints - len(related_constraints)) - # related_variables, related_constraints cs = ConstraintSet() for var in related_variables: @@ -173,7 +183,7 @@ def related_to(self, related_to: Optional[Expression] = None) -> "ConstraintSet" cs.add(constraint) return cs - def to_string(self, replace_constants: bool = True) -> str: + def to_string(self, replace_constants: bool = False) -> str: variables, constraints = self.get_declared_variables(), self.constraints if replace_constants: diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 29cb04603..bddba15f0 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -50,7 +50,7 @@ def result(self): return self._stack[-1] def _method(self, expression, *args): - for cls in expression.__class__.__mro__: + for cls in expression.__class__.__mro__[:-1]: sort = cls.__name__ methodname = "visit_%s" % sort if hasattr(self, methodname): @@ -71,7 +71,8 @@ def visit(self, node, use_fixed_point=False): :param use_fixed_point: if True, it runs _methods until a fixed point is found :type use_fixed_point: Bool """ - + if isinstance(node, ArrayProxy): + node = node.array cache = self._cache visited = set() stack = [] @@ -292,6 +293,7 @@ def __init__(self, **kw): BitVecNot: operator.__not__, BitVecNeg: operator.__invert__, BoolAnd: operator.__and__, + BoolEqual: operator.__eq__, BoolOr: operator.__or__, BoolNot: operator.__not__, UnsignedLessThan: operator.__lt__, @@ -1037,6 +1039,9 @@ def simplify_array_select(array_exp): def get_variables(expression): + if isinstance(expression, ArrayProxy): + expression = expression.array + visitor = GetDeclarations() visitor.visit(expression) return visitor.result From 52d8eb474278e5364230e228d9fb35b86d1eabe1 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 11 Jun 2020 18:59:47 -0300 Subject: [PATCH 059/126] Fix --- manticore/core/smtlib/expression.py | 39 +++++++++++++---------------- manticore/core/smtlib/solver.py | 29 ++++++++++----------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 6f1649f6b..66296ae36 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -967,7 +967,7 @@ def value(self): return self.operands[2] -class ArraySlice(Array): +class ArraySlice(ArrayOperation): def __init__( self, array: Union["Array", "ArrayProxy"], offset: int, size: int, *args, **kwargs ): @@ -975,23 +975,22 @@ def __init__( raise ValueError("Array expected") if isinstance(array, ArrayProxy): array = array._array - super().__init__(array.index_bits, array.index_max, array.value_bits, *args, **kwargs) + super().__init__(array, **kwargs) - self._array = array self._slice_offset = offset self._slice_size = size @property - def underlying_variable(self): - return self._array.underlying_variable + def array(self): + return self.operands[0] @property - def operands(self): - return self._array.operands + def underlying_variable(self): + return self.array.underlying_variable @property def index_bits(self): - return self._array.index_bits + return self.array.index_bits @property def index_max(self): @@ -999,18 +998,14 @@ def index_max(self): @property def value_bits(self): - return self._array.value_bits - - @property - def taint(self): - return self._array.taint + return self.array.value_bits def select(self, index): - return self._array.select(index + self._slice_offset) + return self.array.select(index + self._slice_offset) def store(self, index, value): return ArraySlice( - self._array.store(index + self._slice_offset, value), + self.array.store(index + self._slice_offset, value), self._slice_offset, self._slice_size, ) @@ -1055,7 +1050,7 @@ def name(self): @property def operands(self): - return self._array.operands + return (self._array,) @property def index_bits(self): @@ -1152,13 +1147,13 @@ def written(self): # take out Proxy sleve array = self._array offset = 0 - while isinstance(array, ArraySlice): - # if it is a proxy over a slice take out the slice too - offset += array._slice_offset - array = array._array while not isinstance(array, ArrayVariable): - # The index written to underlaying Array are displaced when sliced - written.add(array.index - offset) + if isinstance(array, ArraySlice): + # if it is a proxy over a slice take out the slice too + offset += array._slice_offset + else: + # The index written to underlaying Array are displaced when sliced + written.add(array.index - offset) array = array.array assert isinstance(array, ArrayVariable) self._written = written diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 5ed6599c3..97e0c3409 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -217,7 +217,6 @@ def __readline_and_count(self): if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") - # lparen, rparen = buf.count("("), buf.count(")") lparen, rparen = map(sum, zip(*((c == "(", c == ")") for c in buf))) return buf, lparen, rparen @@ -246,7 +245,6 @@ def recv(self) -> str: if self._debug: logger.debug("<%s", buf) - print("<", buf) return buf @@ -432,18 +430,20 @@ def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, ma start = time.time() temp_cs = constraints.related_to(x) + X = temp_cs.new_bitvec(x.size) # _getvalue needs a Variable + temp_cs.add(X == x) self._reset(temp_cs.to_string()) # Find one value and use it as currently known min/Max if not self._is_sat(): raise SolverException("UNSAT") - last_value = self._getvalue(x) - self._assert(operation(x, last_value)) + last_value = self._getvalue(X) + self._assert(operation(X, last_value)) # This uses a binary search to find a suitable range for aux # Use known solution as min or max depending on the goal if goal == "maximize": - m, M = last_value, (1 << x.size) - 1 + m, M = last_value, (1 << X.size) - 1 else: m, M = 0, last_value @@ -451,7 +451,7 @@ def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, ma L = None while L not in (M, m): L = (m + M) // 2 - self._assert(operation(x, L)) + self._assert(operation(X, L)) sat = self._is_sat() # depending on the goal move one of the extremes @@ -465,20 +465,22 @@ def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, ma # reset to before the dichotomic search temp_cs = constraints.related_to(x) + X = temp_cs.new_bitvec(x.size) # _getvalue needs a Variable + temp_cs.add(X == x) self._reset(temp_cs.to_string()) # At this point we know aux is inside [m,M] # Lets constrain it to that range - self._assert(Operators.UGE(x, m)) - self._assert(Operators.ULE(x, M)) + self._assert(Operators.UGE(X, m)) + self._assert(Operators.ULE(X, M)) # And now check all remaining possible extremes last_value = None i = 0 while self._is_sat(): - last_value = self._getvalue(x) - self._assert(operation(x, last_value)) - self._assert(x != last_value) + last_value = self._getvalue(X) + self._assert(operation(X, last_value)) + self._assert(X != last_value) i = i + 1 if i > max_iter: raise SolverError("Optimizing error, maximum number of iterations was reached") @@ -525,7 +527,7 @@ def get_all_values( ) temp_cs.add(var == expression) - self._reset(temp_cs.related_to(var).to_string()) + self._reset(temp_cs.to_string()) result = [] start = time.time() while self._is_sat(): @@ -586,7 +588,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): """ values = [] start = time.time() - with constraints as temp_cs: + with constraints.related_to(*expressions) as temp_cs: for expression in expressions: if not issymbolic(expression): values.append(expression) @@ -603,7 +605,6 @@ def get_value(self, constraints: ConstraintSet, *expressions): subvar = temp_cs.new_bitvec(expression.value_bits) var.append(subvar) temp_cs.add(subvar == simplify(expression[i])) - self._reset(temp_cs.to_string()) if not self._is_sat(): raise SolverError( From f1dfb80f682c32e9cb717cfa4879d60cc5cea4f0 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 12 Jun 2020 12:39:26 -0300 Subject: [PATCH 060/126] REmove get_related from the default path and fix arm test --- manticore/core/smtlib/constraints.py | 7 -- manticore/core/smtlib/solver.py | 131 ++++++++++++++------------- tests/native/test_armv7cpu.py | 6 +- 3 files changed, 69 insertions(+), 75 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 3e86443e6..b35519aee 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -31,13 +31,6 @@ logger = logging.getLogger(__name__) -consts = config.get_group("smt") -consts.add( - "related_constraints", - default=False, - description="Try slicing the current path constraint to contain only related items", -) - class ConstraintException(SmtlibError): """ diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 97e0c3409..267ed4a82 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -205,6 +205,8 @@ def stop(self): self._proc.stdout.close() # Kill the process self._proc.kill() + self._proc.wait() + # No need to wait for termination, zombies avoided. self._proc = None @@ -304,8 +306,6 @@ def __init__( self._smtlib.start() # run solver specific initializations - for cfg in self._init: - self._smtlib.send(cfg) def _reset(self, constraints: Optional[str] = None) -> None: """Auxiliary method to reset the smtlib external solver to initial defaults""" @@ -429,66 +429,66 @@ def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, ma last_value: Optional[Union[int, bool, bytes]] = None start = time.time() - temp_cs = constraints.related_to(x) - X = temp_cs.new_bitvec(x.size) # _getvalue needs a Variable - temp_cs.add(X == x) - self._reset(temp_cs.to_string()) - - # Find one value and use it as currently known min/Max - if not self._is_sat(): - raise SolverException("UNSAT") - last_value = self._getvalue(X) - self._assert(operation(X, last_value)) - - # This uses a binary search to find a suitable range for aux - # Use known solution as min or max depending on the goal - if goal == "maximize": - m, M = last_value, (1 << X.size) - 1 - else: - m, M = 0, last_value - - # Iteratively divide the range - L = None - while L not in (M, m): - L = (m + M) // 2 - self._assert(operation(X, L)) - sat = self._is_sat() - - # depending on the goal move one of the extremes - if goal == "maximize" and sat or goal == "minimize" and not sat: - m = L + with constraints as temp_cs: + X = temp_cs.new_bitvec(x.size) # _getvalue needs a Variable + temp_cs.add(X == x) + self._reset(temp_cs.to_string()) + + # Find one value and use it as currently known min/Max + if not self._is_sat(): + raise SolverException("UNSAT") + last_value = self._getvalue(X) + self._assert(operation(X, last_value)) + + # This uses a binary search to find a suitable range for aux + # Use known solution as min or max depending on the goal + if goal == "maximize": + m, M = last_value, (1 << X.size) - 1 else: - M = L + m, M = 0, last_value + + # Iteratively divide the range + L = None + while L not in (M, m): + L = (m + M) // 2 + self._assert(operation(X, L)) + sat = self._is_sat() + + # depending on the goal move one of the extremes + if goal == "maximize" and sat or goal == "minimize" and not sat: + m = L + else: + M = L - if time.time() - start > consts.timeout: - raise SolverError("Timeout") + if time.time() - start > consts.timeout: + raise SolverError("Timeout") # reset to before the dichotomic search - temp_cs = constraints.related_to(x) - X = temp_cs.new_bitvec(x.size) # _getvalue needs a Variable - temp_cs.add(X == x) - self._reset(temp_cs.to_string()) - - # At this point we know aux is inside [m,M] - # Lets constrain it to that range - self._assert(Operators.UGE(X, m)) - self._assert(Operators.ULE(X, M)) - - # And now check all remaining possible extremes - last_value = None - i = 0 - while self._is_sat(): - last_value = self._getvalue(X) - self._assert(operation(X, last_value)) - self._assert(X != last_value) - i = i + 1 - if i > max_iter: - raise SolverError("Optimizing error, maximum number of iterations was reached") - if time.time() - start > consts.timeout: - raise SolverError("Timeout") - if last_value is not None: - return last_value - raise SolverError("Optimizing error, unsat or unknown core") + with constraints as temp_cs: + X = temp_cs.new_bitvec(x.size) # _getvalue needs a Variable + temp_cs.add(X == x) + self._reset(temp_cs.to_string()) + + # At this point we know aux is inside [m,M] + # Lets constrain it to that range + self._assert(Operators.UGE(X, m)) + self._assert(Operators.ULE(X, M)) + + # And now check all remaining possible extremes + last_value = None + i = 0 + while self._is_sat(): + last_value = self._getvalue(X) + self._assert(operation(X, last_value)) + self._assert(X != last_value) + i = i + 1 + if i > max_iter: + raise SolverError("Optimizing error, maximum number of iterations was reached") + if time.time() - start > consts.timeout: + raise SolverError("Timeout") + if last_value is not None: + return last_value + raise SolverError("Optimizing error, unsat or unknown core") @lru_cache(maxsize=32) def get_all_values( @@ -510,7 +510,7 @@ def get_all_values( maxcnt = 2 silent = True - with constraints.related_to(expression) as temp_cs: + with constraints as temp_cs: if isinstance(expression, Bool): var = temp_cs.new_bool() elif isinstance(expression, BitVec): @@ -549,8 +549,9 @@ def get_all_values( return list(result) raise SolverError("Timeout") # Sometimes adding a new contraint after a check-sat eats all the mem - temp_cs.add(var != value) - self._reset(temp_cs.to_string()) + #temp_cs.add(var != value) + #self._reset(temp_cs.to_string()) + self._smtlib.send(f"(assert {translate_to_smtlib(var != value)})") return list(result) def _optimize_fancy(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): @@ -588,7 +589,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): """ values = [] start = time.time() - with constraints.related_to(*expressions) as temp_cs: + with constraints as temp_cs: for expression in expressions: if not issymbolic(expression): values.append(expression) @@ -667,10 +668,10 @@ def __init__(self): command=command, init=init, value_fmt=16, - support_minmax=True, - support_reset=True, + support_minmax=support_minmax, + support_reset=support_reset, support_pushpop=True, - debug=False, + debug=True ) def __autoconfig(self): diff --git a/tests/native/test_armv7cpu.py b/tests/native/test_armv7cpu.py index 144c96d95..1b43bfa7e 100644 --- a/tests/native/test_armv7cpu.py +++ b/tests/native/test_armv7cpu.py @@ -1568,7 +1568,7 @@ def test_uqsub8_concrete_saturated(self): @itest_custom("uqsub8 r3, r1, r2") @itest_setregs("R2=0x01010101") def test_uqsub8_sym(self): - op1 = BitVecVariable(32, "op1") + op1 = self.cpu.memory.constraints.new_bitvec(32, "op1") self.cpu.memory.constraints.add(op1 >= 0x04030201) self.cpu.memory.constraints.add(op1 < 0x04030204) self.cpu.R1 = op1 @@ -2416,7 +2416,7 @@ def test_sxth(self): @itest_custom("blx r1") def test_blx_reg_sym(self): - dest = BitVecVariable(32, "dest") + dest = self.cpu.memory.constraints.new_bitvec(32, "dest") self.cpu.memory.constraints.add(dest >= 0x1000) self.cpu.memory.constraints.add(dest <= 0x1001) self.cpu.R1 = dest @@ -2462,7 +2462,7 @@ def test_symbolic_conditional(self): self._setupCpu(asm, mode=CS_MODE_THUMB) # code starts at 0x1004 # Set R0 as a symbolic value - self.cpu.R0 = BitVecVariable(32, "val") + self.cpu.R0 = self.cpu.memory.constraints.new_bitvec(32, "val") self.cpu.execute() # tst r0, r0 self.cpu.execute() # beq label From d5b577ef5cb5e5b3908f988c6e00df6e4ba5ac4c Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 12 Jun 2020 12:46:24 -0300 Subject: [PATCH 061/126] blkn --- manticore/core/smtlib/solver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 267ed4a82..93120e71d 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -549,8 +549,8 @@ def get_all_values( return list(result) raise SolverError("Timeout") # Sometimes adding a new contraint after a check-sat eats all the mem - #temp_cs.add(var != value) - #self._reset(temp_cs.to_string()) + # temp_cs.add(var != value) + # self._reset(temp_cs.to_string()) self._smtlib.send(f"(assert {translate_to_smtlib(var != value)})") return list(result) @@ -671,7 +671,7 @@ def __init__(self): support_minmax=support_minmax, support_reset=support_reset, support_pushpop=True, - debug=True + debug=True, ) def __autoconfig(self): From 79b42ed12479dc83802d6612f6962d108dbb1cf4 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 12 Jun 2020 13:00:37 -0300 Subject: [PATCH 062/126] fix related to tests --- tests/other/test_smtlibv2.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index ce6ecb377..7ac6a3395 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -36,16 +36,13 @@ def test_related_to(self): # A constraint set and a contraint caught in the act of making related_to fail constraints, constraint = pickle.loads(gzip.open(filename, "rb").read()) - consts = config.get_group("smt") - consts.related_constraints = False Z3Solver.instance().can_be_true.cache_clear() ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) self.assertEqual(ground_truth, False) - consts.related_constraints = True Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, constraint)) + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints.related_to(constraints), constraint)) # Replace new_constraint = Operators.UGE( @@ -54,13 +51,11 @@ def test_related_to(self): ) self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) - consts.related_constraints = False Z3Solver.instance().can_be_true.cache_clear() self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) - consts.related_constraints = True Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) + self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints.related_to(new_constraint), new_constraint)) """ From ec8489ffab90d0bbe0c36b289c7feb546716703a Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 12 Jun 2020 13:14:50 -0300 Subject: [PATCH 063/126] blkn --- tests/other/test_smtlibv2.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 7ac6a3395..c377ddd9e 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -36,13 +36,15 @@ def test_related_to(self): # A constraint set and a contraint caught in the act of making related_to fail constraints, constraint = pickle.loads(gzip.open(filename, "rb").read()) - Z3Solver.instance().can_be_true.cache_clear() ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) self.assertEqual(ground_truth, False) Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints.related_to(constraints), constraint)) + self.assertEqual( + ground_truth, + Z3Solver.instance().can_be_true(constraints.related_to(constraints), constraint), + ) # Replace new_constraint = Operators.UGE( @@ -55,7 +57,10 @@ def test_related_to(self): self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints.related_to(new_constraint), new_constraint)) + self.assertEqual( + ground_truth, + Z3Solver.instance().can_be_true(constraints.related_to(new_constraint), new_constraint), + ) """ From 2113a739b499bddff839ee38670592d043466940 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 12 Jun 2020 15:07:22 -0300 Subject: [PATCH 064/126] Fix bug in test and disable debug messages in Solver --- manticore/core/smtlib/solver.py | 2 +- tests/native/test_linux.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 93120e71d..348ffa6e2 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -671,7 +671,7 @@ def __init__(self): support_minmax=support_minmax, support_reset=support_reset, support_pushpop=True, - debug=True, + debug=False, ) def __autoconfig(self): diff --git a/tests/native/test_linux.py b/tests/native/test_linux.py index a12580192..9f9bd3df9 100644 --- a/tests/native/test_linux.py +++ b/tests/native/test_linux.py @@ -285,7 +285,7 @@ def test_armv7_syscall_openat_concrete(self) -> None: def test_armv7_syscall_openat_symbolic(self) -> None: platform, temp_dir = self._armv7_create_openat_state() try: - platform.current.R0 = BitVecVariable(32, "fd") + platform.current.R0 = platform.constraints.new_bitvec(32, "fd") with self.assertRaises(ConcretizeRegister) as cm: platform.syscall() From c2a21baae07f3247f92da1af05f89111f77a572f Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 12 Jun 2020 18:01:36 -0300 Subject: [PATCH 065/126] smtlib config to disable multiple check-sat in newer z3 --- manticore/core/smtlib/solver.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 348ffa6e2..e0eedba15 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -268,6 +268,7 @@ def __init__( support_reset: bool = False, support_minmax: bool = False, support_pushpop: bool = False, + multiple_check: bool = True, debug: bool = False, ): @@ -294,6 +295,7 @@ def __init__( self._support_minmax = support_minmax self._support_reset = support_reset self._support_pushpop = support_pushpop + self._multiple_check = multiple_check if not self._support_pushpop: setattr(self, "_push", None) @@ -549,9 +551,11 @@ def get_all_values( return list(result) raise SolverError("Timeout") # Sometimes adding a new contraint after a check-sat eats all the mem - # temp_cs.add(var != value) - # self._reset(temp_cs.to_string()) - self._smtlib.send(f"(assert {translate_to_smtlib(var != value)})") + if self._multiple_check: + self._smtlib.send(f"(assert {translate_to_smtlib(var != value)})") + else: + temp_cs.add(var != value) + self._reset(temp_cs.to_string()) return list(result) def _optimize_fancy(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): @@ -663,13 +667,14 @@ def __init__(self): ] command = f"{consts.z3_bin} -t:{consts.timeout * 1000} -memory:{consts.memory} -smt2 -in" - support_minmax, support_reset = self.__autoconfig() + support_minmax, support_reset, multiple_check = self.__autoconfig() super().__init__( command=command, init=init, value_fmt=16, support_minmax=support_minmax, support_reset=support_reset, + multiple_check=multiple_check, support_pushpop=True, debug=False, ) @@ -685,7 +690,11 @@ def __autoconfig(self): support_reset = False else: logger.debug(" Please install Z3 4.4.1 or newer to get optimization support") - return support_minmax, support_reset + + # Certain version of Z3 fails to handle multiple check-sat + # https://gist.github.com/feliam/0f125c00cb99ef05a6939a08c4578902 + multiple_check = self.version < Version(4, 8, 7) + return support_minmax, support_reset, multiple_check def _solver_version(self) -> Version: """ From f87e1e253cbcb9f981ebd25ca1b9a69e78594ee0 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 15 Jun 2020 14:20:10 -0300 Subject: [PATCH 066/126] Disable log test and fix merging poc vs variable migration --- manticore/native/state_merging.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/manticore/native/state_merging.py b/manticore/native/state_merging.py index 2ff9f2e64..1c7b13f61 100644 --- a/manticore/native/state_merging.py +++ b/manticore/native/state_merging.py @@ -32,7 +32,8 @@ def compare_buffers(cs, buffer1, buffer2): if len(buffer1) != len(buffer2): return False for b1, b2 in zip(buffer1, buffer2): - if not SelectedSolver.instance().must_be_true(cs, b1 == b2): + cond = cs.migrate(b1 == b2) + if not SelectedSolver.instance().must_be_true(cs, cond): return False return True @@ -69,7 +70,7 @@ def compare_byte_vals(mem1, mem2, addr, merged_constraint): val2 = mem2.read(addr, 1) # since we only read a single byte value, these lists should only have one entry in them assert len(val1) == 1 and len(val2) == 1 - cond_to_check = val1[0] == val2[0] + cond_to_check = merged_constraint.migrate(val1[0] == val2[0]) if not SelectedSolver.instance().must_be_true(merged_constraint, cond_to_check): return False else: @@ -190,13 +191,15 @@ def merge_cpu(cpu1, cpu2, state, exp1, merged_constraint): if isinstance(val1, BitVec) and isinstance(val2, BitVec): assert val1.size == val2.size if issymbolic(val1) or issymbolic(val2) or val1 != val2: - if SelectedSolver.instance().must_be_true(merged_constraint, val1 != val2): + val1_migrated = merged_constraint.migrate(val1) + val2_migrated = merged_constraint.migrate(val2) + if SelectedSolver.instance().must_be_true(merged_constraint, val1_migrated != val2_migrated): merged_regs.append(reg) if cpu1.regfile.sizeof(reg) == 1: - state.cpu.write_register(reg, Operators.ITE(exp1, val1, val2)) + state.cpu.write_register(reg, Operators.ITE(exp1, val1_migrated, val2_migrated)) else: state.cpu.write_register( - reg, Operators.ITEBV(cpu1.regfile.sizeof(reg), exp1, val1, val2) + reg, Operators.ITEBV(cpu1.regfile.sizeof(reg), exp1, val1_migrated, val2_migrated) ) return merged_regs From 2eb630d39f77729bf4855a77c077a155bc256873 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 15 Jun 2020 14:58:12 -0300 Subject: [PATCH 067/126] blkn --- manticore/native/state_merging.py | 9 +++++++-- manticore/utils/event.py | 14 ++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/manticore/native/state_merging.py b/manticore/native/state_merging.py index 1c7b13f61..332f95b73 100644 --- a/manticore/native/state_merging.py +++ b/manticore/native/state_merging.py @@ -193,13 +193,18 @@ def merge_cpu(cpu1, cpu2, state, exp1, merged_constraint): if issymbolic(val1) or issymbolic(val2) or val1 != val2: val1_migrated = merged_constraint.migrate(val1) val2_migrated = merged_constraint.migrate(val2) - if SelectedSolver.instance().must_be_true(merged_constraint, val1_migrated != val2_migrated): + if SelectedSolver.instance().must_be_true( + merged_constraint, val1_migrated != val2_migrated + ): merged_regs.append(reg) if cpu1.regfile.sizeof(reg) == 1: state.cpu.write_register(reg, Operators.ITE(exp1, val1_migrated, val2_migrated)) else: state.cpu.write_register( - reg, Operators.ITEBV(cpu1.regfile.sizeof(reg), exp1, val1_migrated, val2_migrated) + reg, + Operators.ITEBV( + cpu1.regfile.sizeof(reg), exp1, val1_migrated, val2_migrated + ), ) return merged_regs diff --git a/manticore/utils/event.py b/manticore/utils/event.py index 82268c33c..c6554f814 100644 --- a/manticore/utils/event.py +++ b/manticore/utils/event.py @@ -139,11 +139,17 @@ def _check_event(self, _name): # Wrapper for _publish_impl that also makes sure the event is published from # a class that supports it. # The underscore _name is to avoid naming collisions with callback params - def _publish(self, _name, *args, **kwargs): + def _publish(self, _name, *args, can_raise=False, **kwargs): # only publish if there is at least one subscriber - if _name in self.__sub_events__: - self._check_event(_name) - self._publish_impl(_name, *args, **kwargs) + try: + if _name in self.__sub_events__: + self._check_event(_name) + self._publish_impl(_name, *args, **kwargs) + except Exception as e: + if can_raise: + raise + else: + logger.info("Exception raised at a callback %r", e) # Separate from _publish since the recursive method call to forward an event # shouldn't check the event. From 8818ea3cf12992f54502a54eb16a8fc474ac587f Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 15 Jun 2020 16:55:21 -0300 Subject: [PATCH 068/126] Avoid exception in some callbacks --- manticore/core/manticore.py | 6 +++--- manticore/utils/event.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index c2e0b47a8..92abd8c06 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -525,7 +525,7 @@ def verbosity(level): set_verbosity(level) # State storage - @Eventful.will_did("save_state") + @Eventful.will_did("save_state", can_raise=False) def _save(self, state, state_id=None): """ Store or update a state in secondary storage under state_id. Use a fresh id is None is provided. @@ -538,7 +538,7 @@ def _save(self, state, state_id=None): state._id = self._workspace.save_state(state, state_id=state_id) return state.id - @Eventful.will_did("load_state") + @Eventful.will_did("load_state", can_raise=False) def _load(self, state_id): """ Load the state from the secondary storage @@ -557,7 +557,7 @@ def _load(self, state_id): self.stcache[state_id] = state return state - @Eventful.will_did("remove_state") + @Eventful.will_did("remove_state", can_raise=False) def _remove(self, state_id): """ Remove a state from secondary storage diff --git a/manticore/utils/event.py b/manticore/utils/event.py index c6554f814..13721c8d2 100644 --- a/manticore/utils/event.py +++ b/manticore/utils/event.py @@ -139,7 +139,7 @@ def _check_event(self, _name): # Wrapper for _publish_impl that also makes sure the event is published from # a class that supports it. # The underscore _name is to avoid naming collisions with callback params - def _publish(self, _name, *args, can_raise=False, **kwargs): + def _publish(self, _name, *args, can_raise=True, **kwargs): # only publish if there is at least one subscriber try: if _name in self.__sub_events__: From 27db1b80430189e38ed7064743424dea128071c0 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 15 Jun 2020 17:05:35 -0300 Subject: [PATCH 069/126] can_raise at did_will --- manticore/utils/event.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manticore/utils/event.py b/manticore/utils/event.py index 13721c8d2..2e7e03517 100644 --- a/manticore/utils/event.py +++ b/manticore/utils/event.py @@ -75,15 +75,15 @@ def all_events(cls): return all_evts @staticmethod - def will_did(name): + def will_did(name, can_raise=False): """Pre/pos emiting signal""" def deco(func): @functools.wraps(func) def newFunction(self, *args, **kw): - self._publish(f"will_{name}", *args, **kw) + self._publish(f"will_{name}", *args, can_raise=can_raise, **kw) result = func(self, *args, **kw) - self._publish(f"did_{name}", result) + self._publish(f"did_{name}", result, can_raise=can_raise) return result return newFunction From 937d988d40144a13e732804888c394fea98d0791 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 15 Jun 2020 17:41:20 -0300 Subject: [PATCH 070/126] blkn --- manticore/core/smtlib/expression.py | 1 - 1 file changed, 1 deletion(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 7206c3d53..3821ca1df 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1416,4 +1416,3 @@ def true_value(self): @property def false_value(self): return self.operands[2] - From 6e161d191f2e22767d4343857569a69e115a69d1 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 14 Jul 2020 14:36:28 -0300 Subject: [PATCH 071/126] merge --- manticore/core/smtlib/visitors.py | 41 ++++++++----------------------- manticore/platforms/evm.py | 37 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 9bd8a3bac..37da125e2 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -73,8 +73,7 @@ def visit(self, node, use_fixed_point=False): :param use_fixed_point: if True, it runs _methods until a fixed point is found :type use_fixed_point: Bool """ - if isinstance(node, ArrayProxy): - node = node.array + cache = self._cache visited = set() stack = [] @@ -111,7 +110,7 @@ def visit(self, node, use_fixed_point=False): @staticmethod def _rebuild(expression, operands): if isinstance(expression, Operation): - if not expression.operands is operands: + if any(operands[i] is not expression.operands[i] for i in range(len(operands))): aux = copy.copy(expression) aux._operands = operands return aux @@ -170,14 +169,9 @@ def __init__(self, *args, **kwargs): def visit_Expression(self, expression): return 1 - def _visit_operation(self, expression, *operands): + def visit_Operation(self, expression, *operands): return 1 + max(operands) - visit_ArraySelect = _visit_operation - visit_ArrayOperation = _visit_operation - visit_BoolOperation = _visit_operation - visit_BitVecOperation = _visit_operation - def get_depth(exp): visitor = GetDepth() @@ -220,7 +214,7 @@ def _method(self, expression, *args): return return - def _visit_operation(self, expression, *operands): + def visit_Operation(self, expression, *operands): self._print(expression.__class__.__name__, expression) self.indent += 2 if self.depth is None or self.indent < self.depth * 2: @@ -231,10 +225,6 @@ def _visit_operation(self, expression, *operands): self.indent -= 2 return "" - visit_ArraySelect = _visit_operation - visit_ArrayOperation = _visit_operation - visit_BoolOperation = _visit_operation - visit_BitVecOperation = _visit_operation def visit_BitVecExtract(self, expression): self._print( @@ -392,7 +382,7 @@ def visit_BoolAnd(self, expression, a, b): if isinstance(b, Constant) and b.value == True: return a - def _visit_operation(self, expression, *operands): + def visit_Operation(self, expression, *operands): """ constant folding, if all operands of an expression are a Constant do the math """ operation = self.operations.get(type(expression), None) if operation is not None and all(isinstance(o, Constant) for o in operands): @@ -403,15 +393,9 @@ def _visit_operation(self, expression, *operands): isinstance(expression, Bool) return BoolConstant(value, taint=expression.taint) else: - if any(operands[i] is not expression.operands[i] for i in range(len(operands))): - expression = self._rebuild(expression, operands) + expression = self._rebuild(expression, operands) return expression - visit_ArraySelect = _visit_operation - visit_ArrayOperation = _visit_operation - visit_BoolOperation = _visit_operation - visit_BitVecOperation = _visit_operation - constant_folder_simplifier_cache = CacheDict(max_size=150000, flush_perc=25) @@ -439,18 +423,13 @@ def _changed(expression, operands): arity = len(operands) return any(operands[i] is not expression.operands[i] for i in range(arity)) - def _visit_operation(self, expression, *operands): + def visit_Operation(self, expression, *operands): """ constant folding, if all operands of an expression are a Constant do the math """ + expression = self._rebuild(expression, operands) if all(isinstance(o, Constant) for o in operands): expression = constant_folder(expression) - if self._changed(expression, operands): - expression = self._rebuild(expression, operands) return expression - visit_ArrayOperation = _visit_operation - visit_BoolOperation = _visit_operation - visit_BitVecOperation = _visit_operation - def visit_BitVecZeroExtend(self, expression, *operands): if self._changed(expression, operands): return BitVecZeroExtend(expression.size, *operands, taint=expression.taint) @@ -784,10 +763,10 @@ def visit_ArraySelect(self, expression, *operands): """ ArraySelect (ArrayStore((ArrayStore(x0,v0) ...),xn, vn), x0) -> v0 """ - return self._visit_operation(expression, *operands) + return None arr, index = operands if isinstance(arr, ArrayVariable): - return self._visit_operation(expression, *operands) + return None if isinstance(index, BitVecConstant): ival = index.value diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index db5c70071..4aaabdb7d 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -2140,6 +2140,43 @@ def CREATE(self, value, offset, size): tx = self.world.last_transaction # At this point last and current tx are the same. return tx.return_value + def CREATE2_gas(self, value, offset, size): + return self._get_memfee(offset, size) + + @transact + def CREATE2(self, endowment, memory_start, memory_length, salt): + """Create a new account with associated code""" + + data = self.read_buffer(offset, size) + keccak_init = self.world.symbolic_function(globalsha3, data) + caller = ArrayProxy(msg.caller).read_BE(0,20) + salt = ArrayProxy(salt).read_BE(0, 32) + address = self.world.symbolic_function(b"\xff" + caller + salt + keccak_init) & ((1<<0x20)-1) + + self.world.start_transaction( + "CREATE", + address, + data=data, + caller=self.address, + value=value, + gas=self.gas, + ) + + raise StartTx() + + @CREATE.pos # type: ignore + def CREATE2(self, value, offset, size): + """Create a new account with associated code""" + tx = self.world.last_transaction # At this point last and current tx are the same. + address = tx.address + if tx.result == "RETURN": + self.world.set_code(tx.address, tx.return_data) + else: + self.world.delete_account(address) + address = 0 + return address + + def CALL_gas(self, wanted_gas, address, value, in_offset, in_size, out_offset, out_size): """ Dynamic gas for CALL instruction. _arguably turing complete in itself_ """ GCALLVALUE = 9000 From 685beed6563fad472e0ea16a674c2db53eb4580e Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 15 Jul 2020 23:32:15 -0300 Subject: [PATCH 072/126] merge --- examples/script/concolic.py | 6 +- manticore/core/smtlib/__init__.py | 2 +- manticore/core/smtlib/constraints.py | 17 +- manticore/core/smtlib/expression.py | 1387 ++++++++++++++------------ manticore/core/smtlib/operators.py | 99 +- manticore/core/smtlib/solver.py | 43 +- manticore/core/smtlib/visitors.py | 486 +++++---- manticore/core/state.py | 4 +- manticore/ethereum/abi.py | 58 +- manticore/ethereum/manticore.py | 90 +- manticore/native/cpu/abstractcpu.py | 4 +- manticore/native/cpu/arm.py | 6 +- manticore/native/cpu/bitwise.py | 48 +- manticore/native/cpu/register.py | 4 +- manticore/native/cpu/x86.py | 14 +- manticore/native/memory.py | 14 +- manticore/native/models.py | 8 +- manticore/native/state_merging.py | 4 +- manticore/platforms/evm.py | 153 +-- manticore/wasm/executor.py | 4 +- manticore/wasm/structure.py | 10 +- manticore/wasm/types.py | 10 +- tests/ethereum/test_general.py | 66 +- tests/ethereum/test_regressions.py | 44 +- tests/native/test_cpu_manual.py | 8 +- tests/native/test_driver.py | 4 +- tests/native/test_linux.py | 2 +- tests/native/test_register.py | 8 +- tests/native/test_state.py | 12 +- tests/other/test_smtlibv2.py | 392 +++++++- 30 files changed, 1723 insertions(+), 1284 deletions(-) diff --git a/examples/script/concolic.py b/examples/script/concolic.py index ce3a90292..6be63240b 100755 --- a/examples/script/concolic.py +++ b/examples/script/concolic.py @@ -75,12 +75,12 @@ def flip(constraint): assert len(equal.operands) == 2 # assume they are the equal -> ite form that we produce on standard branches ite, forcepc = equal.operands - if not (isinstance(ite, BitVecITE) and isinstance(forcepc, BitVecConstant)): + if not (isinstance(ite, BitVecITE) and isinstance(forcepc, BitvecConstant)): return constraint - assert isinstance(ite, BitVecITE) and isinstance(forcepc, BitVecConstant) + assert isinstance(ite, BitVecITE) and isinstance(forcepc, BitvecConstant) assert len(ite.operands) == 3 cond, iifpc, eelsepc = ite.operands - assert isinstance(iifpc, BitVecConstant) and isinstance(eelsepc, BitVecConstant) + assert isinstance(iifpc, BitvecConstant) and isinstance(eelsepc, BitvecConstant) equal._operands = (equal.operands[0], eelsepc if forcepc.value == iifpc.value else iifpc) diff --git a/manticore/core/smtlib/__init__.py b/manticore/core/smtlib/__init__.py index 0128c94f6..cc75d1150 100644 --- a/manticore/core/smtlib/__init__.py +++ b/manticore/core/smtlib/__init__.py @@ -1,4 +1,4 @@ -from .expression import Expression, Bool, BitVec, Array, BitVecConstant, issymbolic # noqa +from .expression import Expression, Bool, Bitvec, Array, BitvecConstant, issymbolic # noqa from .constraints import ConstraintSet # noqa from .solver import * # noqa from . import operators as Operators # noqa diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index b96a608f8..feecd3ddd 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -6,12 +6,12 @@ from ...exceptions import SmtlibError from .expression import ( Expression, - BitVecVariable, + BitvecVariable, BoolVariable, ArrayVariable, Array, Bool, - BitVec, + Bitvec, BoolConstant, ArrayProxy, BoolEqual, @@ -185,13 +185,11 @@ def to_string(self, replace_constants: bool = False) -> str: if ( isinstance(expression, BoolEqual) and isinstance(expression.operands[0], Variable) - and isinstance(expression.operands[1], (*Variable, *Constant)) + and not isinstance(expression.operands[1], (Variable, Constant)) ): constant_bindings[expression.operands[0]] = expression.operands[1] - tmp = set() result = "" - translator = TranslatorSmtlib(use_bindings=False) for constraint in constraints: if replace_constants: @@ -210,6 +208,7 @@ def to_string(self, replace_constants: bool = False) -> str: def _declare(self, var): """ Declare the variable `var` """ + print ("declaring", var) if var.name in self._declarations: raise ValueError("Variable already declared") self._declarations[var.name] = var @@ -320,7 +319,7 @@ def migrate(self, expression, name_migration_map=None): # Create and declare a new variable of given type if isinstance(foreign_var, Bool): new_var = self.new_bool(name=migrated_name) - elif isinstance(foreign_var, BitVec): + elif isinstance(foreign_var, Bitvec): new_var = self.new_bitvec(foreign_var.size, name=migrated_name) elif isinstance(foreign_var, Array): # Note that we are discarding the ArrayProxy encapsulation @@ -366,7 +365,7 @@ def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False) :param name: try to assign name to internal variable representation, if not unique, a numeric nonce will be appended :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True - :return: a fresh BitVecVariable + :return: a fresh BitvecVariable """ if size <= 0: raise ValueError(f"Bitvec size ({size}) can't be equal to or less than 0") @@ -377,7 +376,7 @@ def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False) name = self._make_unique_name(name) if not avoid_collisions and name in self._declarations: raise ValueError(f"Name {name} already used") - var = BitVecVariable(size=size, name=name, taint=taint) + var = BitvecVariable(size=size, name=name, taint=taint) return self._declare(var) def new_array( @@ -409,7 +408,7 @@ def new_array( raise ValueError(f"Name {name} already used") var = self._declare( ArrayVariable( - index_bits, index_max, value_bits, name=name, taint=taint, default=default + index_size=index_bits, length=index_max, value_size=value_bits, name=name, taint=taint, default=default ) ) return ArrayProxy(var) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index fcb3a66fa..2e4687a16 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1,5 +1,4 @@ from functools import reduce -from ...exceptions import SmtlibError import uuid import re @@ -7,30 +6,73 @@ from typing import Union, Optional, Dict, Tuple, List -class ExpressionException(SmtlibError): +class ExpressionError(Exception): """ Expression exception """ - pass -class Expression(object): - """ Abstract taintable Expression. """ +class XSlotted(type): + """ Metaclass that will propagate slots on multi-inheritance classes + + class Base(object, metaclass=XSlotted, abstract=True): + pass + + class A(Base, abstract=True): + __xslots__ = ('a',) + pass + + class B(Base, abstract=True): + __xslots__ = ('b',) + pass + + class C(A, B): + pass + + class X(object): + __slots__ = ('a', 'b') + + c = C() + c.a = 1 + c.b = 2 + + x = X() + x.a = 1 + x.b = 2 + + import sys + print (sys.getsizeof(c),sys.getsizeof(x)) + + """ + + def __new__(cls, clsname, bases, attrs, abstract=False): + xslots = set(attrs.get("__xslots__", ())) + for base in bases: + xslots = xslots.union(getattr(base, "__xslots__", ())) + attrs["__xslots__"] : Tuple[str, ...] = tuple(xslots) + if abstract: + attrs["__slots__"] = () + else: + attrs["__slots__"]: Tuple[str, ...] = attrs["__xslots__"] + + return super().__new__(cls, clsname, bases, attrs) + - __slots__: Tuple[str, ...] = () - __xslots__: Tuple[str, ...] = ("_taint",) +class Expression(object, metaclass=XSlotted, abstract=True): + """ Abstract taintable Expression. """ + __xslots__ : Tuple[str, ...] = ("_taint",) def __init__(self, taint: Union[tuple, frozenset] = ()): """ An abstrac Unmutable Taintable Expression :param taint: A frozenzset """ - super().__init__() self._taint = frozenset(taint) + super().__init__() def __repr__(self): - return "<{:s} at {:x}{:s}>".format(type(self).__name__, id(self), self.taint and "-T" or "") + return "<{:s} at {:x}{:s}>".format(type(self).__name__, id(self), self._taint and "-T" or "") @property def is_tainted(self): @@ -40,80 +82,32 @@ def is_tainted(self): def taint(self): return self._taint -def issymbolic(value) -> bool: - """ - Helper to determine whether an object is symbolic (e.g checking - if data read from memory is symbolic) - - :param object value: object to check - :return: whether `value` is symbolic - :rtype: bool - """ - return isinstance(value, (Expression, ArrayProxy)) - - -def istainted(arg, taint=None): - """ - Helper to determine whether an object if tainted. - :param arg: a value or Expression - :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value. - """ - - if not issymbolic(arg): - return False - if taint is None: - return len(arg.taint) != 0 - for arg_taint in arg.taint: - m = re.match(taint, arg_taint, re.DOTALL | re.IGNORECASE) - if m: - return True - return False - - -def get_taints(arg, taint=None): - """ - Helper to list an object taints. - :param arg: a value or Expression - :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value. - """ - - if not issymbolic(arg): - return - for arg_taint in arg.taint: - if taint is not None: - m = re.match(taint, arg_taint, re.DOTALL | re.IGNORECASE) - if m: - yield arg_taint - else: - yield arg_taint - return - + @property + def operands(self): + return () -def taint_with(arg, *taints, value_bits=256, index_bits=256): - """ - Helper to taint a value. - :param arg: a value or Expression - :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value. - """ - tainted_fset = frozenset(tuple(taints)) - if not issymbolic(arg): - if isinstance(arg, int): - arg = BitVecConstant(value_bits, arg) - arg._taint = tainted_fset - else: - raise ValueError("type not supported") + def __getstate__(self): + state = {} + for attr in self.__slots__: + if attr.startswith("__"): + continue + state[attr] = getattr(self, attr) + return state - else: - arg = copy.copy(arg) - arg._taint |= tainted_fset + def __setstate__(self, state): + for attr in self.__slots__: + if attr.startswith("__"): + continue + setattr(self, attr, state[attr]) - return arg + def __hash__(self): + return object.__hash__(self) -class Variable(Expression): +class Variable(Expression, abstract=True): """ Variable is an Expression that has a name """ - __slots__ = () - __xslots__: Tuple[str, ...] = Expression.__xslots__ + ("_name",) + + __xslots__:Tuple[str, ...] = ("_name",) def __init__(self, name: str, **kwargs): """ Variable is an Expression that has a name @@ -122,10 +116,6 @@ def __init__(self, name: str, **kwargs): super().__init__(**kwargs) self._name = name - @property - def declaration(self): - pass - @property def name(self): return self._name @@ -134,11 +124,16 @@ def __repr__(self): return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) -class Constant(Expression): - __slots__ = () - __xslots__: Tuple[str, ...] = Expression.__xslots__ + ("_value",) + def __hash__(self): + return object.__hash__(self) + + +class Constant(Expression, abstract=True): + """ Constants expressions have a concrete python value. """ + + __xslots__:Tuple[str, ...] = ("_value",) - def __init__(self, value, **kwargs): + def __init__(self, value: Union[bool, int, bytes, List[int]], **kwargs): """ A constant expression has a value :param value: The constant value @@ -151,32 +146,43 @@ def value(self): return self._value -class Operation(Expression): - __slots__ = () - __xslots__: Tuple[str, ...] = Expression.__xslots__ + ("_operands",) + def __hash__(self): + return object.__hash__(self) + +class Operation(Expression, abstract=True): + """ Operation expressions contain operands which are also Expressions. """ + + __xslots__:Tuple[str, ...] = ("_operands",) - def __init__(self, operands: Tuple[Expression, ...], taint=None, **kwargs): + def __init__(self, operands: Tuple[Expression, ...], **kwargs): """ An operation has operands :param operands: A tuple of expression operands """ + taint = kwargs.get('taint') + assert isinstance(operands, tuple) + print ("Operation of operands", type(self) ,tuple(map(type,operands))) self._operands = operands - # If taint was not forced by a keyword argument, calculate default if taint is None: - taint = reduce(lambda x, y: x.union(y.taint), operands, frozenset()) - - super().__init__(taint=taint, **kwargs) + operands_taints = map(lambda x: x.taint, operands) + taint = reduce(lambda x, y: x.union(y), operands_taints, frozenset()) + kwargs['taint'] = taint + super().__init__(**kwargs) @property def operands(self): return self._operands + def __hash__(self): + return object.__hash__(self) ############################################################################### # Booleans -class Bool(Expression): - __slots__: Tuple[str, ...] = tuple() +class Bool(Expression, abstract=True): + """Bool expression represent symbolic value of truth""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) def cast(self, value: Union["Bool", int, bool], **kwargs) -> Union["BoolConstant", "Bool"]: if isinstance(value, Bool): @@ -190,6 +196,9 @@ def __invert__(self): return BoolNot(self) def __eq__(self, other): + # A class that overrides __eq__() and does not define __hash__() + # will have its __hash__() implicitly set to None. + #import pdb; pdb.set_trace() return BoolEqual(self, self.cast(other)) def __hash__(self): @@ -216,84 +225,76 @@ def __ror__(self, other): def __rxor__(self, other): return BoolXor(self.cast(other), self) + def __bool__(self): + raise NotImplementedError + """ def __bool__(self): # try to be forgiving. Allow user to use Bool in an IF sometimes from .visitors import simplify - x = simplify(self) if isinstance(x, Constant): return x.value raise NotImplementedError("__bool__ for Bool") + """ class BoolVariable(Bool, Variable): - __slots__ = Bool.__xslots__ + Variable.__xslots__ - __xslots__ = Bool.__xslots__ + Variable.__xslots__ -''' - @property - def declaration(self): - return f"(declare-fun {self.name} () Bool)" -''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) -class BoolConstant(Bool, Constant): - __slots__ = Bool.__xslots__ + Constant.__xslots__ - def __init__(self, value, **kwargs): - super().__init__(value=value, **kwargs) +class BoolConstant(Bool, Constant): + def __init__(self, value: bool, **kwargs): + super().__init__(value=bool(value), **kwargs) def __bool__(self): return self._value + def __eq__(self, other): + # A class that overrides __eq__() and does not define __hash__() + # will have its __hash__() implicitly set to None. + if self.taint: + return super().__eq__(other) + return self.value == other -class BoolOperation(Operation, Bool): - __slots__ = () - __xslots__ = Operation.__xslots__ + Bool.__xslots__ + def __hash__(self): + return object.__hash__(self) -class BoolNot(BoolOperation): - __slots__ = BoolOperation.__xslots__ - __xslots__ = BoolOperation.__xslots__ +class BoolOperation(Bool, Operation, abstract=True): + """ It's an operation that results in a Bool """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) - def __init__(self, *operands, **kwargs): - super().__init__(operands=operands, **kwargs) +class BoolNot(BoolOperation): + def __init__(self, operand: Bool, **kwargs): + super().__init__(operands=(operand,), **kwargs) class BoolAnd(BoolOperation): - __slots__ = BoolOperation.__xslots__ - __xslots__ = BoolOperation.__xslots__ - - def __init__(self, *operands, **kwargs): - super().__init__(operands=operands, **kwargs) + def __init__(self, operanda: Bool, operandb: Bool, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) class BoolOr(BoolOperation): - __slots__ = BoolOperation.__xslots__ - __xslots__ = BoolOperation.__xslots__ - - def __init__(self, *args, **kwargs): - super().__init__(operands=args, **kwargs) + def __init__(self, operanda: Bool, operandb: Bool, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) class BoolXor(BoolOperation): - __slots__ = BoolOperation.__xslots__ - __xslots__ = BoolOperation.__xslots__ - - def __init__(self, *args, **kwargs): - super().__init__(operands=args, **kwargs) + def __init__(self, operanda: Bool, operandb: Bool, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) class BoolITE(BoolOperation): - __slots__ = BoolOperation.__xslots__ - __xslots__ = BoolOperation.__xslots__ - - - def __init__(self, cond: "Bool", true: "Bool", false: "Bool", **kwargs): + def __init__(self, cond: Bool, true: Bool, false: Bool, **kwargs): super().__init__(operands=(cond, true, false), **kwargs) -class BitVec(Expression): - __slots__ = () - __xslots__: Tuple[str, ...] = Expression.__xslots__ + ("_size",) +class Bitvec(Expression, abstract=True): + """ Bitvector expressions have a fixed bit size """ + + __xslots__:Tuple[str, ...] = ("_size",) def __init__(self, size: int, **kwargs): """ This is bit vector expression @@ -316,48 +317,50 @@ def signmask(self): return 1 << (self.size - 1) def cast( - self, value: Union["BitVec", str, int, bytes], **kwargs - ) -> Union["BitVecConstant", "BitVec"]: - if isinstance(value, BitVec): + self, value: Union["Bitvec", str, int, bytes], **kwargs + ) -> "Bitvec": + if isinstance(value, Bitvec): assert value.size == self.size return value if isinstance(value, (str, bytes)) and len(value) == 1: + print ("AAAAAAAAAA"*99) value = ord(value) # Try to support not Integral types that can be casted to int if not isinstance(value, int): + print (value) value = int(value) # FIXME? Assert it fits in the representation - return BitVecConstant(self.size, value, **kwargs) + return BitvecConstant(self.size, value, **kwargs) def __add__(self, other): - return BitVecAdd(self, self.cast(other)) + return BitvecAdd(self, self.cast(other)) def __sub__(self, other): - return BitVecSub(self, self.cast(other)) + return BitvecSub(self, self.cast(other)) def __mul__(self, other): - return BitVecMul(self, self.cast(other)) + return BitvecMul(self, self.cast(other)) def __mod__(self, other): - return BitVecMod(self, self.cast(other)) + return BitvecMod(self, self.cast(other)) # object.__divmod__(self, other) # object.__pow__(self, other[, modulo]) def __lshift__(self, other): - return BitVecShiftLeft(self, self.cast(other)) + return BitvecShiftLeft(self, self.cast(other)) def __rshift__(self, other): - return BitVecShiftRight(self, self.cast(other)) + return BitvecShiftRight(self, self.cast(other)) def __and__(self, other): - return BitVecAnd(self, self.cast(other)) + return BitvecAnd(self, self.cast(other)) def __xor__(self, other): - return BitVecXor(self, self.cast(other)) + return BitvecXor(self, self.cast(other)) def __or__(self, other): - return BitVecOr(self, self.cast(other)) + return BitvecOr(self, self.cast(other)) # The division operator (/) is implemented by these methods. The # __truediv__() method is used when __future__.division is in effect, @@ -366,10 +369,10 @@ def __or__(self, other): # TypeError will be raised instead. def __div__(self, other): - return BitVecDiv(self, self.cast(other)) + return BitvecDiv(self, self.cast(other)) def __truediv__(self, other): - return BitVecDiv(self, self.cast(other)) + return BitvecDiv(self, self.cast(other)) def __floordiv__(self, other): return self / other @@ -383,43 +386,43 @@ def __floordiv__(self, other): # y.__rsub__(x) is called if x.__sub__(y) returns NotImplemented. def __radd__(self, other): - return BitVecAdd(self.cast(other), self) + return BitvecAdd(self.cast(other), self) def __rsub__(self, other): - return BitVecSub(self.cast(other), self) + return BitvecSub(self.cast(other), self) def __rmul__(self, other): - return BitVecMul(self.cast(other), self) + return BitvecMul(self.cast(other), self) def __rmod__(self, other): - return BitVecMod(self.cast(other), self) + return BitvecMod(self.cast(other), self) def __rtruediv__(self, other): - return BitVecDiv(self.cast(other), self) + return BitvecDiv(self.cast(other), self) def __rdiv__(self, other): - return BitVecDiv(self.cast(other), self) + return BitvecDiv(self.cast(other), self) # object.__rdivmod__(self, other) # object.__rpow__(self, other) def __rlshift__(self, other): - return BitVecShiftLeft(self.cast(other), self) + return BitvecShiftLeft(self.cast(other), self) def __rrshift__(self, other): - return BitVecShiftRight(self.cast(other), self) + return BitvecShiftRight(self.cast(other), self) def __rand__(self, other): - return BitVecAnd(self.cast(other), self) + return BitvecAnd(self.cast(other), self) def __rxor__(self, other): - return BitVecXor(self.cast(other), self) + return BitvecXor(self.cast(other), self) def __ror__(self, other): - return BitVecOr(self.cast(other), self) + return BitvecOr(self.cast(other), self) def __invert__(self): - return BitVecXor(self, self.cast(self.mask)) + return BitvecXor(self, self.cast(self.mask)) # These are the so-called "rich comparison" methods, and are called # for comparison operators in preference to __cmp__() below. The @@ -433,12 +436,14 @@ def __invert__(self): # x>=y calls x.__ge__(y). def __lt__(self, other): - return LessThan(self, self.cast(other)) + return BoolLessThan(operanda=self, operandb=self.cast(other)) def __le__(self, other): - return LessOrEqual(self, self.cast(other)) + return BoolLessOrEqualThan(self, self.cast(other)) def __eq__(self, other): + # A class that overrides __eq__() and does not define __hash__() + # will have its __hash__() implicitly set to None. return BoolEqual(self, self.cast(other)) def __hash__(self): @@ -448,82 +453,67 @@ def __ne__(self, other): return BoolNot(BoolEqual(self, self.cast(other))) def __gt__(self, other): - return GreaterThan(self, self.cast(other)) + return BoolGreaterThan(self, self.cast(other)) def __ge__(self, other): - return GreaterOrEqual(self, self.cast(other)) + return BoolGreaterOrEqualThan(self, self.cast(other)) def __neg__(self): - return BitVecNeg(self) + return BitvecNeg(self) # Unsigned comparisons def ugt(self, other): - return UnsignedGreaterThan(self, self.cast(other)) + return BoolUnsignedGreaterThan(self, self.cast(other)) def uge(self, other): - return UnsignedGreaterOrEqual(self, self.cast(other)) + return BoolUnsignedGreaterOrEqualThan(self, self.cast(other)) def ult(self, other): - return UnsignedLessThan(self, self.cast(other)) + return BoolUnsignedLessThan(self, self.cast(other)) def ule(self, other): - return UnsignedLessOrEqual(self, self.cast(other)) + return BoolUnsignedLessOrEqualThan(self, self.cast(other)) def udiv(self, other): - return BitVecUnsignedDiv(self, self.cast(other)) + return BitvecUnsignedDiv(self, self.cast(other)) def rudiv(self, other): - return BitVecUnsignedDiv(self.cast(other), self) + return BitvecUnsignedDiv(self.cast(other), self) def sdiv(self, other): - return BitVecDiv(self, self.cast(other)) + return BitvecDiv(self, self.cast(other)) def rsdiv(self, other): - return BitVecDiv(self.cast(other), self) + return BitvecDiv(self.cast(other), self) def srem(self, other): - return BitVecRem(self, self.cast(other)) + return BitvecRem(self, self.cast(other)) def rsrem(self, other): - return BitVecRem(self.cast(other), self) + return BitvecRem(self.cast(other), self) def urem(self, other): - return BitVecUnsignedRem(self, self.cast(other)) + return BitvecUnsignedRem(self, self.cast(other)) def rurem(self, other): - return BitVecUnsignedRem(self.cast(other), self) + return BitvecUnsignedRem(self.cast(other), self) def sar(self, other): - return BitVecArithmeticShiftRight(self, self.cast(other)) + return BitvecArithmeticShiftRight(self, self.cast(other)) def sal(self, other): - return BitVecArithmeticShiftLeft(self, self.cast(other)) + return BitvecArithmeticShiftLeft(self, self.cast(other)) def Bool(self): return self != 0 -class BitVecVariable(BitVec, Variable): - __slots__ = BitVec.__xslots__ + Variable.__xslots__ - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - @property - def name(self): - return self._name - +class BitvecVariable(Bitvec, Variable): def __repr__(self): return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) - @property - def declaration(self): - return f"(declare-fun {self.name} () (_ BitVec {self.size}))" - - -class BitVecConstant(BitVec, Constant): - __slots__ = BitVec.__xslots__ + Constant.__xslots__ +class BitvecConstant(Bitvec, Constant): def __init__(self, size: int, value: int, **kwargs): value &= (1 << size) - 1 super().__init__(size=size, value=value, **kwargs) @@ -531,17 +521,8 @@ def __init__(self, size: int, value: int, **kwargs): def __bool__(self): return self.value != 0 - def __eq__(self, other): - if self.taint: - return super().__eq__(other) - return self.value == other - - def __hash__(self): - return super().__hash__() - - @property - def value(self): - return self._value + def __int__(self): + return self.value @property def signed_value(self): @@ -550,307 +531,390 @@ def signed_value(self): else: return self._value + def __eq__(self, other): + if self.taint or isinstance(other, Expression) and other.taint: + return super().__eq__(other) + print (self.value, other) + return self.value == other + + def __hash__(self): + return object.__hash__(self) -class BitVecOperation(BitVec, Operation): - __xslots__ = BitVec.__xslots__ + Operation.__xslots__ - __slots__ = () +class BitvecOperation(Bitvec, Operation, abstract=True): + """ Operations that result in a Bitvec """ + pass -class BitVecAdd(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecAdd(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecSub(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecSub(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecMul(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecMul(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecDiv(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecUnsignedDiv(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecDiv(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecMod(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecUnsignedDiv(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecRem(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecUnsignedRem(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecMod(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecShiftLeft(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecRem(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecShiftRight(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecArithmeticShiftLeft(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecUnsignedRem(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecArithmeticShiftRight(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecShiftLeft(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecAnd(BitvecOperation): + def __init__(self, operanda, operandb, *args, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecOr(BitvecOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, *args, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecShiftRight(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecXor(BitvecOperation): + def __init__(self, operanda, operandb, **kwargs): + super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitVecArithmeticShiftLeft(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecNot(BitvecOperation): + def __init__(self, operanda, **kwargs): + super().__init__(size=operanda.size, operands=(operanda,), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BitvecNeg(BitvecOperation): + def __init__(self, operanda, **kwargs): + super().__init__(size=operanda.size, operands=(operanda,), **kwargs) -class BitVecArithmeticShiftRight(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +# Comparing two bitvectors results in a Bool +class BoolLessThan(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) -class BitVecAnd(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BoolLessOrEqualThan(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, *args, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BoolEqual(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + assert isinstance(operanda, Expression) + assert isinstance(operandb, Expression) + super().__init__(operands=(operanda, operandb), **kwargs) -class BitVecOr(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a: BitVec, b: BitVec, *args, **kwargs): - assert a.size == b.size - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BoolGreaterThan(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) -class BitVecXor(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BoolGreaterOrEqualThan(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, *args, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(size=a.size, operands=(a, b), **kwargs) +class BoolUnsignedLessThan(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) -class BitVecNot(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, **kwargs): - super().__init__(size=a.size, operands=(a,), **kwargs) +class BoolUnsignedLessOrEqualThan(BoolOperation): + def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) -class BitVecNeg(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BoolUnsignedGreaterThan(BoolOperation): + def __init__(self, operanda, operandb, *args, **kwargs): + super().__init__(operands=(operanda, operandb), **kwargs) - def __init__(self, a, **kwargs): - super().__init__(size=a.size, operands=(a,), **kwargs) +class BoolUnsignedGreaterOrEqualThan(BoolOperation): + def __init__(self, operanda, operandb, **kwargs): + super(BoolUnsignedGreaterOrEqualThan, self).__init__( + operands=(operanda, operandb), **kwargs + ) -# Comparing two bitvectors results in a Bool -class LessThan(BoolOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, *args, **kwargs): - super().__init__(operands=(a, b), **kwargs) +############################################################################### +# Array BV32 -> BV8 or BV64 -> BV8 +class Array(Expression, abstract=True): + """ And Array expression is a mapping from bitvector to bitvectors + """ + @property + def index_size(self): + """ The bit size of the index part. Must be overloaded by a more specific class""" + raise NotImplementedError + @property + def value_size(self): + """ The bit size of the value part. Must be overloaded by a more specific class""" + raise NotImplementedError + + @property + def index_max(self): + """ Max allowed index. Must be overloaded by a more specific class""" + raise NotImplementedError -class LessOrEqual(BoolOperation): - __slots__ = BitVecOperation.__xslots__ + def get(self, index): + """ Gets an element from the Array """ + raise NotImplementedError - def __init__(self, a, b, *args, **kwargs): - super().__init__(operands=(a, b), **kwargs) + def store(self, index, value): + raise NotImplementedError + ## following methods are implementes on top of the abstract methods ^ + def cast(self, possible_array): + """ Builds an Array from a bytes or bytearray""" + # FIXME This should be related to a constrainSet + arr = ArrayVariable( + index_size=self.index_size, length=len(possible_array), value_size=self.value_size, name="cast{}".format(uuid.uuid1()) + ) + for pos, byte in enumerate(possible_array): + arr = arr.store(pos, byte) + return arr -class BoolEqual(BoolOperation): - __slots__ = BitVecOperation.__xslots__ + def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: + """ Forgiving casting method that will translate compatible values into + a complant BitVec for indexing""" + if isinstance(index, int): + return BitvecConstant(self.index_size, index) + if index.size != self.index_size: + raise ValueError + + return simplify(index) + #return index + + def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: + """ Forgiving casting method that will translate compatible values into + a complant BitVec to ve used as a value""" + if not isinstance(value, (Bitvec, bytes, int)): + raise TypeError + if isinstance(value, Bitvec): + if value.size != self.value_size: + raise ValueError + return value + if isinstance(value, bytes) and len(value) == 1: + value = ord(value) + if not isinstance(value, int): + print (value, type(value)) + value = int(value) + return BitvecConstant(self.value_size, value) + + def __len__(self): + print (self.index_max) + return self.index_max+1 + + def select(self, index): + return self.get(index) + + def write(self, offset, buf): + """ Creates a new Array instance by writing buf at offset """ + arr = self + for i, val in enumerate(buf): + arr = arr.store(offset + i, val) + return arr - def __init__(self, a, b, *args, **kwargs): - if isinstance(a, BitVec) or isinstance(b, BitVec): - assert a.size == b.size - super().__init__(operands=(a, b), **kwargs) + def read(self, offset, size): + return ArraySlice(self, offset=offset, size=size) + + def __getitem__(self, index): + if isinstance(index, slice): + start, stop, size = self._fix_slice(index) + return self.read(start, size) + return self.select(index) + + def __iter__(self): + for i in range(len(self)): + yield self[i] + + def __eq__(self, other): + # FIXME taint + def compare_buffers(a, b): + print (type(a)) + if len(a) != len(b): + return BoolConstant(False) + cond = BoolConstant(True) + for i in range(len(a)): + cond = BoolAnd(cond.cast(a[i] == b[i]), cond) + if cond is BoolConstant(False): + return BoolConstant(False) + return cond + return compare_buffers(self, other) -class GreaterThan(BoolOperation): - __slots__ = BitVecOperation.__xslots__ + def __ne__(self, other): + return BoolNot(self == other) - def __init__(self, a, b, *args, **kwargs): - super().__init__(operands=(a, b), **kwargs) + def _fix_slice(self, index: slice): + """Used to calculate the size of slices""" + stop, start = index.stop, index.start + if start is None: + start = 0 + if stop is None: + stop = len(self) + size = stop - start + if isinstance(size, Bitvec): + from .visitors import simplify + size = simplify(size) + else: + size = BitvecConstant(self.index_size, size) + assert isinstance(size, BitvecConstant) + return start, stop, size.value -class GreaterOrEqual(BoolOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, *args, **kwargs): - assert a.size == b.size - super().__init__(operands=(a, b), **kwargs) +class ArrayConstant(Array, Constant): + __xslots__: Tuple[str, ...] = ( + "_index_size", + "_value_size") -class UnsignedLessThan(BoolOperation): - __slots__ = BitVecOperation.__xslots__ + def __init__( + self, *, + index_size: int, + value_size: int, + **kwargs, + ): + self._index_size = index_size + self._value_size = value_size + super().__init__(**kwargs) - def __init__(self, a, b, **kwargs): - super().__init__(operands=(a, b), **kwargs) + @property + def index_size(self): + return self._index_size + @property + def value_size(self): + return self._value_size -class UnsignedLessOrEqual(BoolOperation): - __slots__ = BitVecOperation.__xslots__ + @property + def index_max(self): + return len(self.value) - def __init__(self, a, b, **kwargs): - assert a.size == b.size - super().__init__(operands=(a, b), **kwargs) + def get(self, index): + index = self.cast_index(index) + if isinstance(index, Constant): + return BitvecConstant(size=self.value_size, value=self.value[index.value], taint=self.taint) + result = BitvecConstant(size=self.value_size, value=0, taint=('out_of_bounds')) + for i, c in enumerate(self.value): + result = BitvecITE(index == i, BitvecConstant(size=self.value_size, value=c), result, taint=self.taint) + return result + def read(self, offset, size): + assert (len(self.value[offset:offset + size]) == size) + return ArrayConstant(index_size=self.index_size, value_size=self.value_size, value=self.value[offset:offset+size]) -class UnsignedGreaterThan(BoolOperation): - __slots__ = BitVecOperation.__xslots__ - def __init__(self, a, b, *args, **kwargs): - assert a.size == b.size - super().__init__(operands=(a, b), **kwargs) +class ArrayVariable(Array, Variable): + """ + An Array expression is a mapping from bitvector of index_size bits + into bitvectors of value_size bits. + If a default value is provided reading from an unused index will return the + default. Otherwise each unused position in the array represents a free bitvector. -class UnsignedGreaterOrEqual(BoolOperation): - __slots__ = BitVecOperation.__xslots__ + If an index_max maximun index is provided accessing over the max is undefined. + Otherwise the array is unbounded. - def __init__(self, a, b, **kwargs): - super(UnsignedGreaterOrEqual, self).__init__(operands=(a, b), **kwargs) + """ + __xslots__: Tuple[str, ...] = ( + "_index_size", + "_value_size", + "_length", + "_default", + ) -############################################################################### -# Array BV32 -> BV8 or BV64 -> BV8 -class Array(Expression): - __slots__ = () - __xslots__ = ( - "_index_bits", - "_index_max", - "_value_bits", - "_default", - "_written", - "_concrete_cache", - ) + Expression.__xslots__ + def __hash__(self): + return object.__hash__(self) def __init__( self, - index_bits: int, - index_max: Optional[int], - value_bits: int, + index_size: int, + value_size: int, + length: Optional[int] = None, default: Optional[int] = None, **kwargs, ): """ This is a mapping from BV to BV. Normally used to represent a memory. - :param index_bits: Number of bits in the addressing side - :param index_max: Max address allowed - :param value_bits: Number of bits in tha value side + :param index_size: Number of bits in the addressing side + :param length: Max address allowed + :param value_size: Number of bits in tha value side :param default: Reading from an uninitialized index will return default if provided. If not the behaivor mimics thtat from smtlib, the returned value is a free symbol. :param kwargs: Used in other parent classes """ - assert index_bits in (32, 64, 256) - assert value_bits in (8, 16, 32, 64, 256) - assert index_max is None or index_max >= 0 and index_max < 2 ** index_bits - self._index_bits = index_bits - self._index_max = index_max - self._value_bits = value_bits + assert index_size in (32, 64, 256) + assert value_size in (8, 16, 32, 64, 256) + assert length is None or length >= 0 and length < 2 ** index_size + self._index_size = index_size + self._length = length + self._value_size = value_size self._default = default - self._written = None # Cache of the known indexs - self._concrete_cache: Dict[int, int] = {} # Cache of concrete indexes super().__init__(**kwargs) - assert type(self) is not Array, "Abstract class" - def _fix_slice(self, index: slice): - """Used to calculate the size of slices""" - stop, start = index.stop, index.start - if start is None: - start = 0 - if stop is None: - stop = len(self) - size = stop - start - if isinstance(size, BitVec): - from .visitors import simplify - - size = simplify(size) - else: - size = BitVecConstant(self.index_bits, size) - assert isinstance(size, BitVecConstant) - return start, stop, size.value - - def cast(self, possible_array): - """ Builds an Array from a bytes or bytearray""" - if isinstance(possible_array, (bytearray, bytes)): - # FIXME This should be related to a constrainSet - arr = ArrayVariable( - self.index_bits, len(possible_array), 8, name="cast{}".format(uuid.uuid1()) - ) - for pos, byte in enumerate(possible_array): - arr = arr.store(pos, byte) - return arr - raise ValueError # cast not implemented - - def cast_index(self, index: Union[int, "BitVec"]) -> Union["BitVecConstant", "BitVec"]: - if isinstance(index, int): - # assert self.index_max is None or index >= 0 and index < self.index_max - return BitVecConstant(self.index_bits, index) - assert index.size == self.index_bits - return index - - def cast_value(self, value: Union["BitVec", str, bytes, int]) -> "BitVec": - if isinstance(value, BitVec): - assert value.size == self.value_bits - return value - if isinstance(value, (str, bytes)) and len(value) == 1: - value = ord(value) - if not isinstance(value, int): - value = int(value) - return BitVecConstant(self.value_bits, value) - - def __len__(self): - if self.index_max is None: - raise ExpressionException("Array max index not set") - return self.index_max @property - def index_bits(self): - return self._index_bits + def index_size(self): + return self._index_size @property - def value_bits(self): - return self._value_bits + def value_size(self): + return self._value_size @property def index_max(self): - return self._index_max + if self._length is None: + return None + return int(self._length) - 1 @property def default(self): @@ -860,55 +924,20 @@ def get(self, index, default=None): """ Gets an element from the Array. If the element was not previously the default is used. """ - index = self.cast_index(index) - - # Emulate list[-1] - if self.index_max is not None: - from .visitors import simplify - - index = simplify( - BitVecITE(self.index_bits, index < 0, self.index_max + index + 1, index) - ) - - if isinstance(index, Constant) and index.value in self._concrete_cache: - return self._concrete_cache[index.value] - value = ArraySelect(self, index) - - # No default. Returns free symbol if default is None: default = self._default - if default is None: - return value - - # If a default is provided calculate check if the index is known - is_known = self.is_known(index) - default = self.cast_value(default) - if isinstance(is_known, Constant): - if is_known.value: - return value - else: - return default - return BitVecITE(self.value_bits, is_known, value, default) + if default is not None: + return default + index = self.cast_index(index) + return ArraySelect(self, index) def select(self, index): return self.get(index) def store(self, index, value): - from .visitors import simplify - - index = simplify(self.cast_index(index)) + index = self.cast_index(index) value = self.cast_value(value) - new_array = ArrayStore(self, index, value) - if isinstance(index, Constant): - new_array._concrete_cache = copy.copy(self._concrete_cache) - new_array._concrete_cache[index.value] = value - else: - # delete all cache as we do not know what this may overwrite. - new_array._concrete_cache = {} - if self.default is not None: - # potentially generate and update .written set - new_array.written.add(index) - return new_array + return ArrayStore(array=self, index=index, value=value) @property def written(self): @@ -924,7 +953,7 @@ def written(self): break if isinstance(array, ArraySlice): # if it is a proxy over a slice take out the slice too - offset += array.slice_offset + offset += array.offset array = array.array else: # The index written to underlaying Array are displaced when sliced @@ -946,53 +975,6 @@ def is_known(self, index): is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) return is_known_index - def write(self, offset, buf): - """ Creates a new Array instance by writing buf at offset """ - if not isinstance(buf, (Array, bytes)): - raise TypeError("Array or bytearray expected got {:s}".format(type(buf))) - arr = self - for i, val in enumerate(buf): - arr = arr.store(offset + i, val) - return arr - - def read(self, offset, size, default: Optional[int] = None): - default = self._default if default is None else default - return ArraySlice(self, offset=offset, size=size, default=default) - - def __getitem__(self, index): - if isinstance(index, slice): - start, stop, size = self._fix_slice(index) - return self.read(start, size, self.default) - else: - if self.index_max is not None: - if not isinstance(index, Expression) and index >= self.index_max: - raise IndexError - return self.get(index, self._default) - - def __iter__(self): - for i in range(len(self)): - yield self[i] - - def __eq__(self, other): - # FIXME taint - def compare_buffers(a, b): - if len(a) != len(b): - return BoolConstant(False) - cond = BoolConstant(True) - for i in range(len(a)): - cond = BoolAnd(cond.cast(a[i] == b[i]), cond) - if cond is BoolConstant(False): - return BoolConstant(False) - return cond - - return compare_buffers(self, other) - - def __ne__(self, other): - return BoolNot(self == other) - - def __hash__(self): - return super().__hash__() - @property def underlying_variable(self): array = self @@ -1005,34 +987,34 @@ def read_BE(self, address, size): bytes = [] for offset in range(size): bytes.append(self.get(address + offset, self._default)) - return BitVecConcat(size * self.value_bits, *bytes) + return BitvecConcat(*bytes) def read_LE(self, address, size): address = self.cast_index(address) bytes = [] for offset in range(size): bytes.append(self.get(address + offset, self._default)) - return BitVecConcat(size * self.value_bits, *reversed(bytes)) + return BitvecConcat(size * self.value_size, *reversed(bytes)) def write_BE(self, address, value, size): address = self.cast_index(address) - value = BitVecConstant(size=size * self.value_bits, value=0).cast(value) + value = BitvecConstant(size=size * self.value_size, value=0).cast(value) array = self for offset in range(size): array = array.store( address + offset, - BitVecExtract(value, (size - 1 - offset) * self.value_bits, self.value_bits), + BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), ) return array def write_LE(self, address, value, size): address = self.cast_index(address) - value = BitVec(size * self.value_bits).cast(value) + value = Bitvec(size * self.value_size).cast(value) array = self for offset in reversed(range(size)): array = array.store( address + offset, - BitVecExtract(value, (size - 1 - offset) * self.value_bits, self.value_bits), + BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), ) return array @@ -1040,16 +1022,14 @@ def __add__(self, other): if not isinstance(other, (Array, bytes)): raise TypeError("can't concat Array to {}".format(type(other))) if isinstance(other, Array): - if self.index_bits != other.index_bits or self.value_bits != other.value_bits: + if self.index_size != other.index_size or self.value_size != other.value_size: raise ValueError("Array sizes do not match for concatenation") - from .visitors import simplify - # FIXME This should be related to a constrainSet new_arr = ArrayVariable( - self.index_bits, + self.index_size, self.index_max + len(other), - self.value_bits, + self.value_size, default=self._default, name="concatenation{}".format(uuid.uuid1()), ) @@ -1064,16 +1044,16 @@ def __radd__(self, other): if not isinstance(other, (Array, bytes)): raise TypeError("can't concat Array to {}".format(type(other))) if isinstance(other, Array): - if self.index_bits != other.index_bits or self.value_bits != other.value_bits: + if self.index_size != other.index_size or self.value_size != other.value_size: raise ValueError("Array sizes do not match for concatenation") from .visitors import simplify # FIXME This should be related to a constrainSet new_arr = ArrayVariable( - self.index_bits, + self.index_size, self.index_max + len(other), - self.value_bits, + self.value_size, default=self._default, name="concatenation{}".format(uuid.uuid1()), ) @@ -1085,34 +1065,49 @@ def __radd__(self, other): return new_arr -class ArrayVariable(Array, Variable): - __slots__ = Array.__xslots__ + Variable.__xslots__ - - @property - def declaration(self): - return f"(declare-fun {self.name} () (Array (_ BitVec {self.index_bits}) (_ BitVec {self.value_bits})))" - -class ArrayOperation(Array, Operation): - __slots__ = () - __xslots__ = Array.__xslots__ + Operation.__xslots__ +class ArrayOperation(Array, Operation, abstract=True): + """ It's an operation that results in an Array""" + pass class ArrayStore(ArrayOperation): - __slots__ = ArrayOperation.__xslots__ - - def __init__(self, array: "Array", index: "BitVec", value: "BitVec", **kwargs): - assert index.size == array.index_bits - assert value.size == array.value_bits + __xslots__: Tuple[str, ...] = ( + "_written", + "_concrete_cache", + ) + def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): + assert index.size == array.index_size + assert value.size == array.value_size + self._written = None # Cache of the known indexs + self._concrete_cache = {} + if isinstance(index, Constant): + self._concrete_cache.update(getattr(array, "_concrete_cache", ())) # Cache of concrete indexes + self._concrete_cache[index.value] = value super().__init__( - index_bits=array.index_bits, - value_bits=array.value_bits, - index_max=array.index_max, - default=array.default, operands=(array, index, value), **kwargs, ) + @property + def default(self): + return self.array.default + + @property + def index_size(self): + return self.index.size + + @property + def value_size(self): + return self.value.size + + @property + def index_max(self): + return self.array.index_max + + def __hash__(self): + return object.__hash__(self) + @property def array(self): return self.operands[0] @@ -1129,6 +1124,53 @@ def index(self): def value(self): return self.operands[2] + def get(self, index, default=None): + """ Gets an element from the Array. + If the element was not previously the default is used. + """ + index = simplify(self.cast_index(index)) + + # Emulate list[-1] + if self.index_max is not None: + index = simplify( + BitvecITE(index < 0, self.index_max + index + 1, index) + ) + + if isinstance(index, Constant): + if self.index_max is not None and index.value > self.index_max: + raise IndexError + if index.value in self._concrete_cache: + return self._concrete_cache[index.value] + + # take out Proxy sleve + if default is None: + default = self.default + if default is None: + # No default. Returns normal array select + return ArraySelect(self, index) + + # build a big ITE expression that would en in the default + array, offset, items = self, 0, [] + while not isinstance(array, ArrayVariable): + if isinstance(array, ArraySlice): + # jump over array slices + offset += array.offset + else: + # The index written to underlaying Array are displaced when sliced + items.insert(0, (index == (array.index - offset), array.value)) + array = array.array + + result = self.cast_value(default) + for cond_i, value_i in items: + result = BitvecITE(cond_i, value_i, result) + return result + + def store(self, index, value): + index = simplify(self.cast_index(index)) + value = self.cast_value(value) + new_array = ArrayStore(self, index, value) + return new_array + class ArraySlice(ArrayOperation): """ Provides a projection of an underlying array. @@ -1136,46 +1178,52 @@ class ArraySlice(ArrayOperation): (It needs to be simplified out before translating it smtlib) """ - __slots__ = ArrayOperation.__xslots__ + ("_slice_offset", "_slice_size") - def __init__( - self, array: "Array", offset: int, size: int, default: Optional[int] = None, **kwargs - ): - print (self.__slots__) - assert size + def __init__(self, array: "Array", offset: int, size: int, **kwargs): if not isinstance(array, Array): raise ValueError("Array expected") + super().__init__( - index_bits=array.index_bits, - value_bits=array.value_bits, - index_max=size, - operands=(array,), + operands=(array, array.cast_index(offset), array.cast_index(size)), **kwargs, ) - self._slice_offset = offset - self._slice_size = size - @property def array(self): return self.operands[0] @property - def underlying_variable(self): - return self.array.underlying_variable + def offset(self): + return self.operands[1] @property - def slice_offset(self): - return self._slice_offset + def index_max(self): + return self.operands[2].value-1 + + @property + def index_size(self): + return self.array.index_size + + @property + def value_size(self): + return self.array.value_size - def get(self, index, default): - return self.array.get(index + self._slice_offset, default) + @property + def underlying_variable(self): + return self.array.underlying_variable + + def get(self, index, default=None): + index = self.cast_index(index) + if isinstance(index, Constant): + if self.index_max is not None and index.value >= len(self): + raise IndexError + return self.array.get(simplify(index + self.offset), default) def store(self, index, value): return ArraySlice( - self.array.store(index + self._slice_offset, value), - self._slice_offset, - self._slice_size, + self.array.store(index + self.offset, value), + offset=self.offset, + size=len(self), ) @@ -1191,10 +1239,8 @@ class ArrayProxy: """ - def __hash__(self): - return hash(self.array) - def __init__(self, array: Array): + assert isinstance(array, Array) self._array = array @property @@ -1214,16 +1260,16 @@ def operands(self): return (self._array,) @property - def index_bits(self): - return self._array.index_bits + def index_size(self): + return self._array.index_size @property def index_max(self): return self._array.index_max @property - def value_bits(self): - return self._array.value_bits + def value_size(self): + return self._array.value_size @property def taint(self): @@ -1236,7 +1282,9 @@ def select(self, index): return self._array.select(index) def store(self, index, value): - return self._array.store(index, value) + self._array = self._array.store(index, value) + assert self._array is not None + return self @property def written(self): @@ -1256,16 +1304,17 @@ def __setitem__(self, index, value): self._array = self._array.store(start + i, value[i]) else: self._array = self._array.store(index, value) + assert self._array is not None - def __copy__(self, memo=None): - print ("AAAA") - return ArrayProxy(self.array) def get(self, index, default=None): - return self._array.get(index, default) + x = self._array.get(index, default) + print ("A"*199, type(self._array), "X:", x) + return x def write_BE(self, address, value, size): self._array = self._array.write_BE(address, value, size) + assert self._array is not None return self def read_BE(self, address, size): @@ -1273,71 +1322,49 @@ def read_BE(self, address, size): def write(self, offset, buf): self._array = self._array.write(offset, buf) + assert self._array is not None + return self def read(self, offset, size): return ArrayProxy(self._array[offset : offset + size]) def __eq__(self, other): - return other == self.array + print ("type:"*99, type(self.array ), type(other)) + return self.array == other def __ne__(self, other): return BoolNot(self == other) - def __add__(self, other): - if not isinstance(other, (Array, ArrayProxy, bytes, bytearray)): - raise TypeError("can't concat Array to {}".format(type(other))) - if isinstance(other, Array): - if self.index_bits != other.index_bits or self.value_bits != other.value_bits: - raise ValueError("Array sizes do not match for concatenation") + def _concatenate(self, array_a, array_b): from .visitors import simplify - - # FIXME This should be related to a constrainSet + # FIXME/Research This should be related to a constrainSet new_arr = ArrayProxy( ArrayVariable( - self.index_bits, - self.index_max + len(other), - self.value_bits, + index_size = self.index_size, + length=len(array_a) + len(array_b), + value_size = self.value_size, name="concatenation{}".format(uuid.uuid1()), ) ) - for index in range(self.index_max): - new_arr[index] = simplify(self[index]) - for index in range(len(other)): - new_arr[index + self.index_max] = simplify(other[index]) + for index in range(len(array_a)): + new_arr[index] = simplify(array_a[index]) + for index in range(len(array_b)): + new_arr[index + len(array_a)] = simplify(array_b[index]) return new_arr - def __radd__(self, other): - if not isinstance(other, (Array, bytearray, bytes)): - raise TypeError("can't concat Array to {}".format(type(other))) - if isinstance(other, Array): - if self.index_bits != other.index_bits or self.value_bits != other.value_bits: - raise ValueError("Array sizes do not match for concatenation") - - from .visitors import simplify - - # FIXME This should be related to a constrainSet - new_arr = ArrayProxy( - ArrayVariable( - self.index_bits, - self.index_max + len(other), - self.value_bits, - name="concatenation{}".format(uuid.uuid1()), - ) - ) - for index in range(len(other)): - new_arr[index] = simplify(other[index]) - for index in range(self.index_max): - new_arr[index + len(other)] = simplify(self[index]) - return new_arr + def __add__(self, other): + return self._concatenate(self, other) + def __radd__(self, other): + return self._concatenate(other, self) -class ArraySelect(BitVec, Operation): - __slots__ = BitVec.__xslots__ + Operation.__xslots__ +class ArraySelect(BitvecOperation): + __xslots__ = BitvecOperation.__xslots__ - def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): - assert index.size == array.index_bits - super().__init__(size=array.value_bits, operands=(array, index), **kwargs) + def __init__(self, array: "Array", index: "Bitvec", *args, **kwargs): + assert index.size == array.index_size + super().__init__(size=array.value_size, operands=(array, index), **kwargs) @property def array(self): @@ -1347,40 +1374,36 @@ def array(self): def index(self): return self.operands[1] - @property - def operands(self): - return self._operands - def __repr__(self): - return f"" + return f"" + +class BitvecSignExtend(BitvecOperation): + def __init__(self, operand: Bitvec, size: int, *args, **kwargs): + super().__init__(size=size, operands=(operand,), **kwargs) -class BitVecSignExtend(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ + ("extend",) + @property + def extend(self): + return self.size - self.operands[0].size - def __init__(self, operand: "BitVec", size_dest: int, *args, **kwargs): - assert size_dest >= operand.size - super().__init__(size=size_dest, operands=(operand,), **kwargs) - self.extend = size_dest - operand.size -class BitVecZeroExtend(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ + ("extend",) +class BitvecZeroExtend(BitvecOperation): + def __init__(self, size: int, operand: Bitvec, *args, **kwargs): + super().__init__(size=size, operands=(operand,), **kwargs) - def __init__(self, size_dest: int, operand: "BitVec", *args, **kwargs): - assert size_dest >= operand.size - super().__init__(size=size_dest, operands=(operand,), **kwargs) - self.extend = size_dest - operand.size + @property + def extend(self): + return self.size - self.operands[0].size -class BitVecExtract(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ + ("_begining", "_end") +class BitvecExtract(BitvecOperation): + __xslots__ = ("_offset",) - def __init__(self, operand: "BitVec", offset: int, size: int, *args, **kwargs): + def __init__(self, operand: "Bitvec", offset: int, size: int, *args, **kwargs): assert offset >= 0 and offset + size <= operand.size super().__init__(size=size, operands=(operand,), **kwargs) - self._begining = offset - self._end = offset + size - 1 + self._offset = offset @property def value(self): @@ -1388,36 +1411,32 @@ def value(self): @property def begining(self): - return self._begining + return self._offset @property def end(self): - return self._end + return self.begining + self.size - 1 -class BitVecConcat(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecConcat(BitvecOperation): - def __init__(self, size_dest: int, *operands, **kwargs): - assert all(isinstance(x, BitVec) for x in operands) - assert size_dest == sum(x.size for x in operands) - super().__init__(size=size_dest, operands=operands, **kwargs) + def __init__(self, operands: Tuple[Bitvec, ...], **kwargs): + size = sum(x.size for x in operands) + super().__init__(size=size, operands=operands, **kwargs) -class BitVecITE(BitVecOperation): - __slots__ = BitVecOperation.__xslots__ +class BitvecITE(BitvecOperation): + __xslots__ = BitvecOperation.__xslots__ def __init__( self, - size: int, - condition: Union["Bool", bool], - true_value: "BitVec", - false_value: "BitVec", + condition: Bool, + true_value: Bitvec, + false_value: Bitvec, **kwargs, ): - assert true_value.size == size - assert false_value.size == size - super().__init__(size=size, operands=(condition, true_value, false_value), **kwargs) + + super().__init__(size=true_value.size, operands=(condition, true_value, false_value), **kwargs) @property def condition(self): @@ -1430,3 +1449,77 @@ def true_value(self): @property def false_value(self): return self.operands[2] + + +# auxiliar functions maybe move to operators +def issymbolic(value) -> bool: + """ + Helper to determine whether an object is symbolic (e.g checking + if data read from memory is symbolic) + + :param object value: object to check + :return: whether `value` is symbolic + :rtype: bool + """ + return isinstance(value, (Expression, ArrayProxy)) + + +def istainted(arg, taint=None): + """ + Helper to determine whether an object if tainted. + :param arg: a value or Expression + :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value. + """ + + if not issymbolic(arg): + return False + if taint is None: + return len(arg.taint) != 0 + for arg_taint in arg.taint: + m = re.match(taint, arg_taint, re.DOTALL | re.IGNORECASE) + if m: + return True + return False + + +def get_taints(arg, taint=None): + """ + Helper to list an object taints. + :param arg: a value or Expression + :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value. + """ + + if not issymbolic(arg): + return + for arg_taint in arg.taint: + if taint is not None: + m = re.match(taint, arg_taint, re.DOTALL | re.IGNORECASE) + if m: + yield arg_taint + else: + yield arg_taint + return + + +def taint_with(arg, *taints, value_size=256, index_size=256): + """ + Helper to taint a value. + :param arg: a value or Expression + :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value. + """ + tainted_fset = frozenset(tuple(taints)) + if not issymbolic(arg): + if isinstance(arg, int): + arg = BitvecConstant(value_size, arg) + arg._taint = tainted_fset + else: + raise ValueError("type not supported") + + else: + arg = copy.copy(arg) + arg._taint |= tainted_fset + + return arg + + +from .visitors import simplify diff --git a/manticore/core/smtlib/operators.py b/manticore/core/smtlib/operators.py index cbc5e4e3e..ca123f650 100644 --- a/manticore/core/smtlib/operators.py +++ b/manticore/core/smtlib/operators.py @@ -1,12 +1,12 @@ from .expression import ( - BitVec, - BitVecExtract, - BitVecSignExtend, - BitVecZeroExtend, - BitVecConstant, - BitVecConcat, + Bitvec, + BitvecExtract, + BitvecSignExtend, + BitvecZeroExtend, + BitvecConstant, + BitvecConcat, Bool, - BitVecITE, + BitvecITE, BoolConstant, BoolITE, ) @@ -15,11 +15,11 @@ def ORD(s): - if isinstance(s, BitVec): + if isinstance(s, Bitvec): if s.size == 8: return s else: - return BitVecExtract(s, 0, 8) + return BitvecExtract(s, 0, 8) elif isinstance(s, int): return s & 0xFF else: @@ -27,11 +27,11 @@ def ORD(s): def CHR(s): - if isinstance(s, BitVec): + if isinstance(s, Bitvec): if s.size == 8: return s else: - return BitVecExtract(s, 0, 8) + return BitvecExtract(s, 0, 8) elif isinstance(s, int): return bytes([s & 0xFF]) else: @@ -66,15 +66,15 @@ def OR(a, b, *others): if isinstance(b, Bool): return b | a result = a | b - if isinstance(result, (BitVec, int)): + if isinstance(result, (Bitvec, int)): result = ITE(result != 0, True, False) return result def UGT(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.ugt(b) - if isinstance(b, BitVec): + if isinstance(b, Bitvec): return b.ult(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -85,9 +85,9 @@ def UGT(a, b): def UGE(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.uge(b) - if isinstance(b, BitVec): + if isinstance(b, Bitvec): return b.ule(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -98,9 +98,9 @@ def UGE(a, b): def ULT(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.ult(b) - if isinstance(b, BitVec): + if isinstance(b, Bitvec): return b.ugt(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -111,9 +111,9 @@ def ULT(a, b): def ULE(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.ule(b) - if isinstance(b, BitVec): + if isinstance(b, Bitvec): return b.uge(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -124,10 +124,10 @@ def ULE(a, b): def EXTRACT(x, offset, size): - if isinstance(x, BitVec): + if isinstance(x, Bitvec): if offset == 0 and size == x.size: return x - return BitVecExtract(x, offset, size) + return BitvecExtract(x, offset, size) else: return (x >> offset) & ((1 << size) - 1) @@ -138,15 +138,15 @@ def SEXTEND(x, size_src, size_dest): x -= 1 << size_src return x & ((1 << size_dest) - 1) assert x.size == size_src - return BitVecSignExtend(x, size_dest) + return BitvecSignExtend(x, size_dest) def ZEXTEND(x, size): if isinstance(x, int): return x & ((1 << size) - 1) - assert isinstance(x, BitVec) and size - x.size >= 0 + assert isinstance(x, Bitvec) and size - x.size >= 0 if size - x.size > 0: - return BitVecZeroExtend(size, x) + return BitvecZeroExtend(size, x) else: return x @@ -158,10 +158,10 @@ def CONCAT(total_size, *args): def cast(x): if isinstance(x, int): - return BitVecConstant(arg_size, x) + return BitvecConstant(arg_size, x) return x - return BitVecConcat(total_size, *list(map(cast, args))) + return BitvecConcat(operands=tuple(map(cast, args))) else: return args[0] else: @@ -172,8 +172,8 @@ def cast(x): def ITE(cond, true_value, false_value): - assert isinstance(true_value, (Bool, bool, BitVec, int)) - assert isinstance(false_value, (Bool, bool, BitVec, int)) + assert isinstance(true_value, (Bool, bool, Bitvec, int)) + assert isinstance(false_value, (Bool, bool, Bitvec, int)) assert isinstance(cond, (Bool, bool)) if isinstance(cond, bool): if cond: @@ -191,14 +191,15 @@ def ITE(cond, true_value, false_value): def ITEBV(size, cond, true_value, false_value): - if isinstance(cond, BitVec): + print ("ITE", cond) + if isinstance(cond, Bitvec): cond = cond.Bool() if isinstance(cond, int): cond = cond != 0 assert isinstance(cond, (Bool, bool)) - assert isinstance(true_value, (BitVec, int)) - assert isinstance(false_value, (BitVec, int)) + assert isinstance(true_value, (Bitvec, int)) + assert isinstance(false_value, (Bitvec, int)) assert isinstance(size, int) if isinstance(cond, BoolConstant) and not cond.taint: @@ -211,42 +212,42 @@ def ITEBV(size, cond, true_value, false_value): return false_value if isinstance(true_value, int): - true_value = BitVecConstant(size, true_value) + true_value = BitvecConstant(size, true_value) if isinstance(false_value, int): - false_value = BitVecConstant(size, false_value) - return BitVecITE(size, cond, true_value, false_value) + false_value = BitvecConstant(size, false_value) + return BitvecITE(cond, true_value, false_value) def UDIV(dividend, divisor): - if isinstance(dividend, BitVec): + if isinstance(dividend, Bitvec): return dividend.udiv(divisor) - elif isinstance(divisor, BitVec): + elif isinstance(divisor, Bitvec): return divisor.rudiv(dividend) assert dividend >= 0 or divisor > 0 # unsigned-es return dividend // divisor def SDIV(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.sdiv(b) - elif isinstance(b, BitVec): + elif isinstance(b, Bitvec): return b.rsdiv(a) return int(math.trunc(float(a) / float(b))) def SMOD(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.smod(b) - elif isinstance(b, BitVec): + elif isinstance(b, Bitvec): return b.rsmod(a) return int(math.fmod(a, b)) def SREM(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.srem(b) - elif isinstance(b, BitVec): + elif isinstance(b, Bitvec): return b.rsrem(a) elif isinstance(a, int) and isinstance(b, int): return a - int(a / b) * b @@ -254,22 +255,22 @@ def SREM(a, b): def UREM(a, b): - if isinstance(a, BitVec): + if isinstance(a, Bitvec): return a.urem(b) - elif isinstance(b, BitVec): + elif isinstance(b, Bitvec): return b.rurem(a) return a % b def SAR(size, a, b): assert isinstance(size, int) - if isinstance(b, BitVec) and b.size != size: + if isinstance(b, Bitvec) and b.size != size: b = ZEXTEND(b, size) - if isinstance(a, BitVec): + if isinstance(a, Bitvec): assert size == a.size return a.sar(b) - elif isinstance(b, BitVec): - return BitVecConstant(size, a).sar(b) + elif isinstance(b, Bitvec): + return BitvecConstant(size, a).sar(b) else: tempDest = a tempCount = b diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 604ae6217..4c016af95 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -134,16 +134,16 @@ def get_value(self, constraints, expression): """Ask the solver for one possible result of given expression using given set of constraints.""" raise SolverException("Abstract method not implemented") - def max(self, constraints, X: BitVec, M=10000): + def max(self, constraints, X: Bitvec, M=10000): """ Iteratively finds the maximum value for a symbol within given constraints. :param X: a symbol or expression :param M: maximum number of iterations allowed """ - assert isinstance(X, BitVec) + assert isinstance(X, Bitvec) return self.optimize(constraints, X, "maximize", M) - def min(self, constraints, X: BitVec, M=10000): + def min(self, constraints, X: Bitvec, M=10000): """ Iteratively finds the minimum value for a symbol within given constraints. @@ -151,7 +151,7 @@ def min(self, constraints, X: BitVec, M=10000): :param X: a symbol or expression :param M: maximum number of iterations allowed """ - assert isinstance(X, BitVec) + assert isinstance(X, Bitvec) return self.optimize(constraints, X, "minimize", M) def minmax(self, constraints, x, iters=10000): @@ -216,6 +216,7 @@ def __readline_and_count(self): buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error # Error messages may contain an unbalanced parenthesis situation + print (buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -382,11 +383,11 @@ def _getvalue(self, expression) -> Union[int, bool, bytes]: else: if isinstance(expression, BoolVariable): return self.__getvalue_bool(expression.name) - elif isinstance(expression, BitVecVariable): + elif isinstance(expression, BitvecVariable): return self.__getvalue_bv(expression.name) raise NotImplementedError( - f"_getvalue only implemented for Bool, BitVec and Array. Got {type(expression)}" + f"_getvalue only implemented for Bool, Bitvec and Array. Got {type(expression)}" ) # push pop @@ -415,7 +416,7 @@ def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool] return self._is_sat() # get-all-values min max minmax - def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): + def _optimize_generic(self, constraints: ConstraintSet, x: Bitvec, goal: str, max_iter=10000): """ Iteratively finds the maximum or minimum value for the operation (Normally Operators.UGT or Operators.ULT) @@ -502,9 +503,9 @@ def get_all_values( silent: bool = False, ): """Returns a list with all the possible values for the symbol x""" - if not issymbolic(expression): + if not isinstance(expression, Expression): return [expression] - expression = simplify(expression) + assert isinstance(expression, Expression) if maxcnt is None: maxcnt = consts.maxsolutions if isinstance(expression, Bool) and consts.maxsolutions > 1: @@ -515,9 +516,9 @@ def get_all_values( with constraints as temp_cs: if isinstance(expression, Bool): var = temp_cs.new_bool() - elif isinstance(expression, BitVec): + elif isinstance(expression, Bitvec): var = temp_cs.new_bitvec(expression.size) - elif isinstance(expression, (Array, ArrayProxy)): + elif isinstance(expression, Array): var = temp_cs.new_array( index_max=expression.index_max, value_bits=expression.value_bits, @@ -558,7 +559,7 @@ def get_all_values( self._reset(temp_cs.to_string()) return list(result) - def _optimize_fancy(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): + def _optimize_fancy(self, constraints: ConstraintSet, x: Bitvec, goal: str, max_iter=10000): """ Iteratively finds the maximum or minimum value for the operation (Normally Operators.UGT or Operators.ULT) @@ -576,9 +577,11 @@ def _optimize_fancy(self, constraints: ConstraintSet, x: BitVec, goal: str, max_ X = temp_cs.new_bitvec(x.size) temp_cs.add(X == x) aux = temp_cs.new_bitvec(X.size, name="optimized_") + + temp_cs.add(operation(X, aux)) self._reset(temp_cs.to_string()) - self._assert(operation(X, aux)) + #self._assert(operation(X, aux)) self._smtlib.send("(%s %s)" % (goal, aux.name)) self._smtlib.send("(check-sat)") _status = self._smtlib.recv() @@ -591,19 +594,23 @@ def get_value(self, constraints: ConstraintSet, *expressions): Ask the solver for one possible result of given expressions using given set of constraints. """ + + + + ####################33 values = [] start = time.time() - with constraints as temp_cs: + with constraints.related_to(*expressions) as temp_cs: for expression in expressions: if not issymbolic(expression): values.append(expression) continue - assert isinstance(expression, (Bool, BitVec, Array, ArrayProxy)) + assert isinstance(expression, (Bool, Bitvec, Array)) if isinstance(expression, Bool): var = temp_cs.new_bool() - elif isinstance(expression, BitVec): + elif isinstance(expression, Bitvec): var = temp_cs.new_bitvec(expression.size) - elif isinstance(expression, (Array, ArrayProxy)): + elif isinstance(expression, Array): var = [] result = [] for i in range(expression.index_max): @@ -634,7 +641,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): if isinstance(expression, Bool): values.append(self.__getvalue_bool(var.name)) - if isinstance(expression, BitVec): + if isinstance(expression, Bitvec): values.append(self.__getvalue_bv(var.name)) if time.time() - start > consts.timeout: raise SolverError("Timeout") diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 37da125e2..1c1461bd9 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -10,6 +10,8 @@ logger = logging.getLogger(__name__) +class VisitorException(Exception): + pass class Visitor: """ Class/Type Visitor @@ -17,17 +19,15 @@ class Visitor: Inherit your class visitor from this one and get called on a different visiting function for each type of expression. It will call the first implemented method for the __mro__ class order. - For example for a BitVecAdd it will try - visit_BitVecAdd() if not defined then it will try with - visit_BitVecOperation() if not defined then it will try with - visit_BitVec() if not defined then it will try with + For example for a BitvecAdd expression this will try + visit_BitvecAdd() if not defined(*) then it will try with + visit_BitvecOperation() if not defined(*) then it will try with + visit_Bitvec() if not defined(*) then it will try with visit_Expression() - Other class named visitors are: - - visit_BitVec() - visit_Bool() - visit_Array() + (*) Or it is defined and it returns None for the node. + You can overload the visiting method to react to different semantic + aspects of an Exrpession. """ @@ -43,116 +43,130 @@ def push(self, value): def pop(self): if len(self._stack) == 0: return None - result = self._stack.pop() - return result + return self._stack.pop() @property def result(self): assert len(self._stack) == 1 return self._stack[-1] - def _method(self, expression, *args): - for cls in expression.__class__.__mro__[:-1]: - sort = cls.__name__ - methodname = "visit_%s" % sort - if hasattr(self, methodname): - value = getattr(self, methodname)(expression, *args) - if value is not None: - return value - return self._rebuild(expression, args) - def visit(self, node, use_fixed_point=False): + assert isinstance(node, Expression) """ The entry point of the visitor. The exploration algorithm is a DFS post-order traversal The implementation used two stacks instead of a recursion - The final result is store in self.result + The final result is store in result or can be taken from the resultant + stack via pop() + + #TODO: paste example :param node: Node to explore :type node: Expression :param use_fixed_point: if True, it runs _methods until a fixed point is found - :type use_fixed_point: Bool """ - cache = self._cache visited = set() - stack = [] - stack.append(node) - while stack: - node = stack.pop() - if node in cache: + local_stack = [] + local_stack.append(node) # initially the stack contains only the visiting node + while local_stack: + node = local_stack.pop() + if node in cache: # If seen we do not really need to visit this self.push(cache[node]) - elif isinstance(node, Operation): - if node in visited: - operands = [self.pop() for _ in range(len(node.operands))] - value = self._method(node, *operands) - - visited.remove(node) - self.push(value) - cache[node] = value - else: - visited.add(node) - stack.append(node) - stack.extend(node.operands) + continue + if node in visited: + visited.remove(node) + # Visited! Then there is a visited version of the operands in the stack + operands = (self.pop() for _ in range(len(node.operands))) + # Actually process the node + value = self._method(node, *operands) + self.push(value) + cache[node] = value else: - self.push(self._method(node)) + visited.add(node) + local_stack.append(node) + local_stack.extend(node.operands) + # Repeat until the result is not changed if use_fixed_point: - old_value = None + old_value = node new_value = self.pop() while old_value is not new_value: self.visit(new_value) old_value = new_value new_value = self.pop() - self.push(new_value) - @staticmethod - def _rebuild(expression, operands): - if isinstance(expression, Operation): - if any(operands[i] is not expression.operands[i] for i in range(len(operands))): - aux = copy.copy(expression) - aux._operands = operands - return aux - return expression - - -class Translator(Visitor): - """ Simple visitor to translate an expression into something else - """ + def _method(self, expression, *operands): + """ + Magic method to walk the mro looking for the first overloaded + visiting method that returns something. - def _method(self, expression, *args): - # Special case. Need to get the unsleeved version of the array - if isinstance(expression, ArrayProxy): - expression = expression.array + If no visiting methods are found the expression gets _rebuild using + the processed operands. + Iff the operands did not change the expression is left unchanged. + :param expression: Expression node + :param operands: The already processed operands + :return: Optional resulting Expression + """ assert expression.__class__.__mro__[-1] is object - for cls in expression.__class__.__mro__: + for cls in expression.__class__.__mro__[:-1]: sort = cls.__name__ methodname = f"visit_{sort:s}" if hasattr(self, methodname): - value = getattr(self, methodname)(expression, *args) + value = getattr(self, methodname)(expression, *operands) if value is not None: return value - raise SmtlibError(f"No translation for this {expression}") + return self._rebuild(expression, operands) + + def _changed(self, expression:Expression, operands): + return any(x is not y for x, y in zip(expression.operands, operands)) + + def _rebuild(self, expression:Expression, operands): + """ Default operation used when no visiting method was successful for + this expression. If he operands have changed this reubild the curren expression + with the new operands. + Assumes the stack is used for Expresisons + """ + if self._changed(expression, operands): + aux = copy.copy(expression) + aux._operands = operands + return aux + # The expression is not modified in any way iff: + # - no visitor method is defined for the expression type + # - no operands were modified + return expression + +class Translator(Visitor): + """ Simple visitor to translate an expression into something else + """ + def _rebuild(self, expression, operands): + """ The stack holds the translation of the expression. + There is no default action -class GetDeclarations(Visitor): + :param expression: Current expression + :param operands: The translations of the nodes + :return: No + """ + raise VisitorException(f"No translation for this {expression}") + + +class GetDeclarations(Translator): """ Simple visitor to collect all variables in an expression or set of expressions """ + def _rebuild(self, expression, operands): + return expression def __init__(self, **kwargs): super().__init__(**kwargs) self.variables = set() - def _visit_variable(self, expression): + def visit_Variable(self, expression): self.variables.add(expression) - visit_ArrayVariable = _visit_variable - visit_BitVecVariable = _visit_variable - visit_BoolVariable = _visit_variable - @property def result(self): return self.variables @@ -162,6 +176,8 @@ class GetDepth(Translator): """ Simple visitor to collect all variables in an expression or set of expressions """ + def _rebuild(self, expression, operands): + return expression def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -199,21 +215,6 @@ def visit(self, expression): """ self._method(expression) - def _method(self, expression, *args): - """ - Overload Visitor._method because we want to stop to iterate over the - visit_ functions as soon as a valid visit_ function is found - """ - assert expression.__class__.__mro__[-1] is object - for cls in expression.__class__.__mro__: - sort = cls.__name__ - methodname = "visit_%s" % sort - method = getattr(self, methodname, None) - if method is not None: - method(expression, *args) - return - return - def visit_Operation(self, expression, *operands): self._print(expression.__class__.__name__, expression) self.indent += 2 @@ -223,10 +224,9 @@ def visit_Operation(self, expression, *operands): else: self._print("...") self.indent -= 2 - return "" + return True - - def visit_BitVecExtract(self, expression): + def visit_BitvecExtract(self, expression): self._print( expression.__class__.__name__ + "{%d:%d}" % (expression.begining, expression.end), expression, @@ -238,22 +238,16 @@ def visit_BitVecExtract(self, expression): else: self._print("...") self.indent -= 2 - return "" + return True - def _visit_constant(self, expression): + def visit_Constant(self, expression): self._print(expression.value) - return "" + return True - visit_BitVecConstant = _visit_constant - visit_BoolConstant = _visit_constant - def _visit_variable(self, expression): + def visit_Variable(self, expression): self._print(expression.name) - return "" - - visit_ArrayVariable = _visit_variable - visit_BitVecVariable = _visit_variable - visit_BoolVariable = _visit_variable + return True @property def result(self): @@ -269,32 +263,30 @@ def pretty_print(expression, **kwargs): class ConstantFolderSimplifier(Visitor): - def __init__(self, **kw): - super().__init__(**kw) operations = { - BitVecMod: operator.__mod__, - BitVecAdd: operator.__add__, - BitVecSub: operator.__sub__, - BitVecMul: operator.__mul__, - BitVecShiftLeft: operator.__lshift__, - BitVecShiftRight: operator.__rshift__, - BitVecAnd: operator.__and__, - BitVecOr: operator.__or__, - BitVecXor: operator.__xor__, - BitVecNot: operator.__not__, - BitVecNeg: operator.__invert__, + BitvecMod: operator.__mod__, + BitvecAdd: operator.__add__, + BitvecSub: operator.__sub__, + BitvecMul: operator.__mul__, + BitvecShiftLeft: operator.__lshift__, + BitvecShiftRight: operator.__rshift__, + BitvecAnd: operator.__and__, + BitvecOr: operator.__or__, + BitvecXor: operator.__xor__, + BitvecNot: operator.__not__, + BitvecNeg: operator.__invert__, BoolAnd: operator.__and__, BoolEqual: operator.__eq__, BoolOr: operator.__or__, BoolNot: operator.__not__, - UnsignedLessThan: operator.__lt__, - UnsignedLessOrEqual: operator.__le__, - UnsignedGreaterThan: operator.__gt__, - UnsignedGreaterOrEqual: operator.__ge__, + BoolUnsignedLessThan: operator.__lt__, + BoolUnsignedLessOrEqualThan: operator.__le__, + BoolUnsignedGreaterThan: operator.__gt__, + BoolUnsignedGreaterOrEqualThan: operator.__ge__, } - def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConstant]: + def visit_BitvecUnsignedDiv(self, expression, *operands) -> Optional[BitvecConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].value b = operands[1].value @@ -302,38 +294,38 @@ def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConst ret = 0 else: ret = math.trunc(Decimal(a) / Decimal(b)) - return BitVecConstant(expression.size, ret, taint=expression.taint) + return BitvecConstant(expression.size, ret, taint=expression.taint) return None - def visit_LessThan(self, expression, *operands) -> Optional[BoolConstant]: + def visit_BoolLessThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a < b, taint=expression.taint) return None - def visit_LessOrEqual(self, expression, *operands) -> Optional[BoolConstant]: + def visit_BoolLessOrEqual(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a <= b, taint=expression.taint) return None - def visit_GreaterThan(self, expression, *operands) -> Optional[BoolConstant]: + def visit_BoolGreaterThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a > b, taint=expression.taint) return None - def visit_GreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: + def visit_BoolGreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value return BoolConstant(a >= b, taint=expression.taint) return None - def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: + def visit_BitvecDiv(self, expression, *operands) -> Optional[BitvecConstant]: if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask mask = operands[0].mask @@ -347,26 +339,26 @@ def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: result = 0 else: result = math.trunc(Decimal(numeral) / Decimal(dividend)) - return BitVecConstant(expression.size, result, taint=expression.taint) + return BitvecConstant(expression.size, result, taint=expression.taint) return None - def visit_BitVecConcat(self, expression, *operands): + def visit_BitvecConcat(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): result = 0 for o in operands: result <<= o.size result |= o.value - return BitVecConstant(expression.size, result, taint=expression.taint) + return BitvecConstant(expression.size, result, taint=expression.taint) - def visit_BitVecZeroExtend(self, expression, *operands): + def visit_BitvecZeroExtend(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): - return BitVecConstant(expression.size, operands[0].value, taint=expression.taint) + return BitvecConstant(expression.size, operands[0].value, taint=expression.taint) - def visit_BitVecSignExtend(self, expression, *operands): + def visit_BitvecSignExtend(self, expression, *operands): if expression.extend == 0: return operands[0] - def visit_BitVecExtract(self, expression, *operands): + def visit_BitvecExtract(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): value = operands[0].value begining = expression.begining @@ -374,7 +366,7 @@ def visit_BitVecExtract(self, expression, *operands): value = value >> begining mask = (1 << (end - begining)) - 1 value = value & mask - return BitVecConstant(expression.size, value, taint=expression.taint) + return BitvecConstant(expression.size, value, taint=expression.taint) def visit_BoolAnd(self, expression, a, b): if isinstance(a, Constant) and a.value == True: @@ -387,19 +379,15 @@ def visit_Operation(self, expression, *operands): operation = self.operations.get(type(expression), None) if operation is not None and all(isinstance(o, Constant) for o in operands): value = operation(*(x.value for x in operands)) - if isinstance(expression, BitVec): - return BitVecConstant(expression.size, value, taint=expression.taint) + if isinstance(expression, Bitvec): + return BitvecConstant(expression.size, value, taint=expression.taint) else: isinstance(expression, Bool) return BoolConstant(value, taint=expression.taint) - else: - expression = self._rebuild(expression, operands) - return expression constant_folder_simplifier_cache = CacheDict(max_size=150000, flush_perc=25) - @lru_cache(maxsize=128, typed=True) def constant_folder(expression): global constant_folder_simplifier_cache @@ -409,20 +397,11 @@ def constant_folder(expression): class ArithmeticSimplifier(Visitor): - def __init__(self, parent=None, **kw): - super().__init__(**kw) @staticmethod def _same_constant(a, b): return isinstance(a, Constant) and isinstance(b, Constant) and a.value == b.value or a is b - @staticmethod - def _changed(expression, operands): - if isinstance(expression, Constant) and len(operands) > 0: - return True - arity = len(operands) - return any(operands[i] is not expression.operands[i] for i in range(arity)) - def visit_Operation(self, expression, *operands): """ constant folding, if all operands of an expression are a Constant do the math """ expression = self._rebuild(expression, operands) @@ -430,9 +409,9 @@ def visit_Operation(self, expression, *operands): expression = constant_folder(expression) return expression - def visit_BitVecZeroExtend(self, expression, *operands): + def visit_BitvecZeroExtend(self, expression, *operands): if self._changed(expression, operands): - return BitVecZeroExtend(expression.size, *operands, taint=expression.taint) + return BitvecZeroExtend(expression.size, *operands, taint=expression.taint) else: return expression @@ -455,10 +434,10 @@ def visit_BoolAnd(self, expression, *operands): operand_1_1 = operand_1.operands[1] if ( - isinstance(operand_0_0, BitVecExtract) - and isinstance(operand_0_1, BitVecExtract) - and isinstance(operand_1_0, BitVecExtract) - and isinstance(operand_1_1, BitVecExtract) + isinstance(operand_0_0, BitvecExtract) + and isinstance(operand_0_1, BitvecExtract) + and isinstance(operand_1_0, BitvecExtract) + and isinstance(operand_1_1, BitvecExtract) ): if ( @@ -477,7 +456,7 @@ def visit_BoolAnd(self, expression, *operands): value1 = operand_0_1.value beg = min(operand_0_0.begining, operand_1_0.begining) end = max(operand_0_0.end, operand_1_0.end) - return BitVecExtract(value0, beg, end - beg + 1) == BitVecExtract( + return BitvecExtract(value0, beg, end - beg + 1) == BitvecExtract( value1, beg, end - beg + 1 ) @@ -490,7 +469,7 @@ def visit_BoolEqual(self, expression, *operands): (EQ, ITE(cond, constant1, constant2), constant2) -> NOT cond (EQ (extract a, b, c) (extract a, b, c)) """ - if isinstance(operands[0], BitVecITE) and isinstance(operands[1], Constant): + if isinstance(operands[0], BitvecITE) and isinstance(operands[1], Constant): if isinstance(operands[0].operands[1], Constant) and isinstance( operands[0].operands[2], Constant ): @@ -507,7 +486,7 @@ def visit_BoolEqual(self, expression, *operands): if operands[0] is operands[1]: return BoolConstant(True, taint=expression.taint) - if isinstance(operands[0], BitVecExtract) and isinstance(operands[1], BitVecExtract): + if isinstance(operands[0], BitvecExtract) and isinstance(operands[1], BitvecExtract): if ( operands[0].value is operands[1].value and operands[0].end == operands[1].end @@ -530,7 +509,7 @@ def visit_BoolOr(self, expression, a, b): if a is b: return a - def visit_BitVecITE(self, expression, *operands): + def visit_BitvecITE(self, expression, *operands): if isinstance(operands[0], Constant): if operands[0].value: result = operands[1] @@ -543,9 +522,9 @@ def visit_BitVecITE(self, expression, *operands): return result if self._changed(expression, operands): - return BitVecITE(expression.size, *operands, taint=expression.taint) + return BitvecITE(*operands, taint=expression.taint) - def visit_BitVecConcat(self, expression, *operands): + def visit_BitvecConcat(self, expression, *operands): """ concat( extract(k1, 0, a), extract(sizeof(a)-k1, k1, a)) ==> a concat( extract(k1, beg, a), extract(end, k1, a)) ==> extract(beg, end, a) concat( x , extract(k1, beg, a), extract(end, k1, a), z) ==> concat( x , extract(k1, beg, a), extract(end, k1, a), z) @@ -557,12 +536,12 @@ def visit_BitVecConcat(self, expression, *operands): last_o = None new_operands = [] for o in operands: - if isinstance(o, BitVecExtract): + if isinstance(o, BitvecExtract): if last_o is None: last_o = o else: if last_o.value is o.value and last_o.begining == o.end + 1: - last_o = BitVecExtract( + last_o = BitvecExtract( o.value, o.begining, last_o.end - o.begining + 1, taint=expression.taint ) changed = True @@ -577,15 +556,15 @@ def visit_BitVecConcat(self, expression, *operands): if last_o is not None: new_operands.append(last_o) if changed: - return BitVecConcat(expression.size, *new_operands) + return BitvecConcat(operands=tuple(new_operands)) op = operands[0] value = None end = None begining = None for o in operands: - # If found a non BitVecExtract, do not apply - if not isinstance(o, BitVecExtract): + # If found a non BitvecExtract, do not apply + if not isinstance(o, BitvecExtract): value = None break # Set the value for the first item @@ -607,11 +586,11 @@ def visit_BitVecConcat(self, expression, *operands): if value is not None: if end + 1 != value.size or begining != 0: - return BitVecExtract(value, begining, end - begining + 1, taint=expression.taint) + return BitvecExtract(value, begining, end - begining + 1, taint=expression.taint) return value - def visit_BitVecExtract(self, expression, *operands): + def visit_BitvecExtract(self, expression, *operands): """ extract(sizeof(a), 0)(a) ==> a extract(16, 0)( concat(a,b,c,d) ) => concat(c, d) extract(m,M)(and/or/xor a b ) => and/or/xor((extract(m,M) a) (extract(m,M) a) @@ -623,15 +602,15 @@ def visit_BitVecExtract(self, expression, *operands): # extract(sizeof(a), 0)(a) ==> a if begining == 0 and end + 1 == op.size: return op - elif isinstance(op, BitVecExtract): - return BitVecExtract(op.value, op.begining + begining, size, taint=expression.taint) - elif isinstance(op, BitVecConcat): + elif isinstance(op, BitvecExtract): + return BitvecExtract(op.value, op.begining + begining, size, taint=expression.taint) + elif isinstance(op, BitvecConcat): new_operands = [] for item in reversed(op.operands): if size == 0: assert expression.size == sum([x.size for x in new_operands]) - return BitVecConcat( - expression.size, *reversed(new_operands), taint=expression.taint + return BitvecConcat( + operands=tuple(reversed(new_operands)), taint=expression.taint ) if begining >= item.size: @@ -643,62 +622,62 @@ def visit_BitVecExtract(self, expression, *operands): size = 0 else: if size <= item.size - begining: - new_operands.append(BitVecExtract(item, begining, size)) + new_operands.append(BitvecExtract(item, begining, size)) size = 0 else: - new_operands.append(BitVecExtract(item, begining, item.size - begining)) + new_operands.append(BitvecExtract(item, begining, item.size - begining)) size -= item.size - begining begining = 0 - elif isinstance(op, BitVecConstant): - return BitVecConstant(size, (op.value >> begining) & ~(1 << size)) + elif isinstance(op, BitvecConstant): + return BitvecConstant(size, (op.value >> begining) & ~(1 << size)) - if isinstance(op, (BitVecAnd, BitVecOr, BitVecXor)): + if isinstance(op, (BitvecAnd, BitvecOr, BitvecXor)): bitoperand_a, bitoperand_b = op.operands return op.__class__( - BitVecExtract(bitoperand_a, begining, expression.size), - BitVecExtract(bitoperand_b, begining, expression.size), + BitvecExtract(bitoperand_a, begining, expression.size), + BitvecExtract(bitoperand_b, begining, expression.size), taint=expression.taint, ) - def visit_BitVecAdd(self, expression, *operands): + def visit_BitvecAdd(self, expression, *operands): """ a + 0 ==> a 0 + a ==> a """ left = operands[0] right = operands[1] - if isinstance(right, BitVecConstant): + if isinstance(right, BitvecConstant): if right.value == 0: return left - if isinstance(left, BitVecConstant): + if isinstance(left, BitvecConstant): if left.value == 0: return right - def visit_BitVecSub(self, expression, *operands): + def visit_BitvecSub(self, expression, *operands): """ a - 0 ==> 0 (a + b) - b ==> a (b + a) - b ==> a """ left = operands[0] right = operands[1] - if isinstance(left, BitVecAdd): + if isinstance(left, BitvecAdd): if self._same_constant(left.operands[0], right): return left.operands[1] elif self._same_constant(left.operands[1], right): return left.operands[0] - elif isinstance(left, BitVecSub) and isinstance(right, Constant): + elif isinstance(left, BitvecSub) and isinstance(right, Constant): subleft = left.operands[0] subright = left.operands[1] if isinstance(subright, Constant): - return BitVecSub( + return BitvecSub( subleft, - BitVecConstant( + BitvecConstant( subleft.size, subright.value + right.value, taint=subright.taint | right.taint, ), ) - def visit_BitVecOr(self, expression, *operands): + def visit_BitvecOr(self, expression, *operands): """ a | 0 => a 0 | a => a 0xffffffff & a => 0xffffffff @@ -707,20 +686,20 @@ def visit_BitVecOr(self, expression, *operands): """ left = operands[0] right = operands[1] - if isinstance(right, BitVecConstant): + if isinstance(right, BitvecConstant): if right.value == 0: return left elif right.value == left.mask: return right - elif isinstance(left, BitVecOr): + elif isinstance(left, BitvecOr): left_left = left.operands[0] left_right = left.operands[1] if isinstance(right, Constant): - return BitVecOr(left_left, (left_right | right), taint=expression.taint) - elif isinstance(left, BitVecConstant): - return BitVecOr(right, left, taint=expression.taint) + return BitvecOr(left_left, (left_right | right), taint=expression.taint) + elif isinstance(left, BitvecConstant): + return BitvecOr(right, left, taint=expression.taint) - def visit_BitVecAnd(self, expression, *operands): + def visit_BitvecAnd(self, expression, *operands): """ ct & x => x & ct move constants to the right a & 0 => 0 remove zero a & 0xffffffff => a remove full mask @@ -729,31 +708,31 @@ def visit_BitVecAnd(self, expression, *operands): """ left = operands[0] right = operands[1] - if isinstance(right, BitVecConstant): + if isinstance(right, BitvecConstant): if right.value == 0: return right elif right.value == right.mask: return left - elif isinstance(left, BitVecAnd): + elif isinstance(left, BitvecAnd): left_left = left.operands[0] left_right = left.operands[1] if isinstance(right, Constant): - return BitVecAnd(left_left, left_right & right, taint=expression.taint) - elif isinstance(left, BitVecOr): + return BitvecAnd(left_left, left_right & right, taint=expression.taint) + elif isinstance(left, BitvecOr): left_left = left.operands[0] left_right = left.operands[1] - return BitVecOr(right & left_left, right & left_right, taint=expression.taint) + return BitvecOr(right & left_left, right & left_right, taint=expression.taint) - elif isinstance(left, BitVecConstant): - return BitVecAnd(right, left, taint=expression.taint) + elif isinstance(left, BitvecConstant): + return BitvecAnd(right, left, taint=expression.taint) - def visit_BitVecShiftLeft(self, expression, *operands): + def visit_BitvecShiftLeft(self, expression, *operands): """ a << 0 => a remove zero a << ct => 0 if ct > sizeof(a) remove big constant shift """ left = operands[0] right = operands[1] - if isinstance(right, BitVecConstant): + if isinstance(right, BitvecConstant): if right.value == 0: return left elif right.value >= right.size: @@ -766,24 +745,24 @@ def visit_ArraySelect(self, expression, *operands): return None arr, index = operands if isinstance(arr, ArrayVariable): - return None + return self._visit_Operation(expression, *operands) - if isinstance(index, BitVecConstant): + if isinstance(index, BitvecConstant): ival = index.value # props are slow and using them in tight loops should be avoided, esp when they offer no additional validation # arr._operands[1] = arr.index, arr._operands[0] = arr.array while ( isinstance(arr, ArrayStore) - and isinstance(arr._operands[1], BitVecConstant) + and isinstance(arr._operands[1], BitvecConstant) and arr._operands[1]._value != ival ): arr = arr._operands[0] # arr.array if ( - isinstance(index, BitVecConstant) + isinstance(index, BitvecConstant) and isinstance(arr, ArrayStore) - and isinstance(arr.index, BitVecConstant) + and isinstance(arr.index, BitvecConstant) and arr.index.value == index.value ): if arr.value is not None: @@ -804,8 +783,10 @@ def visit_Expression(self, expression, *operands): arithmetic_simplifier_cache = CacheDict(max_size=250000, flush_perc=25) -@lru_cache(maxsize=128, typed=True) +#@lru_cache(maxsize=128, typed=True) def arithmetic_simplify(expression): + if not isinstance(expression, Expression): + return expression global arithmetic_simplifier_cache simp = ArithmeticSimplifier(cache=arithmetic_simplifier_cache) simp.visit(expression, use_fixed_point=True) @@ -838,7 +819,7 @@ def to_constant(expression): return value -@lru_cache(maxsize=128, typed=True) +#@lru_cache(maxsize=128, typed=True) def simplify(expression): expression = arithmetic_simplify(expression) return expression @@ -885,42 +866,42 @@ def bindings(self): BoolOr: "or", BoolXor: "xor", BoolITE: "ite", - BitVecAdd: "bvadd", - BitVecSub: "bvsub", - BitVecMul: "bvmul", - BitVecDiv: "bvsdiv", - BitVecUnsignedDiv: "bvudiv", - BitVecMod: "bvsmod", - BitVecRem: "bvsrem", - BitVecUnsignedRem: "bvurem", - BitVecShiftLeft: "bvshl", - BitVecShiftRight: "bvlshr", - BitVecArithmeticShiftLeft: "bvashl", - BitVecArithmeticShiftRight: "bvashr", - BitVecAnd: "bvand", - BitVecOr: "bvor", - BitVecXor: "bvxor", - BitVecNot: "bvnot", - BitVecNeg: "bvneg", - LessThan: "bvslt", - LessOrEqual: "bvsle", - GreaterThan: "bvsgt", - GreaterOrEqual: "bvsge", - UnsignedLessThan: "bvult", - UnsignedLessOrEqual: "bvule", - UnsignedGreaterThan: "bvugt", - UnsignedGreaterOrEqual: "bvuge", - BitVecSignExtend: "(_ sign_extend %d)", - BitVecZeroExtend: "(_ zero_extend %d)", - BitVecExtract: "(_ extract %d %d)", - BitVecConcat: "concat", - BitVecITE: "ite", + BitvecAdd: "bvadd", + BitvecSub: "bvsub", + BitvecMul: "bvmul", + BitvecDiv: "bvsdiv", + BitvecUnsignedDiv: "bvudiv", + BitvecMod: "bvsmod", + BitvecRem: "bvsrem", + BitvecUnsignedRem: "bvurem", + BitvecShiftLeft: "bvshl", + BitvecShiftRight: "bvlshr", + BitvecArithmeticShiftLeft: "bvashl", + BitvecArithmeticShiftRight: "bvashr", + BitvecAnd: "bvand", + BitvecOr: "bvor", + BitvecXor: "bvxor", + BitvecNot: "bvnot", + BitvecNeg: "bvneg", + BoolLessThan: "bvslt", + BoolLessOrEqualThan: "bvsle", + BoolGreaterThan: "bvsgt", + BoolGreaterOrEqualThan: "bvsge", + BoolUnsignedLessThan: "bvult", + BoolUnsignedLessOrEqualThan: "bvule", + BoolUnsignedGreaterThan: "bvugt", + BoolUnsignedGreaterOrEqualThan: "bvuge", + BitvecSignExtend: "(_ sign_extend %d)", + BitvecZeroExtend: "(_ zero_extend %d)", + BitvecExtract: "(_ extract %d %d)", + BitvecConcat: "concat", + BitvecITE: "ite", ArrayStore: "store", ArraySelect: "select", } - def visit_BitVecConstant(self, expression): - assert isinstance(expression, BitVecConstant) + def visit_BitvecConstant(self, expression): + assert isinstance(expression, BitvecConstant) if expression.size == 1: return "#" + bin(expression.value & expression.mask)[1:] else: @@ -939,20 +920,16 @@ def visit_ArraySelect(self, expression, *operands): array_smt = self._add_binding(expression.array, array_smt) return "(select %s %s)" % (array_smt, index_smt) - def _visit_operation(self, expression, *operands): + def visit_Operation(self, expression, *operands): operation = self.translation_table[type(expression)] - if isinstance(expression, (BitVecSignExtend, BitVecZeroExtend)): + if isinstance(expression, (BitvecSignExtend, BitvecZeroExtend)): operation = operation % expression.extend - elif isinstance(expression, BitVecExtract): + elif isinstance(expression, BitvecExtract): operation = operation % (expression.end, expression.begining) operands = [self._add_binding(*x) for x in zip(expression.operands, operands)] return "(%s %s)" % (operation, " ".join(operands)) - visit_ArrayOperation = _visit_operation - visit_BoolOperation = _visit_operation - visit_BitVecOperation = _visit_operation - @property def result(self): output = super().result @@ -964,12 +941,12 @@ def result(self): def declarations(self): result = '' for exp in self._variables: - if isinstance(exp, BitVec): + if isinstance(exp, Bitvec): result += f"(declare-fun {exp.name} () (_ BitVec {exp.size}))\n" elif isinstance(exp, Bool): result += f"(declare-fun {exp.name} () Bool)\n" elif isinstance(exp, Array): - result += f"(declare-fun {exp.name} () (Array (_ BitVec {exp.index_bits}) (_ BitVec {exp.value_bits})))\n" + result += f"(declare-fun {exp.name} () (Array (_ BitVec {exp.index_size}) (_ BitVec {exp.value_size})))\n" else: raise ConstraintException(f"Type not supported {exp!r}") return result @@ -999,15 +976,14 @@ def __init__(self, bindings=None, **kwargs): raise ValueError("bindings needed in replace") self._replace_bindings = bindings - def _visit_variable(self, expression): + def visit(self, *args, **kwargs): + return super().visit(*args, **kwargs) + + def visit_Variable(self, expression): if expression in self._replace_bindings: return self._replace_bindings[expression] return expression - visit_ArrayVariable = _visit_variable - visit_BitVecVariable = _visit_variable - visit_BoolVariable = _visit_variable - def replace(expression, bindings): if not bindings: @@ -1031,7 +1007,7 @@ def __init__(self, target_index, **kwargs): self.stores = [] def visit_ArrayStore(self, exp, target, where, what): - if not isinstance(what, BitVecConstant): + if not isinstance(what, BitvecConstant): raise self.ExpressionNotSimple if where.value == self._target_index: diff --git a/manticore/core/state.py b/manticore/core/state.py index 93067d772..d0e99de0c 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,7 +1,7 @@ import copy import logging -from .smtlib import solver, Bool, issymbolic, BitVecConstant +from .smtlib import solver, Bool, issymbolic, BitvecConstant from ..utils.event import Eventful from ..utils.helpers import PickleSerializer @@ -68,7 +68,7 @@ def _setstate(self, state, _value): def __init__(self, filename, **kwargs): super().__init__( f"Saving state to {filename}", - BitVecConstant(32, 0), + BitvecConstant(32, 0), setstate=self._setstate, policy="ONE", **kwargs, diff --git a/manticore/ethereum/abi.py b/manticore/ethereum/abi.py index 88a6a481f..35b5d5238 100644 --- a/manticore/ethereum/abi.py +++ b/manticore/ethereum/abi.py @@ -6,7 +6,15 @@ import sha3 from . import abitypes -from ..core.smtlib import Array, Operators, BitVec, ArrayVariable, to_constant, issymbolic +from ..core.smtlib import ( + Array, + Operators, + Bitvec, + ArrayVariable, + ArrayProxy, + to_constant, + issymbolic, +) from ..exceptions import EthereumError logger = logging.getLogger(__name__) @@ -102,8 +110,8 @@ def _serialize(ty, value, dyn_offset=None): if dyn_offset is None: dyn_offset = ABI._type_size(ty) - result = bytes() - dyn_result = bytes() + result = bytearray() + dyn_result = bytearray() if ty[0] == "int": result += ABI._serialize_int(value, size=ty[1] // 8, padding=32 - ty[1] // 8) @@ -150,8 +158,8 @@ def _serialize_bytes(value): @staticmethod def _serialize_tuple(types, value, dyn_offset=None): - result = bytes() - dyn_result = bytes() + result = bytearray() + dyn_result = bytearray() if len(types) != len(value): raise ValueError( f"The number of values to serialize is {'less' if len(value) < len(types) else 'greater'} than the number of types" @@ -198,10 +206,10 @@ def function_selector(method_name_and_signature): def deserialize(type_spec, data): try: if isinstance(data, str): - data = bytes(data.encode()) + data = bytearray(data.encode()) elif isinstance(data, bytes): - data = bytes(data) - assert isinstance(data, (bytes, Array)) + data = bytearray(data) + assert isinstance(data, (bytearray, Array)) m = re.match(r"(?P[a-zA-Z_0-9]+)(?P\(.*\))", type_spec) if m and m.group("name"): @@ -221,7 +229,7 @@ def deserialize(type_spec, data): @staticmethod def _deserialize(ty, buf: typing.Union[bytearray, bytes, Array], offset=0): - assert isinstance(buf, (bytes, Array)) + assert isinstance(buf, (bytearray, bytes, Array)) result = None if ty[0] == "int": result = ABI._deserialize_int(buf[offset : offset + 32], nbytes=ty[1] // 8) @@ -263,26 +271,26 @@ def _deserialize(ty, buf: typing.Union[bytearray, bytes, Array], offset=0): @staticmethod def _serialize_uint(value, size=32, padding=0): """ - Translates a python integral or a BitVec into a 32 byte string, MSB first + Translates a python integral or a Bitvec into a 32 byte string, MSB first """ if size <= 0 or size > 32: raise ValueError from .account import EVMAccount # because of circular import - if not isinstance(value, (int, BitVec, EVMAccount)): + if not isinstance(value, (int, Bitvec, EVMAccount)): raise ValueError if issymbolic(value): # Help mypy out. Can remove this by teaching it how issymbolic works - assert isinstance(value, BitVec) + assert isinstance(value, Bitvec) # FIXME This temporary array variable should be obtained from a specific constraint store buffer = ArrayVariable( - index_bits=256, index_max=32, value_bits=8, name="temp{}".format(uuid.uuid1()) + index_bits=256, length=32, value_bits=8, name="temp{}".format(uuid.uuid1()) ) if value.size <= size * 8: value = Operators.ZEXTEND(value, size * 8) else: - # automatically truncate, e.g. if they passed a BitVec(256) for an `address` argument (160 bits) + # automatically truncate, e.g. if they passed a Bitvec(256) for an `address` argument (160 bits) value = Operators.EXTRACT(value, 0, size * 8) buffer = buffer.write_BE(padding, value, size) else: @@ -297,22 +305,22 @@ def _serialize_uint(value, size=32, padding=0): return buffer @staticmethod - def _serialize_int(value: typing.Union[int, BitVec], size=32, padding=0): + def _serialize_int(value: typing.Union[int, Bitvec], size=32, padding=0): """ - Translates a signed python integral or a BitVec into a 32 byte string, MSB first + Translates a signed python integral or a Bitvec into a 32 byte string, MSB first """ if size <= 0 or size > 32: raise ValueError - if not isinstance(value, (int, BitVec)): + if not isinstance(value, (int, Bitvec)): raise ValueError if issymbolic(value): # Help mypy out. Can remove this by teaching it how issymbolic works - assert isinstance(value, BitVec) + assert isinstance(value, Bitvec) buf = ArrayVariable( - index_bits=256, index_max=32, value_bits=8, name="temp{}".format(uuid.uuid1()) + index_bits=256, length=32, value_bits=8, name="temp{}".format(uuid.uuid1()) ) value = Operators.SEXTEND(value, value.size, size * 8) - return buf.write_BE(padding, value, size) + return ArrayProxy(buf.write_BE(padding, value, size)) else: buf_arr = bytearray() for _ in range(padding): @@ -350,7 +358,9 @@ def _readBE(data, nbytes, padding=False, offset=0): return Operators.CONCAT(nbytes * 8, *values) @staticmethod - def _deserialize_uint(data: typing.Union[bytes, Array], nbytes=32, padding=0, offset=0): + def _deserialize_uint( + data: typing.Union[bytearray, bytes, Array], nbytes=32, padding=0, offset=0 + ): """ Read a `nbytes` bytes long big endian unsigned integer from `data` starting at `offset` @@ -358,13 +368,13 @@ def _deserialize_uint(data: typing.Union[bytes, Array], nbytes=32, padding=0, of :param nbytes: number of bytes to read starting from least significant byte :rtype: int or Expression """ - assert isinstance(data, (bytes, Array)) + assert isinstance(data, (bytearray, bytes, Array)) value = ABI._readBE(data, nbytes, padding=True, offset=offset) value = Operators.ZEXTEND(value, (nbytes + padding) * 8) return value @staticmethod - def _deserialize_int(data: typing.Union[bytes, Array], nbytes=32, padding=0): + def _deserialize_int(data: typing.Union[bytearray, bytes, Array], nbytes=32, padding=0): """ Read a `nbytes` bytes long big endian signed integer from `data` starting at `offset` @@ -372,7 +382,7 @@ def _deserialize_int(data: typing.Union[bytes, Array], nbytes=32, padding=0): :param nbytes: number of bytes to read starting from least significant byte :rtype: int or Expression """ - assert isinstance(data, (bytes, Array)) + assert isinstance(data, (bytearray, bytes, Array)) value = ABI._readBE(data, nbytes, padding=True) value = Operators.SEXTEND(value, nbytes * 8, (nbytes + padding) * 8) if not issymbolic(value): diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 23a937a90..40da37395 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -13,12 +13,13 @@ from crytic_compile import CryticCompile, InvalidCompilation, is_supported -from ..core.manticore import ManticoreBase +from ..core.manticore import ManticoreBase, ManticoreError + from ..core.smtlib import ( ConstraintSet, Array, ArrayProxy, - BitVec, + Bitvec, Operators, BoolConstant, BoolOperation, @@ -189,7 +190,7 @@ def make_symbolic_address(self, *accounts, name=None, select="both"): :param name: Name of the symbolic variable. Defaults to 'TXADDR' and later to 'TXADDR_' :param select: Whether to select contracts or normal accounts. Not implemented for now. - :return: Symbolic address in form of a BitVecVariable. + :return: Symbolic address in form of a BitvecVariable. """ if select not in ("both", "normal", "contract"): raise EthereumError("Wrong selection type") @@ -308,7 +309,6 @@ def _compile_through_crytic_compile(filename, contract_name, libraries, crytic_c filename = crytic_compile.filename_of_contract(name).absolute with open(filename) as f: source_code = f.read() - return name, source_code, bytecode, runtime, srcmap, srcmap_runtime, hashes, abi except InvalidCompilation as e: @@ -547,7 +547,7 @@ def solidity_create_contract( :param contract_name: Name of the contract to analyze (optional if there is a single one in the source code) :type contract_name: str :param balance: balance to be transferred on creation - :type balance: int or BitVecVariable + :type balance: int or BitvecVariable :param address: the address for the new contract (optional) :type address: int or EVMAccount :param tuple args: constructor arguments @@ -666,7 +666,7 @@ def create_contract(self, owner, balance=0, address=None, init=None, name=None, :param owner: owner account (will be default caller in any transactions) :type owner: int or EVMAccount :param balance: balance to be transferred on creation - :type balance: int or BitVecVariable + :type balance: int or BitvecVariable :param int address: the address for the new contract (optional) :param str init: initializing evm bytecode and arguments :param str name: a unique name for reference @@ -754,7 +754,7 @@ def transaction(self, caller, address, value, data, gas=None, price=1): :param address: the address of the contract to call :type address: int or EVMAccount :param value: balance to be transfered on creation - :type value: int or BitVecVariable + :type value: int or BitvecVariable :param data: initial data :param gas: gas budget :param price: gas unit price @@ -768,7 +768,7 @@ def create_account(self, balance=0, address=None, code=None, name=None, nonce=No """ Low level creates an account. This won't generate a transaction. :param balance: balance to be set on creation (optional) - :type balance: int or BitVecVariable + :type balance: int or BitvecVariable :param address: the address for the new account (optional) :type address: int :param code: the runtime code for the new account (None means normal account), str or bytes (optional) @@ -877,7 +877,7 @@ def _transaction(self, sort, caller, value=0, address=None, data=None, gas=None, :param int address: the address for the transaction (optional) :param value: value to be transferred :param price: the price of gas for this transaction. - :type value: int or BitVecVariable + :type value: int or BitvecVariable :param str data: initializing evm bytecode and arguments or transaction call data :param gas: gas budget for current transaction :rtype: EVMAccount @@ -894,17 +894,17 @@ def _transaction(self, sort, caller, value=0, address=None, data=None, gas=None, data = b"" if isinstance(data, str): data = bytes(data) - if not isinstance(data, (bytes, Array, ArrayProxy)): + if not isinstance(data, (bytes, Array)): raise TypeError("code bad type") # Check types - if not isinstance(caller, (int, BitVec)): + if not isinstance(caller, (int, Bitvec)): raise TypeError("Caller invalid type") - if not isinstance(value, (int, BitVec)): + if not isinstance(value, (int, Bitvec)): raise TypeError("Value invalid type") - if not isinstance(address, (int, BitVec)): + if not isinstance(address, (int, Bitvec)): raise TypeError("address invalid type") if not isinstance(price, int): @@ -1204,8 +1204,6 @@ def _on_unsound_symbolication(self, state, func, data, result): name = func.__name__ value = func(data) # If func returns None then result should considered unknown/symbolic - if isinstance(data, ArrayProxy): - data = data.array # Value is known. Let's add it to our concrete database if value is not None: with self.locked_context("ethereum", dict) as ethereum_context: @@ -1233,7 +1231,7 @@ def _on_unsound_symbolication(self, state, func, data, result): data_var = state.new_symbolic_buffer(len(data)) # FIXME: generalize to bitvec state.constrain(data_var == data) - data = data_var.array + data = data_var # symbolic_pairs list of symbolic applications of func in sate symbolic_pairs = state.context.get(f"symbolic_func_sym_{name}", []) @@ -1256,7 +1254,6 @@ def _on_unsound_symbolication(self, state, func, data, result): else: constraint = y != value state.constrain(constraint) - print(type(data), type(value)) symbolic_pairs.append((data, value)) state.context[f"symbolic_func_sym_{name}"] = symbolic_pairs @@ -1539,10 +1536,6 @@ def current_location(self, state): def generate_testcase(self, state, message="", only_if=None, name="user"): """ - Generate a testcase to the workspace for the given program state. The details of what - a testcase is depends on the type of Platform the state is, but involves serializing the state, - and generating an input (concretizing symbolic variables) to trigger this state. - The only_if parameter should be a symbolic expression. If this argument is provided, and the expression *can be true* in this state, a testcase is generated such that the expression will be true in the state. If it *is impossible* for the expression to be true in the state, a testcase is not generated. @@ -1555,29 +1548,38 @@ def generate_testcase(self, state, message="", only_if=None, name="user"): m.generate_testcase(state, 'balance CAN be 0', only_if=balance == 0) # testcase generated with an input that will violate invariant (make balance == 0) + """ + try: + with state as temp_state: + if only_if is not None: + temp_state.constrain(only_if) + return self.generate_testcase_ex(temp_state, message, name=name) + except ManticoreError: + return None + + def generate_testcase_ex(self, state, message="", name="user"): + """ + Generate a testcase in the outputspace for the given program state. + An exception is raised if the state is unsound or unfeasible + + On return you get a Testcase containing all the informations about the state. + A Testcase is a collection of files living at the OutputSpace. + Registered plugins and other parts of Manticore add files to the Testcase. + User can add more streams to is like this: + + testcase = m.generate_testcase_ex(state) + with testcase.open_stream("txt") as descf: + descf.write("This testcase is interesting!") :param manticore.core.state.State state: :param str message: longer description of the testcase condition - :param manticore.core.smtlib.Bool only_if: only if this expr can be true, generate testcase. if is None, generate testcase unconditionally. - :param str name: short string used as the prefix for the workspace key (e.g. filename prefix for testcase files) - :return: If a testcase was generated + :param str name: short string used as the prefix for the outputspace key (e.g. filename prefix for testcase files) + :return: a Testcase :rtype: bool """ - """ - Create a serialized description of a given state. - :param state: The state to generate information about - :param message: Accompanying message - """ + # Refuse to generate a testcase from an unsound state if not self.fix_unsound_symbolication(state): - return False - - if only_if is not None: - with state as temp_state: - temp_state.constrain(only_if) - if temp_state.is_feasible(): - return self.generate_testcase(temp_state, message, only_if=None, name=name) - else: - return False + raise ManticoreError("Trying to generate a testcase out of an unsound state path") blockchain = state.platform @@ -1587,8 +1589,6 @@ def generate_testcase(self, state, message="", only_if=None, name="user"): testcase = super().generate_testcase( state, message + f"({len(blockchain.human_transactions)} txs)", name=name ) - # TODO(mark): Refactor ManticoreOutput to let the platform be more in control - # so this function can be fully ported to EVMWorld.generate_workspace_files. local_findings = set() for detector in self.detectors.values(): @@ -1719,11 +1719,7 @@ def finalizer(state_id): if self.fix_unsound_symbolication(st): last_tx = st.platform.last_transaction # Do not generate killed state if only_alive_states is True - if ( - not last_tx - or only_alive_states - and last_tx.result in {"REVERT", "THROW", "TXERROR"} - ): + if only_alive_states and last_tx.result in {"REVERT", "THROW", "TXERROR"}: return logger.debug("Generating testcase for state_id %d", state_id) message = last_tx.result if last_tx else "NO STATE RESULT (?)" @@ -1872,8 +1868,8 @@ def ready_sound_states(self): This tries to solve any symbolic imprecision added by unsound_symbolication and then iterates over the resultant set. - This is the recommended to iterate over resultant steas after an exploration - that included unsound symbolication + This is the recommended way to iterate over the resultant states after + an exploration that included unsound symbolication """ self.fix_unsound_all() diff --git a/manticore/native/cpu/abstractcpu.py b/manticore/native/cpu/abstractcpu.py index 57489de5c..d8d353601 100644 --- a/manticore/native/cpu/abstractcpu.py +++ b/manticore/native/cpu/abstractcpu.py @@ -655,7 +655,7 @@ def write_int(self, where, expression, size=None, force=False): :param int where: address to write to :param expr: value to write - :type expr: int or BitVec + :type expr: int or Bitvec :param size: bit size of `expr` :param force: whether to ignore memory permissions """ @@ -713,7 +713,7 @@ def read_int(self, where, size=None, force=False): :param int where: address to read from :param size: number of bits to read :return: the value read - :rtype: int or BitVec + :rtype: int or Bitvec :param force: whether to ignore memory permissions """ if size is None: diff --git a/manticore/native/cpu/arm.py b/manticore/native/cpu/arm.py index 6d87aea84..10c6c611d 100644 --- a/manticore/native/cpu/arm.py +++ b/manticore/native/cpu/arm.py @@ -10,7 +10,7 @@ from .abstractcpu import instruction as abstract_instruction from .bitwise import * from .register import Register -from ...core.smtlib import Operators, BitVecConstant, issymbolic +from ...core.smtlib import Operators, BitvecConstant, issymbolic from typing import NamedTuple @@ -405,7 +405,7 @@ def _read_APSR(self): def make_apsr_flag(flag_expr, offset): """Helper for constructing an expression for the APSR register""" return Operators.ITEBV( - 32, flag_expr, BitVecConstant(32, 1 << offset), BitVecConstant(32, 0) + 32, flag_expr, BitvecConstant(32, 1 << offset), BitvecConstant(32, 0) ) apsr = 0 @@ -719,7 +719,7 @@ def stack_push(self, data, nbytes=None): nbytes = nbytes or self.address_bit_size // 8 self.SP -= nbytes self.write_int(self.SP, data, nbytes * 8) - elif isinstance(data, BitVec): + elif isinstance(data, Bitvec): self.SP -= data.size // 8 self.write_int(self.SP, data, data.size) elif isinstance(data, str): diff --git a/manticore/native/cpu/bitwise.py b/manticore/native/cpu/bitwise.py index d00dba1d5..35702306d 100644 --- a/manticore/native/cpu/bitwise.py +++ b/manticore/native/cpu/bitwise.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from ...core.smtlib import Operators -from ...core.smtlib.expression import BitVec +from ...core.smtlib.expression import Bitvec def Mask(width): @@ -19,7 +19,7 @@ def Bit(value, idx): Extract `idx` bit from `value`. :param value: Source value from which to extract. - :type value: int or long or BitVec + :type value: int or long or Bitvec :param idx: Bit index :return: int or long or BitVex """ @@ -31,15 +31,15 @@ def GetNBits(value, nbits): Get the first `nbits` from `value`. :param value: Source value from which to extract - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int nbits: How many bits to extract :return: Low `nbits` bits of `value`. - :rtype int or long or BitVec + :rtype int or long or Bitvec """ # NOP if sizes are the same if isinstance(value, int): return Operators.EXTRACT(value, 0, nbits) - elif isinstance(value, BitVec): + elif isinstance(value, Bitvec): if value.size < nbits: return Operators.ZEXTEND(value, nbits) else: @@ -52,10 +52,10 @@ def SInt(value, width): representation. :param value: The value to convert. - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int width: The width of the bitstring to consider :return: The converted value - :rtype int or long or BitVec + :rtype int or long or Bitvec """ return Operators.ITEBV( width, @@ -70,10 +70,10 @@ def UInt(value, width): Return integer value of `value` as a bitstring of `width` width. :param value: The value to convert. - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int width: The width of the bitstring to consider :return: The integer value - :rtype int or long or BitVec + :rtype int or long or Bitvec """ return GetNBits(value, width) @@ -83,7 +83,7 @@ def LSL_C(value, amount, width, with_carry=True): The ARM LSL_C (logical left shift with carry) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value and the carry result @@ -107,11 +107,11 @@ def LSL(value, amount, width): The ARM LSL (logical left shift) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value - :rtype int or BitVec + :rtype int or Bitvec """ if isinstance(amount, int) and amount == 0: return value @@ -125,7 +125,7 @@ def LSR_C(value, amount, width, with_carry=True): The ARM LSR_C (logical shift right with carry) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value and carry result @@ -146,11 +146,11 @@ def LSR(value, amount, width): The ARM LSR (logical shift right) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value - :rtype int or BitVec + :rtype int or Bitvec """ if isinstance(amount, int) and amount == 0: return value @@ -163,7 +163,7 @@ def ASR_C(value, amount, width, with_carry=True): The ARM ASR_C (arithmetic shift right with carry) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value and carry result @@ -190,11 +190,11 @@ def ASR(value, amount, width): The ARM ASR (arithmetic shift right) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value - :rtype int or BitVec + :rtype int or Bitvec """ if isinstance(amount, int) and amount == 0: return value @@ -208,7 +208,7 @@ def ROR_C(value, amount, width, with_carry=True): The ARM ROR_C (rotate right with carry) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value and carry result @@ -234,11 +234,11 @@ def ROR(value, amount, width): The ARM ROR (rotate right) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value - :rtype int or BitVec + :rtype int or Bitvec """ if isinstance(amount, int) and amount == 0: return value @@ -251,7 +251,7 @@ def RRX_C(value, carry, width, with_carry=True): The ARM RRX (rotate right with extend and with carry) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value and carry result @@ -270,11 +270,11 @@ def RRX(value, carry, width): The ARM RRX (rotate right with extend) operation. :param value: Value to shift - :type value: int or long or BitVec + :type value: int or long or Bitvec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value - :rtype int or BitVec + :rtype int or Bitvec """ result = RRX_C(value, carry, width, with_carry=False) return result diff --git a/manticore/native/cpu/register.py b/manticore/native/cpu/register.py index 8cac2ff4d..150de95dc 100644 --- a/manticore/native/cpu/register.py +++ b/manticore/native/cpu/register.py @@ -1,4 +1,4 @@ -from ...core.smtlib import Operators, BitVec, Bool +from ...core.smtlib import Operators, Bitvec, Bool class Register: @@ -20,7 +20,7 @@ def read(self): def write(self, val): if isinstance(val, (Bool, bool)): self.value = val - elif isinstance(val, BitVec): + elif isinstance(val, Bitvec): self.value = val.Bool() if self.is_flag() else val elif isinstance(val, int): self.value = Operators.EXTRACT(val, 0, self.width) diff --git a/manticore/native/cpu/x86.py b/manticore/native/cpu/x86.py index e839d7268..dc5e0ee75 100644 --- a/manticore/native/cpu/x86.py +++ b/manticore/native/cpu/x86.py @@ -20,7 +20,7 @@ ) -from ...core.smtlib import Operators, BitVec, Bool, BitVecConstant, operator, visitors, issymbolic +from ...core.smtlib import Operators, Bitvec, Bool, BitvecConstant, operator, visitors, issymbolic from ..memory import Memory, ConcretizeMemory from functools import reduce @@ -585,7 +585,7 @@ def _set_bv(self, register_id, register_size, offset, size, reset, value): # if (value & ~((1< Union[int, BitVec]: +def strlen(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: """ strlen symbolic model. @@ -188,7 +188,7 @@ def can_be_NULL(byte, constrs) -> bool: return byte == 0 -def strcpy(state: State, dst: Union[int, BitVec], src: Union[int, BitVec]) -> Union[int, BitVec]: +def strcpy(state: State, dst: Union[int, Bitvec], src: Union[int, Bitvec]) -> Union[int, Bitvec]: """ strcpy symbolic model diff --git a/manticore/native/state_merging.py b/manticore/native/state_merging.py index 332f95b73..0f3230439 100644 --- a/manticore/native/state_merging.py +++ b/manticore/native/state_merging.py @@ -1,4 +1,4 @@ -from ..core.smtlib import SelectedSolver, ConstraintSet, Operators, issymbolic, BitVec +from ..core.smtlib import SelectedSolver, ConstraintSet, Operators, issymbolic, Bitvec def compare_sockets(cs, socket1, socket2): @@ -188,7 +188,7 @@ def merge_cpu(cpu1, cpu2, state, exp1, merged_constraint): for reg in cpu1.canonical_registers: val1 = cpu1.read_register(reg) val2 = cpu2.read_register(reg) - if isinstance(val1, BitVec) and isinstance(val2, BitVec): + if isinstance(val1, Bitvec) and isinstance(val2, Bitvec): assert val1.size == val2.size if issymbolic(val1) or issymbolic(val2) or val1 != val2: val1_migrated = merged_constraint.migrate(val1) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 4aaabdb7d..475c21e64 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -1,4 +1,5 @@ """Symbolic EVM implementation based on the yellow paper: http://gavwood.com/paper.pdf""" +import time import uuid import binascii import random @@ -10,14 +11,14 @@ from ..platforms.platform import * from ..core.smtlib import ( SelectedSolver, - BitVec, + Bitvec, Array, ArrayProxy, Operators, Constant, ArrayVariable, ArrayStore, - BitVecConstant, + BitvecConstant, translate_to_smtlib, to_constant, simplify, @@ -102,6 +103,10 @@ def globalfakesha3(data): default=-1, description="Max calldata size to explore in each CALLDATACOPY. Iff size in a calldata related instruction are symbolic it will be constrained to be less than this constant. -1 means free(only use when gas is being tracked)", ) +consts.add( + "ignore_balance", default=False, description="Do not try to solve symbolic balances", +) + # Auxiliary constants and functions TT256 = 2 ** 256 @@ -123,7 +128,7 @@ def globalfakesha3(data): def ceil32(x): size = 256 - if isinstance(x, BitVec): + if isinstance(x, Bitvec): size = x.size return Operators.ITEBV(size, Operators.UREM(x, 32) == 0, x, x + 32 - Operators.UREM(x, 32)) @@ -251,6 +256,8 @@ def dump(self, stream, state, mevm, conc_tx=None): stream.write("Gas used: %d %s\n" % (conc_tx.gas, flagged(issymbolic(self.gas)))) tx_data = conc_tx.data + if len(tx_data) > 80: + tx_data = tx_data.rstrip(conc_tx.data[-3:-1]) stream.write( "Data: 0x{} {}\n".format( @@ -262,9 +269,9 @@ def dump(self, stream, state, mevm, conc_tx=None): return_data = conc_tx.return_data stream.write( - "Return_data: 0x{} ({}) {}\n".format( + "Return_data: 0x{} {} {}\n".format( binascii.hexlify(return_data).decode(), - printable_bytes(return_data), + f"({printable_bytes(return_data)})" if conc_tx.sort != "CREATE" else "", flagged(issymbolic(self.return_data)), ) ) @@ -375,12 +382,12 @@ def used_gas(self): def set_result(self, result, return_data=None, used_gas=None): if getattr(self, "result", None) is not None: raise EVMException("Transaction result already set") - if not isinstance(used_gas, (int, BitVec, type(None))): + if not isinstance(used_gas, (int, Bitvec, type(None))): raise EVMException("Invalid used gas in Transaction") if result not in {None, "TXERROR", "REVERT", "RETURN", "THROW", "STOP", "SELFDESTRUCT"}: raise EVMException("Invalid transaction result") if result in {"RETURN", "REVERT"}: - if not isinstance(return_data, (bytes, bytearray, Array, ArrayProxy)): + if not isinstance(return_data, (bytes, bytearray, Array)): raise EVMException( "Invalid transaction return_data type:", type(return_data).__name__ ) @@ -482,7 +489,7 @@ def __init__(self, result, data=None): raise EVMException("Invalid end transaction result") if result is None and data is not None: raise EVMException("Invalid end transaction result") - if not isinstance(data, (type(None), Array, ArrayProxy, bytes)): + if not isinstance(data, (type(None), Array, bytes)): raise EVMException("Invalid end transaction data type") self.result = result self.data = data @@ -743,7 +750,7 @@ def __init__( # This is a very cornered corner case in which code is actually symbolic # We should simply not allow to jump to unconstrained(*) symbolic code. # (*) bytecode that could take more than a single value - self._check_jumpdest = False + self._need_check_jumpdest = False self._valid_jumpdests = set() # Compile the list of valid jumpdests via linear dissassembly @@ -876,7 +883,7 @@ def __getstate__(self): state["_published_pre_instruction_events"] = self._published_pre_instruction_events state["_used_calldata_size"] = self._used_calldata_size state["_valid_jumpdests"] = self._valid_jumpdests - state["_check_jumpdest"] = self._check_jumpdest + state["_need_check_jumpdest"] = self._need_check_jumpdest state["_return_data"] = self._return_data state["evmfork"] = self.evmfork state["_refund"] = self._refund @@ -905,7 +912,7 @@ def __setstate__(self, state): self.suicides = state["suicides"] self._used_calldata_size = state["_used_calldata_size"] self._valid_jumpdests = state["_valid_jumpdests"] - self._check_jumpdest = state["_check_jumpdest"] + self._need_check_jumpdest = state["_need_check_jumpdest"] self._return_data = state["_return_data"] self.evmfork = state["evmfork"] self._refund = state["_refund"] @@ -1026,14 +1033,14 @@ def _push(self, value): ITEM2 sp-> {empty} """ - assert isinstance(value, int) or isinstance(value, BitVec) and value.size == 256 + assert isinstance(value, int) or isinstance(value, Bitvec) and value.size == 256 if len(self.stack) >= 1024: raise StackOverflow() if isinstance(value, int): value = value & TT256M1 - value = simplify(value) + #value = simplify(value) if isinstance(value, Constant) and not value.taint: value = value.value @@ -1056,7 +1063,7 @@ def _consume(self, fee): if isinstance(fee, int): if fee > (1 << 512) - 1: raise ValueError - elif isinstance(fee, BitVec): + elif isinstance(fee, Bitvec): if fee.size != 512: raise ValueError("Fees should be 512 bit long") # This configuration variable allows the user to control and perhaps relax the gas calculation @@ -1158,11 +1165,16 @@ def _push_results(self, instruction, result): assert result is None def _calculate_gas(self, *arguments): - current = self.instruction - implementation = getattr(self, f"{current.semantics}_gas", None) - if implementation is None: - return current.fee - return current.fee + implementation(*arguments) + start= time.time() + try: + current = self.instruction + implementation = getattr(self, f"{current.semantics}_gas", None) + if implementation is None: + return current.fee + return current.fee + implementation(*arguments) + finally: + print (f"calculate gas for {current} took {time.time()-start:0.4f}") + def _handler(self, *arguments): current = self.instruction @@ -1213,23 +1225,23 @@ def _set_check_jmpdest(self, flag=True): Note that at this point `flag` can be the conditional from a JUMPI instruction hence potentially a symbolic value. """ - self._check_jumpdest = flag + self._need_check_jumpdest = flag def _check_jmpdest(self): """ If the previous instruction was a JUMP/JUMPI and the conditional was True, this checks that the current instruction must be a JUMPDEST. - Here, if symbolic, the conditional `self._check_jumpdest` would be + Here, if symbolic, the conditional `self._need_check_jumpdest` would be already constrained to a single concrete value. """ # If pc is already pointing to a JUMPDEST thre is no need to check. pc = self.pc.value if isinstance(self.pc, Constant) else self.pc if pc in self._valid_jumpdests: - self._check_jumpdest = False + self._need_check_jumpdest = False return - should_check_jumpdest = simplify(self._check_jumpdest) + should_check_jumpdest = simplify(self._need_check_jumpdest) if isinstance(should_check_jumpdest, Constant): should_check_jumpdest = should_check_jumpdest.value elif issymbolic(should_check_jumpdest): @@ -1242,7 +1254,7 @@ def _check_jmpdest(self): # If it can be solved only to False just set it False. If it can be solved # only to True, process it and also set it to False - self._check_jumpdest = False + self._need_check_jumpdest = False if should_check_jumpdest: if pc not in self._valid_jumpdests: @@ -1287,16 +1299,26 @@ def execute(self): def setstate(state, value): if taints: - state.platform.current_vm.pc = BitVecConstant(256, value, taint=taints) + state.platform.current_vm.pc = BitvecConstant(256, value, taint=taints) else: state.platform.current_vm.pc = value raise Concretize("Symbolic PC", expression=expression, setstate=setstate, policy="ALL") try: - self._check_jmpdest() - last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() - result = self._handler(*arguments) - self._advance(result) + import time + start = time.time() + i = self.instruction + try: + #print (self) + self._check_jmpdest() + last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() + result = self._handler(*arguments) + self._advance(result) + except: + print ("Excaption!") + raise + finally: + print (f"Elapsed {i}: {time.time()-start:0.4f}") except ConcretizeGas as ex: def setstate(state, value): @@ -1395,17 +1417,10 @@ def write_buffer(self, offset, data): self._store(offset + i, Operators.ORD(c)) def _load(self, offset, size=1): - if size == 1: - value = self.memory[offset] - else: - value = self.memory.read_BE(offset, size) - try: - value = simplify(value) - if not value.taint: - value = value.value - except Exception: - pass - + value = self.memory.read_BE(offset, size) + #value = simplify(value) + if isinstance(value, Constant) and not value.taint: + value = value.value self._publish("did_evm_read_memory", offset, value, size) return value @@ -1536,7 +1551,7 @@ def EXP(self, base, exponent): :param base: exponential base, concretized with sampled values :param exponent: exponent value, concretized with sampled values - :return: BitVec* EXP result + :return: Bitvec* EXP result """ if exponent == 0: return 1 @@ -1778,23 +1793,20 @@ def CODECOPY_gas(self, mem_offset, code_offset, size): @concretized_args(code_offset="SAMPLED", size="SAMPLED") def CODECOPY(self, mem_offset, code_offset, size): """Copy code running in current environment to memory""" - + import pdb; pdb.set_trace() self._allocate(mem_offset, size) GCOPY = 3 # cost to copy one 32 byte word copyfee = self.safe_mul(GCOPY, Operators.UDIV(self.safe_add(size, 31), 32)) self._consume(copyfee) if issymbolic(size): - size = simplify(size) - if isinstance(size, Constant): - max_size = size.value - else: - max_size = SelectedSolver.instance().max(self.constraints, size) + max_size = SelectedSolver.instance().max(self.constraints, size) else: max_size = size for i in range(max_size): if issymbolic(i < size): + print("SYMBOLICCCCCCCCCCCCCCCCCC"*100) default = Operators.ITEBV( 8, i < size, 0, self._load(mem_offset + i, 1) ) # Fixme. unnecessary memory read @@ -2043,7 +2055,7 @@ def AND(*args): SstoreCleanRefund, 0, ) - self._refund += simplify(refund) + self._refund += refund return gascost def SSTORE(self, offset, value): @@ -2854,22 +2866,24 @@ def get_storage_data(self, storage_address, offset): :param storage_address: an account address :param offset: the storage slot to use. - :type offset: int or BitVec + :type offset: int or Bitvec :return: the value - :rtype: int or BitVec + :rtype: int or Bitvec """ - value = self._world_state[storage_address]["storage"].get(offset, 0) - return simplify(value) - + start = time.time() + try: + return self._world_state[storage_address]["storage"].get(offset, 0) + finally: + print (f"Ellapseeeeeeeeeeeeeeeeeeeeeeeeeeeeeed in get_storage_data: {time.time()-start:04f}") def set_storage_data(self, storage_address, offset, value): """ Writes a value to a storage slot in specified account :param storage_address: an account address :param offset: the storage slot to use. - :type offset: int or BitVec + :type offset: int or Bitvec :param value: the value to write - :type value: int or BitVec + :type value: int or Bitvec """ self._world_state[storage_address]["storage"][offset] = value @@ -2930,7 +2944,7 @@ def increase_nonce(self, address): return new_nonce def set_balance(self, address, value): - if isinstance(value, BitVec): + if isinstance(value, Bitvec): value = Operators.ZEXTEND(value, 512) self._world_state[int(address)]["balance"] = value @@ -2940,17 +2954,17 @@ def get_balance(self, address): return Operators.EXTRACT(self._world_state[address]["balance"], 0, 256) def add_to_balance(self, address, value): - if isinstance(value, BitVec): + if isinstance(value, Bitvec): value = Operators.ZEXTEND(value, 512) self._world_state[address]["balance"] += value def sub_from_balance(self, address, value): - if isinstance(value, BitVec): + if isinstance(value, Bitvec): value = Operators.ZEXTEND(value, 512) self._world_state[address]["balance"] -= value def send_funds(self, sender, recipient, value): - if isinstance(value, BitVec): + if isinstance(value, Bitvec): value = Operators.ZEXTEND(value, 512) self._world_state[sender]["balance"] -= value self._world_state[recipient]["balance"] += value @@ -2961,7 +2975,7 @@ def get_code(self, address): return self._world_state[address]["code"] def set_code(self, address, data): - assert data is not None and isinstance(data, (bytes, Array, ArrayProxy)) + assert data is not None and isinstance(data, (bytes, Array)) if self._world_state[address]["code"]: raise EVMException("Code already set") self._world_state[address]["code"] = data @@ -3419,21 +3433,24 @@ def dump(self, stream, state, mevm, message): stream.write("\n") # Accounts summary + assert state.can_be_true(True) is_something_symbolic = False stream.write("%d accounts.\n" % len(blockchain.accounts)) for account_address in blockchain.accounts: is_account_address_symbolic = issymbolic(account_address) - account_address = state.solve_one(account_address) + account_address = state.solve_one(account_address, constrain=True) stream.write("* %s::\n" % mevm.account_name(account_address)) stream.write( "Address: 0x%x %s\n" % (account_address, flagged(is_account_address_symbolic)) ) balance = blockchain.get_balance(account_address) - is_balance_symbolic = issymbolic(balance) - is_something_symbolic = is_something_symbolic or is_balance_symbolic - balance = state.solve_one(balance, constrain=True) - stream.write("Balance: %d %s\n" % (balance, flagged(is_balance_symbolic))) + + if not consts.ignore_balance: + is_balance_symbolic = issymbolic(balance) + is_something_symbolic = is_something_symbolic or is_balance_symbolic + balance = state.solve_one(balance, constrain=True) + stream.write("Balance: %d %s\n" % (balance, flagged(is_balance_symbolic))) storage = blockchain.get_storage(account_address) concrete_indexes = set() @@ -3442,7 +3459,7 @@ def dump(self, stream, state, mevm, message): for index in concrete_indexes: stream.write( - f"storage[{index:x}] = {state.solve_one(storage[index], constrain=True):x}" + f"storage[{index:x}] = {state.solve_one(storage[index], constrain=True):x}\n" ) storage = blockchain.get_storage(account_address) stream.write("Storage: %s\n" % translate_to_smtlib(storage, use_bindings=False)) @@ -3468,8 +3485,8 @@ def dump(self, stream, state, mevm, message): stream.write( "storage[%x] = %x %s\n" % ( - state.solve_one(i), - state.solve_one(value), + state.solve_one(i, constrain=True), + state.solve_one(value, constrain=True), flagged(is_storage_symbolic), ) ) diff --git a/manticore/wasm/executor.py b/manticore/wasm/executor.py index dd6eeede7..ba7b48f19 100644 --- a/manticore/wasm/executor.py +++ b/manticore/wasm/executor.py @@ -21,7 +21,7 @@ Value_t, ZeroDivisionTrap, ) -from ..core.smtlib import Operators, BitVec, issymbolic +from ..core.smtlib import Operators, Bitvec, issymbolic from ..utils.event import Eventful from decimal import Decimal, InvalidOperation @@ -289,7 +289,7 @@ def select(self, store, stack): c = stack.pop() v2 = stack.pop() v1 = stack.pop() - assert isinstance(c, (I32, BitVec)), f"{type(c)} is not I32" + assert isinstance(c, (I32, Bitvec)), f"{type(c)} is not I32" if not issymbolic(v2) and not issymbolic(v1): assert type(v2) == type(v1), f"{type(v2)} is not the same as {type(v1)}" diff --git a/manticore/wasm/structure.py b/manticore/wasm/structure.py index 667b052f2..6f6414438 100644 --- a/manticore/wasm/structure.py +++ b/manticore/wasm/structure.py @@ -44,7 +44,7 @@ ZeroDivisionTrap, ) from .state import State -from ..core.smtlib import BitVec, Bool, issymbolic, Operators, Expression +from ..core.smtlib import Bitvec, Bool, issymbolic, Operators, Expression from ..core.state import Concretize from ..utils.event import Eventful from ..utils import config @@ -1022,7 +1022,7 @@ def invoke_by_name(self, name: str, stack, store, argv): passing argv :param name: Name of the function to look for - :param argv: Arguments to pass to the function. Can be BitVecs or Values + :param argv: Arguments to pass to the function. Can be Bitvecs or Values """ for export in self.exports: if export.name == name and isinstance(export.value, FuncAddr): @@ -1041,7 +1041,7 @@ def invoke(self, stack: "Stack", funcaddr: FuncAddr, store: Store, argv: typing. https://www.w3.org/TR/wasm-core-1/#invocation%E2%91%A1 :param funcaddr: Address (in Store) of the function to call - :param argv: Arguments to pass to the function. Can be BitVecs or Values + :param argv: Arguments to pass to the function. Can be Bitvecs or Values """ assert funcaddr in range(len(store.funcs)) funcinst = store.funcs[funcaddr] @@ -1796,7 +1796,7 @@ def empty(self) -> bool: def has_type_on_top(self, t: typing.Union[type, typing.Tuple[type, ...]], n: int): """ - *Asserts* that the stack has at least n values of type t or type BitVec on the top + *Asserts* that the stack has at least n values of type t or type Bitvec on the top :param t: type of value to look for (Bitvec is always included as an option) :param n: Number of values to check @@ -1804,7 +1804,7 @@ def has_type_on_top(self, t: typing.Union[type, typing.Tuple[type, ...]], n: int """ for i in range(1, n + 1): assert isinstance( - self.data[i * -1], (t, BitVec) + self.data[i * -1], (t, Bitvec) ), f"{type(self.data[i * -1])} is not an {t}!" return True diff --git a/manticore/wasm/types.py b/manticore/wasm/types.py index fdecc8034..47e6425be 100644 --- a/manticore/wasm/types.py +++ b/manticore/wasm/types.py @@ -1,6 +1,6 @@ import typing from dataclasses import dataclass -from ..core.smtlib import issymbolic, BitVec +from ..core.smtlib import issymbolic, Bitvec from ctypes import * import wasm import struct @@ -165,9 +165,9 @@ def cast(cls, other): ValType = type #: https://www.w3.org/TR/wasm-core-1/#syntax-valtype -Value_t = (I32, I64, F32, F64, BitVec) -# Value = typing.TypeVar('Value', I32, I64, F32, F64, BitVec) #: https://www.w3.org/TR/wasm-core-1/#syntax-val -Value = typing.Union[I32, I64, F32, F64, BitVec] #: https://www.w3.org/TR/wasm-core-1/#syntax-val +Value_t = (I32, I64, F32, F64, Bitvec) +# Value = typing.TypeVar('Value', I32, I64, F32, F64, Bitvec) #: https://www.w3.org/TR/wasm-core-1/#syntax-val +Value = typing.Union[I32, I64, F32, F64, Bitvec] #: https://www.w3.org/TR/wasm-core-1/#syntax-val class Name(str): @@ -450,7 +450,7 @@ def __init__(self, depth: int, ty: type, message: str, expression, policy=None, :param depth: Index in the stack (should typically be negative) :param ty: The type to cast the :param message: Debug message describing the reason for concretization - :param expression: The expression to concretize, either a Value or a BitVec + :param expression: The expression to concretize, either a Value or a Bitvec """ def setstate(state, value): diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index 4446e451b..de494bef4 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -1,5 +1,7 @@ import binascii import unittest +import subprocess +import pkg_resources from contextlib import contextmanager from pathlib import Path @@ -14,7 +16,7 @@ from manticore.core.plugin import Plugin from manticore.core.smtlib import ConstraintSet, operators from manticore.core.smtlib import Z3Solver -from manticore.core.smtlib.expression import BitVecVariable +from manticore.core.smtlib.expression import Bitvec from manticore.core.smtlib.visitors import to_constant from manticore.core.state import TerminateState from manticore.ethereum import ( @@ -27,12 +29,17 @@ ABI, EthereumError, EVMContract, + verifier, ) from manticore.ethereum.plugins import FilterFunctions from manticore.ethereum.solidity import SolidityMetadata from manticore.platforms import evm from manticore.platforms.evm import EVMWorld, ConcretizeArgument, concretized_args, Return, Stop from manticore.utils.deprecated import ManticoreDeprecationWarning +from manticore.utils import config +import io +import contextlib + solver = Z3Solver.instance() @@ -62,6 +69,29 @@ def test_int_ovf(self): self.assertIn("Unsigned integer overflow at MUL instruction", all_findings) +class EthVerifierIntegrationTest(unittest.TestCase): + def test_propverif(self): + smtcfg = config.get_group("smt") + smtcfg.solver = smtcfg.solver.yices + with smtcfg.temp_vals(): + smtcfg.solver = smtcfg.solver.yices + + filename = os.path.join(THIS_DIR, "contracts/prop_verifier.sol") + f = io.StringIO() + with contextlib.redirect_stdout(f): + verifier.manticore_verifier(filename, "TestToken") + self.assertIsNotNone( + re.compile( + r".*crytic_test_balance\s*\|\s*failed\s*\([0-9a-f]+\).*", re.DOTALL + ).match(f.getvalue()) + ) + + def test_propverif_external(self) -> None: + cli_version = subprocess.check_output(("manticore-verifier", "--version")).decode("utf-8") + py_version = f"Manticore {pkg_resources.get_distribution('manticore').version}\n" + self.assertEqual(cli_version, py_version) + + class EthAbiTests(unittest.TestCase): _multiprocess_can_split = True @@ -93,7 +123,7 @@ def test_parse_tx(self): } } """ - user_account = m.create_account(balance=1000000, name="user_account") + user_account = m.create_account(balance=10 ** 10, name="user_account") contract_account = m.solidity_create_contract( source_code, owner=user_account, name="contract_account", gas=36225 ) @@ -125,7 +155,7 @@ def test_dyn_address(self): def test_dyn_bytes(self): d = [ b"AAAA", # function hash - self._pack_int_to_32(32), # offset to data start1350 + self._pack_int_to_32(32), # offset to data start self._pack_int_to_32(30), # data start; # of elements b"Z" * 30, b"\x00" * 2, @@ -446,7 +476,7 @@ def setUp(self): def tearDown(self): workspace = self.mevm.workspace del self.mevm - # shutil.rmtree(workspace) + shutil.rmtree(workspace) def test_solidity_create_contract_no_args(self): source_code = "contract A { constructor() {} }" @@ -684,7 +714,7 @@ def test_function_name_collision(self): def test_check_jumpdest_symbolic_pc(self): """ In Manticore 0.2.4 (up to 6804661) when run with DetectIntegerOverflow, - the EVM.pc is tainted and so it becomes a Constant and so a check in EVM._check_jumpdest: + the EVM.pc is tainted and so it becomes a Constant and so a check in EVM._need_check_jumpdest: self.pc in self._valid_jumpdests failed (because we checked if the object is in a list of integers...). @@ -993,20 +1023,16 @@ class TestPlugin(Plugin): """ def did_evm_execute_instruction_callback(self, state, instruction, arguments, result): - try: - world = state.platform - if world.current_transaction.sort == "CREATE": - name = "init" - else: - name = "rt" + world = state.platform + if world.current_transaction.sort == "CREATE": + name = "init" + else: + name = "rt" - # collect all end instructions based on whether they are in init or rt - if instruction.is_endtx: - with self.locked_context(name) as d: - d.append(instruction.pc) - except Exception as e: - print(e) - raise + # collect all end instructions based on whether they are in init or rt + if instruction.is_endtx: + with self.locked_context(name) as d: + d.append(instruction.pc) mevm = self.mevm p = TestPlugin() @@ -1015,7 +1041,7 @@ def did_evm_execute_instruction_callback(self, state, instruction, arguments, re filename = os.path.join(THIS_DIR, "contracts/int_overflow.sol") mevm.multi_tx_analysis(filename, tx_limit=1) - mevm.finalize(only_alive_states=True) + mevm.finalize() worksp = mevm.workspace listdir = os.listdir(worksp) @@ -1350,7 +1376,7 @@ def will_evm_execute_instruction_callback(self, state, i, *args, **kwargs): class EthHelpersTest(unittest.TestCase): def setUp(self): - self.bv = BitVecVariable(size=256, name="bv") + self.bv = Bitvec(256) def test_concretizer(self): policy = "SOME_NONSTANDARD_POLICY" diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index 26ba1c5d0..64c393ed9 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -247,17 +247,17 @@ def test_705(self): def test_addmod(self): """ - (declare-fun BV () (_ BitVec 256)) - (declare-fun BV_2 () (_ BitVec 256)) - (declare-fun BV_1 () (_ BitVec 256)) - (declare-fun a_1 () (_ BitVec 256))(assert (= a_1 (bvmul BV BV_1))) - (declare-fun a_2 () (_ BitVec 512))(assert (= a_2 ((_ zero_extend 256) BV))) - (declare-fun a_3 () (_ BitVec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) - (declare-fun a_4 () (_ BitVec 512))(assert (= a_4 (bvmul a_2 a_3))) - (declare-fun a_5 () (_ BitVec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) - (declare-fun a_6 () (_ BitVec 512))(assert (= a_6 (bvsmod a_4 a_5))) - (declare-fun a_7 () (_ BitVec 256))(assert (= a_7 ((_ extract 255 0) a_6))) - (declare-fun a_8 () (_ BitVec 256))(assert (= a_8 (bvsmod a_1 BV_2))) + (declare-fun BV () (_ Bitvec 256)) + (declare-fun BV_2 () (_ Bitvec 256)) + (declare-fun BV_1 () (_ Bitvec 256)) + (declare-fun a_1 () (_ Bitvec 256))(assert (= a_1 (bvmul BV BV_1))) + (declare-fun a_2 () (_ Bitvec 512))(assert (= a_2 ((_ zero_extend 256) BV))) + (declare-fun a_3 () (_ Bitvec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) + (declare-fun a_4 () (_ Bitvec 512))(assert (= a_4 (bvmul a_2 a_3))) + (declare-fun a_5 () (_ Bitvec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) + (declare-fun a_6 () (_ Bitvec 512))(assert (= a_6 (bvsmod a_4 a_5))) + (declare-fun a_7 () (_ Bitvec 256))(assert (= a_7 ((_ extract 255 0) a_6))) + (declare-fun a_8 () (_ Bitvec 256))(assert (= a_8 (bvsmod a_1 BV_2))) (declare-fun a_9 () Bool)(assert (= a_9 (= a_7 a_8))) (assert (not a_9)) @@ -304,17 +304,17 @@ def test_addmod(self): def test_mulmod(self): """ - (declare-fun BV () (_ BitVec 256)) - (declare-fun BV_2 () (_ BitVec 256)) - (declare-fun BV_1 () (_ BitVec 256)) - (declare-fun a_1 () (_ BitVec 256))(assert (= a_1 (bvmul BV BV_1))) - (declare-fun a_2 () (_ BitVec 512))(assert (= a_2 ((_ zero_extend 256) BV))) - (declare-fun a_3 () (_ BitVec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) - (declare-fun a_4 () (_ BitVec 512))(assert (= a_4 (bvmul a_2 a_3))) - (declare-fun a_5 () (_ BitVec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) - (declare-fun a_6 () (_ BitVec 512))(assert (= a_6 (bvsmod a_4 a_5))) - (declare-fun a_7 () (_ BitVec 256))(assert (= a_7 ((_ extract 255 0) a_6))) - (declare-fun a_8 () (_ BitVec 256))(assert (= a_8 (bvsmod a_1 BV_2))) + (declare-fun BV () (_ Bitvec 256)) + (declare-fun BV_2 () (_ Bitvec 256)) + (declare-fun BV_1 () (_ Bitvec 256)) + (declare-fun a_1 () (_ Bitvec 256))(assert (= a_1 (bvmul BV BV_1))) + (declare-fun a_2 () (_ Bitvec 512))(assert (= a_2 ((_ zero_extend 256) BV))) + (declare-fun a_3 () (_ Bitvec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) + (declare-fun a_4 () (_ Bitvec 512))(assert (= a_4 (bvmul a_2 a_3))) + (declare-fun a_5 () (_ Bitvec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) + (declare-fun a_6 () (_ Bitvec 512))(assert (= a_6 (bvsmod a_4 a_5))) + (declare-fun a_7 () (_ Bitvec 256))(assert (= a_7 ((_ extract 255 0) a_6))) + (declare-fun a_8 () (_ Bitvec 256))(assert (= a_8 (bvsmod a_1 BV_2))) (declare-fun a_9 () Bool)(assert (= a_9 (= a_7 a_8))) (assert (not a_9)) diff --git a/tests/native/test_cpu_manual.py b/tests/native/test_cpu_manual.py index 853a77785..606989368 100644 --- a/tests/native/test_cpu_manual.py +++ b/tests/native/test_cpu_manual.py @@ -309,7 +309,7 @@ def _construct_flag_bitfield(self, flags): return reduce(operator.or_, (self._flags[f] for f in flags)) def _construct_sym_flag_bitfield(self, flags): - return reduce(operator.or_, (BitVecConstant(32, self._flags[f]) for f in flags)) + return reduce(operator.or_, (BitvecConstant(32, self._flags[f]) for f in flags)) def test_set_eflags(self) -> None: cpu = I386Cpu(Memory32()) @@ -371,8 +371,8 @@ def flatten_ors(x: BitVecOr) -> List: cpu.CF = 1 cpu.AF = 1 - a = BitVecConstant(32, 1) != 0 - b = BitVecConstant(32, 0) != 0 + a = BitvecConstant(32, 1) != 0 + b = BitvecConstant(32, 0) != 0 cpu.ZF = a cpu.SF = b @@ -1292,7 +1292,7 @@ def test_symbolic_instruction(self): code = mem.mmap(0x1000, 0x1000, "rwx") stack = mem.mmap(0xF000, 0x1000, "rw") - mem[code] = BitVecConstant(8, 0x90) + mem[code] = BitvecConstant(8, 0x90) cpu.EIP = code cpu.EAX = 116 cpu.EBP = stack + 0x700 diff --git a/tests/native/test_driver.py b/tests/native/test_driver.py index a22eb2f44..3aa1d80b4 100644 --- a/tests/native/test_driver.py +++ b/tests/native/test_driver.py @@ -5,7 +5,7 @@ import tempfile from manticore import issymbolic -from manticore.core.smtlib import BitVecVariable +from manticore.core.smtlib import BitvecVariable from manticore.native import Manticore @@ -26,7 +26,7 @@ def testCreating(self): m.log_file = "/dev/null" def test_issymbolic(self): - v = BitVecVariable(size=32, name="sym") + v = BitvecVariable(32, "sym") self.assertTrue(issymbolic(v)) def test_issymbolic_neg(self): diff --git a/tests/native/test_linux.py b/tests/native/test_linux.py index 9f9bd3df9..e1c41b12d 100644 --- a/tests/native/test_linux.py +++ b/tests/native/test_linux.py @@ -10,7 +10,7 @@ from manticore.native.cpu.abstractcpu import ConcretizeRegister from manticore.core.smtlib.solver import Z3Solver -from manticore.core.smtlib import BitVecVariable, issymbolic +from manticore.core.smtlib import BitvecVariable, issymbolic from manticore.native import Manticore from manticore.platforms import linux, linux_syscalls from manticore.utils.helpers import pickle_dumps diff --git a/tests/native/test_register.py b/tests/native/test_register.py index 0663e811b..b38cef3e5 100644 --- a/tests/native/test_register.py +++ b/tests/native/test_register.py @@ -1,6 +1,6 @@ import unittest -from manticore.core.smtlib import Bool, BoolVariable, BitVecConstant +from manticore.core.smtlib import Bool, BitvecConstant from manticore.native.cpu.register import Register @@ -47,20 +47,20 @@ def test_bool_write_nonflag(self): def test_Bool(self): r = Register(32) - b = BoolVariable(name="B") + b = Bool() r.write(b) self.assertIs(r.read(), b) def test_bitvec_flag(self): r = Register(1) - b = BitVecConstant(32, 0) + b = BitvecConstant(32, 0) r.write(b) # __nonzero__ (==) currently unimplemented for Bool self.assertTrue(isinstance(r.read(), Bool)) def test_bitvec(self): r = Register(32) - b = BitVecConstant(32, 0) + b = BitvecConstant(32, 0) r.write(b) self.assertIs(r.read(), b) diff --git a/tests/native/test_state.py b/tests/native/test_state.py index daf8035b1..3c7bd814b 100644 --- a/tests/native/test_state.py +++ b/tests/native/test_state.py @@ -4,7 +4,7 @@ from manticore.utils.event import Eventful from manticore.platforms import linux from manticore.native.state import State -from manticore.core.smtlib import BitVecVariable, ConstraintSet +from manticore.core.smtlib import BitvecVariable, ConstraintSet from manticore.native import Manticore from manticore.native.plugins import Merger from manticore.core.plugin import Plugin @@ -76,27 +76,27 @@ def setUp(self): def test_solve_one(self): val = 42 - expr = BitVecVariable(size=32, name="tmp") + expr = BitvecVariable(32, "tmp") self.state.constrain(expr == val) solved = self.state.solve_one(expr) self.assertEqual(solved, val) def test_solve_n(self): - expr = BitVecVariable(size=32, name="tmp") + expr = BitvecVariable(32, "tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) solved = sorted(self.state.solve_n(expr, 2)) self.assertEqual(solved, [5, 6]) def test_solve_n2(self): - expr = BitVecVariable(size=32, name="tmp") + expr = BitvecVariable(32, "tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 100) solved = self.state.solve_n(expr, 5) self.assertEqual(len(solved), 5) def test_solve_min_max(self): - expr = BitVecVariable(size=32, name="tmp") + expr = BitvecVariable(32, "tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) self.assertEqual(self.state.solve_min(expr), 5) @@ -104,7 +104,7 @@ def test_solve_min_max(self): self.assertEqual(self.state.solve_minmax(expr), (5, 6)) def test_policy_one(self): - expr = BitVecVariable(size=32, name="tmp") + expr = BitvecVariable(32, "tmp") self.state.constrain(expr > 0) self.state.constrain(expr < 100) solved = self.state.concretize(expr, "ONE") diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index d73e6aff0..91f8c3b4f 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -1,6 +1,7 @@ import unittest import os - +import sys +import pickle from manticore.core.smtlib import ( ConstraintSet, Version, @@ -12,7 +13,7 @@ arithmetic_simplify, constant_folder, replace, - BitVecConstant, + BitvecConstant, ) from manticore.core.smtlib.solver import Z3Solver, YicesSolver, CVC4Solver from manticore.core.smtlib.expression import * @@ -26,10 +27,263 @@ DIRPATH = os.path.dirname(__file__) +class ExpressionTestNew(unittest.TestCase): + _multiprocess_can_split_ = True + + def setUp(self): + self.solver = Z3Solver.instance() + + def assertItemsEqual(self, a, b): + # Required for Python3 compatibility + self.assertEqual(sorted(a), sorted(b)) + + def test_xslotted(self): + """ Test that XSlotted multi inheritance classes uses same amount + of memory than a single class object with slots + """ + + class Base(object, metaclass=XSlotted, abstract=True): + __xslots__ = ("t",) + pass + + class A(Base, abstract=True): + __xslots__ = ("a",) + pass + + class B(Base, abstract=True): + __xslots__ = ("b",) + pass + + class C(A, B): + pass + + class X(object): + __slots__ = ("t", "a", "b") + + c = C() + c.a = 1 + c.t = 10 + + c.b = 2 + c.t = 10 + + x = X() + x.a = 1 + x.b = 2 + x.t = 20 + + self.assertEqual(sys.getsizeof(c), sys.getsizeof(x)) + + def test_Bitvec_ops(self): + a = BitvecVariable(size=32, name='BV') + b = BitvecVariable(size=32, name='BV1') + c = BitvecVariable(size=32, name='BV2') + x = BitvecConstant(size=32, value=100, taint=("T",)) + z = ((b + 1) % b < a * x / c - 5) + self.assertSetEqual(z.taint, set(("T",))) + self.assertEqual(translate_to_smtlib(z), + "(bvslt (bvsmod (bvadd BV1 #x00000001) BV1) (bvsub (bvsdiv (bvmul BV #x00000064) BV2) #x00000005))") + z = ((1 + b) / b <= a - x * 5 + c) + self.assertSetEqual(z.taint, set(("T",))) + self.assertEqual(translate_to_smtlib(z), + "(bvsle (bvsdiv (bvadd #x00000001 BV1) BV1) (bvadd (bvsub BV (bvmul #x00000064 #x00000005)) BV2))") + + + def test_ConstantArrayBitvec(self): + c = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") + self.assertEqual(c[0], "A") + self.assertEqual(c[1], "B") + self.assertEqual(c[2], "C") + self.assertEqual(c[3], "D") + self.assertEqual(c[4], "E") + self.assertRaises(IndexError, c.get, 5) + + + def test_ConstantArrayBitvec(self): + c = ArrayProxy(ArrayVariable(index_size=32, value_size=8, length=5, name="ARR")) + c[1] = 10 + c[2] = 20 + c[3] = 30 + self.assertEqual(c[1], 10) + self.assertEqual(c[2], 20) + self.assertEqual(c[3], 30) + self.assertRaises(IndexError, c.get, 25) + + def test_Expression(self): + # Used to check if all Expression have test + checked = set() + + def check(ty, pickle_size=None, sizeof=None, **kwargs): + x = ty(**kwargs) + print( + type(x), + "\n Pickle size:", + len(pickle_dumps(x)), + "\n GetSizeOf:", + sys.getsizeof(x), + "\n Slotted:", + not hasattr(x, "__dict__"), + ) + self.assertEqual(len(pickle_dumps(x)), pickle_size) + self.assertEqual(sys.getsizeof(x), sizeof) + self.assertFalse(hasattr(x, "__dict__")) # slots! + self.assertTrue(hasattr(x, "_taint")) # taint! + checked.add(ty) + + # Can not instantiate an Expression + for ty in ( + Expression, + Constant, + Variable, + Operation, + BoolOperation, + BitvecOperation, + ArrayOperation, + Bitvec, + Bool, + Array, + ): + self.assertRaises(Exception, ty, ()) + self.assertTrue(hasattr(ty, "__doc__")) + self.assertTrue(ty.__doc__, ty) + checked.add(ty) + + check(BitvecVariable, size=32, name="name", pickle_size=84, sizeof=64) + check(BoolVariable, name="name", pickle_size=80, sizeof=56) + check( + ArrayVariable, + index_bits=32, + value_bits=8, + index_max=100, + name="name", + pickle_size=92, + sizeof=104, + ) + check(BitvecConstant, size=32, value=10, pickle_size=79, sizeof=64) + check(BoolConstant, value=False, pickle_size=74, sizeof=56) + + # TODO! But you can instantiate an ArraConstant + """ + x = ArrayConstant(index_bits=32, value_bits=8, b"AAAAAAAAAAAAAAA") + self.assertLessEqual(len(pickle_dumps(x)), 76) #master 71 + self.assertLessEqual(sys.getsizeof(x), 64) #master 56 + self.assertFalse(hasattr(x, '__dict__')) #slots! + """ + + # But you can instantiate a BoolOr + x = BoolVariable(name="x") + y = BoolVariable(name="y") + z = BoolVariable(name="z") + check(BoolEqual, operanda=x, operandb=y, pickle_size=118, sizeof=56) + check(BoolAnd, operanda=x, operandb=y, pickle_size=116, sizeof=56) + check(BoolOr, operanda=x, operandb=y, pickle_size=115, sizeof=56) + check(BoolXor, operanda=x, operandb=y, pickle_size=116, sizeof=56) + check(BoolNot, operand=x, pickle_size=102, sizeof=56) + check(BoolITE, cond=z, true=x, false=y, pickle_size=130, sizeof=56) + + bvx = BitvecVariable(size=32, name="bvx") + bvy = BitvecVariable(size=32, name="bvy") + + check(BoolUnsignedGreaterThan, operanda=bvx, operandb=bvy, pickle_size=142, sizeof=56) + check(BoolGreaterThan, operanda=bvx, operandb=bvy, pickle_size=134, sizeof=56) + check( + BoolUnsignedGreaterOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=149, sizeof=56 + ) + check(BoolGreaterOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=141, sizeof=56) + check(BoolUnsignedLessThan, operanda=bvx, operandb=bvy, pickle_size=139, sizeof=56) + check(BoolLessThan, operanda=bvx, operandb=bvy, pickle_size=131, sizeof=56) + check(BoolUnsignedLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=56) + check(BoolLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) + + check(BitvecOr, operanda=bvx, operandb=bvy, pickle_size=129, sizeof=64) + check(BitvecXor, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecAnd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecNot, operanda=bvx, pickle_size=112, sizeof=64) + check(BitvecNeg, operanda=bvx, pickle_size=112, sizeof=64) + check(BitvecAdd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecMul, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecSub, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecDiv, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecMod, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecUnsignedDiv, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=64) + check(BitvecRem, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) + check(BitvecUnsignedRem, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=64) + + check(BitvecShiftLeft, operanda=bvx, operandb=bvy, pickle_size=136, sizeof=64) + check(BitvecShiftRight, operanda=bvx, operandb=bvy, pickle_size=137, sizeof=64) + check(BitvecArithmeticShiftLeft, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=64) + check(BitvecArithmeticShiftRight, operanda=bvx, operandb=bvy, pickle_size=147, sizeof=64) + + + check(BitvecZeroExtend, operand=bvx, size=122, pickle_size=119, sizeof=64) + check(BitvecSignExtend, operand=bvx, size=122, pickle_size=119, sizeof=64) + check(BitvecExtract, operand=bvx, offset=0, size=8, pickle_size=119, sizeof=72) + check(BitvecConcat, operands=(bvx, bvy), pickle_size=133, sizeof=64) + check(BitvecITE, condition=x, true_value=bvx, false_value=bvy, pickle_size=161, sizeof=64) + + a = ArrayVariable(index_bits=32, value_bits=32, length=324, name="name") + check(ArraySlice, array=a, offset=0, size=10 , pickle_size=136, sizeof=120) + check(ArraySelect, array=a, index=bvx, pickle_size=161, sizeof=64) + check(ArrayStore, array=a, index=bvx, value=bvy, pickle_size=188, sizeof=104) + + + def all_subclasses(cls): + return set((Expression,)).union( + set(cls.__subclasses__()).union( + [s for c in cls.__subclasses__() for s in all_subclasses(c)] + ) + ) + + all_types = all_subclasses(Expression) + self.assertSetEqual(checked, all_types) + def test_Expression_BitvecOp(self): + a = BitvecConstant(size=32,value=100) + b = BitvecConstant(size=32,value=101) + + + def test_Expression_BoolTaint(self): + #Bool can not be instantiaated + self.assertRaises(Exception, Bool, ()) + + x = BoolConstant(value=True, taint=('red',)) + y = BoolConstant(value=False, taint=('blue',)) + z = BoolOr(x,y) + self.assertIn('red', x.taint) + self.assertIn('blue', y.taint) + self.assertIn('red', z.taint) + self.assertIn('blue', z.taint) + + def test_Expression_BitvecTaint(self): + #Bool can not be instantiaated + self.assertRaises(Exception, Bitvec, ()) + + x = BitvecConstant(size=32, value=123, taint=('red',)) + y = BitvecConstant(size=32, value=124, taint=('blue',)) + z = BoolGreaterOrEqualThan(x,y) + self.assertIn('red', x.taint) + self.assertIn('blue', y.taint) + self.assertIn('red', z.taint) + self.assertIn('blue', z.taint) + + + def test_Expression_Array(self): + #Bool can not be instantiaated + self.assertRaises(Exception, Array, ()) + + a = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") + a[0] == ord('A') + + x = BitvecConstant(size=32, value=123, taint=('red',)) + y = BitvecConstant(size=32, value=124, taint=('blue',)) + z = BoolGreaterOrEqualThan(x,y) + self.assertIn('red', x.taint) + self.assertIn('blue', y.taint) + self.assertIn('red', z.taint) + self.assertIn('blue', z.taint) + class RegressionTest(unittest.TestCase): def test_related_to(self): import gzip - import pickle, sys filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) @@ -48,7 +302,7 @@ def test_related_to(self): # Replace new_constraint = Operators.UGE( - Operators.SEXTEND(BitVecConstant(256, 0x1A), 256, 512) * BitVecConstant(512, 1), + Operators.SEXTEND(BitvecConstant(256, 0x1A), 256, 512) * BitvecConstant(512, 1), 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000, ) self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) @@ -100,9 +354,54 @@ def test_check_solver_undefined(self): """ -class ExpressionTest(unittest.TestCase): +class ExpressionTestLoco(unittest.TestCase): _multiprocess_can_split_ = True + def setUp(self): + self.solver = Z3Solver.instance() + cs = ConstraintSet() + self.assertTrue(self.solver.check(cs)) + + + def assertItemsEqual(self, a, b): + # Required for Python3 compatibility + self.assertEqual(sorted(a), sorted(b)) + + def tearDown(self): + del self.solver + + def test_signed_unsigned_LT_(self): + mask = (1 << 32) - 1 + + cs = ConstraintSet() + _a = cs.new_bitvec(32) + _b = cs.new_bitvec(32) + + cs.add(_a == 0x1) + cs.add(_b == (0x80000000 - 1)) + + self.assertFalse(self.solver.can_be_true(cs, ult)) + self.assertTrue(self.solver.must_be_true(cs, lt)) + + def test_signed_unsigned_LT_simple(self): + cs = ConstraintSet() + a = cs.new_bitvec(32) + b = cs.new_bitvec(32) + + cs.add(a == 0x1) + cs.add(b == 0x80000000) + + lt = b < a + ult = b.ult(a) + + self.assertFalse(self.solver.can_be_true(cs, ult)) + self.assertTrue(self.solver.must_be_true(cs, lt)) + + + +class ExpressionTest(unittest.TestCase): + _multiprocess_can_split_ = False + def setUp(self): self.solver = Z3Solver.instance() @@ -116,9 +415,9 @@ def tearDown(self): def test_no_variable_expression_can_be_true(self): """ Tests if solver.can_be_true is correct when the expression has no nodes that subclass - from Variable (e.g. BitVecConstant) + from Variable (e.g. BitvecConstant) """ - x = BitVecConstant(32, 10) + x = BitvecConstant(32, 10) cs = ConstraintSet() self.assertFalse(self.solver.can_be_true(cs, x == False)) @@ -126,12 +425,12 @@ def test_constant_bitvec(self): """ Tests if higher bits are masked out """ - x = BitVecConstant(32, 0xFF00000000) + x = BitvecConstant(32, 0xFF00000000) self.assertTrue(x.value == 0) def testBasicAST_001(self): """ Can't build abstract classes """ - a = BitVecConstant(32, 100) + a = BitvecConstant(32, 100) self.assertRaises(Exception, Expression, ()) self.assertRaises(Exception, Constant, 123) @@ -140,28 +439,28 @@ def testBasicAST_001(self): def testBasicOperation(self): """ Add """ - a = BitVecConstant(size=32, value=100) - b = BitVecVariable(size=32, name="VAR") + a = BitvecConstant(size=32, value=100) + b = BitvecVariable(size=32, name="VAR") c = a + b - self.assertIsInstance(c, BitVecAdd) + self.assertIsInstance(c, BitvecAdd) self.assertIsInstance(c, Operation) self.assertIsInstance(c, Expression) def testBasicTaint(self): - a = BitVecConstant(32, 100, taint=("SOURCE1",)) - b = BitVecConstant(32, 200, taint=("SOURCE2",)) + a = BitvecConstant(32, 100, taint=("SOURCE1",)) + b = BitvecConstant(32, 200, taint=("SOURCE2",)) c = a + b - self.assertIsInstance(c, BitVecAdd) + self.assertIsInstance(c, BitvecAdd) self.assertIsInstance(c, Operation) self.assertIsInstance(c, Expression) self.assertTrue("SOURCE1" in c.taint) self.assertTrue("SOURCE2" in c.taint) def testBasicITETaint(self): - a = BitVecConstant(32, 100, taint=("SOURCE1",)) - b = BitVecConstant(32, 200, taint=("SOURCE2",)) - c = BitVecConstant(32, 300, taint=("SOURCE3",)) - d = BitVecConstant(32, 400, taint=("SOURCE4",)) + a = BitvecConstant(32, 100, taint=("SOURCE1",)) + b = BitvecConstant(32, 200, taint=("SOURCE2",)) + c = BitvecConstant(32, 300, taint=("SOURCE3",)) + d = BitvecConstant(32, 400, taint=("SOURCE4",)) x = Operators.ITEBV(32, a > b, c, d) self.assertTrue("SOURCE1" in x.taint) self.assertTrue("SOURCE2" in x.taint) @@ -384,12 +683,13 @@ def testBasicArrayConcatSlice(self): hw = b"Hello world!" cs = ConstraintSet() # make array of 32->8 bits - array = cs.new_array(32, index_max=12) - + array = cs.new_array(32, index_max=len(hw)) array = array.write(0, hw) + self.assertEqual(len(array), len(hw)) self.assertTrue(self.solver.must_be_true(cs, array == hw)) - + self.assertEqual(len(array.read(0, 12)), 12) self.assertTrue(self.solver.must_be_true(cs, array.read(0, 12) == hw)) + cs.add(array.read(6, 6) == hw[6:12]) self.assertTrue(self.solver.must_be_true(cs, array.read(6, 6) == hw[6:12])) self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 6) == hw)) @@ -406,10 +706,12 @@ def testBasicArrayConcatSlice(self): self.assertTrue(len(array[0:12]) == 12) + self.assertEqual(len(array[6:11]), 5) + results = [] for c in array[6:11]: results.append(c) - self.assertTrue(len(results) == 5) + self.assertEqual(len(results), 5) def testBasicArraySlice(self): hw = b"Hello world!" @@ -435,7 +737,7 @@ def testBasicArraySlice(self): def testBasicArrayProxySymbIdx(self): cs = ConstraintSet() - array = ArrayProxy(cs.new_array(index_bits=32, value_bits=32, name="array", default=0)) + array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") @@ -450,7 +752,7 @@ def testBasicArrayProxySymbIdx(self): def testBasicArrayProxySymbIdx2(self): cs = ConstraintSet() - array = ArrayProxy(cs.new_array(index_bits=32, value_bits=32, name="array")) + array = cs.new_array(index_bits=32, value_bits=32, name="array") key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") @@ -459,7 +761,6 @@ def testBasicArrayProxySymbIdx2(self): solutions = self.solver.get_all_values(cs, array[0]) # get a concrete solution for index self.assertItemsEqual(solutions, (1, 2)) - solutions = self.solver.get_all_values( cs, array.get(0, 100) ) # get a concrete solution for index 0 @@ -474,6 +775,18 @@ def testBasicArrayProxySymbIdx2(self): self.solver.can_be_true(cs, array[1] == 12345) ) # no default so it can be anything + def testBasicConstatArray(self): + cs = ConstraintSet() + array1 = cs.new_array(index_bits=32, value_bits=32, index_max=10, name="array1", default=0) + array2 = cs.new_array(index_bits=32, value_bits=32, index_max=10, name="array2", default=0) + array1[0:10] = range(10) + self.assertTrue(array1[0] == 0) + #yeah right self.assertTrue(array1[0:10] == range(10)) + array_slice = array1[0:10] + self.assertTrue(array_slice[0] == 0) + + + def testBasicPickle(self): import pickle @@ -599,7 +912,7 @@ def test_visitors(self): self.assertEqual(len(values), 2) self.assertItemsEqual(solver.get_all_values(cs, values[0]), [ord("a")]) self.assertItemsEqual(solver.get_all_values(cs, values[1]), [ord("b")]) - arr[1:3] = "cd" + arr[1:3] = b"cd" values = arr[0:3] self.assertEqual(len(values), 3) @@ -607,15 +920,15 @@ def test_visitors(self): self.assertItemsEqual(solver.get_all_values(cs, values[1]), [ord("c")]) self.assertItemsEqual(solver.get_all_values(cs, values[2]), [ord("d")]) self.assertEqual( - pretty_print(aux, depth=2), "ArraySelect\n ArrayStore\n ...\n BitVecAdd\n ...\n" + pretty_print(aux, depth=2), "ArraySelect\n ArrayStore\n ...\n BitvecAdd\n ...\n" ) self.assertEqual( - pretty_print(Operators.EXTRACT(a, 0, 8), depth=1), "BitVecExtract{0:7}\n ...\n" + pretty_print(Operators.EXTRACT(a, 0, 8), depth=1), "BitvecExtract{0:7}\n ...\n" ) self.assertEqual(pretty_print(a, depth=2), "VAR\n") - x = BitVecConstant(32, 100, taint=("important",)) - y = BitVecConstant(32, 200, taint=("stuff",)) + x = BitvecConstant(32, 100, taint=("important",)) + y = BitvecConstant(32, 200, taint=("stuff",)) z = constant_folder(x + y) self.assertItemsEqual(z.taint, ("important", "stuff")) self.assertEqual(z.value, 300) @@ -685,27 +998,27 @@ def test_arithmetic_simplify_extract(self): def test_arithmetic_simplify_udiv(self): cs = ConstraintSet() a = cs.new_bitvec(32, name="VARA") - b = a + Operators.UDIV(BitVecConstant(32, 0), BitVecConstant(32, 2)) + b = a + Operators.UDIV(BitvecConstant(32, 0), BitvecConstant(32, 2)) self.assertEqual(translate_to_smtlib(b), "(bvadd VARA (bvudiv #x00000000 #x00000002))") self.assertEqual(translate_to_smtlib(simplify(b)), "VARA") - c = a + Operators.UDIV(BitVecConstant(32, 2), BitVecConstant(32, 2)) + c = a + Operators.UDIV(BitvecConstant(32, 2), BitvecConstant(32, 2)) self.assertEqual(translate_to_smtlib(c), "(bvadd VARA (bvudiv #x00000002 #x00000002))") self.assertEqual(translate_to_smtlib(simplify(c)), "(bvadd VARA #x00000001)") def test_constant_folding_udiv(self): cs = ConstraintSet() - x = BitVecConstant(32, 0xFFFFFFFF, taint=("important",)) - y = BitVecConstant(32, 2, taint=("stuff",)) + x = BitvecConstant(32, 0xFFFFFFFF, taint=("important",)) + y = BitvecConstant(32, 2, taint=("stuff",)) z = constant_folder(x.udiv(y)) self.assertItemsEqual(z.taint, ("important", "stuff")) self.assertEqual(z.value, 0x7FFFFFFF) def testBasicReplace(self): """ Add """ - a = BitVecConstant(size=32, value=100) - b1 = BitVecVariable(size=32, name="VAR1") - b2 = BitVecVariable(size=32, name="VAR2") + a = BitvecConstant(size=32, value=100) + b1 = BitvecVariable(size=32, name="VAR1") + b2 = BitvecVariable(size=32, name="VAR2") c = a + b1 @@ -873,6 +1186,7 @@ def test_ORD_proper_extract(self): def test_CHR(self): solver = Z3Solver.instance() cs = ConstraintSet() + self.assertTrue(solver.check(cs)) a = cs.new_bitvec(8) cs.add(Operators.CHR(a) == Operators.CHR(0x41)) @@ -1121,7 +1435,7 @@ def test_signed_unsigned_LT_simple(self): self.assertFalse(self.solver.can_be_true(cs, ult)) self.assertTrue(self.solver.must_be_true(cs, lt)) - def test_signed_unsigned_LT_complex(self): + def test_signed_unsigned_LT_(self): mask = (1 << 32) - 1 cs = ConstraintSet() From df4d457d6d056d191be4e83b9f8e2715b2ef1e3f Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 17 Jul 2020 10:34:58 -0300 Subject: [PATCH 073/126] WIP --- manticore/core/smtlib/constraints.py | 6 ++ manticore/core/smtlib/expression.py | 90 ++++++++++--------- manticore/core/smtlib/solver.py | 30 ++++++- manticore/core/smtlib/visitors.py | 1 + tests/other/data/ErrRelated.pkl.gz | Bin 17638 -> 0 bytes tests/other/test_smtlibv2.py | 125 ++++++--------------------- 6 files changed, 110 insertions(+), 142 deletions(-) delete mode 100644 tests/other/data/ErrRelated.pkl.gz diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index feecd3ddd..1b99802e2 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -39,6 +39,8 @@ class ConstraintException(SmtlibError): pass +class Model(): + pass class ConstraintSet: """ Constraint Sets @@ -214,6 +216,10 @@ def _declare(self, var): self._declarations[var.name] = var return var + @property + def variables(self): + return self._declarations.values() + def get_declared_variables(self): """ Returns the variable expressions of this constraint set """ return self._declarations.values() diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 2e4687a16..1d6e63ce1 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -61,11 +61,11 @@ def __new__(cls, clsname, bases, attrs, abstract=False): class Expression(object, metaclass=XSlotted, abstract=True): """ Abstract taintable Expression. """ - __xslots__ : Tuple[str, ...] = ("_taint",) + def __init__(self, taint: Union[tuple, frozenset] = ()): """ - An abstrac Unmutable Taintable Expression + An abstract Unmutable Taintable Expression :param taint: A frozenzset """ self._taint = frozenset(taint) @@ -89,21 +89,14 @@ def operands(self): def __getstate__(self): state = {} for attr in self.__slots__: - if attr.startswith("__"): - continue state[attr] = getattr(self, attr) return state def __setstate__(self, state): for attr in self.__slots__: - if attr.startswith("__"): - continue setattr(self, attr, state[attr]) - def __hash__(self): - return object.__hash__(self) - class Variable(Expression, abstract=True): """ Variable is an Expression that has a name """ @@ -198,7 +191,6 @@ def __invert__(self): def __eq__(self, other): # A class that overrides __eq__() and does not define __hash__() # will have its __hash__() implicitly set to None. - #import pdb; pdb.set_trace() return BoolEqual(self, self.cast(other)) def __hash__(self): @@ -534,7 +526,6 @@ def signed_value(self): def __eq__(self, other): if self.taint or isinstance(other, Expression) and other.taint: return super().__eq__(other) - print (self.value, other) return self.value == other def __hash__(self): @@ -569,32 +560,26 @@ class BitvecUnsignedDiv(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecMod(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecRem(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecUnsignedRem(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecShiftLeft(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecShiftRight(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecArithmeticShiftLeft(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) @@ -604,7 +589,6 @@ class BitvecArithmeticShiftRight(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) - class BitvecAnd(BitvecOperation): def __init__(self, operanda, operandb, *args, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) @@ -635,12 +619,10 @@ class BoolLessThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) - class BoolLessOrEqualThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) - class BoolEqual(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): assert isinstance(operanda, Expression) @@ -808,6 +790,27 @@ def _fix_slice(self, index: slice): return start, stop, size.value + def _concatenate(self, array_a, array_b): + # FIXME/Research This should be related to a constrainSet + new_arr = ArrayVariable( + index_size = self.index_size, + length=len(array_a) + len(array_b), + value_size = self.value_size, + name="concatenation{}".format(uuid.uuid1()), + ) + + for index in range(len(array_a)): + new_arr = new_arr.store(index, simplify(array_a[index])) + for index in range(len(array_b)): + new_arr = new_arr.store(index + len(array_a), simplify(array_b[index])) + return new_arr + + def __add__(self, other): + return self._concatenate(self, other) + + def __radd__(self, other): + print ("RADD!"*9) + return self._concatenate(other, self) class ArrayConstant(Array, Constant): __xslots__: Tuple[str, ...] = ( @@ -869,6 +872,9 @@ class ArrayVariable(Array, Variable): "_default", ) + @property + def index_max(self): + return len(self.value) def __hash__(self): return object.__hash__(self) @@ -901,7 +907,6 @@ def __init__( self._default = default super().__init__(**kwargs) - @property def index_size(self): return self._index_size @@ -1076,6 +1081,22 @@ class ArrayStore(ArrayOperation): "_written", "_concrete_cache", ) + def __getstate__(self): + #Overload serialization so _written and _concrete_cache are not actually saved + state = {} + for attr in self.__slots__: + if attr in ('_written', '_concrete_cache'): + continue + state[attr] = getattr(self, attr) + return state + + def __setstate__(self, state): + #Overload serialization so _written and _concrete_cache are not actually saved + for attr in self.__slots__: + if attr in ('_written', '_concrete_cache'): + continue + setattr(self, attr, state[attr]) + def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size @@ -1330,34 +1351,20 @@ def read(self, offset, size): return ArrayProxy(self._array[offset : offset + size]) def __eq__(self, other): - print ("type:"*99, type(self.array ), type(other)) return self.array == other def __ne__(self, other): return BoolNot(self == other) - def _concatenate(self, array_a, array_b): - from .visitors import simplify - # FIXME/Research This should be related to a constrainSet - new_arr = ArrayProxy( - ArrayVariable( - index_size = self.index_size, - length=len(array_a) + len(array_b), - value_size = self.value_size, - name="concatenation{}".format(uuid.uuid1()), - ) - ) - for index in range(len(array_a)): - new_arr[index] = simplify(array_a[index]) - for index in range(len(array_b)): - new_arr[index + len(array_a)] = simplify(array_b[index]) - return new_arr - def __add__(self, other): - return self._concatenate(self, other) + if isinstance(other, ArrayProxy): + other = other.array + return ArrayProxy(self.array + other) def __radd__(self, other): - return self._concatenate(other, self) + if isinstance(other, ArrayProxy): + other = other.array + return ArrayProxy(other+self.array) class ArraySelect(BitvecOperation): __xslots__ = BitvecOperation.__xslots__ @@ -1523,3 +1530,4 @@ def taint_with(arg, *taints, value_size=256, index_size=256): from .visitors import simplify + diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 4c016af95..40cbaf1c9 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -216,7 +216,7 @@ def __readline_and_count(self): buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error # Error messages may contain an unbalanced parenthesis situation - print (buf) + print (">",buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -229,7 +229,7 @@ def send(self, cmd: str) -> None: :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ - print (cmd) + print ("<",cmd) if self._debug: logger.debug(">%s", cmd) self._proc.stdout.flush() # type: ignore @@ -399,6 +399,30 @@ def _pop(self): """Recall the last pushed constraint store and state.""" self._smtlib.send("(pop 1)") + def get_model(self, constraints: ConstraintSet): + self._reset(constraints.to_string()) + self._smtlib.send("(check-sat)") + self._smtlib.recv() + + model = {} + for variable in constraints.variables: + print (variable) + if isinstance(variable, Bool): + value = self.__getvalue_bool(variable.name) + elif isinstance(variable, Bitvec): + value = self.__getvalue_bv(variable.name) + else: + try: + #Only works if we know the max index of the arrray + value = [] + for i in range(len(variable)): + value.append(self.__getvalue_bv(variable[i])) + except: + value = None #We failed to get the model from the solver + + model[variable.name] = value + return model + @lru_cache(maxsize=32) def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool] = True) -> bool: """Check if two potentially symbolic values can be equal""" @@ -615,7 +639,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): result = [] for i in range(expression.index_max): subvar = temp_cs.new_bitvec(expression.value_bits) - var.append(subvar) + var.append(expression[i]) #subvar) temp_cs.add(subvar == simplify(expression[i])) self._reset(temp_cs.to_string()) if not self._is_sat(): diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 1c1461bd9..3b38b0f74 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -139,6 +139,7 @@ def _rebuild(self, expression:Expression, operands): # - no operands were modified return expression + class Translator(Visitor): """ Simple visitor to translate an expression into something else """ diff --git a/tests/other/data/ErrRelated.pkl.gz b/tests/other/data/ErrRelated.pkl.gz deleted file mode 100644 index eb1036567ca2ce788a1c79c0ea6973bfe1b2240d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17638 zcmYIvbyO72`!(I&ow{`IE(_9KOSq)uN;gUfh%DV50t*TP(kY-K2$E7uw}9jlqQru* zi0seL_x7I z-}r{s*Hb>c)i@&<0$IbQ^;Ywk494|s6M{Vgo;`b3AVc`RFX2am?=lIe5&Qrgx&L|} zeGzejI(gCc3^Bdic@@z!wU0(;wPc*7a{Cw>K$D;$3SOzI4@s&4fMx7XWuAF~1iuLA zepzS4+CnzRmxwwWo0maoPPSas*RR#z>59j*^TuvV9{nnv6puDmnnX^%3*&J&aaT~v zB%HM|p4zcH6#uaQL7cWSgrs8C)O%iY^^XscL&1Eu*ee~WG`o% z(Tu|ebEhzO)=zba}C_AFot4exg*{U61l|1`VLsHWI%jtBxF$0 zb7P*>t_%I&VY?V9e{BzvH;vVu{D>XP0iAE~Ne){w=(6H@3NlSQy);(S`RcSzC2yt#A z#O{6NN2W!-z`~5t+gb2#mJZh}WV&e7)lB~hegnDVVxCm$*!;U+`Xc0nIb2VEVll>& zpCjcmJSOQSWaVw_zHiW2J-ui#PqgGmHv1P%K1yaCxTB#C;fWH3Y+{EQDdJl zGRnt{HYRa0ev9K{;h57`Miam78Z9LgB^I`FSQ}_hrVE~hUOC#eG8>wdLu|EbB*Y<- z_ar5O0{1p7<&_{x^DL~aTwE-VKKN;cLW+go=jcbAyZrpAJTq=7JA|rwv;tIl{zo_8YCW&0{-R+O*m1_25aY>&#NaZu3u3yYCBr?}x`0&jaj-y}Mp_zrPJG8f;!O zl7tjkS+nKZ|2p^^vqDn+UfX{+5bVh;50MK&qx{|vjlF;R8{B0*FHvQ4^-`ju(WmQl zwU71ZJd5v@U1!y6v1*CcJ(+J36TeT}uDskYNr}*bft-qUYM?gjKi<)jB)u(NljmL~C$FOdp%(I2Bs1uLKBXapa6Jsj-)scP8 z8~Sn8YsZFSe^L^we;WHbDhw=@E|7RWA5ys6?RIbe_@&eimdu|z0nn-)?aHibPyLS90 zUwL~gGo<=A8kfAo`@=sn{U(kua=5ei<4UvTZ%$eStGph|PZI2o>IUB}V0pxGj$4Kv z_alAkq`rEaC2TnLZ|%B(3H1jR|KDpU2kRBXjpO`%<1mR|dGZl6j=0BxtO!ONiuG=Q zwdXG@r7~imU5|}p^L%uB2j<%#Mv0q|S>GE++)^@pPj&XK#g|l7yTxzCV<;TWEU@QJ z&akE#rO*Sby(8RSd$X?B#6?D^RpUc>EBoo``LxPRekv{?A*sIMZD&VN+2lm*UKqI? zZ~v{LT|*a~OTUcuGs**BzN|G#R{RF^zBTu@54m?-1V&N7jZENb* zKIW6W*GSd|>IW;zC64(}x;+^4X_kH96sFsM9M{q|No87O|E2w*n^pb_^>`Pf8*Q9& zu$yr{JE5D@*m0xm_r7zLxE0o|E=9S5v13v<#8`W+(Sw&?8Qi!u5+^4bWq10x)QXJr zi=y@j$ph*7wO$qJy3&xyCClbopy|Zj;|?_n%jGb26Xw!$!htno5wzop^yq*%!5}x4 zd5pOVc`$Drllfnb2N#x{6jP#zmt9Z10C8j?pu(XS$-$?1b1F1%=K6Q z`FdO@>6XJQ8za|Wgn_EAJf=mYt+#Y=9-UAvcNVK6-l(uVSyl5QUO4t^9G1M_B-o8S zKa6T(CofFHH8NqZlo8EJW^y!h+!eWtcx117rix9@>|h+q5@I`-OE8l7iQ&ByEEPwn z$2uN1d>?H1KGdKp{iC+zlB(@)-rZ*795*7gx$o3dHc=3RHIR0U#p=l72r&M%ZI*58 ziDVdF>F-`f(U`M^0GkL>2O*xBT(%R?e>5lq@*Q3##uCb}X$4h(AwUFPY=t zVYlC|2|sevEgV=nZ7p3Qrwe|XqS$FF^r5+?O0>mN4D3zU_!j<4V&VR2gzpEhn5u^UZ4N6c3DK9=HnBph$%S=Up&w z+OcF&nV-K!5PYegMHjXxNctFWmP}Qm_MKk5SuW8EN?DCaz7VTGdG5Mj$ZpM}#m7qI zR!R!Mtau*}*3k@j1#eNz@IZ@Yy8#79!(#t0pPH7})PizX*EGVvd0-^YPcCHP^Jto} zHPFghF8B&RbaNg394|5pOdF39gkuCJvHGh@BtMcLW8}&mYf#O42)WhUj9c2zM#3Fo zDY8>WV1z;OK3?~!ABG{yY8Ktm5+>+#*ROyuEJl0@uV22tP8Lq(Z9UUI+IMd`)Ac*k zEc!GLs4o@HMNN6+u1`7V5=P;vHiH%LNA~txM}JnY?4Hf}oZln|o@2v&&SNdUNDWl+ zm21pbNzV%C6qk>r9=dAOK%eA^z~708_!{7wu69A$@x}~_pY~UEx76Ylr_5F{Sm!er zizd}QELN*rNCPiqFHMgV!HSW>uyGfd?dm)}-$)l!8jpXrs;PyX84PbBF96d!Ex>#P z=ohNN)LCHQDo|s?T8dH27g=Tm%Z28(dy=RHo`5Crl3s!(;-O;jIS#OICDgWMRRsK# zKDZ3bogE^(8lDhl*Gh&jM5SJ*r$5 z*J+Pi71#bW;nBv0rTA<_?bXKMYFXrCX;=)xYq<1lqlvnlb&G9T5%vN1xwW;g5G+~_ zxp9{NoOdIgeWg=uKl1E{YaOguS-5SNSMzAnO4@XAW)`ldpw_@QQlscApjc=Y?T#0D=7$l(>(wv*9*^SfM~nEp z)-P7=3vXKPR=dbq?bf(i^}vu|9m9)6=h6Oo*sm@a%hl`a9nGd*D^5oQ{wd~a`TCDH zm<53X)b~P!ihI8T9#|{f%J~q`?Oh0gJG0Qg<(GHWUt3ARidJ_ui*E09iV5y`ievDY z&i!C)Q3A77H7z0lpOk^BxONe8OOc@}V7cskmH6yao#OKmu>)68v*HwC*r*H4X4M?O ztnOK{8eW}#@lk(OLQ5@q@uS%)uyrs~F`ri7Jb=j4z5*(IF8$d^+lw(nmL5?zL{ygB{-1P{af#WNlvRx!nG10GbWzHG>rR2QIW3j=yYDn+H-z9R$Ic`0 zXXABn+9Rw#K;x#k`@(JW^G}P8db!e=2NC!CE!h3x+r1pSAi((}E^DE`OO!b3euUty zygm+9_)h!mdx{6=<@b!B#2JLpxqUclb{-)=b}ZAeR)2Fs==?9&h&pu|E`EK@OMq=j zjvyjc$-2{x$0hdT1g{a|e^vYPOqg-bNf|I4qn@hV4I<-~^0S-B5rV z*mD~}vbB=t8*^Psi0w{}z$cY7z!~=A-ktUm;Ob{_990pG*NE5y#mnC4i>etM%lWr( zRL(rkHV->Ff1YuKU`=~Ih*Rk&W#^CYZ@kcm@G^;@zs9*kn7?iuc&|O5pX5+69Jwj8{T2A8SNq=riM35EO)_i2)f(Yw+OPTd5c?3 z`gk*3Kqb8nE__7r5~*E_c#-Gnv*FyBi#W(}3-s=HhNa$tP>obT8(`yaWP z_?-pGYn-GT*>^AE5RO|o$=g5rxVyr~u-l(Wk82TJ4-;opd9Zk&-QY(jT17;#@#s1n zpEN@<&M8j+KW0rx!jh6F0>0e;3yVY4otnhrOb>B9RaCIsL_rSqzq$h4t`{b{cRdbm zbeFv8vd%^lU+eC6zOb3x3h@QG`-l%B1V(a=c1I!7+BYW5cXM&LpzEouJ2@HGm)okG zGh9citcOY!xQJcK3nXzZP9x98#jDf$(ve{J;dU?8)_Gcb{B<*4*xTf83Q|kMf9H*8 zJar_+UCts1t9;CF<@r~Q{(j(F_qzC=5Ygq(hIjcrTJ`}{;#uuoSKqSc?Ule|AQrtv`Ugu5s<-_J-ciw(AJ%gV_ql;jl^by7ooEkcU^f9w2P z|6y$W%BG_VHv#qc`|N-9^O5Df2bV*5OmmuJyp%PGiH4Ds0VcE!?7EZwmu0R@v`Z47 zmEvx}9)rK>FkkW6mSRoBAKrzUX84;LN0BmwYoSec-|U$j|Lfc8C6y(r>~=YnSQwsx z$i4|Nu*3Tf_+{?Md-x@$dO-G&+SZK@;~u925J9*_*(hBP3hYf$?D>%F9f$If?oa7s z1I_N%MoJSg`|+vvBPY}&x|1iA^HZQ+)l<9|(BcwRx^W`*_f#yH{aBqy5d?k|vGRWy zQbc?GBzpp(Wy1SXI@na$9m9xY4yGW+(32m*uI9RTyBGEH8OjkqRQm-~N&#_S9JbQp z$}EGh@G#Fkw<5J3_}`w8rcJJc(V4H_3F7e>zyrN*F*2{G3;08BJu63dGBNO<3oD5%x@?*5m8 zBkxDMx8p4nPX}I#4c>~qX*C$@<=8~lFs9}qB=VNn^Okm^sY?)od*8LN@`5;)$zEp3 zZ*jNZ@PB4uPKZ8@gK&PE&o4mAySL9}Zc;Pc3-2KghOOW#@w%MtuK0%^68QZM6laPl4 ze6juCc6^1u%LsoTBD{j&UX0r3B%CeFV)#y!Tzf7so^I;XF+5 zu&h6oZ4Wc1H+J~B2#z$0x5D_}tu-;d^D(`PA?+U~nz)Q zBQdlIY5K*(J1RW6NB;`}qP+={J@HUJvHd9>Y#Z#3Yos(6vmayV$B*Dq3){O5Dcfkm zzKHsBEU`mmonncpE)UmUB)@IN`qak7ZTWJW94a%I#>4l3wXODGh9@077QW zq}LL^fZ^$Kj+$X-f&_JUS`Q3Z;8-6Os4!r#*Fk!BY3phba=1w~(?eesL+r1F!&;fR z-$yYx2D2V{tab-u7NM{5aZY=^n_>*yjRUVN+HrU5t<`OHJs<9dfD0|b!DTDg_U>Om zZx_As8+^Csif)_a1QtPB4Z2g+zqRUCXBxbuJS`qH*RGD*FGTj&u4aZ=bS;{7P^wsT z9a(f3A}qRaW*x<97F~>%9UlnIe=5^kJa^Vx*x-hBd?2&<`Qo9)^ET^+4H>JBOltF= zZHyLw@bni{v|$~Y3>H5z<`#cUZ5C8)tU4T}4mKN19$>zg`Ksw^YbuIzY}I%)xUwDi z^lt$j$j&s9SxFoq1_7t^Eb1jygPtVDfdS^|c0Oan?RJ7{5X84|?j_`B(hgCzGK9Nu zE*tVHX^y=_ax=))SbjT5$e1=DJZ3Ic;dAm_BIHHiA*nF}O~p5ObTzO0Vdf3hGXP16n2v3q9KfZMMRS>Xe0hf^KBzScSVR?pMXXG1di6y3l#X|UZu;0}U#7q-2G{77o!DWTr< zahU{c`v4|c0^Y~8r7L_(Zi|IH=vyJ4^g|!>PdaQL6S}KHNDBkAAc;wVY$bx5T&|Nc z+gw7E>k5IrC=- zK`aB9(4q8SbxM{u2l0g@1GnjgRGu(I<sIOacvFU;*Bp4`5FC_QmT~sb}iOUuiPrAu1DR z_cM5(-1d><31um$50V3nRRYdqVNnXi$*?3ye4hw`u^F04(D=!A9$xi5$lF2~65^T! zV<~yMndfdyylsBZ_S=kcp0A(w@)QV4a@99@ad zCeY28aXXGqh^a(6pucE`&zR^5GG>md1U+GvJQo9z=<_5t_C~k!8{2NT6MiK(evS^} zGuGV>BB+*w*cZ;_L)Me#xJ!68FI4~liomox`|a8lMo84}F< zq{0z#{X+IuxXHySwcZgw$LPr1nG-d{!9w0G;opmSzj}nHJAK4b3+98V9Z+n8u&w-K zP=C5}5Osa-kExY&wpcMe`cuS;%@%@`Sw?1;5YIBaLF9RJH z`ti($%2p1)$h#HGYt7m z6RNO`=t*nK8%ll4@I`UUPteQu?b{@O-jGKf9p)(Qg+wNf^ZqO-XSqi!y+j5Z6av_O zboRTbf#-YK>!^7~4Q2Xzp@eV-d*#N|@P{=*xeK|R^RTRNgPQ8cm-D73J8oC)rY1u5 z)$%p&e3xmaQ-eoEZ(a}GTv-`f*0|>`v~kV{W`!Hq?9V0$(5|dWk2nwY)$(qHMXv^% zKWIA4RAsGwm!H`;R?u+kv^S+rP{x+Tx7uBDk4!(t6&pc&{AL7ozpLIy@woBqfF0HTJ$^g&W|If&S#a&m{53Yq z1UR#Tu|=#m=xHO(5(if`VBXsX`$|p!^Lrhx;|9U{B~Z{{pycU;pVZ`vrZE|M)5+fssR|_h9M6njN zpq*h6WyEWXYY(kVmG5=yP2zk7@c}k*jV$Bb5;6_%^AcXdJrtg`6S#@32enfi)hl2) zVox72IFN2#Zu&dj=@23q&*Wo!wU0f-LRJ7l?cCEeOBo>q-(B-f5Xc?BBM;in?lH#JwUr*Gf| zVQ_6JO0kxK?GH;nIE8T4++T%AwkUXZ$3xf z6SdN5zLO}v#Ai{05D1?%P*{3RetzmJR_(bTmK2%eSF#I&BxM@5s?6S?{bW^SyV@>lh%e$f}w zm>e3#Fd2)hFm09QElk3TjsE>t7T0j`>+ZxU(#X*lz}9OmmQwgxJdIgnNOd_$SIrsf`1 zBKyx%Lsg#Fr&p9&I({oHQFi;+vW0I|{N5iX@R^8)iuM*P?M2E!bzJB~(leG*k^6u0 z{;g}OxNgBRUW^S?$DChLMyH(C-uq(?KC9JGvDkt^>HV1M2kH`T)0T)H8R~XaBk!&Q zyZDP2Wn>x+t1g`$l@5TSLNtd^63&8^-r}4f8HL}PBf?BNa5mb4uiSv=I@6)BGr zz2(NJX6k8$3g=Ov%Vj6YWmu&O?-8Z-3;ualk^YY8vL3c0!@QC4oVO%fuJ^jR`o#{k z%eccTdl-)D*J_$A4|LBt!mc_H7D5%R~++K**B3jyiS*~rmDMWtru)Zx-OR$B$v*Wkz7YYgL4>twvwDF z^6KcMH4#)#U*|VYvNufTE)ZMW@A4Itg|4jYV<~d%^7oqAcbju>sYkvKPD_Lx zyoVpqP_GY7D^soSJ6}d`-_8Hc0`S7mTN$uA>x`9QbypSk3pJ?PQulMoDQLT6)^9_P z-t8&}%|Gb^Yxrc^pNCSnUL9jpMBBEimq^dCWThs0I2#_^<9BE)4qU?72^)?-69;^P z)86To;X$Pi<1i`5DxV6T@31+nx)@2VF*(h~3P}KRUjsddD!1=<|JdBq-dpW1KkK4t z#UH}2^zzrJS$%+q@FVW7%sVsU+R1e$fB-I1A|x5rAG9ugckI!p6BS7`K=1X8?7CzV zu!zVMr3k@_#S0PubB|?TS?h)$7pT_=3YHm#b3b};t%V=TB5(HY@f)9~t}D><-Ll4~ zkL&I1Z2YcwpX}tas)W$|@_oFe$q@+;^_OPPciD~6WE$I^o6p4(wjn(c_+=MoJYs7nT|f=FAsj0F;ENzy&>t!F_ixp^fgj>m*hEe@Q3!!XPJ#e&ST1Nku8md zlO6qc(i~s3seor}T0F5^GDNwWVNVP7;|DyS5&?8mm^1ogD&i@=;&J?Vv<2DRxfJh_ ziSq)OCNhq?G57$S`$V25!l38SinBwu(mw9VxD#=C-7TN%i7f#%(t1KL729GRH5#wf zIa1j_1neZrH5{TSfcNfc#EaoI1~mpSpDu*Q<3En~uIcY^X`=!EXx${~>`w<^!86xx zY5rRjANrpa7gKx~a3^JLof~}2YIB@0#AEbpZ}bI#`B(KQVWE<+WhmJnZWd7`f`c0G z_^(mCOJe_yOi7*j^}S+Pi$Xf_)Cluy(CD*_;VI`^0;^E^o`MSs=T7rbb``6SN%a7h zrIBq#aX+VXfXRkz?I{?#(YI^g+W&m{q{noNP>t4JEQ08^)d9}m+An0j1$!4pQoL_{ z8gI8{m>bdTaSIy$9uV)p^=A@gcFm*0`!OjKz*0K0EhpaNbS`ERDOY>?0DAXj*QB-I zarxwfDICZd={bUa2oKtn@u_L4v4$IQzSwF>kL>-b17_irIPlCx^nOLFlxplauzgq& z5KE)}4td{e#m1^~=98^GDBr55#zGFyCd*9@lTAjwh}ZX&We4Pbi=%)!_2#{=p@88V z{CouG*z~-lFic{us-T$5g_11048|1gQpgqWj%)1L$h=M$o1pp5LXhF@s3%A&Dn^Ampl3+zF>YZzhn^;S7TG>{SE(Ph_E}an# zvlv&x-vmnL3_Rd$N59z@-5<=B^kQ6^%o+EKpR}-N5(|wm8v{q>lZSRQ>n(xf57spS3rt;>X|2_AV+v=au((T#5ScB;t$MC+OVV z(1V48@WlW3i5%EJ@vnu-J@0tmJKTyjy09{m$*ECq97$ zgs;srPtAcnb3s^kyU3TBr(D04ahX!Pl*{`9yn}B&-hVW*wd-TCda?)VIS&WwhQ>B^ zCyV)y>V|S+*0VECCCi`R2zF=QMnn3C&*@bv1tYY+DGh^7q&H!gWTo0W&y5RuzwAk2 z$sFO|mvE4N`Q?2H-oY-9_n{`X5not%pX>?soCgARuj89;Q^a~lb+5TGf9dw`tB~U4 zT7NfJ^wwmnV0i>%R8Q}06@RLJR7}u*wkpZe-8vjI=LaYa5l;Cdr*2PMfams-VJey=XDmd48=puE)0aa18iSwo&_&?V+?&_>+y*v$;+7EO(|@_Zk6i zXMn`+v-fe-P zH>^fNYW4h$gzASd$O>BD*TDnx@}!dXS7G!0qtsE_?B2Un5xlUnkDY(( z4COyW`IR+f)V7;2O{3#m!r-PUHI9$9UtKU&jBVG5_c_Ynmw}5IS)1WZs^7LgV^FTT3kUXhNhB^G(ts0i4RRv_$X z7Jbnh+-9O~`~ET-j|G_P|~?ze>5_wZ#fmoKWbF+SH< zqE>>yw{f?4mLmf^3-wH`d39ISuDhYWmZyFWO-@5s1y!ORz%i3$L1z92Rzk|_HSZpX zK0k#XCX+u;E@p6i1`BsSQHe-<@oMcoukSbBZ!~J()ciPiUA{SaamKuV6L)aQDnpgD zrAAet!XHhod!A5HvK9dJg^+M&EUd#LBO zjG2P}7p<=7_x?=ZhpfNCW-%c?< z43dw9><}OAbWV@PI%wsmnsvtUPQ5J*C<;}wujGDd+s^yVaynS-$+~FTJEZ>=hfuY( zq3@6B_dLFwAAKuweT!>+Bpr8kU+d3v zZW*sd5r^84!>WyJ`409D)v5r>!f)oK;O#1Htnx0+i>#WK{ zie%q<120n|%Isen{8{3kS7r##tDZZWwUR{73K0IiFKu>VEyf__W8(4o_taN?d!${INL8 zwHDn-_``nbIDa8F>p#o7|4d6&t5W$=mHux(p`5_v$Fl1VF4fug>zga#$ho$+>9>i1 z6nHwv%1}!A{a8WgNBPa8|Gsxz=M8$(9-WQXF!!d_QSQw36SYh8vjApU6gRR8mklsB z4Q3Q#0#^0E?hjLUb}j7QA9nkDWtjE~Y!ZXYeLUuJlsDQq`J%D9Y%H(F$ThfouyQ!g zvtihu(DRUMb|(1ni9JbLjv4p;+}sM6sxk4JiIg!PFV_$uu|QK-V{>%QiKQPNCdpQC zBkwhjtYJES62Xl1o4-*t zw0kLnl4@^P2|@SXm4yq(mnh}+2MKFW+vfR)$xN88)&22Rr!pmwudMv^Oho8P68gMA z-POkYW!t%`j>3v%^r~oiV@||zesryXUvtQATtiCts%qAPj2Shv&D?0K*NkbNRrhCw!|9*z);ns4T<=N*buD z$IvxC@V7Wx2gCaFS91NMW0vMC9QnScE0(X`y~@r1S^JJqRIo9_UxvGJ^+TcY?1#e1 zZJRFb??1b0xSq#7a{xK8HE_^ZR_IN1)>nYBqPH`Lvkm4T$TMzHLZaJeXT zK-Io!wL`bxj~c&+zIk8kQJnDAL-7MD;GHj^1EulJr+$ZYQvp)h38{o5dYy*g69daA#dogxxKLpGgfNX~}NE$7`+_M^TMn{d!ohmF3dcEC_#{bh{QEij}5g8>WPc6Qvz!lux<;`&;^dokw0 zC&3=upJq#?gBO5j_MT7zt3&|*Cx%WPPX@tvA4NID9$Hzw`Y*EBs-o&z;cQ*%xp5$U zv#bfbTrOe}>)ae*tzIq%64;=h+7y%nP%fvc0|?Fi}X2qi)kK?ob-0 zUEtEL#=p9eU4E#;>~1VT)mv$LPsFpEF07t|x3VWZo%bn1#wekNlTqS z5DI5vfBA_Is|~fWO`meAc*<|;s_HXDA&-3<|_A zOy&w@^X%83<3h4DcqTWI>CaJ^d0Whpc-nrh;y(ZVn9b2*4u3$41ITSNM^5|-1+q7t z&IhgQ&AkFDmmQ8G9S)8pJa71p>b-9Ge>N{ssi~M!?NoS?zzh3GP;`q_+I`Z2r~sxk zCzRf7ngGhfW4Ogl{L`>L@+=8ZgG|;FIt1lro6rx0)+1}#+>q*WuRwC9(sOoll*DZ| za*4n|eK}erw!tQ?8jePcPI7)VRd`Q}%&W$+1#&f#iG0KanL1w8zyH9bk zK?p@M>l5e3=nby$POV%?@hGla`RA5$Xk)}m%A{4-t)#<$vjLymw9yj(L4in3PkFs* za?o_1G@5~NAl;c@qggK9AdQ6*t#E^8h4kArwqebMk|F7RDkP?%Gb$z(jWiVgIDr(a z>@+}nPusbpsDsw?fFJ1AEkVz+HWs{p@%Kt4j5d4mI29(6pad6{bov+rQ7|SYXH?*9 zHy_H-!=V)o`m~{k{E)O!i@efPJpl1$BNzsDVW3i(3D#J&n0L>c&8|)~DMjQvG^=N1 zE}9l7v&O~2Du0x6$nrKhUif`+d2%eUso0p2HY`G(D%Wb^UO)w$;nN`~d5sG>DBSX~ z$g4p(AUM!tZS7#{s>_q3n6t694M6@W#{lV)lv9sn(t{m=YO|_Y26oLrp3+8-VgcOV zvlV=?od7`!*;as@=esO`7;wcpm)l<99@JQ?(;Vto8Fqo}9>FBTpVlL9*@g$o&Ro)2 zDY{W072#9YbY9O}eMUSepEpHWeOaYQTun=zNKAW;1B@!{iHwn?1&m-I59ndgS_PyS zEA$>KFXQqa%dH~t6xDzwgYsuAv}et=&7lkCx{oxQYm>WQvH_~=YSZCIY`|a}7%lN7 z3Pf!>odrtQn=1rD%RI-BZU+=Oz`i66bExpF^k)E9SfGp%jl~?nKs(Swz;lyd3eEw(q7OC$*=L8$d4Cc-;ZGQC@(O=k=s*JXU%XIPcMOE8 zGF`Z%@Mfv*q4+#V+T6%KB^V&#NeT5xdOh7EP(@ZI%YX|EPw-9s5^wV9)QDg!T&og@3n@OO1cESp=jdRrDK)wE^Vu8oM; z(55X7IIiDDfXr{OiEmoSaicEpsB`0~aoC+nA#ORBbC|EB(ku;bpg`m5dnf zh82CJedP)@if8dydr3vVjdup$0>e8S8;;`NO&pRgvOv*O1u6-aK!9v{r`29Msl@C0ukmUa>zu*XIVT7qJu$a|w=lh9iw z2hHfIqKSzIInG={XCrhkG#75QZVa2T>9?6|fY`d^E#rZYAAf*k*13_^zpnY0*3B77 ziJ$zNh5kYUdb+CgH;ug+K0N>?PvZ z6{hI7W;8~kD?Ra{hKpKd_4$>a={4zfo~iAx9Oih!NM(#fEf|yGiX6aK)^qH{_7fm` zv*~ouy8c`&Q2EW_5OU+-NYL{};HcW`Mv%K~iAqaFk1F@!0@bjzCKd8c1A9Wu78*$} z%r=TNt7L~FWv0_yQIMeY0w5zD)ExBhO|j_x&!_#5v^QQ=@WMk96xHH{R-bqv%71I0TJnH`5K=vjU09Oo+z1fxH*XoEkgbxG5If z+z36vIc|)=Vw*N%d)xacu+}#V6iwxWOXNS3b}Brl7OBkU)~w!^3HoCe@Wl?pOF{GQ zKRM1tlL2*Y0;%l3-bayZ30a^-DyuJ%awAhI@SHkiARD?{Jun)?Y8EhR7syU*WgVCg zT+$2V{$EbfWSfHqmp`aOOU^zIN4?mk9hBBLM`==FM#{n_kV3+tUQJ;MG5ZZ+>d1o1 zFk+PVVz>H|tv-fk=D7t1|4{ccn(=(l6nU%P09spbp0c$g|dMf=>WZp0W&V<%lWE_|SkPU6t_X+e67j9WK0yPN- zv?&08fuYWAo*WN=`qsHj_M`WpPqjqMp$3(;7s&aMxg_|S1@sYB?qr$%6p~7K#G`36 zE@q-(R0G*iIZ6TzS=?2#GS<(fH1o5_CG~Xp4B$P#eat3ru@enfqUMlV46XR`)ov%1 zKy~pt7Z3%Eo@R1_ss}OW0ORSI%|P1OEOXu|f>QhmPb@(9WsF2(tAO5PWn~SWRa_Kf zZU1R;P89<%-v&l(Z*d9Qdfd5k&i~aQ{fGDRWu2;-qZ8W&=*WV=_R0+y5J%Z z0A-v7##ga>$4oPeduq_d*CC}Vx5A-;yAp%aEEaPN16pjrM(a5aV&4gnui114=%@Z% zG%$NQ$OS4C6#NosPZw+fTF(ku@($-a5lPT*@Ct_)ZjeBE7caEcRRf_2CMqYC+ANg- z%E=?4#ZCBAfsfyAGr4rxJi@;*SRUrQir2-c8dq4$<0x2RBc0LD`QAs;bp(3mFSpP4J+En^~x1O zl*8h&=2E17Al?k3?!Q8sa1?x3cToDw0%b-OI8mlBiDVRpcs41-$ILY-Xd>Gx6-ZHG zi*{;D+WLXyGp-ha#D^B2(H!SL9v6nzp-B=tJ*U?mqkNgxWR}*n`;-^ggiwUDN^xj< zZ=Q-D$(3O#2Ss&Z85i*m7zV7EjFFuMhGU>x#$)3hnww&y*juG+Q_x!_8!hMAqlU=` zeV@LxpS@MPfGyl=+}Ji_W0exP2L{bRk|7*`D*NVS*JmD&zny6|{ZnIB;z=u(Rou;lqGdm2~(uB zRzW-sva0R@-&(^s>|O6cZM62xp<$KG7s!+mv1GWNInH$vW6+F5>sEp~2d7`+*f9)yBm^U*_4 z5Xz&6(dgkcdUK540&80Gnh09ciq^Eonl@O|me;hSHSK9l2ds(2nvT4t6Rqh?Yr0@f zSFGvAYr4~#9<-(>*7U-f-n^y{t?5f^`e99ftQo*-2GW{Av}Q2Y48fYAyk;1!8BS|P zV9iLZ8O3Wx)0#1~W-Qi>!-+^6qw5}sWfRc z>0++HT!pztbDidfn42)SU~bdgp}8yO9?X5144O=u2Vx$=Jc4;l^MvN9m}fB0VP4R@ zqVM7+V-Sn9($2 zXvT^e2V)O2o@N5gL@^F9lVB#(OrdcUGZn@OW*Ut%&2%v?Ff(9g(#)cnEoKhPT$p(@ z^Jx}{SqQTTW-*N`%@Q$7VV1!xr*WfMA!a4aDwx$YYiQPrafew4vz}%H%|xr$>n z$Hkn0ISF%$=Co?;G+lwbv7Gb zqyPHiYO8T$9L!yu>>O;h9sL!aB2m;n1Oe*BX7?coWK2GhWSSuExB4VsS}O#rV?0!@ z%VXmq*r8DFP?)+yu8!eq;NM60BXUiO#H1)ridHm*atsrnSL9DZjut7g{EaT;j2@RO zdOV^hFnXdyPh#|B&26Ohm(QQ+FEaflxkXZPWxkBesmz>4pZ@F1;3t3KgxBWh000+E BKM4Q; diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 91f8c3b4f..c9a0cfc8a 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -88,7 +88,6 @@ def test_Bitvec_ops(self): self.assertEqual(translate_to_smtlib(z), "(bvsle (bvsdiv (bvadd #x00000001 BV1) BV1) (bvadd (bvsub BV (bvmul #x00000064 #x00000005)) BV2))") - def test_ConstantArrayBitvec(self): c = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") self.assertEqual(c[0], "A") @@ -98,7 +97,6 @@ def test_ConstantArrayBitvec(self): self.assertEqual(c[4], "E") self.assertRaises(IndexError, c.get, 5) - def test_ConstantArrayBitvec(self): c = ArrayProxy(ArrayVariable(index_size=32, value_size=8, length=5, name="ARR")) c[1] = 10 @@ -124,7 +122,7 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): "\n Slotted:", not hasattr(x, "__dict__"), ) - self.assertEqual(len(pickle_dumps(x)), pickle_size) + #self.assertEqual(len(pickle_dumps(x)), pickle_size) self.assertEqual(sys.getsizeof(x), sizeof) self.assertFalse(hasattr(x, "__dict__")) # slots! self.assertTrue(hasattr(x, "_taint")) # taint! @@ -148,19 +146,19 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): self.assertTrue(ty.__doc__, ty) checked.add(ty) - check(BitvecVariable, size=32, name="name", pickle_size=84, sizeof=64) - check(BoolVariable, name="name", pickle_size=80, sizeof=56) + check(BitvecVariable, size=32, name="name", pickle_size=111, sizeof=64) + check(BoolVariable, name="name", pickle_size=99, sizeof=56) check( ArrayVariable, - index_bits=32, - value_bits=8, - index_max=100, + index_size=32, + value_size=8, + length=100, name="name", - pickle_size=92, - sizeof=104, + pickle_size=156, + sizeof=88, ) - check(BitvecConstant, size=32, value=10, pickle_size=79, sizeof=64) - check(BoolConstant, value=False, pickle_size=74, sizeof=56) + check(BitvecConstant, size=32, value=10, pickle_size=107, sizeof=64) + check(BoolConstant, value=False, pickle_size=94, sizeof=56) # TODO! But you can instantiate an ArraConstant """ @@ -174,11 +172,11 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): x = BoolVariable(name="x") y = BoolVariable(name="y") z = BoolVariable(name="z") - check(BoolEqual, operanda=x, operandb=y, pickle_size=118, sizeof=56) - check(BoolAnd, operanda=x, operandb=y, pickle_size=116, sizeof=56) - check(BoolOr, operanda=x, operandb=y, pickle_size=115, sizeof=56) - check(BoolXor, operanda=x, operandb=y, pickle_size=116, sizeof=56) - check(BoolNot, operand=x, pickle_size=102, sizeof=56) + check(BoolEqual, operanda=x, operandb=y, pickle_size=159, sizeof=56) + check(BoolAnd, operanda=x, operandb=y, pickle_size=157, sizeof=56) + check(BoolOr, operanda=x, operandb=y, pickle_size=156, sizeof=56) + check(BoolXor, operanda=x, operandb=y, pickle_size=157, sizeof=56) + check(BoolNot, operand=x, pickle_size=137, sizeof=56) check(BoolITE, cond=z, true=x, false=y, pickle_size=130, sizeof=56) bvx = BitvecVariable(size=32, name="bvx") @@ -221,10 +219,11 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): check(BitvecConcat, operands=(bvx, bvy), pickle_size=133, sizeof=64) check(BitvecITE, condition=x, true_value=bvx, false_value=bvy, pickle_size=161, sizeof=64) - a = ArrayVariable(index_bits=32, value_bits=32, length=324, name="name") - check(ArraySlice, array=a, offset=0, size=10 , pickle_size=136, sizeof=120) + a = ArrayVariable(index_size=32, value_size=32, length=324, name="name") + check(ArrayConstant, index_size=32, value_size=8, value=b"A", pickle_size=136, sizeof=72) + check(ArraySlice, array=a, offset=0, size=10 , pickle_size=136, sizeof=56) check(ArraySelect, array=a, index=bvx, pickle_size=161, sizeof=64) - check(ArrayStore, array=a, index=bvx, value=bvy, pickle_size=188, sizeof=104) + check(ArrayStore, array=a, index=bvx, value=bvy, pickle_size=188, sizeof=72) def all_subclasses(cls): @@ -281,78 +280,6 @@ def test_Expression_Array(self): self.assertIn('red', z.taint) self.assertIn('blue', z.taint) -class RegressionTest(unittest.TestCase): - def test_related_to(self): - import gzip - - filename = os.path.abspath(os.path.join(DIRPATH, "data", "ErrRelated.pkl.gz")) - - # A constraint set and a contraint caught in the act of making related_to fail - constraints, constraint = pickle.loads(gzip.open(filename, "rb").read()) - - Z3Solver.instance().can_be_true.cache_clear() - ground_truth = Z3Solver.instance().can_be_true(constraints, constraint) - self.assertEqual(ground_truth, False) - - Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual( - ground_truth, - Z3Solver.instance().can_be_true(constraints.related_to(constraints), constraint), - ) - - # Replace - new_constraint = Operators.UGE( - Operators.SEXTEND(BitvecConstant(256, 0x1A), 256, 512) * BitvecConstant(512, 1), - 0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000, - ) - self.assertEqual(translate_to_smtlib(constraint), translate_to_smtlib(new_constraint)) - - Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual(ground_truth, Z3Solver.instance().can_be_true(constraints, new_constraint)) - - Z3Solver.instance().can_be_true.cache_clear() - self.assertEqual( - ground_truth, - Z3Solver.instance().can_be_true(constraints.related_to(new_constraint), new_constraint), - ) - - -""" -class Z3Specific(unittest.TestCase): - _multiprocess_can_split_ = True - - def setUp(self): - self.solver = Z3Solver.instance() - - - @patch('subprocess.check_output', mock_open()) - def test_check_solver_min(self, mock_check_output): - mock_check_output.return_value = ("output", "Error") - #mock_check_output.return_value='(:version "4.4.1")' - #mock_function = create_autospec(function, return_value='(:version "4.4.1")') - #with patch.object(subprocess, 'check_output' , return_value='(:version "4.4.1")'): - #test_patch.return_value = '(:version "4.4.1")' - print (self.solver._solver_version()) - self.assertTrue(self.solver._solver_version() == Version(major=4, minor=4, patch=1)) - - def test_check_solver_newer(self): - self.solver._received_version = '(:version "4.5.0")' - self.assertTrue(self.solver._solver_version() > Version(major=4, minor=4, patch=1)) - - def test_check_solver_long_format(self): - self.solver._received_version = '(:version "4.8.6 - build hashcode 78ed71b8de7d")' - self.assertTrue(self.solver._solver_version() == Version(major=4, minor=8, patch=6)) - - def test_check_solver_undefined(self): - self.solver._received_version = '(:version "78ed71b8de7d")' - self.assertTrue( - - self.solver._solver_version() - == Version(major=float("inf"), minor=float("inf"), patch=float("inf")) - ) - self.assertTrue(self.solver._solver_version() > Version(major=4, minor=4, patch=1)) -""" - class ExpressionTestLoco(unittest.TestCase): _multiprocess_can_split_ = True @@ -374,14 +301,17 @@ def test_signed_unsigned_LT_(self): mask = (1 << 32) - 1 cs = ConstraintSet() - _a = cs.new_bitvec(32) - _b = cs.new_bitvec(32) + a = cs.new_bitvec(32) + b = cs.new_bitvec(32) - cs.add(_a == 0x1) - cs.add(_b == (0x80000000 - 1)) + cs.add(a == 0x1) + cs.add(b == (0x80000000 - 1)) + + lt = b < a + ult = b.ult(a) self.assertFalse(self.solver.can_be_true(cs, ult)) - self.assertTrue(self.solver.must_be_true(cs, lt)) + self.assertTrue(self.solver.must_be_true(cs, Operators.NOT(lt))) def test_signed_unsigned_LT_simple(self): cs = ConstraintSet() @@ -691,7 +621,6 @@ def testBasicArrayConcatSlice(self): self.assertTrue(self.solver.must_be_true(cs, array.read(0, 12) == hw)) cs.add(array.read(6, 6) == hw[6:12]) self.assertTrue(self.solver.must_be_true(cs, array.read(6, 6) == hw[6:12])) - self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 6) == hw)) self.assertTrue(self.solver.must_be_true(cs, b"Hello " + array.read(6, 5) + b"!" == hw)) From 5690dc576ba00e94215d06823ae2bd29163b6d07 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 17 Aug 2020 13:00:08 -0300 Subject: [PATCH 074/126] OMG expression refactor AHHH! --- CHANGELOG.md | 2 +- examples/evm/asm_to_smtlib.py | 6 +- examples/evm/minimal.py | 4 +- manticore/core/manticore.py | 2 +- manticore/core/smtlib/constraints.py | 35 +- manticore/core/smtlib/expression.py | 513 ++++++++++++++------------- manticore/core/smtlib/operators.py | 1 - manticore/core/smtlib/solver.py | 85 +++-- manticore/core/smtlib/visitors.py | 17 +- manticore/core/state.py | 13 +- manticore/ethereum/abi.py | 10 +- manticore/ethereum/manticore.py | 32 +- manticore/native/memory.py | 9 +- manticore/native/models.py | 10 +- manticore/platforms/evm.py | 148 ++++---- manticore/platforms/linux.py | 10 +- tests/ethereum/test_general.py | 14 +- tests/other/test_smtlibv2.py | 103 ++++-- 18 files changed, 560 insertions(+), 454 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44d3e7ee0..8b2acb84f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -409,7 +409,7 @@ Thanks to our external contributors! - Ethereum: Support for Solidity `bytesM` and `bytes` types - Ethereum: Beta API for preconstraining inputs (`ManticoreEVM.constrain`) - Improved performance for smtlib module -- Ability to transparently operate on bytearray and symbolic buffer (ArrayProxy) types (e.g: concatenate, slice) +- Ability to transparently operate on bytearray and symbolic buffer (MutableArray) types (e.g: concatenate, slice) ### Changed diff --git a/examples/evm/asm_to_smtlib.py b/examples/evm/asm_to_smtlib.py index 3af7944f7..c74913dce 100644 --- a/examples/evm/asm_to_smtlib.py +++ b/examples/evm/asm_to_smtlib.py @@ -42,7 +42,7 @@ def printi(instruction): ) -data = constraints.new_array(index_bits=256, name="array") +data = constraints.new_array(index_size=256, name="array") class callbacks: @@ -57,8 +57,8 @@ def will_execute_instruction(self, pc, instr): class DummyWorld: def __init__(self, constraints): - self.balances = constraints.new_array(index_bits=256, value_bits=256, name="balances") - self.storage = constraints.new_array(index_bits=256, value_bits=256, name="storage") + self.balances = constraints.new_array(index_size=256, value_size=256, name="balances") + self.storage = constraints.new_array(index_size=256, value_size=256, name="storage") self.origin = constraints.new_bitvec(256, name="origin") self.price = constraints.new_bitvec(256, name="price") self.timestamp = constraints.new_bitvec(256, name="timestamp") diff --git a/examples/evm/minimal.py b/examples/evm/minimal.py index df3fc3670..ac1442c7d 100644 --- a/examples/evm/minimal.py +++ b/examples/evm/minimal.py @@ -25,13 +25,13 @@ } """ -user_account = m.create_account(balance=1000, name="user_account") +user_account = m.create_account(balance=10**10, name="user_account") print("[+] Creating a user account", user_account.name_) contract_account = m.solidity_create_contract( source_code, owner=user_account, name="contract_account" ) -print("[+] Creating a contract account", contract_account.name_) +print("[+] Creating a contract account", contract_account) contract_account.named_func(1) print("[+] Now the symbolic values") diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index f924008ce..74a43754a 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -62,7 +62,7 @@ def to_class(self): ) consts.add("procs", default=10, description="Number of parallel processes to spawn") -proc_type = MProcessingType.multiprocessing +proc_type = MProcessingType.single if sys.platform != "linux": logger.warning("Manticore is only supported on Linux. Proceed at your own risk!") proc_type = MProcessingType.threading diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 1b99802e2..817e8e5f3 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -13,7 +13,7 @@ Bool, Bitvec, BoolConstant, - ArrayProxy, + MutableArray, BoolEqual, Variable, Constant, @@ -23,8 +23,7 @@ TranslatorSmtlib, get_variables, simplify, - replace, - pretty_print, + replace ) from ...utils import config import logging @@ -180,7 +179,6 @@ def related_to(self, *related_to) -> "ConstraintSet": def to_string(self, replace_constants: bool = False) -> str: variables, constraints = self.get_declared_variables(), self.constraints - if replace_constants: constant_bindings = {} for expression in constraints: @@ -193,6 +191,7 @@ def to_string(self, replace_constants: bool = False) -> str: result = "" translator = TranslatorSmtlib(use_bindings=False) + tuple(translator.visit_Variable(v) for v in variables) for constraint in constraints: if replace_constants: constraint = simplify(replace(constraint, constant_bindings)) @@ -210,7 +209,6 @@ def to_string(self, replace_constants: bool = False) -> str: def _declare(self, var): """ Declare the variable `var` """ - print ("declaring", var) if var.name in self._declarations: raise ValueError("Variable already declared") self._declarations[var.name] = var @@ -330,11 +328,11 @@ def migrate(self, expression, name_migration_map=None): elif isinstance(foreign_var, Array): # Note that we are discarding the ArrayProxy encapsulation new_var = self.new_array( - index_max=foreign_var.index_max, - index_bits=foreign_var.index_bits, - value_bits=foreign_var.value_bits, + length=foreign_var.length, + index_size=foreign_var.index_size, + value_size=foreign_var.value_size, name=migrated_name, - ).array + ) else: raise NotImplementedError( f"Unknown expression type {type(foreign_var)} encountered during expression migration" @@ -387,20 +385,20 @@ def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False) def new_array( self, - index_bits=32, + index_size=32, name=None, - index_max=None, - value_bits=8, + length=None, + value_size=8, taint=frozenset(), avoid_collisions=False, default=None, ): - """ Declares a free symbolic array of value_bits long bitvectors in the constraint store. - :param index_bits: size in bits for the array indexes one of [32, 64] - :param value_bits: size in bits for the array values + """ Declares a free symbolic array of value_size long bitvectors in the constraint store. + :param index_size: size in bits for the array indexes one of [32, 64] + :param value_size: size in bits for the array values :param name: try to assign name to internal variable representation, if not unique, a numeric nonce will be appended - :param index_max: upper limit for indexes on this array (#FIXME) + :param length: upper limit for indexes on this array (#FIXME) :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True :param default: default for not initialized values :return: a fresh ArrayProxy @@ -414,7 +412,6 @@ def new_array( raise ValueError(f"Name {name} already used") var = self._declare( ArrayVariable( - index_size=index_bits, length=index_max, value_size=value_bits, name=name, taint=taint, default=default - ) + index_size=index_size, length=length, value_size=value_size, name=name, taint=taint, default=default ) ) - return ArrayProxy(var) + return var diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 1d6e63ce1..403b5120b 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1,9 +1,28 @@ +""" Module for Symbolic Expression + +ConstraintSets are considered a factory for new symbolic variables of type: +BoolVariable, BitvecVariable and ArrayVariable. + +Normal python operators are overloaded in each class, complex expressions trees +are built operating over expression variables and constants + + cs = ConstraintSet() + x = cs.new_bitvec(name="SOMEVARNAME", size=32) + y = x + 199 + condition1 = y < 1000 + condition1 = x > 0 + + cs.add( condition1 ) + cs.add( condition2 ) + +""" + from functools import reduce import uuid import re import copy -from typing import Union, Optional, Dict, Tuple, List +from typing import Union, Optional, Dict, Tuple, List, AnyStr class ExpressionError(Exception): @@ -15,6 +34,7 @@ class ExpressionError(Exception): class XSlotted(type): """ Metaclass that will propagate slots on multi-inheritance classes + Every class should define __xslots__ (instead of __slots__) class Base(object, metaclass=XSlotted, abstract=True): pass @@ -30,6 +50,7 @@ class B(Base, abstract=True): class C(A, B): pass + # Normal case / baseline class X(object): __slots__ = ('a', 'b') @@ -42,26 +63,34 @@ class X(object): x.b = 2 import sys - print (sys.getsizeof(c),sys.getsizeof(x)) + print (sys.getsizeof(c),sys.getsizeof(x)) #same value """ + @staticmethod + def _remove_mod(attr:str) -> str: + """ xlots attrivutes could have modifficators after a # symbol + attribute#v means attribute is _volatile_ and should not be saved to storage + """ + return attr.split('#')[0] def __new__(cls, clsname, bases, attrs, abstract=False): + xslots = set(attrs.get("__xslots__", ())) + # merge the xslots of all the bases with the one defined here for base in bases: xslots = xslots.union(getattr(base, "__xslots__", ())) - attrs["__xslots__"] : Tuple[str, ...] = tuple(xslots) + attrs["__xslots__"] : Tuple[str] = tuple(xslots) if abstract: attrs["__slots__"] = () else: - attrs["__slots__"]: Tuple[str, ...] = attrs["__xslots__"] + attrs["__slots__"]: Tuple[str] = tuple(map(cls._remove_mod, attrs["__xslots__"])) return super().__new__(cls, clsname, bases, attrs) class Expression(object, metaclass=XSlotted, abstract=True): """ Abstract taintable Expression. """ - __xslots__ : Tuple[str, ...] = ("_taint",) + __xslots__ : Tuple[str, ...] = ("_taint", ) def __init__(self, taint: Union[tuple, frozenset] = ()): """ @@ -74,6 +103,7 @@ def __init__(self, taint: Union[tuple, frozenset] = ()): def __repr__(self): return "<{:s} at {:x}{:s}>".format(type(self).__name__, id(self), self._taint and "-T" or "") + @property def is_tainted(self): return len(self._taint) != 0 @@ -84,6 +114,7 @@ def taint(self): @property def operands(self): + """ Hack so we can use any Expression as a node """ return () def __getstate__(self): @@ -117,13 +148,8 @@ def __repr__(self): return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) - def __hash__(self): - return object.__hash__(self) - - class Constant(Expression, abstract=True): """ Constants expressions have a concrete python value. """ - __xslots__:Tuple[str, ...] = ("_value",) def __init__(self, value: Union[bool, int, bytes, List[int]], **kwargs): @@ -139,12 +165,8 @@ def value(self): return self._value - def __hash__(self): - return object.__hash__(self) - class Operation(Expression, abstract=True): """ Operation expressions contain operands which are also Expressions. """ - __xslots__:Tuple[str, ...] = ("_operands",) def __init__(self, operands: Tuple[Expression, ...], **kwargs): @@ -152,23 +174,19 @@ def __init__(self, operands: Tuple[Expression, ...], **kwargs): :param operands: A tuple of expression operands """ - taint = kwargs.get('taint') - assert isinstance(operands, tuple) - print ("Operation of operands", type(self) ,tuple(map(type,operands))) self._operands = operands + taint = kwargs.get('taint') # If taint was not forced by a keyword argument, calculate default if taint is None: operands_taints = map(lambda x: x.taint, operands) taint = reduce(lambda x, y: x.union(y), operands_taints, frozenset()) - kwargs['taint'] = taint + kwargs['taint'] = taint super().__init__(**kwargs) @property def operands(self): return self._operands - def __hash__(self): - return object.__hash__(self) ############################################################################### # Booleans @@ -178,6 +196,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def cast(self, value: Union["Bool", int, bool], **kwargs) -> Union["BoolConstant", "Bool"]: + """ Cast any type into a Bool or fail """ if isinstance(value, Bool): return value return BoolConstant(bool(value), **kwargs) @@ -219,15 +238,6 @@ def __rxor__(self, other): def __bool__(self): raise NotImplementedError - """ - def __bool__(self): - # try to be forgiving. Allow user to use Bool in an IF sometimes - from .visitors import simplify - x = simplify(self) - if isinstance(x, Constant): - return x.value - raise NotImplementedError("__bool__ for Bool") - """ class BoolVariable(Bool, Variable): @@ -285,7 +295,6 @@ def __init__(self, cond: Bool, true: Bool, false: Bool, **kwargs): class Bitvec(Expression, abstract=True): """ Bitvector expressions have a fixed bit size """ - __xslots__:Tuple[str, ...] = ("_size",) def __init__(self, size: int, **kwargs): @@ -311,17 +320,17 @@ def signmask(self): def cast( self, value: Union["Bitvec", str, int, bytes], **kwargs ) -> "Bitvec": + """ Cast a value int a Bitvec """ if isinstance(value, Bitvec): - assert value.size == self.size + if value.size != self.size: + raise ExpressionError("Bitvector of unexpected size") return value if isinstance(value, (str, bytes)) and len(value) == 1: - print ("AAAAAAAAAA"*99) value = ord(value) # Try to support not Integral types that can be casted to int + value = int(value) & self.mask if not isinstance(value, int): - print (value) - value = int(value) - # FIXME? Assert it fits in the representation + raise ExpressionError("Not cast-able to Bitvec") return BitvecConstant(self.size, value, **kwargs) def __add__(self, other): @@ -501,13 +510,12 @@ def Bool(self): class BitvecVariable(Bitvec, Variable): - def __repr__(self): - return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) - + pass class BitvecConstant(Bitvec, Constant): def __init__(self, size: int, value: int, **kwargs): - value &= (1 << size) - 1 + """ A bitvector constant """ + value &= (1 << size) - 1 # Can not use self.mask yet super().__init__(size=size, value=value, **kwargs) def __bool__(self): @@ -518,17 +526,20 @@ def __int__(self): @property def signed_value(self): + """ Gives signed python int representation """ if self._value & self.signmask: return self._value - (1 << self.size) else: return self._value def __eq__(self, other): - if self.taint or isinstance(other, Expression) and other.taint: - return super().__eq__(other) - return self.value == other + # If not tainted use the concrete value + if not self.taint: + return self.value == other + return super().__eq__(other) def __hash__(self): + # need to overload because we defined an __eq__ return object.__hash__(self) class BitvecOperation(Bitvec, Operation, abstract=True): @@ -629,6 +640,11 @@ def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): assert isinstance(operandb, Expression) super().__init__(operands=(operanda, operandb), **kwargs) + def __bool__(self): + simplified = simplify(self) + if isinstance(simplified, Constant): + return simplified.value + raise NotImplementedError class BoolGreaterThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): @@ -662,10 +678,13 @@ def __init__(self, operanda, operandb, **kwargs): ) -############################################################################### -# Array BV32 -> BV8 or BV64 -> BV8 class Array(Expression, abstract=True): - """ And Array expression is a mapping from bitvector to bitvectors + """ And Array expression is a mapping from bitvector to bitvector + + array.index_size is the number of bits used for addressing a value + array.value_size is the number of bits used in the values + array.length counts the valid indexes starting at 0. Accessing outside the bound is undefined + """ @property def index_size(self): @@ -678,42 +697,75 @@ def value_size(self): raise NotImplementedError @property - def index_max(self): - """ Max allowed index. Must be overloaded by a more specific class""" + def length(self): + """ Number of defined items. Must be overloaded by a more specific class""" raise NotImplementedError - def get(self, index): - """ Gets an element from the Array """ + def select(self, index): + """ Gets a bitvector element from the Array que la""" raise NotImplementedError def store(self, index, value): + """ Create a new array that contains the updated value""" + raise NotImplementedError + + @property + def default(self): + """ If defined, reading from an uninitialized index return the default value. + Otherwise, reading from an uninitialized index gives a symbol (normal Array behavior) + """ + raise NotImplementedError + + @property + def written(self): + """ Returns the set of potentially symbolic indexes that were written in + this array. + + Note that as you could overwrite an index this could have more elements + than total elements in the array. + """ + raise NotImplementedError + + def is_known(self, index): raise NotImplementedError ## following methods are implementes on top of the abstract methods ^ - def cast(self, possible_array): + def in_bounds(self, index:Union[Bitvec, int]) -> Union[Bool, bool]: + """ True if the index points inside the array """ + if self.length is not None: + return (0 <= index) & (index < self.length) + return True + + def __len__(self): + """ Number o defined values. """ + return self.length + + def get(self, index): + """ Should this exist?""" + return self.select(index) + + def cast(self, array) -> "Array": """ Builds an Array from a bytes or bytearray""" - # FIXME This should be related to a constrainSet - arr = ArrayVariable( - index_size=self.index_size, length=len(possible_array), value_size=self.value_size, name="cast{}".format(uuid.uuid1()) + if isinstance(array, Array): + return array + arr = self._constraints.new_array(index_size=self.index_size, length=len(array), default=0, value_size=self.value_size, name="cast{}".format(uuid.uuid1()) ) - for pos, byte in enumerate(possible_array): + for pos, byte in enumerate(array): arr = arr.store(pos, byte) return arr def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: """ Forgiving casting method that will translate compatible values into - a complant BitVec for indexing""" + a compliant BitVec for indexing""" if isinstance(index, int): return BitvecConstant(self.index_size, index) - if index.size != self.index_size: - raise ValueError - + if not isinstance(index, Bitvec) or index.size != self.index_size: + raise ExpressionError(f"Expected Bitvector of size {self.index_size}") return simplify(index) - #return index def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: """ Forgiving casting method that will translate compatible values into - a complant BitVec to ve used as a value""" + a compliant Bitvec to ve used as a value""" if not isinstance(value, (Bitvec, bytes, int)): raise TypeError if isinstance(value, Bitvec): @@ -723,41 +775,39 @@ def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: if isinstance(value, bytes) and len(value) == 1: value = ord(value) if not isinstance(value, int): - print (value, type(value)) value = int(value) return BitvecConstant(self.value_size, value) - def __len__(self): - print (self.index_max) - return self.index_max+1 - - def select(self, index): - return self.get(index) - def write(self, offset, buf): - """ Creates a new Array instance by writing buf at offset """ - arr = self - for i, val in enumerate(buf): - arr = arr.store(offset + i, val) - return arr + """ Builds a new Array instance by writing buf at offset """ + array = self + for i, value in enumerate(buf): + array = array.store(offset + i, value) + return array def read(self, offset, size): + """ A proyection of the current array. """ return ArraySlice(self, offset=offset, size=size) def __getitem__(self, index): + """ __getitem__ allows for pythonic access + A = ArrayVariable(index_size=32, value_size=8) + A[10] := a symbol representing the value under index 10 in array A + A[10:20] := a symbol representing a slice of array A + """ if isinstance(index, slice): start, stop, size = self._fix_slice(index) return self.read(start, size) return self.select(index) def __iter__(self): + """ Iterations """ for i in range(len(self)): yield self[i] def __eq__(self, other): # FIXME taint def compare_buffers(a, b): - print (type(a)) if len(a) != len(b): return BoolConstant(False) cond = BoolConstant(True) @@ -766,7 +816,6 @@ def compare_buffers(a, b): if cond is BoolConstant(False): return BoolConstant(False) return cond - return compare_buffers(self, other) def __ne__(self, other): @@ -789,14 +838,12 @@ def _fix_slice(self, index: slice): assert isinstance(size, BitvecConstant) return start, stop, size.value - def _concatenate(self, array_a, array_b): - # FIXME/Research This should be related to a constrainSet new_arr = ArrayVariable( - index_size = self.index_size, + index_size=self.index_size, length=len(array_a) + len(array_b), - value_size = self.value_size, - name="concatenation{}".format(uuid.uuid1()), + value_size=self.value_size, + name="concatenation", ) for index in range(len(array_a)): @@ -809,9 +856,45 @@ def __add__(self, other): return self._concatenate(self, other) def __radd__(self, other): - print ("RADD!"*9) return self._concatenate(other, self) + def read_BE(self, address, size): + address = self.cast_index(address) + bytes = [] + for offset in range(size): + bytes.append(self.cast_value(self.get(address + offset))) + return BitvecConcat(operands=tuple(bytes)) + + def read_LE(self, address, size): + address = self.cast_index(address) + bytes = [] + for offset in range(size): + bytes.append(self.get(address + offset, self._default)) + return BitvecConcat(operands=reversed(bytes)) + + def write_BE(self, address, value, size): + address = self.cast_index(address) + value = BitvecConstant(size=size * self.value_size, value=0).cast(value) + array = self + for offset in range(size): + array = array.store( + address + offset, + BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), + ) + return array + + def write_LE(self, address, value, size): + address = self.cast_index(address) + value = Bitvec(size * self.value_size).cast(value) + array = self + for offset in reversed(range(size)): + array = array.store( + address + offset, + BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), + ) + return array + + class ArrayConstant(Array, Constant): __xslots__: Tuple[str, ...] = ( "_index_size", @@ -836,13 +919,16 @@ def value_size(self): return self._value_size @property - def index_max(self): + def length(self): return len(self.value) - def get(self, index): + def select(self, index): + """ ArrayConstant get """ index = self.cast_index(index) if isinstance(index, Constant): return BitvecConstant(size=self.value_size, value=self.value[index.value], taint=self.taint) + + # Index being symbolic generates a sybolic result ! result = BitvecConstant(size=self.value_size, value=0, taint=('out_of_bounds')) for i, c in enumerate(self.value): result = BitvecITE(index == i, BitvecConstant(size=self.value_size, value=c), result, taint=self.taint) @@ -861,7 +947,7 @@ class ArrayVariable(Array, Variable): If a default value is provided reading from an unused index will return the default. Otherwise each unused position in the array represents a free bitvector. - If an index_max maximun index is provided accessing over the max is undefined. + If an length maximun index is provided accessing over the max is undefined. Otherwise the array is unbounded. """ @@ -873,8 +959,8 @@ class ArrayVariable(Array, Variable): ) @property - def index_max(self): - return len(self.value) + def length(self): + return self._length def __hash__(self): return object.__hash__(self) @@ -919,21 +1005,20 @@ def value_size(self): def index_max(self): if self._length is None: return None - return int(self._length) - 1 + return self._length - 1 @property def default(self): return self._default def get(self, index, default=None): - """ Gets an element from the Array. - If the element was not previously the default is used. + """ Gets an element from an empty Array. """ + index = self.cast_index(index) if default is None: default = self._default if default is not None: return default - index = self.cast_index(index) return ArraySelect(self, index) def select(self, index): @@ -944,9 +1029,51 @@ def store(self, index, value): value = self.cast_value(value) return ArrayStore(array=self, index=index, value=value) + @property + def written(self): + return set() + + def is_known(self, index): + return False + + @property + def underlying_variable(self): + array = self + while not isinstance(array, ArrayVariable): + array = array.array + return array + +class ArrayOperation(Array, Operation, abstract=True): + """ It's an operation that results in an Array""" + pass + + +class ArrayStore(ArrayOperation): + __xslots__: Tuple[str, ...] = ( + "_written#v", + "_concrete_cache#v", + ) + + @property + def length(self): + return self.array.length + + @property + def concrete_cache(self): + if self._concrete_cache is not None: + return self._concrete_cache + index = self.index + self._concrete_cache = {} + if isinstance(index, Constant): + self._concrete_cache.update(getattr(self.array, "concrete_cache", + ())) # Cache of concrete indexes + self._concrete_cache[index.value] = self.value + return self._concrete_cache + @property def written(self): # Calculate only first time + # This can have repeated and reused written indexes. if self._written is None: written = set() # take out Proxy sleve @@ -968,7 +1095,7 @@ def written(self): return self._written def is_known(self, index): - if isinstance(index, Constant) and index.value in self._concrete_cache: + if isinstance(index, Constant) and index.value in self.concrete_cache: return BoolConstant(True) is_known_index = BoolConstant(False) @@ -980,131 +1107,11 @@ def is_known(self, index): is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) return is_known_index - @property - def underlying_variable(self): - array = self - while not isinstance(array, ArrayVariable): - array = array.array - return array - - def read_BE(self, address, size): - address = self.cast_index(address) - bytes = [] - for offset in range(size): - bytes.append(self.get(address + offset, self._default)) - return BitvecConcat(*bytes) - - def read_LE(self, address, size): - address = self.cast_index(address) - bytes = [] - for offset in range(size): - bytes.append(self.get(address + offset, self._default)) - return BitvecConcat(size * self.value_size, *reversed(bytes)) - - def write_BE(self, address, value, size): - address = self.cast_index(address) - value = BitvecConstant(size=size * self.value_size, value=0).cast(value) - array = self - for offset in range(size): - array = array.store( - address + offset, - BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), - ) - return array - - def write_LE(self, address, value, size): - address = self.cast_index(address) - value = Bitvec(size * self.value_size).cast(value) - array = self - for offset in reversed(range(size)): - array = array.store( - address + offset, - BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), - ) - return array - - def __add__(self, other): - if not isinstance(other, (Array, bytes)): - raise TypeError("can't concat Array to {}".format(type(other))) - if isinstance(other, Array): - if self.index_size != other.index_size or self.value_size != other.value_size: - raise ValueError("Array sizes do not match for concatenation") - - # FIXME This should be related to a constrainSet - new_arr = ArrayVariable( - self.index_size, - self.index_max + len(other), - self.value_size, - default=self._default, - name="concatenation{}".format(uuid.uuid1()), - ) - - for index in range(self.index_max): - new_arr = new_arr.store(index, simplify(self[index])) - for index in range(len(other)): - new_arr = new_arr.store(index + self.index_max, simplify(other[index])) - return new_arr - - def __radd__(self, other): - if not isinstance(other, (Array, bytes)): - raise TypeError("can't concat Array to {}".format(type(other))) - if isinstance(other, Array): - if self.index_size != other.index_size or self.value_size != other.value_size: - raise ValueError("Array sizes do not match for concatenation") - - from .visitors import simplify - - # FIXME This should be related to a constrainSet - new_arr = ArrayVariable( - self.index_size, - self.index_max + len(other), - self.value_size, - default=self._default, - name="concatenation{}".format(uuid.uuid1()), - ) - - for index in range(len(other)): - new_arr = new_arr.store(index, simplify(other[index])) - for index in range(self.index_max): - new_arr = new_arr.store(index + len(other), simplify(self[index])) - return new_arr - - - -class ArrayOperation(Array, Operation, abstract=True): - """ It's an operation that results in an Array""" - pass - - -class ArrayStore(ArrayOperation): - __xslots__: Tuple[str, ...] = ( - "_written", - "_concrete_cache", - ) - def __getstate__(self): - #Overload serialization so _written and _concrete_cache are not actually saved - state = {} - for attr in self.__slots__: - if attr in ('_written', '_concrete_cache'): - continue - state[attr] = getattr(self, attr) - return state - - def __setstate__(self, state): - #Overload serialization so _written and _concrete_cache are not actually saved - for attr in self.__slots__: - if attr in ('_written', '_concrete_cache'): - continue - setattr(self, attr, state[attr]) - def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size self._written = None # Cache of the known indexs - self._concrete_cache = {} - if isinstance(index, Constant): - self._concrete_cache.update(getattr(array, "_concrete_cache", ())) # Cache of concrete indexes - self._concrete_cache[index.value] = value + self._concrete_cache = None super().__init__( operands=(array, index, value), **kwargs, @@ -1145,7 +1152,8 @@ def index(self): def value(self): return self.operands[2] - def get(self, index, default=None): + def select(self, index): + """ Gets an element from the Array. If the element was not previously the default is used. """ @@ -1160,25 +1168,35 @@ def get(self, index, default=None): if isinstance(index, Constant): if self.index_max is not None and index.value > self.index_max: raise IndexError - if index.value in self._concrete_cache: - return self._concrete_cache[index.value] + if index.value in self.concrete_cache: + return self.concrete_cache[index.value] - # take out Proxy sleve - if default is None: - default = self.default + default = self.default if default is None: # No default. Returns normal array select return ArraySelect(self, index) - # build a big ITE expression that would en in the default + # if a default is defined we need to check if the index was previously written + return BitvecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default)) + + # build a big ITE expression array, offset, items = self, 0, [] while not isinstance(array, ArrayVariable): if isinstance(array, ArraySlice): # jump over array slices offset += array.offset else: + assert isinstance(array, ArrayStore) # The index written to underlaying Array are displaced when sliced - items.insert(0, (index == (array.index - offset), array.value)) + cond = index == (array.index - offset) + if isinstance(cond, Constant): + if cond.value == True: + items.insert(0, (cond, array.value)) + break + else: + array = array.array + continue + items.insert(0, (cond, array.value)) array = array.array result = self.cast_value(default) @@ -1209,6 +1227,9 @@ def __init__(self, array: "Array", offset: int, size: int, **kwargs): **kwargs, ) + def __hash__(self): + return object.__hash__(self) + @property def array(self): return self.operands[0] @@ -1218,8 +1239,8 @@ def offset(self): return self.operands[1] @property - def index_max(self): - return self.operands[2].value-1 + def length(self): + return self.operands[2].value @property def index_size(self): @@ -1233,12 +1254,13 @@ def value_size(self): def underlying_variable(self): return self.array.underlying_variable - def get(self, index, default=None): + def select(self, index): index = self.cast_index(index) if isinstance(index, Constant): - if self.index_max is not None and index.value >= len(self): + length = self.length + if length is not None and index.value >= length: raise IndexError - return self.array.get(simplify(index + self.offset), default) + return self.array.select(simplify(index + self.offset)) def store(self, index, value): return ArraySlice( @@ -1247,22 +1269,27 @@ def store(self, index, value): size=len(self), ) + @property + def default(self): + return self.array.default -class ArrayProxy: +class MutableArray: """ Arrayproxy is a layer on top of an array that provides mutability and some simple optimizations for concrete indexes. It is not hasheable. Think: - bytearray <-> ArrayProxy ::: not hasheable, mutable + bytearray <-> MutableArray ::: not hasheable, mutable bytes <-> Array (ArraySlice, ArrayVariable, ArrayStore) ::: hasheable, notmutable """ def __init__(self, array: Array): - assert isinstance(array, Array) - self._array = array + if isinstance(array, MutableArray): + array = array._array + + self._array: Array = array @property def underlying_variable(self): @@ -1285,8 +1312,8 @@ def index_size(self): return self._array.index_size @property - def index_max(self): - return self._array.index_max + def length(self): + return self._array.length @property def value_size(self): @@ -1296,6 +1323,10 @@ def value_size(self): def taint(self): return self._array.taint + @property + def default(self): + return self._array.default + def __len__(self): return len(self._array) @@ -1314,7 +1345,7 @@ def written(self): def __getitem__(self, index): result = self._array[index] if isinstance(index, slice): - return ArrayProxy(result) + return MutableArray(result) return result def __setitem__(self, index, value): @@ -1326,13 +1357,9 @@ def __setitem__(self, index, value): else: self._array = self._array.store(index, value) assert self._array is not None + return self - def get(self, index, default=None): - x = self._array.get(index, default) - print ("A"*199, type(self._array), "X:", x) - return x - def write_BE(self, address, value, size): self._array = self._array.write_BE(address, value, size) assert self._array is not None @@ -1348,7 +1375,7 @@ def write(self, offset, buf): return self def read(self, offset, size): - return ArrayProxy(self._array[offset : offset + size]) + return MutableArray(self._array[offset: offset + size]) def __eq__(self, other): return self.array == other @@ -1357,14 +1384,14 @@ def __ne__(self, other): return BoolNot(self == other) def __add__(self, other): - if isinstance(other, ArrayProxy): + if isinstance(other, MutableArray): other = other.array - return ArrayProxy(self.array + other) + return MutableArray(self.array + other) def __radd__(self, other): - if isinstance(other, ArrayProxy): + if isinstance(other, MutableArray): other = other.array - return ArrayProxy(other+self.array) + return MutableArray(other + self.array) class ArraySelect(BitvecOperation): __xslots__ = BitvecOperation.__xslots__ @@ -1468,7 +1495,7 @@ def issymbolic(value) -> bool: :return: whether `value` is symbolic :rtype: bool """ - return isinstance(value, (Expression, ArrayProxy)) + return isinstance(value, (Expression, MutableArray)) def istainted(arg, taint=None): diff --git a/manticore/core/smtlib/operators.py b/manticore/core/smtlib/operators.py index ca123f650..f1bd648cd 100644 --- a/manticore/core/smtlib/operators.py +++ b/manticore/core/smtlib/operators.py @@ -191,7 +191,6 @@ def ITE(cond, true_value, false_value): def ITEBV(size, cond, true_value, false_value): - print ("ITE", cond) if isinstance(cond, Bitvec): cond = cond.Bool() if isinstance(cond, int): diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 55e08a66c..72039b3b6 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -63,7 +63,7 @@ class SolverType(config.ConfigEnum): consts.add( "solver", - default=SolverType.z3, + default=SolverType.yices, description="Choose default smtlib2 solver (z3, yices, cvc4, auto)", ) @@ -214,8 +214,7 @@ def __readline_and_count(self): assert self._proc.stdout buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error - # Error messages may contain an unbalanced parenthesis situation - print (">",buf) + #print (">",buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -228,7 +227,7 @@ def send(self, cmd: str) -> None: :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ - print ("<",cmd) + #print ("<",cmd) if self._debug: logger.debug(">%s", cmd) self._proc.stdout.flush() # type: ignore @@ -398,6 +397,7 @@ def _pop(self): """Recall the last pushed constraint store and state.""" self._smtlib.send("(pop 1)") + @lru_cache(maxsize=32) def get_model(self, constraints: ConstraintSet): self._reset(constraints.to_string()) self._smtlib.send("(check-sat)") @@ -405,18 +405,36 @@ def get_model(self, constraints: ConstraintSet): model = {} for variable in constraints.variables: - print (variable) + value = None if isinstance(variable, Bool): value = self.__getvalue_bool(variable.name) elif isinstance(variable, Bitvec): value = self.__getvalue_bv(variable.name) - else: + elif isinstance(variable, Array): try: - #Only works if we know the max index of the arrray - value = [] - for i in range(len(variable)): - value.append(self.__getvalue_bv(variable[i])) - except: + if variable.length is not None: + value = [] + for i in range(len(variable)): + variable_i = variable[i] + if issymbolic(variable_i): + value.append(self.__getvalue_bv(translate_to_smtlib(variable_i))) + else: + value.append(variable_i) + value = bytes(value) + else: + #Only works if we know the max index of the arrray + used_indexes = map(self.__getvalue_bv, variable.written) + valued = {} + for i in used_indexes: + valued[i] = self.__getvalue_bv(variable[i]) + class A: + def __init__(self, d, default): + self._d = d + self._default = default + def __getitem__(self, index): + return self._d.get(index, self._default) + value = A(valued,variable.default) + except Exception as e: value = None #We failed to get the model from the solver model[variable.name] = value @@ -543,8 +561,8 @@ def get_all_values( var = temp_cs.new_bitvec(expression.size) elif isinstance(expression, Array): var = temp_cs.new_array( - index_max=expression.index_max, - value_bits=expression.value_bits, + length=expression.length, + value_size=expression.value_size, taint=expression.taint, ).array else: @@ -617,14 +635,25 @@ def get_value(self, constraints: ConstraintSet, *expressions): Ask the solver for one possible result of given expressions using given set of constraints. """ - - + self._cache = getattr(self, '_cache', {}) + model = self.get_model(constraints) ####################33 values = [] start = time.time() - with constraints.related_to(*expressions) as temp_cs: + #with constraints.related_to(*expressions) as temp_cs: + with constraints as temp_cs: for expression in expressions: + bucket = self._cache.setdefault(hash(constraints), {}) + cached_result = bucket.get(hash(expression)) + if cached_result is not None: + values.append(cached_result) + continue + elif isinstance(expression, Variable): + if model[expression.name] is not None: + values.append(model[expression.name]) + continue + if not issymbolic(expression): values.append(expression) continue @@ -636,19 +665,27 @@ def get_value(self, constraints: ConstraintSet, *expressions): elif isinstance(expression, Array): var = [] result = [] - for i in range(expression.index_max): - subvar = temp_cs.new_bitvec(expression.value_bits) - var.append(expression[i]) #subvar) - temp_cs.add(subvar == simplify(expression[i])) + for i in range(len(expression)): + expression_i = expression[i] + if issymbolic(expression_i): + subvar = temp_cs.new_bitvec(expression.value_size) + temp_cs.add(subvar == simplify(expression[i])) + var.append(subvar) + else: + var.append(expression_i) self._reset(temp_cs.to_string()) if not self._is_sat(): raise SolverError( "Solver could not find a value for expression under current constraint set" ) + for i in range(expression.length): + if issymbolic(var[i]): + result.append(self.__getvalue_bv(var[i].name)) + else: + result.append(var[i]) - for i in range(expression.index_max): - result.append(self.__getvalue_bv(var[i].name)) values.append(bytes(result)) + bucket[hash(expression)] = values[-1] if time.time() - start > consts.timeout: raise SolverError("Timeout") continue @@ -666,9 +703,12 @@ def get_value(self, constraints: ConstraintSet, *expressions): values.append(self.__getvalue_bool(var.name)) if isinstance(expression, Bitvec): values.append(self.__getvalue_bv(var.name)) + bucket[hash(expression)] = values[-1] if time.time() - start > consts.timeout: raise SolverError("Timeout") + + if len(expressions) == 1: return values[0] else: @@ -792,4 +832,5 @@ def instance(cls): cls.choice = consts.solver SelectedSolver = {"cvc4": CVC4Solver, "yices": YicesSolver, "z3": Z3Solver}[cls.choice.name] + return YicesSolver.instance() return SelectedSolver.instance() diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 3b38b0f74..df1a3ba3a 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -123,7 +123,7 @@ def _method(self, expression, *operands): def _changed(self, expression:Expression, operands): return any(x is not y for x, y in zip(expression.operands, operands)) - def _rebuild(self, expression:Expression, operands): + def _rebuild(self, expression:Operation, operands): """ Default operation used when no visiting method was successful for this expression. If he operands have changed this reubild the curren expression with the new operands. @@ -784,7 +784,7 @@ def visit_Expression(self, expression, *operands): arithmetic_simplifier_cache = CacheDict(max_size=250000, flush_perc=25) -#@lru_cache(maxsize=128, typed=True) +@lru_cache(maxsize=128, typed=True) def arithmetic_simplify(expression): if not isinstance(expression, Expression): return expression @@ -799,7 +799,7 @@ def to_constant(expression): Iff the expression can be simplified to a Constant get the actual concrete value. This discards/ignore any taint """ - if isinstance(expression, ArrayProxy): + if isinstance(expression, MutableArray): expression = expression.array value = simplify(expression) if isinstance(value, Expression) and value.taint: @@ -820,10 +820,9 @@ def to_constant(expression): return value -#@lru_cache(maxsize=128, typed=True) +@lru_cache(maxsize=128, typed=True) def simplify(expression): - expression = arithmetic_simplify(expression) - return expression + return arithmetic_simplify(expression) class TranslatorSmtlib(Translator): @@ -961,7 +960,7 @@ def smtlib(self): def translate_to_smtlib(expression, **kwargs): - if isinstance(expression, ArrayProxy): + if isinstance(expression, MutableArray): expression = expression.array translator = TranslatorSmtlib(**kwargs) translator.visit(expression) @@ -989,7 +988,7 @@ def visit_Variable(self, expression): def replace(expression, bindings): if not bindings: return expression - if isinstance(expression, ArrayProxy): + if isinstance(expression, MutableArray): expression = expression.array visitor = Replace(bindings) @@ -1023,7 +1022,7 @@ def simplify_array_select(array_exp): def get_variables(expression): - if isinstance(expression, ArrayProxy): + if isinstance(expression, MutableArray): expression = expression.array visitor = GetDeclarations() diff --git a/manticore/core/state.py b/manticore/core/state.py index d0e99de0c..3b1e9d532 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,7 +1,7 @@ import copy import logging -from .smtlib import solver, Bool, issymbolic, BitvecConstant +from .smtlib import solver, Bool, issymbolic, BitvecConstant, MutableArray from ..utils.event import Eventful from ..utils.helpers import PickleSerializer @@ -224,8 +224,8 @@ def new_symbolic_buffer(self, nbytes, **options): taint = options.get("taint", frozenset()) expr = self._constraints.new_array( name=label, - index_max=nbytes, - value_bits=8, + length=nbytes, + value_size=8, taint=taint, avoid_collisions=avoid_collisions, ) @@ -325,6 +325,8 @@ def _solver(self): return SelectedSolver.instance() # solver def migrate_expression(self, expression): + if isinstance(expression, MutableArray): + expression=expression.array if not issymbolic(expression): return expression migration_map = self.context.get("migration_map") @@ -373,6 +375,7 @@ def solve_one_n(self, *exprs, constrain=False): values.append(expr) else: expr = self.migrate_expression(expr) + print ("SOLVEONE", expr) value = self._solver.get_value(self._constraints, expr) if constrain: self.constrain(expr == value) @@ -380,6 +383,8 @@ def solve_one_n(self, *exprs, constrain=False): if isinstance(value, bytearray): value = bytes(value) values.append(value) + assert any(issymbolic for x in values) + print (exprs, values) return values def solve_n(self, expr, nsolves): @@ -479,7 +484,7 @@ def symbolicate_buffer( if wildcard in data: size = len(data) symb = self._constraints.new_array( - name=label, index_max=size, taint=taint, avoid_collisions=True + name=label, length=size, taint=taint, avoid_collisions=True ) self._input_symbols.append(symb) diff --git a/manticore/ethereum/abi.py b/manticore/ethereum/abi.py index 35b5d5238..381f26782 100644 --- a/manticore/ethereum/abi.py +++ b/manticore/ethereum/abi.py @@ -11,7 +11,7 @@ Operators, Bitvec, ArrayVariable, - ArrayProxy, + MutableArray, to_constant, issymbolic, ) @@ -285,7 +285,7 @@ def _serialize_uint(value, size=32, padding=0): assert isinstance(value, Bitvec) # FIXME This temporary array variable should be obtained from a specific constraint store buffer = ArrayVariable( - index_bits=256, length=32, value_bits=8, name="temp{}".format(uuid.uuid1()) + index_size=256, length=32, value_size=8, name="temp{}".format(uuid.uuid1()) ) if value.size <= size * 8: value = Operators.ZEXTEND(value, size * 8) @@ -316,11 +316,11 @@ def _serialize_int(value: typing.Union[int, Bitvec], size=32, padding=0): if issymbolic(value): # Help mypy out. Can remove this by teaching it how issymbolic works assert isinstance(value, Bitvec) - buf = ArrayVariable( - index_bits=256, length=32, value_bits=8, name="temp{}".format(uuid.uuid1()) + buf = value.constraints.new_array( + index_size=256, length=32, value_size=8, name="temp{}".format(uuid.uuid1()) ) value = Operators.SEXTEND(value, value.size, size * 8) - return ArrayProxy(buf.write_BE(padding, value, size)) + return MutableArray(buf.write_BE(padding, value, size)) else: buf_arr = bytearray() for _ in range(padding): diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 2d900e31a..c80667acb 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -18,7 +18,7 @@ from ..core.smtlib import ( ConstraintSet, Array, - ArrayProxy, + MutableArray, Bitvec, Operators, BoolConstant, @@ -135,7 +135,7 @@ class ManticoreEVM(ManticoreBase): m.finalize() """ - def make_symbolic_buffer(self, size, name=None, avoid_collisions=False): + def make_symbolic_buffer(self, size, name=None, avoid_collisions=False, default=None): """ Creates a symbolic buffer of size bytes to be used in transactions. You can operate on it normally and add constraints to manticore.constraints via manticore.constrain(constraint_expression) @@ -154,12 +154,13 @@ def make_symbolic_buffer(self, size, name=None, avoid_collisions=False): avoid_collisions = True return self.constraints.new_array( - index_bits=256, + index_size=256, name=name, - index_max=size, - value_bits=8, + length=size, + value_size=8, taint=frozenset(), avoid_collisions=avoid_collisions, + default=default ) def make_symbolic_value(self, nbits=256, name=None): @@ -595,11 +596,11 @@ def solidity_create_contract( for state in self.ready_states: world = state.platform - - if not SelectedSolver.instance().can_be_true( - self.constraints, - Operators.UGE(world.get_balance(owner.address), balance), - ): + if not state.can_be_true(Operators.UGE(world.get_balance(owner.address), balance)): + #if not SelectedSolver.instance().can_be_true( + # self.constraints, + # Operators.UGE(world.get_balance(owner.address), balance), + #): raise EthereumError( f"Can't create solidity contract with balance ({balance}) " f"because the owner account ({owner}) has insufficient balance." @@ -760,6 +761,8 @@ def transaction(self, caller, address, value, data, gas=None, price=1): :param price: gas unit price :raises NoAliveStates: if there are no alive states to execute """ + if isinstance(data, MutableArray): + data = data.array self._transaction( "CALL", caller, value=value, address=address, data=data, gas=gas, price=price ) @@ -852,11 +855,12 @@ def _migrate_tx_expressions(self, state, caller, address, value, data, gas, pric value = state.migrate_expression(value) if issymbolic(data): - if isinstance(data, ArrayProxy): # FIXME is this necessary here? + if isinstance(data, MutableArray): # FIXME is this necessary here? data = data.array + print ("data:"*10, data) data = state.migrate_expression(data) - if isinstance(data, Array): - data = ArrayProxy(data) + #if isinstance(data, Array): + # data = MutableArray(data) if issymbolic(gas): gas = state.migrate_expression(gas) @@ -1083,7 +1087,7 @@ def multi_tx_analysis( logger.info("Starting symbolic transaction: %d", tx_no) # run_symbolic_tx - symbolic_data = self.make_symbolic_buffer(320) + symbolic_data = self.make_symbolic_buffer(320, default=0) if tx_send_ether: value = self.make_symbolic_value() else: diff --git a/manticore/native/memory.py b/manticore/native/memory.py index 89032ff4b..1eb34a36a 100644 --- a/manticore/native/memory.py +++ b/manticore/native/memory.py @@ -345,7 +345,7 @@ def __init__( if backing_array is not None: self._array = backing_array else: - self._array = expression.ArrayProxy( + self._array = expression.MutableArray( expression.ArrayVariable(index_bits, length=size, value_bits=8, name=name) ) @@ -380,10 +380,10 @@ def split(self, address: int): left_size, right_size = address - self.start, self.end - address left_name, right_name = ["{}_{:d}".format(self._array.name, i) for i in range(2)] - head_arr = expression.ArrayProxy( + head_arr = expression.MutableArray( expression.ArrayVariable(index_bits, left_size, value_bits, name=left_name) ) - tail_arr = expression.ArrayProxy( + tail_arr = expression.MutableArray( expression.ArrayVariable(index_bits, right_size, value_bits, name=right_name) ) @@ -1364,7 +1364,8 @@ class LazySMemory(SMemory): def __init__(self, constraints, *args, **kwargs): super(LazySMemory, self).__init__(constraints, *args, **kwargs) - self.backing_array = constraints.new_array(index_bits=self.memory_bit_size) + self.backing_array = constraints.new_array( + index_size=self.memory_bit_size) self.backed_by_symbolic_store = set() def __reduce__(self): diff --git a/manticore/native/models.py b/manticore/native/models.py index 1d00697a6..80df8bfcb 100644 --- a/manticore/native/models.py +++ b/manticore/native/models.py @@ -77,7 +77,7 @@ def can_be_NULL(byte, constrs) -> bool: return byte == 0 -def _find_zero(cpu, constrs, ptr: Union[int, BitVec]) -> int: +def _find_zero(cpu, constrs, ptr: Union[int, Bitvec]) -> int: """ Helper for finding the closest NULL or, effectively NULL byte from a starting address. @@ -153,7 +153,7 @@ def strcmp(state: State, s1: Union[int, Bitvec], s2: Union[int, Bitvec]): return ret -def strlen_exact(state: State, s: Union[int, BitVec]) -> Union[int, BitVec]: +def strlen_exact(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: """ strlen symbolic model @@ -192,7 +192,7 @@ def strlen_exact(state: State, s: Union[int, BitVec]) -> Union[int, BitVec]: return offset -def strlen_approx(state: State, s: Union[int, BitVec]) -> Union[int, BitVec]: +def strlen_approx(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: """ strlen symbolic model @@ -272,8 +272,8 @@ def strcpy(state: State, dst: Union[int, Bitvec], src: Union[int, Bitvec]) -> Un def strncpy( - state: State, dst: Union[int, BitVec], src: Union[int, BitVec], n: Union[int, BitVec] -) -> Union[int, BitVec]: + state: State, dst: Union[int, Bitvec], src: Union[int, Bitvec], n: Union[int, Bitvec] +) -> Union[int, Bitvec]: """ strncpy symbolic model diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 475c21e64..2f3d5c2e4 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -13,7 +13,7 @@ SelectedSolver, Bitvec, Array, - ArrayProxy, + MutableArray, Operators, Constant, ArrayVariable, @@ -181,13 +181,7 @@ def concretize(self, state, constrain=False): :param state: a manticore state :param bool constrain: If True, constrain expr to concretized value """ - conc_caller = state.solve_one(self.caller, constrain=constrain) - conc_address = state.solve_one(self.address, constrain=constrain) - conc_value = state.solve_one(self.value, constrain=constrain) - conc_gas = state.solve_one(self.gas, constrain=constrain) - conc_data = state.solve_one(self.data, constrain=constrain) - conc_return_data = state.solve_one(self.return_data, constrain=constrain) - conc_used_gas = state.solve_one(self.used_gas, constrain=constrain) + conc_caller, conc_address, conc_value, conc_gas, conc_data, conc_return_data,conc_used_gas = state.solve_one_n(self.caller,self.address,self.value,self.gas,self.data,self.return_data,self.used_gas,constrain=constrain) return Transaction( self.sort, conc_address, @@ -489,8 +483,10 @@ def __init__(self, result, data=None): raise EVMException("Invalid end transaction result") if result is None and data is not None: raise EVMException("Invalid end transaction result") + if isinstance(data, MutableArray): + data = data.array if not isinstance(data, (type(None), Array, bytes)): - raise EVMException("Invalid end transaction data type") + raise EVMException("Invalid end transaction data type") self.result = result self.data = data @@ -722,28 +718,26 @@ def __init__( if data is not None and not issymbolic(data): data_size = len(data) data_symbolic = constraints.new_array( - index_bits=256, - value_bits=8, - index_max=data_size, + index_size=256, + value_size=8, + length=data_size, name=f"DATA_{address:x}", avoid_collisions=True, default=0, ) - data_symbolic[0:data_size] = data - data = data_symbolic + data = data_symbolic.write(0, data) if bytecode is not None and not issymbolic(bytecode): bytecode_size = len(bytecode) bytecode_symbolic = constraints.new_array( - index_bits=256, - value_bits=8, - index_max=bytecode_size, + index_size=256, + value_size=8, + length=bytecode_size, name=f"BYTECODE_{address:x}", avoid_collisions=True, default=0, ) - bytecode_symbolic[0:bytecode_size] = bytecode - bytecode = bytecode_symbolic + data = bytecode_symbolic.write(0, bytecode) # TODO: Handle the case in which bytecode is symbolic (This happens at # CREATE instructions that has the arguments appended to the bytecode) @@ -777,13 +771,13 @@ def extend_with_zeroes(b): # raise EVMException("Need code") self._constraints = constraints # Uninitialized values in memory are 0 by spec - self.memory = constraints.new_array( - index_bits=256, - value_bits=8, + self.memory = MutableArray(constraints.new_array( + index_size=256, + value_size=8, name=f"EMPTY_MEMORY_{address:x}", avoid_collisions=True, default=0, - ) + )) self.address = address self.caller = ( caller # address of the account that is directly responsible for this execution @@ -855,7 +849,7 @@ def constraints(self): @constraints.setter def constraints(self, constraints): self._constraints = constraints - self.memory.constraints = constraints + #self.memory.constraints = constraints @property def gas(self): @@ -987,7 +981,11 @@ def PC(self): def _getcode(self, pc): bytecode = self.bytecode for pc_i in range(pc, len(bytecode)): - yield simplify(bytecode[pc_i]).value + c = bytecode[pc_i] + if issymbolic(c): + yield simplify(c).value + else: + yield c while True: yield 0 # STOP opcode @@ -1166,14 +1164,11 @@ def _push_results(self, instruction, result): def _calculate_gas(self, *arguments): start= time.time() - try: - current = self.instruction - implementation = getattr(self, f"{current.semantics}_gas", None) - if implementation is None: - return current.fee - return current.fee + implementation(*arguments) - finally: - print (f"calculate gas for {current} took {time.time()-start:0.4f}") + current = self.instruction + implementation = getattr(self, f"{current.semantics}_gas", None) + if implementation is None: + return current.fee + return current.fee + implementation(*arguments) def _handler(self, *arguments): @@ -1305,20 +1300,10 @@ def setstate(state, value): raise Concretize("Symbolic PC", expression=expression, setstate=setstate, policy="ALL") try: - import time - start = time.time() - i = self.instruction - try: - #print (self) - self._check_jmpdest() - last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() - result = self._handler(*arguments) - self._advance(result) - except: - print ("Excaption!") - raise - finally: - print (f"Elapsed {i}: {time.time()-start:0.4f}") + self._check_jmpdest() + last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() + result = self._handler(*arguments) + self._advance(result) except ConcretizeGas as ex: def setstate(state, value): @@ -1408,8 +1393,7 @@ def read_buffer(self, offset, size): if size == 0: return b"" self._allocate(offset, size) - data = self.memory[offset : offset + size] - return ArrayProxy(data) + return self.memory[offset : offset + size] def write_buffer(self, offset, data): self._allocate(offset, len(data)) @@ -1793,12 +1777,12 @@ def CODECOPY_gas(self, mem_offset, code_offset, size): @concretized_args(code_offset="SAMPLED", size="SAMPLED") def CODECOPY(self, mem_offset, code_offset, size): """Copy code running in current environment to memory""" - import pdb; pdb.set_trace() self._allocate(mem_offset, size) GCOPY = 3 # cost to copy one 32 byte word copyfee = self.safe_mul(GCOPY, Operators.UDIV(self.safe_add(size, 31), 32)) self._consume(copyfee) + if issymbolic(size): max_size = SelectedSolver.instance().max(self.constraints, size) else: @@ -1806,7 +1790,6 @@ def CODECOPY(self, mem_offset, code_offset, size): for i in range(max_size): if issymbolic(i < size): - print("SYMBOLICCCCCCCCCCCCCCCCCC"*100) default = Operators.ITEBV( 8, i < size, 0, self._load(mem_offset + i, 1) ) # Fixme. unnecessary memory read @@ -1980,11 +1963,7 @@ def SSTORE_gas(self, offset, value): self.fail_if(Operators.ULT(self.gas, SSSTORESENTRYGAS)) # Get the storage from the snapshot took before this call - try: - original_value = self.world._callstack[-1][-2].get(offset, 0) - except IndexError: - original_value = 0 - + original_value = self.world._callstack[-1][-2].select(offset) current_value = self.world.get_storage_data(storage_address, offset) def ITE(*args): @@ -2161,8 +2140,8 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): data = self.read_buffer(offset, size) keccak_init = self.world.symbolic_function(globalsha3, data) - caller = ArrayProxy(msg.caller).read_BE(0,20) - salt = ArrayProxy(salt).read_BE(0, 32) + caller = MutableArray(msg.caller).read_BE(0, 20) + salt = MutableArray(salt).read_BE(0, 32) address = self.world.symbolic_function(b"\xff" + caller + salt + keccak_init) & ((1<<0x20)-1) self.world.start_transaction( @@ -2454,7 +2433,7 @@ def __init__(self, constraints, fork=DEFAULT_FORK, **kwargs): self._world_state = {} self._constraints = constraints self._callstack: List[ - Tuple[Transaction, List[EVMLog], Set[int], Union[bytearray, ArrayProxy], EVM] + Tuple[Transaction, List[EVMLog], Set[int], Union[bytearray, MutableArray], EVM] ] = [] self._deleted_accounts: Set[int] = set() self._logs: List[EVMLog] = list() @@ -2583,21 +2562,19 @@ def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value else: tx_fee = GTRANSACTION # Simple transaction fee + ## This popcnt like thing is expensive when the bytecode or + # data has symbolic content + zerocount = 0 nonzerocount = 0 - if isinstance(bytecode_or_data, (Array, ArrayProxy)): - # if nothing was written we can assume all elements are default to zero - if len(bytecode_or_data.written) == 0: - zerocount = len(bytecode_or_data) - else: - for index in range(len(bytecode_or_data)): - try: - c = bytecode_or_data.get(index, 0) - except AttributeError: - c = bytecode_or_data[index] + for index in range(len(bytecode_or_data)): + try: + c = bytecode_or_data.get(index) + except AttributeError: + c = bytecode_or_data[index] - zerocount += Operators.ITEBV(256, c == 0, 1, 0) - nonzerocount += Operators.ITEBV(256, c == 0, 0, 1) + zerocount += Operators.ITEBV(512, c == 0, 1, 0) + nonzerocount += Operators.ITEBV(512, c == 0, 0, 1) tx_fee += zerocount * GTXDATAZERO tx_fee += nonzerocount * GTXDATANONZERO @@ -2870,11 +2847,8 @@ def get_storage_data(self, storage_address, offset): :return: the value :rtype: int or Bitvec """ - start = time.time() - try: - return self._world_state[storage_address]["storage"].get(offset, 0) - finally: - print (f"Ellapseeeeeeeeeeeeeeeeeeeeeeeeeeeeeed in get_storage_data: {time.time()-start:04f}") + return self._world_state[storage_address]["storage"].select(offset) + def set_storage_data(self, storage_address, offset, value): """ Writes a value to a storage slot in specified account @@ -2923,12 +2897,14 @@ def get_storage(self, address): :param address: account address :return: account storage - :rtype: bytearray or ArrayProxy + :rtype: bytearray or MutableArray """ return self._world_state[address]["storage"] def _set_storage(self, address, storage): """Private auxiliary function to replace the storage""" + if not isinstance(storage, MutableArray) or storage.default != 0: + raise TypeError self._world_state[address]["storage"] = storage def get_nonce(self, address): @@ -2946,7 +2922,7 @@ def increase_nonce(self, address): def set_balance(self, address, value): if isinstance(value, Bitvec): value = Operators.ZEXTEND(value, 512) - self._world_state[int(address)]["balance"] = value + self._world_state[int(address)]["balance"] = simplify(value) def get_balance(self, address): if address not in self._world_state: @@ -3156,17 +3132,17 @@ def create_account(self, address=None, balance=0, code=None, storage=None, nonce if storage is None: # Uninitialized values in a storage are 0 by spec - storage = self.constraints.new_array( - index_bits=256, - value_bits=256, + storage = MutableArray(self.constraints.new_array( + index_size=256, + value_size=256, name=f"STORAGE_{address:x}", avoid_collisions=True, default=0, - ) + )) else: - if isinstance(storage, ArrayProxy): + if isinstance(storage, MutableArray): if storage.index_bits != 256 or storage.value_bits != 256: - raise TypeError("An ArrayProxy 256bits -> 256bits is needed") + raise TypeError("An MutableArray 256bits -> 256bits is needed") else: if any((k < 0 or k >= 1 << 256 for k, v in storage.items())): raise TypeError( @@ -3237,6 +3213,10 @@ def start_transaction( """ assert self._pending_transaction is None, "Already started tx" assert caller is not None + if issymbolic(data ): + assert data.index_max is not None + assert data.value_size == 8 + self._pending_transaction = PendingTransaction( sort, address, price, data, caller, value, gas, None ) diff --git a/manticore/platforms/linux.py b/manticore/platforms/linux.py index bc2e88f01..7978421fb 100644 --- a/manticore/platforms/linux.py +++ b/manticore/platforms/linux.py @@ -26,7 +26,7 @@ from . import linux_syscalls from .linux_syscall_stubs import SyscallStubs from ..core.state import TerminateState -from ..core.smtlib import ConstraintSet, Operators, Expression, issymbolic, ArrayProxy +from ..core.smtlib import ConstraintSet, Operators, Expression, issymbolic, MutableArray from ..core.smtlib.solver import SelectedSolver from ..exceptions import SolverError from ..native.cpu.abstractcpu import Cpu, Syscall, ConcretizeArgument, Interruption @@ -422,7 +422,7 @@ def __init__( # build the constraints array size = len(data) - self.array = constraints.new_array(name=self.name, index_max=size) + self.array = constraints.new_array(name=self.name, length=size) symbols_cnt = 0 for i in range(size): @@ -690,7 +690,7 @@ def __init__( self.symb_name = name self.max_recv_symbolic = max_recv_symbolic # 0 for unlimited. Unlimited is not tested # Keep track of the symbolic inputs we create - self.inputs_recvd: List[ArrayProxy] = [] + self.inputs_recvd: List[MutableArray] = [] self.recv_pos = 0 def __getstate__(self): @@ -719,7 +719,7 @@ def _next_symb_name(self) -> str: """ return f"{self.symb_name}-{len(self.inputs_recvd)}" - def receive(self, size: int) -> Union[ArrayProxy, List[bytes]]: + def receive(self, size: int) -> Union[MutableArray, List[bytes]]: """ Return a symbolic array of either `size` or rest of remaining symbolic bytes :param size: Size of receive @@ -735,7 +735,7 @@ def receive(self, size: int) -> Union[ArrayProxy, List[bytes]]: if rx_bytes == 0: # If no symbolic bytes left, return empty list return [] - ret = self._constraints.new_array(name=self._next_symb_name(), index_max=rx_bytes) + ret = self._constraints.new_array(name=self._next_symb_name(), length=rx_bytes) self.recv_pos += rx_bytes self.inputs_recvd.append(ret) return ret diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index ee92eec1c..826032590 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -16,7 +16,7 @@ from manticore.core.plugin import Plugin from manticore.core.smtlib import ConstraintSet, operators from manticore.core.smtlib import Z3Solver -from manticore.core.smtlib.expression import Bitvec +from manticore.core.smtlib.expression import BitvecVariable from manticore.core.smtlib.visitors import to_constant from manticore.core.state import TerminateState from manticore.ethereum import ( @@ -377,7 +377,7 @@ def test_serialize_fixed_bytes_too_big(self): # test serializing symbolic buffer with bytesM def test_serialize_bytesM_symbolic(self): cs = ConstraintSet() - buf = cs.new_array(index_max=17) + buf = cs.new_array(length=17) ret = ABI.serialize("bytes32", buf) self.assertEqual(solver.minmax(cs, ret[0]), (0, 255)) self.assertEqual(solver.minmax(cs, ret[17]), (0, 0)) @@ -385,7 +385,7 @@ def test_serialize_bytesM_symbolic(self): # test serializing symbolic buffer with bytes def test_serialize_bytes_symbolic(self): cs = ConstraintSet() - buf = cs.new_array(index_max=17) + buf = cs.new_array(length=17) ret = ABI.serialize("bytes", buf) # does the offset field look right? @@ -410,7 +410,7 @@ def _make(self): price = 0 value = 10000 bytecode = b"\x05" - data = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + data = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" gas = 1000000 new_vm = evm.EVM(constraints, address, data, caller, value, bytecode, gas=gas, world=world) @@ -815,9 +815,11 @@ def test_function_name_with_signature(self): self.mevm.make_symbolic_value(), signature="(uint256,uint256)", ) + z = None for st in self.mevm.all_states: z = st.solve_one(st.platform.transactions[1].return_data) break + self.assertIsNot(z, None) self.assertEqual(ABI.deserialize("(uint256)", z)[0], 2) def test_migrate_integration(self): @@ -1380,7 +1382,7 @@ def will_evm_execute_instruction_callback(self, state, i, *args, **kwargs): class EthHelpersTest(unittest.TestCase): def setUp(self): - self.bv = Bitvec(256) + self.bv = BitvecVariable(size=256, name="A") def test_concretizer(self): policy = "SOME_NONSTANDARD_POLICY" @@ -1723,6 +1725,8 @@ def test_delegatecall_env(self): # check balances self.assertEqual(world.get_balance(0x111111111111111111111111111111111111111), 0) self.assertEqual(world.get_balance(0x222222222222222222222222222222222222222), 10) + from manticore.core.smtlib.visitors import translate_to_smtlib, simplify + print ( translate_to_smtlib(simplify(world.get_balance(0x333333333333333333333333333333333333333)))) self.assertEqual( world.get_balance(0x333333333333333333333333333333333333333), 100000000000000000000000 - 10, diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 490bc7eaa..d4e373308 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -15,7 +15,7 @@ replace, BitvecConstant, ) -from manticore.core.smtlib.solver import Z3Solver, YicesSolver, CVC4Solver +from manticore.core.smtlib.solver import Z3Solver, YicesSolver, CVC4Solver, SelectedSolver from manticore.core.smtlib.expression import * from manticore.utils.helpers import pickle_dumps from manticore import config @@ -98,14 +98,64 @@ def test_ConstantArrayBitvec(self): self.assertRaises(IndexError, c.get, 5) def test_ConstantArrayBitvec(self): - c = ArrayProxy(ArrayVariable(index_size=32, value_size=8, length=5, name="ARR")) + c = MutableArray(ArrayVariable(index_size=32, value_size=8, length=5, name="ARR")) c[1] = 10 c[2] = 20 c[3] = 30 self.assertEqual(c[1], 10) self.assertEqual(c[2], 20) self.assertEqual(c[3], 30) - self.assertRaises(IndexError, c.get, 25) + + + def test_ArrayDefault3(self): + c = MutableArray(ArrayVariable(index_size=32, value_size=8, length=5, default=0, name="ARR")) + self.assertEqual(c[1], 0) + self.assertEqual(c[2], 0) + self.assertEqual(c[3], 0) + + c[1] = 10 + c[3] = 30 + self.assertEqual(c[1], 10) + self.assertEqual(c[2], 0) + self.assertEqual(c[3], 30) + + def test_ArrayDefault4(self): + cs = ConstraintSet() + a = MutableArray(cs.new_array(index_size=32, value_size=8, length=4, default=0, name="ARR")) + i = cs.new_bitvec(size = a.index_size) + SelectedSolver.instance().must_be_true(cs, 0 == a.default) + SelectedSolver.instance().must_be_true(cs, a[i] == a.default) + cs.add(i==2) + SelectedSolver.instance().must_be_true(cs, 0 == a.default) + SelectedSolver.instance().must_be_true(cs, a[i] == a.default) + + b = a[:] + i = cs.new_bitvec(size = a.index_size) + SelectedSolver.instance().must_be_true(cs, 0 == b.default) + SelectedSolver.instance().must_be_true(cs, b[i] == b.default) + + a[1] = 10 + a[2] = 20 + a[3] = 30 + # a := 0 10 20 30 0 0 x x x x (x undefined) + SelectedSolver.instance().must_be_true(cs, a.default == 0) + SelectedSolver.instance().must_be_true(cs, a[0] == 0) + SelectedSolver.instance().must_be_true(cs, a[1] == 10) + SelectedSolver.instance().must_be_true(cs, a[2] == 20) + SelectedSolver.instance().must_be_true(cs, a[3] == 30) + # SelectedSolver.instance().must_be_true(cs, a[4] == 0) #undefined! + + + b = a[:] + # b := 0 10 20 30 0 0 x x x x (x undefined) + SelectedSolver.instance().must_be_true(cs, b.default == 0) + SelectedSolver.instance().must_be_true(cs, b[0] == 0) + SelectedSolver.instance().must_be_true(cs, b[1] == 10) + SelectedSolver.instance().must_be_true(cs, b[2] == 20) + SelectedSolver.instance().must_be_true(cs, b[3] == 30) + + + def test_Expression(self): # Used to check if all Expression have test @@ -113,6 +163,7 @@ def test_Expression(self): def check(ty, pickle_size=None, sizeof=None, **kwargs): x = ty(**kwargs) + """ print( type(x), "\n Pickle size:", @@ -122,6 +173,7 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): "\n Slotted:", not hasattr(x, "__dict__"), ) + """ #self.assertEqual(len(pickle_dumps(x)), pickle_size) self.assertEqual(sys.getsizeof(x), sizeof) self.assertFalse(hasattr(x, "__dict__")) # slots! @@ -162,7 +214,7 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): # TODO! But you can instantiate an ArraConstant """ - x = ArrayConstant(index_bits=32, value_bits=8, b"AAAAAAAAAAAAAAA") + x = ArrayConstant(index_size=32, value_size=8, b"AAAAAAAAAAAAAAA") self.assertLessEqual(len(pickle_dumps(x)), 76) #master 71 self.assertLessEqual(sys.getsizeof(x), 64) #master 56 self.assertFalse(hasattr(x, '__dict__')) #slots! @@ -491,7 +543,7 @@ def testBasicArray(self): def testBasicArray256(self): cs = ConstraintSet() # make array of 32->8 bits - array = cs.new_array(32, value_bits=256) + array = cs.new_array(32, value_size=256) # make free 32bit bitvector key = cs.new_bitvec(32) @@ -559,13 +611,13 @@ def testBasicArrayStore(self): def testBasicArraySymbIdx(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array") + array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=0)) key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") array[key] = 1 # Write 1 to a single location - cs.add(array.get(index, default=0) != 0) # Constrain index so it selects that location + cs.add(array.select(index) != 0) # Constrain index so it selects that location cs.add(index != key) # key and index are the same there is only one slot in 1 @@ -573,14 +625,14 @@ def testBasicArraySymbIdx(self): def testBasicArraySymbIdx2(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array") + array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=0)) key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") array[key] = 1 # Write 1 to a single location - cs.add(array.get(index, 0) != 0) # Constrain index so it selects that location + cs.add(array.select(index) != 0) # Constrain index so it selects that location a_index = self.solver.get_value(cs, index) # get a concrete solution for index - cs.add(array.get(a_index, 0) != 0) # now storage must have something at that location + cs.add(array.select(a_index) != 0) # now storage must have something at that location cs.add(a_index != index) # remove it from the solutions # It should not be another solution for index @@ -588,13 +640,13 @@ def testBasicArraySymbIdx2(self): def testBasicArrayDefault(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + array = cs.new_array(index_size=32, value_size=32, name="array", default=0) key = cs.new_bitvec(32, name="key") self.assertTrue(self.solver.must_be_true(cs, array[key] == 0)) def testBasicArrayDefault2(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=0)) index1 = cs.new_bitvec(32) index2 = cs.new_bitvec(32) value = cs.new_bitvec(32) @@ -605,7 +657,7 @@ def testBasicArrayDefault2(self): def testBasicArrayIndexConcrete(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=0)) array[0] = 100 self.assertTrue(array[0] == 100) @@ -613,7 +665,7 @@ def testBasicArrayConcatSlice(self): hw = b"Hello world!" cs = ConstraintSet() # make array of 32->8 bits - array = cs.new_array(32, index_max=len(hw)) + array = cs.new_array(32, length=len(hw)) array = array.write(0, hw) self.assertEqual(len(array), len(hw)) self.assertTrue(self.solver.must_be_true(cs, array == hw)) @@ -646,7 +698,7 @@ def testBasicArraySlice(self): hw = b"Hello world!" cs = ConstraintSet() # make array of 32->8 bits - array = cs.new_array(32, index_max=12) + array = MutableArray(cs.new_array(32, length=12)) array = array.write(0, hw) array_slice = array[0:2] self.assertTrue(self.solver.must_be_true(cs, array == hw)) @@ -666,22 +718,22 @@ def testBasicArraySlice(self): def testBasicArrayProxySymbIdx(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array", default=0) + array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=0)) key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") array[key] = 1 # Write 1 to a single location - cs.add(array.get(index) != 0) # Constrain index so it selects that location + cs.add(array.select(index) != 0) # Constrain index so it selects that location a_index = self.solver.get_value(cs, index) # get a concrete solution for index - cs.add(array.get(a_index) != 0) # now storage must have something at that location + cs.add(array.select(a_index) != 0) # now storage must have something at that location cs.add(a_index != index) # remove it from the solutions # It should not be another solution for index self.assertFalse(self.solver.check(cs)) def testBasicArrayProxySymbIdx2(self): cs = ConstraintSet() - array = cs.new_array(index_bits=32, value_bits=32, name="array") + array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=100)) key = cs.new_bitvec(32, name="key") index = cs.new_bitvec(32, name="index") @@ -691,23 +743,20 @@ def testBasicArrayProxySymbIdx2(self): solutions = self.solver.get_all_values(cs, array[0]) # get a concrete solution for index self.assertItemsEqual(solutions, (1, 2)) solutions = self.solver.get_all_values( - cs, array.get(0, 100) + cs, array.select(0) ) # get a concrete solution for index 0 self.assertItemsEqual(solutions, (1, 2)) solutions = self.solver.get_all_values( - cs, array.get(1, 100) + cs, array.select(1) ) # get a concrete solution for index 1 (default 100) self.assertItemsEqual(solutions, (100, 2)) - self.assertTrue( - self.solver.can_be_true(cs, array[1] == 12345) - ) # no default so it can be anything def testBasicConstatArray(self): cs = ConstraintSet() - array1 = cs.new_array(index_bits=32, value_bits=32, index_max=10, name="array1", default=0) - array2 = cs.new_array(index_bits=32, value_bits=32, index_max=10, name="array2", default=0) + array1 = MutableArray(cs.new_array(index_size=32, value_size=32, length=10, name="array1", default=0)) + array2 = MutableArray(cs.new_array(index_size=32, value_size=32, length=10, name="array2", default=0)) array1[0:10] = range(10) self.assertTrue(array1[0] == 0) #yeah right self.assertTrue(array1[0:10] == range(10)) @@ -813,7 +862,7 @@ def testBool_nonzero(self): def test_visitors(self): solver = Z3Solver.instance() cs = ConstraintSet() - arr = cs.new_array(name="MEM") + arr = MutableArray(cs.new_array(name="MEM")) a = cs.new_bitvec(32, name="VAR") self.assertEqual(get_depth(a), 1) cond = Operators.AND(a < 200, a > 100) From 21be70614be233443c0ded460935d3ecb168d8ee Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 17 Aug 2020 13:50:16 -0300 Subject: [PATCH 075/126] mypy + Z3->SelectedSolver --- manticore/core/smtlib/expression.py | 3 +-- manticore/core/smtlib/solver.py | 4 ++-- manticore/ethereum/abi.py | 2 +- manticore/native/memory.py | 2 +- tests/ethereum/test_general.py | 4 ++-- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 403b5120b..246aaa92f 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -748,8 +748,7 @@ def cast(self, array) -> "Array": """ Builds an Array from a bytes or bytearray""" if isinstance(array, Array): return array - arr = self._constraints.new_array(index_size=self.index_size, length=len(array), default=0, value_size=self.value_size, name="cast{}".format(uuid.uuid1()) - ) + arr = ArrayVariable(index_size=self.index_size, length=len(array), default=0, value_size=self.value_size, name="cast{}".format(uuid.uuid1())) for pos, byte in enumerate(array): arr = arr.store(pos, byte) return arr diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 72039b3b6..74a81972c 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -406,9 +406,9 @@ def get_model(self, constraints: ConstraintSet): model = {} for variable in constraints.variables: value = None - if isinstance(variable, Bool): + if isinstance(variable, BoolVariable): value = self.__getvalue_bool(variable.name) - elif isinstance(variable, Bitvec): + elif isinstance(variable, BitvecVariable): value = self.__getvalue_bv(variable.name) elif isinstance(variable, Array): try: diff --git a/manticore/ethereum/abi.py b/manticore/ethereum/abi.py index 381f26782..112a2c9d9 100644 --- a/manticore/ethereum/abi.py +++ b/manticore/ethereum/abi.py @@ -316,7 +316,7 @@ def _serialize_int(value: typing.Union[int, Bitvec], size=32, padding=0): if issymbolic(value): # Help mypy out. Can remove this by teaching it how issymbolic works assert isinstance(value, Bitvec) - buf = value.constraints.new_array( + buf = ArrayVariable( index_size=256, length=32, value_size=8, name="temp{}".format(uuid.uuid1()) ) value = Operators.SEXTEND(value, value.size, size * 8) diff --git a/manticore/native/memory.py b/manticore/native/memory.py index 1eb34a36a..7fc2774f7 100644 --- a/manticore/native/memory.py +++ b/manticore/native/memory.py @@ -346,7 +346,7 @@ def __init__( self._array = backing_array else: self._array = expression.MutableArray( - expression.ArrayVariable(index_bits, length=size, value_bits=8, name=name) + expression.ArrayVariable(index_size=index_bits, length=size, value_size=8, name=name) ) def __reduce__(self): diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index f45f795ac..cb68be0de 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -15,7 +15,7 @@ from manticore import ManticoreError from manticore.core.plugin import Plugin from manticore.core.smtlib import ConstraintSet, operators -from manticore.core.smtlib import Z3Solver +from manticore.core.smtlib import SelectedSolver from manticore.core.smtlib.expression import BitvecVariable from manticore.core.smtlib.visitors import to_constant from manticore.core.state import TerminateState @@ -41,7 +41,7 @@ import contextlib -solver = Z3Solver.instance() +solver = SelectedSolver.instance() THIS_DIR = os.path.dirname(os.path.abspath(__file__)) From 5ac21a4605adff4106cf6ef7e3ad23cff8a88ec8 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 17 Aug 2020 13:58:19 -0300 Subject: [PATCH 076/126] lint --- tests/native/test_cpu_manual.py | 58 ++++++++++++++++----------------- tests/native/test_models.py | 2 +- tests/other/test_smtlibv2.py | 35 +------------------- 3 files changed, 31 insertions(+), 64 deletions(-) diff --git a/tests/native/test_cpu_manual.py b/tests/native/test_cpu_manual.py index 606989368..4cc2595dd 100644 --- a/tests/native/test_cpu_manual.py +++ b/tests/native/test_cpu_manual.py @@ -6,13 +6,13 @@ from manticore.native.cpu.abstractcpu import ConcretizeRegister from manticore.native.cpu.x86 import AMD64Cpu from manticore.native.memory import * -from manticore.core.smtlib import BitVecOr, operator, Bool -from manticore.core.smtlib.solver import Z3Solver +from manticore.core.smtlib import BitvecOr, operator, Bool +from manticore.core.smtlib.solver import SelectedSolver from functools import reduce from typing import List -solver = Z3Solver.instance() +solver = SelectedSolver.instance() sizes = { @@ -351,15 +351,15 @@ def check_flag(obj, flag): check_flag(cpu.ZF, "ZF") def test_get_sym_eflags(self): - def flatten_ors(x: BitVecOr) -> List: + def flatten_ors(x: BitvecOr) -> List: """ - Retrieve all nodes of a BitVecOr expression tree + Retrieve all nodes of a BitvecOr expression tree """ - assert isinstance(x, BitVecOr) - if any(isinstance(op, BitVecOr) for op in x.operands): + assert isinstance(x, BitvecOr) + if any(isinstance(op, BitvecOr) for op in x.operands): ret: List = [] for op in x.operands: - if isinstance(op, BitVecOr): + if isinstance(op, BitvecOr): ret += flatten_ors(op) else: ret.append(op) @@ -378,7 +378,7 @@ def flatten_ors(x: BitVecOr) -> List: flags = flatten_ors(cpu.EFLAGS) - self.assertTrue(isinstance(cpu.EFLAGS, BitVecOr)) + self.assertTrue(isinstance(cpu.EFLAGS, BitvecOr)) self.assertEqual(len(flags), 8) self.assertEqual(cpu.CF, 1) @@ -497,13 +497,13 @@ def test_le_or(self): cpu.write_int(0x1000, 0x4142434445464748, 64) cpu.write_int(0x1000, cpu.read_int(0x1000, 32) | 0, 32) - addr1 = cs.new_bitvec(64) + addr1 = cs.new_Bitvec(64) cs.add(addr1 == 0x1004) cpu.write_int(addr1, 0x58, 8) self.assertEqual(cpu.read_int(0x1000, 32), 0x45464748) - addr1 = cs.new_bitvec(64) + addr1 = cs.new_Bitvec(64) cs.add(addr1 == 0x1000) cpu.write_int(addr1, 0x59, 8) @@ -592,7 +592,7 @@ def test_cache_003(self): for i in range(8): self.assertEqual(cpu.read_int(0x1008 + i, 8), ord("hgfedcba"[i])) - addr1 = cs.new_bitvec(64) + addr1 = cs.new_Bitvec(64) cs.add(addr1 == 0x1004) cpu.write_int(addr1, 0x58, 8) @@ -601,7 +601,7 @@ def test_cache_003(self): value = cpu.read_int(0x1004, 16) self.assertItemsEqual(solver.get_all_values(cs, value), [0x4358]) - addr2 = cs.new_bitvec(64) + addr2 = cs.new_Bitvec(64) cs.add(Operators.AND(addr2 >= 0x1000, addr2 <= 0x100C)) cpu.write_int(addr2, 0x5959, 16) @@ -728,11 +728,11 @@ def test_IDIV_symbolic(self): mem[code : code + 3] = "\xf7\x7d\xf4" cpu.EIP = code - cpu.EAX = cs.new_bitvec(32, "EAX") + cpu.EAX = cs.new_Bitvec(32, "EAX") cs.add(cpu.EAX == 116) - cpu.EBP = cs.new_bitvec(32, "EBP") + cpu.EBP = cs.new_Bitvec(32, "EBP") cs.add(cpu.EBP == stack + 0x700) - value = cs.new_bitvec(32, "VALUE") + value = cs.new_Bitvec(32, "VALUE") cpu.write_int(cpu.EBP - 0xC, value, 32) cs.add(value == 100) cpu.execute() @@ -765,11 +765,11 @@ def test_IDIV_grr001_symbolic(self): mem[code : code + 2] = "\xf7\xf9" cpu.EIP = code - cpu.EAX = cs.new_bitvec(32, "EAX") + cpu.EAX = cs.new_Bitvec(32, "EAX") cs.add(cpu.EAX == 0xFFFFFFFF) - cpu.EDX = cs.new_bitvec(32, "EDX") + cpu.EDX = cs.new_Bitvec(32, "EDX") cs.add(cpu.EDX == 0xFFFFFFFF) - cpu.ECX = cs.new_bitvec(32, "ECX") + cpu.ECX = cs.new_Bitvec(32, "ECX") cs.add(cpu.ECX == 0x32) cpu.execute() @@ -808,9 +808,9 @@ def test_ADC_001_symbolic(self): mem[code : code + 2] = "\x13\xf2" cpu.EIP = code - cpu.ESI = cs.new_bitvec(32, "ESI") + cpu.ESI = cs.new_Bitvec(32, "ESI") cs.add(cpu.ESI == 0) - cpu.EDX = cs.new_bitvec(32, "EDX") + cpu.EDX = cs.new_Bitvec(32, "EDX") cs.add(cpu.EDX == 0xFFFFFFFF) cpu.CF = cs.new_bool("CF") cs.add(cpu.CF) @@ -874,7 +874,7 @@ def test_CMPXCHG8B_symbolic(self): mem[code : code + 5] = "\xf0\x0f\xc7\x0f;" cpu.EIP = code - cpu.EDI = cs.new_bitvec(32, "EDI") + cpu.EDI = cs.new_Bitvec(32, "EDI") cs.add(Operators.OR(cpu.EDI == 0x2000, cpu.EDI == 0x2100, cpu.EDI == 0x2200)) self.assertEqual(sorted(solver.get_all_values(cs, cpu.EDI)), [0x2000, 0x2100, 0x2200]) self.assertEqual(cpu.read_int(0x2000, 64), 0) @@ -885,7 +885,7 @@ def test_CMPXCHG8B_symbolic(self): cpu.write_int(0x2100, 0x4142434445464748, 64) - cpu.EAX = cs.new_bitvec(32, "EAX") + cpu.EAX = cs.new_Bitvec(32, "EAX") cs.add(Operators.OR(cpu.EAX == 0x41424344, cpu.EAX == 0x0BADF00D, cpu.EAX == 0xF7F7F7F7)) cpu.EDX = 0x45464748 @@ -1060,7 +1060,7 @@ def test_SAR_1_symbolic(self): cs.add(cpu.AF == False) cpu.OF = cs.new_bool() cs.add(cpu.OF == False) - cpu.EAX = cs.new_bitvec(32) + cpu.EAX = cs.new_Bitvec(32) cs.add(cpu.EAX == 0xFFFFFFF7) done = False @@ -1138,7 +1138,7 @@ def test_SAR_2_symbolicsa(self): cs.add(cpu.AF == False) cpu.OF = cs.new_bool() cs.add(cpu.OF == False) - cpu.EAX = cs.new_bitvec(32) + cpu.EAX = cs.new_Bitvec(32) cs.add(cpu.EAX == 0xFFFFFFFF) done = False @@ -1192,15 +1192,15 @@ def test_SAR_3_symbolic(self): mem[0x804D601] = "\x78" mem[0x804D602] = "\x00" mem[0x804D603] = "\xff" - addr = cs.new_bitvec(32) + addr = cs.new_Bitvec(32) cs.add(addr == 0xFFFFB000) - value = cs.new_bitvec(8) + value = cs.new_Bitvec(8) cs.add(value == 0x8F) mem[addr] = value - cpu.EAX = cs.new_bitvec(32) + cpu.EAX = cs.new_Bitvec(32) cs.add(cpu.EAX == 0xFFFFB000) - cpu.CL = cs.new_bitvec(8) + cpu.CL = cs.new_Bitvec(8) cs.add(cpu.CL == 0xFF) cpu.EIP = 0x804D600 diff --git a/tests/native/test_models.py b/tests/native/test_models.py index a3e5f498c..c5499470e 100644 --- a/tests/native/test_models.py +++ b/tests/native/test_models.py @@ -11,7 +11,7 @@ Z3Solver, issymbolic, ArraySelect, - BitVecITE, + BitvecITE, ) from manticore.native.state import State from manticore.platforms import linux diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index d4e373308..5f0d3d2a4 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -97,7 +97,7 @@ def test_ConstantArrayBitvec(self): self.assertEqual(c[4], "E") self.assertRaises(IndexError, c.get, 5) - def test_ConstantArrayBitvec(self): + def test_ConstantArrayBitvec2(self): c = MutableArray(ArrayVariable(index_size=32, value_size=8, length=5, name="ARR")) c[1] = 10 c[2] = 20 @@ -1422,39 +1422,6 @@ def test_signed_unsigned_LT_simple(self): self.assertFalse(self.solver.can_be_true(cs, ult)) self.assertTrue(self.solver.must_be_true(cs, lt)) - def test_signed_unsigned_LT_(self): - mask = (1 << 32) - 1 - - cs = ConstraintSet() - _a = cs.new_bitvec(32) - _b = cs.new_bitvec(32) - - cs.add(_a == 0x1) - cs.add(_b == (0x80000000 - 1)) - - a = _a & mask - b = (_b + 1) & mask - - lt = b < a - ult = b.ult(a) - - self.assertFalse(self.solver.can_be_true(cs, ult)) - self.assertTrue(self.solver.must_be_true(cs, lt)) - - def test_signed_unsigned_LT_simple(self): - cs = ConstraintSet() - a = cs.new_bitvec(32) - b = cs.new_bitvec(32) - - cs.add(a == 0x1) - cs.add(b == 0x80000000) - - lt = b < a - ult = b.ult(a) - - self.assertFalse(self.solver.can_be_true(cs, ult)) - self.assertTrue(self.solver.must_be_true(cs, lt)) - def test_signed_unsigned_LT_complex(self): mask = (1 << 32) - 1 From 19f4d435db6600c59819338db1b4873b23215e54 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 17 Aug 2020 14:37:48 -0300 Subject: [PATCH 077/126] BitVec Bitvec --- examples/script/concolic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/script/concolic.py b/examples/script/concolic.py index 6be63240b..37eff5cff 100755 --- a/examples/script/concolic.py +++ b/examples/script/concolic.py @@ -66,18 +66,18 @@ def flip(constraint): """ flips a constraint (Equal) - (Equal (BitVecITE Cond IfC ElseC) IfC) + (Equal (BitvecITE Cond IfC ElseC) IfC) -> - (Equal (BitVecITE Cond IfC ElseC) ElseC) + (Equal (BitvecITE Cond IfC ElseC) ElseC) """ equal = copy.copy(constraint) assert len(equal.operands) == 2 # assume they are the equal -> ite form that we produce on standard branches ite, forcepc = equal.operands - if not (isinstance(ite, BitVecITE) and isinstance(forcepc, BitvecConstant)): + if not (isinstance(ite, BitvecITE) and isinstance(forcepc, BitvecConstant)): return constraint - assert isinstance(ite, BitVecITE) and isinstance(forcepc, BitvecConstant) + assert isinstance(ite, BitvecITE) and isinstance(forcepc, BitvecConstant) assert len(ite.operands) == 3 cond, iifpc, eelsepc = ite.operands assert isinstance(iifpc, BitvecConstant) and isinstance(eelsepc, BitvecConstant) From cbe21749b8c436d980a7152d72b6891fa623e1aa Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 17 Aug 2020 15:50:05 -0300 Subject: [PATCH 078/126] fix tests --- tests/other/test_smtlibv2.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 5f0d3d2a4..ba4fc6a25 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -90,11 +90,11 @@ def test_Bitvec_ops(self): def test_ConstantArrayBitvec(self): c = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") - self.assertEqual(c[0], "A") - self.assertEqual(c[1], "B") - self.assertEqual(c[2], "C") - self.assertEqual(c[3], "D") - self.assertEqual(c[4], "E") + self.assertEqual(c[0], ord("A")) + self.assertEqual(c[1], ord("B")) + self.assertEqual(c[2], ord("C")) + self.assertEqual(c[3], ord("D")) + self.assertEqual(c[4], ord("E")) self.assertRaises(IndexError, c.get, 5) def test_ConstantArrayBitvec2(self): From 354cec362c23bc5e4dcf056507ae921e7cb2604d Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 8 Sep 2020 12:51:39 -0300 Subject: [PATCH 079/126] Various Expression refactors and docs --- manticore/core/smtlib/expression.py | 152 +++++++++++++++++----------- manticore/core/state.py | 3 - manticore/ethereum/manticore.py | 3 - manticore/platforms/evm.py | 40 ++++---- tests/other/test_smtlibv2.py | 105 +++++++++---------- 5 files changed, 172 insertions(+), 131 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 246aaa92f..dbbf43c3e 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -679,7 +679,7 @@ def __init__(self, operanda, operandb, **kwargs): class Array(Expression, abstract=True): - """ And Array expression is a mapping from bitvector to bitvector + """ An Array expression is an unmutable mapping from bitvector to bitvector array.index_size is the number of bits used for addressing a value array.value_size is the number of bits used in the values @@ -721,31 +721,37 @@ def written(self): """ Returns the set of potentially symbolic indexes that were written in this array. - Note that as you could overwrite an index this could have more elements + Note that as you could overwrite an index so this could have more elements than total elements in the array. """ raise NotImplementedError - def is_known(self, index): + def is_known(self, index) -> Union[Bool, bool]: + """ Returned Boolean Expression holds when the index was used""" raise NotImplementedError - ## following methods are implementes on top of the abstract methods ^ + ########################################################################### + ## following methods are implemented on top of the abstract methods ^ def in_bounds(self, index:Union[Bitvec, int]) -> Union[Bool, bool]: - """ True if the index points inside the array """ + """ True if the index points inside the array (or array is unbounded)""" if self.length is not None: return (0 <= index) & (index < self.length) return True def __len__(self): - """ Number o defined values. """ + """ Number of values. """ return self.length def get(self, index): - """ Should this exist?""" + """ FIXME: Should this exist?""" return self.select(index) def cast(self, array) -> "Array": - """ Builds an Array from a bytes or bytearray""" + """ Builds an Array from bytes or bytearray + FIXME: this assigns a random name to a new variable and does not use + a ConstraintSet as a Factory + """ + logger.error("THis is creating a variable out of band FTAG4985732") if isinstance(array, Array): return array arr = ArrayVariable(index_size=self.index_size, length=len(array), default=0, value_size=self.value_size, name="cast{}".format(uuid.uuid1())) @@ -764,7 +770,7 @@ def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: """ Forgiving casting method that will translate compatible values into - a compliant Bitvec to ve used as a value""" + a compliant Bitvec to be used as a value""" if not isinstance(value, (Bitvec, bytes, int)): raise TypeError if isinstance(value, Bitvec): @@ -778,14 +784,15 @@ def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: return BitvecConstant(self.value_size, value) def write(self, offset, buf): - """ Builds a new Array instance by writing buf at offset """ + """ Builds a new unmutable Array instance on top of current array by + writing buf at offset """ array = self for i, value in enumerate(buf): array = array.store(offset + i, value) return array def read(self, offset, size): - """ A proyection of the current array. """ + """ A projection of the current array. """ return ArraySlice(self, offset=offset, size=size) def __getitem__(self, index): @@ -800,22 +807,26 @@ def __getitem__(self, index): return self.select(index) def __iter__(self): - """ Iterations """ + """ In a bounded array iterates over all elements. """ for i in range(len(self)): yield self[i] - def __eq__(self, other): - # FIXME taint - def compare_buffers(a, b): - if len(a) != len(b): + @staticmethod + def _compare_buffers(a, b): + """ Builds an expression that represents equality between the two arrays.""" + if len(a) != len(b): + return BoolConstant(False) + cond = BoolConstant(True) + for i in range(len(a)): + cond = BoolAnd(cond.cast(a[i] == b[i]), cond) + if cond is BoolConstant(False): return BoolConstant(False) - cond = BoolConstant(True) - for i in range(len(a)): - cond = BoolAnd(cond.cast(a[i] == b[i]), cond) - if cond is BoolConstant(False): - return BoolConstant(False) - return cond - return compare_buffers(self, other) + return cond + + def __eq__(self, other): + """ If both arrays has the same elements they are equal. + The difference in taints are ignored.""" + return self._compare_buffers(self, other) def __ne__(self, other): return BoolNot(self == other) @@ -830,11 +841,11 @@ def _fix_slice(self, index: slice): size = stop - start if isinstance(size, Bitvec): from .visitors import simplify - size = simplify(size) else: size = BitvecConstant(self.index_size, size) - assert isinstance(size, BitvecConstant) + if not isinstance(size, BitvecConstant): + raise ExpressionError("Size could not be simplified to a constant in a slice operation") return start, stop, size.value def _concatenate(self, array_a, array_b): @@ -1025,7 +1036,7 @@ def select(self, index): def store(self, index, value): index = self.cast_index(index) - value = self.cast_value(value) + value = simplify(self.cast_value(value)) return ArrayStore(array=self, index=index, value=value) @property @@ -1046,27 +1057,56 @@ class ArrayOperation(Array, Operation, abstract=True): """ It's an operation that results in an Array""" pass +def get_items(array): + if isinstance(array, ArrayStore): + yield from get_items_array_store(array) + elif isinstance(array, ArraySlice): + yield from get_items_array_slice(array) + elif isinstance(array, ArrayConstant): + yield from get_items_array_constant(array) + return + +def get_items_array_slice(array): + assert isinstance(array, ArraySlice) + for offset, value in get_items(array.array): + yield offset+array.offset, value + + +def get_items_array_store(array): + assert isinstance(array, ArrayStore) + while isinstance(array, ArrayStore): + yield array.index, array.value + array = array.array + yield from get_items(array) + +def get_items_array_constant(array): + assert isinstance(array, ArrayConstant) + for index, value in enumerate(array.value): + yield index, value + +def get_items_array_variable(array): + assert isinstance(array, ArrayVariable) + raise GeneratorExit class ArrayStore(ArrayOperation): __xslots__: Tuple[str, ...] = ( "_written#v", "_concrete_cache#v", + "_length#v", + "_default#v", ) - @property - def length(self): - return self.array.length @property def concrete_cache(self): if self._concrete_cache is not None: return self._concrete_cache - index = self.index self._concrete_cache = {} - if isinstance(index, Constant): - self._concrete_cache.update(getattr(self.array, "concrete_cache", - ())) # Cache of concrete indexes - self._concrete_cache[index.value] = self.value + for index, value in get_items(self): + if not isinstance(index, Constant): + break + if index.value not in self._concrete_cache: + self._concrete_cache[index.value] = value return self._concrete_cache @property @@ -1075,21 +1115,8 @@ def written(self): # This can have repeated and reused written indexes. if self._written is None: written = set() - # take out Proxy sleve - array = self - offset = 0 - while not isinstance(array, ArrayVariable): - if array._written is not None: - written = written.union((x - offset for x in array.written)) - break - if isinstance(array, ArraySlice): - # if it is a proxy over a slice take out the slice too - offset += array.offset - array = array.array - else: - # The index written to underlaying Array are displaced when sliced - written.add(array.index - offset) - array = array.array + for offset, value in get_items(self): + written.add(offset) self._written = written return self._written @@ -1109,16 +1136,28 @@ def is_known(self, index): def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size - self._written = None # Cache of the known indexs + self._written = None # Cache of the known indexes self._concrete_cache = None + self._length = array.length + self._default = array.default + + #recreate and reuse cache + #if isinstance(index, Constant) and isinstance(array, ArrayStore) and array._concrete_cache is not None: + # self._concrete_cache = dict(array._concrete_cache) + # self._concrete_cache[index.value] = value + super().__init__( operands=(array, index, value), **kwargs, ) + @property + def length(self): + return self._length + @property def default(self): - return self.array.default + return self._default @property def index_size(self): @@ -1128,9 +1167,6 @@ def index_size(self): def value_size(self): return self.value.size - @property - def index_max(self): - return self.array.index_max def __hash__(self): return object.__hash__(self) @@ -1159,13 +1195,14 @@ def select(self, index): index = simplify(self.cast_index(index)) # Emulate list[-1] - if self.index_max is not None: + has_length = self.length is not None + if has_length: index = simplify( - BitvecITE(index < 0, self.index_max + index + 1, index) + BitvecITE(index < 0, self.length + index, index) ) if isinstance(index, Constant): - if self.index_max is not None and index.value > self.index_max: + if has_length and index.value >= self.length: raise IndexError if index.value in self.concrete_cache: return self.concrete_cache[index.value] @@ -1210,6 +1247,7 @@ def store(self, index, value): return new_array + class ArraySlice(ArrayOperation): """ Provides a projection of an underlying array. Lets you slice an array without copying it. diff --git a/manticore/core/state.py b/manticore/core/state.py index 3b1e9d532..03d8c218c 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -375,7 +375,6 @@ def solve_one_n(self, *exprs, constrain=False): values.append(expr) else: expr = self.migrate_expression(expr) - print ("SOLVEONE", expr) value = self._solver.get_value(self._constraints, expr) if constrain: self.constrain(expr == value) @@ -383,8 +382,6 @@ def solve_one_n(self, *exprs, constrain=False): if isinstance(value, bytearray): value = bytes(value) values.append(value) - assert any(issymbolic for x in values) - print (exprs, values) return values def solve_n(self, expr, nsolves): diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index c80667acb..ad2c4cd3a 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -857,10 +857,7 @@ def _migrate_tx_expressions(self, state, caller, address, value, data, gas, pric if issymbolic(data): if isinstance(data, MutableArray): # FIXME is this necessary here? data = data.array - print ("data:"*10, data) data = state.migrate_expression(data) - #if isinstance(data, Array): - # data = MutableArray(data) if issymbolic(gas): gas = state.migrate_expression(gas) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 2f3d5c2e4..8bf332d1c 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -972,13 +972,13 @@ def read_code(self, address, size=1): return value def disassemble(self): - return EVMAsm.disassemble(self.bytecode) + return EVMAsm.disassemble(self._getcode(zerotail=False)) @property def PC(self): return self.pc - def _getcode(self, pc): + def _getcode(self, pc:int = 0, zerotail:bool = True): bytecode = self.bytecode for pc_i in range(pc, len(bytecode)): c = bytecode[pc_i] @@ -986,14 +986,23 @@ def _getcode(self, pc): yield simplify(c).value else: yield c - while True: - yield 0 # STOP opcode + if zerotail: + while True: + yield 0 # STOP opcode + @property def instruction(self): """ Current instruction pointed by self.pc """ + return self.get_instruction(pc=self.pc) + + def get_instruction(self, pc: Union[Bitvec, int]): + """ + Current instruction pointed by self.pc + """ + # FIXME check if pc points to invalid instruction # if self.pc >= len(self.bytecode): # return InvalidOpcode('Code out of range') @@ -1005,7 +1014,6 @@ def instruction(self): self._decoding_cache = {} _decoding_cache = self._decoding_cache - pc = self.pc if isinstance(pc, Constant): pc = pc.value @@ -1674,19 +1682,12 @@ def CALLDATALOAD(self, offset): self.constraints.add(self.safe_add(offset, 32) <= len(self.data) + calldata_overflow) self._use_calldata(offset, 32) - data_length = len(self.data) bytes = [] for i in range(32): try: - c = simplify( - Operators.ITEBV( - 8, - Operators.ULT(self.safe_add(offset, i), data_length), - self.data[offset + i], - 0, - ) - ) + c=Operators.ITEBV(8, Operators.ULT(self.safe_add(offset, i), data_length), self.data[offset + i], 0, ) + c = simplify(c) except IndexError: # offset + i is concrete and outside data c = 0 @@ -1745,6 +1746,7 @@ def CALLDATACOPY(self, mem_offset, data_offset, size): max_size = cap self.constraints.add(Operators.ULE(size, cap)) + for i in range(max_size): try: c1 = Operators.ITEBV( @@ -1762,7 +1764,7 @@ def CALLDATACOPY(self, mem_offset, data_offset, size): if not issymbolic(c) or get_depth(c) < 3: x = c else: - # if te expression is deep enough lets replace it by a binding + # if the expression is deep enough lets replace it by a binding x = self.constraints.new_bitvec(8, name="temp{}".format(uuid.uuid1())) self.constraints.add(x == c) self._store(mem_offset + i, x) @@ -1811,7 +1813,11 @@ def CODECOPY(self, mem_offset, code_offset, size): value = default else: value = self.bytecode[code_offset + i] + self._store(mem_offset + i, value) + + assert SelectedSolver.instance().must_be_true(self.constraints,self.memory[0:max_size] == self.bytecode[code_offset:code_offset + max_size]) + self._publish("did_evm_read_code", self.address, code_offset, size) def GASPRICE(self): @@ -2140,8 +2146,8 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): data = self.read_buffer(offset, size) keccak_init = self.world.symbolic_function(globalsha3, data) - caller = MutableArray(msg.caller).read_BE(0, 20) - salt = MutableArray(salt).read_BE(0, 32) + caller = msg.caller.read_BE(0, 20) + salt = salt.read_BE(0, 32) address = self.world.symbolic_function(b"\xff" + caller + salt + keccak_init) & ((1<<0x20)-1) self.world.start_transaction( diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index ba4fc6a25..ba90fca68 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -105,6 +105,8 @@ def test_ConstantArrayBitvec2(self): self.assertEqual(c[1], 10) self.assertEqual(c[2], 20) self.assertEqual(c[3], 30) + c[2] = 100 + self.assertEqual(c[2], 100) def test_ArrayDefault3(self): @@ -198,8 +200,8 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): self.assertTrue(ty.__doc__, ty) checked.add(ty) - check(BitvecVariable, size=32, name="name", pickle_size=111, sizeof=64) - check(BoolVariable, name="name", pickle_size=99, sizeof=56) + check(BitvecVariable, size=32, name="name", pickle_size=111, sizeof=56) + check(BoolVariable, name="name", pickle_size=99, sizeof=48) check( ArrayVariable, index_size=32, @@ -207,10 +209,10 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): length=100, name="name", pickle_size=156, - sizeof=88, + sizeof=80, ) - check(BitvecConstant, size=32, value=10, pickle_size=107, sizeof=64) - check(BoolConstant, value=False, pickle_size=94, sizeof=56) + check(BitvecConstant, size=32, value=10, pickle_size=107, sizeof=56) + check(BoolConstant, value=False, pickle_size=94, sizeof=48) # TODO! But you can instantiate an ArraConstant """ @@ -224,58 +226,58 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): x = BoolVariable(name="x") y = BoolVariable(name="y") z = BoolVariable(name="z") - check(BoolEqual, operanda=x, operandb=y, pickle_size=159, sizeof=56) - check(BoolAnd, operanda=x, operandb=y, pickle_size=157, sizeof=56) - check(BoolOr, operanda=x, operandb=y, pickle_size=156, sizeof=56) - check(BoolXor, operanda=x, operandb=y, pickle_size=157, sizeof=56) - check(BoolNot, operand=x, pickle_size=137, sizeof=56) - check(BoolITE, cond=z, true=x, false=y, pickle_size=130, sizeof=56) + check(BoolEqual, operanda=x, operandb=y, pickle_size=159, sizeof=48) + check(BoolAnd, operanda=x, operandb=y, pickle_size=157, sizeof=48) + check(BoolOr, operanda=x, operandb=y, pickle_size=156, sizeof=48) + check(BoolXor, operanda=x, operandb=y, pickle_size=157, sizeof=48) + check(BoolNot, operand=x, pickle_size=137, sizeof=48) + check(BoolITE, cond=z, true=x, false=y, pickle_size=130, sizeof=48) bvx = BitvecVariable(size=32, name="bvx") bvy = BitvecVariable(size=32, name="bvy") - check(BoolUnsignedGreaterThan, operanda=bvx, operandb=bvy, pickle_size=142, sizeof=56) - check(BoolGreaterThan, operanda=bvx, operandb=bvy, pickle_size=134, sizeof=56) + check(BoolUnsignedGreaterThan, operanda=bvx, operandb=bvy, pickle_size=142, sizeof=48) + check(BoolGreaterThan, operanda=bvx, operandb=bvy, pickle_size=134, sizeof=48) check( - BoolUnsignedGreaterOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=149, sizeof=56 + BoolUnsignedGreaterOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=149, sizeof=48 ) - check(BoolGreaterOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=141, sizeof=56) - check(BoolUnsignedLessThan, operanda=bvx, operandb=bvy, pickle_size=139, sizeof=56) - check(BoolLessThan, operanda=bvx, operandb=bvy, pickle_size=131, sizeof=56) - check(BoolUnsignedLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=56) - check(BoolLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) - - check(BitvecOr, operanda=bvx, operandb=bvy, pickle_size=129, sizeof=64) - check(BitvecXor, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecAnd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecNot, operanda=bvx, pickle_size=112, sizeof=64) - check(BitvecNeg, operanda=bvx, pickle_size=112, sizeof=64) - check(BitvecAdd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecMul, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecSub, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecDiv, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecMod, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecUnsignedDiv, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=64) - check(BitvecRem, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=64) - check(BitvecUnsignedRem, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=64) - - check(BitvecShiftLeft, operanda=bvx, operandb=bvy, pickle_size=136, sizeof=64) - check(BitvecShiftRight, operanda=bvx, operandb=bvy, pickle_size=137, sizeof=64) - check(BitvecArithmeticShiftLeft, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=64) - check(BitvecArithmeticShiftRight, operanda=bvx, operandb=bvy, pickle_size=147, sizeof=64) - - - check(BitvecZeroExtend, operand=bvx, size=122, pickle_size=119, sizeof=64) - check(BitvecSignExtend, operand=bvx, size=122, pickle_size=119, sizeof=64) - check(BitvecExtract, operand=bvx, offset=0, size=8, pickle_size=119, sizeof=72) - check(BitvecConcat, operands=(bvx, bvy), pickle_size=133, sizeof=64) - check(BitvecITE, condition=x, true_value=bvx, false_value=bvy, pickle_size=161, sizeof=64) + check(BoolGreaterOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=141, sizeof=48) + check(BoolUnsignedLessThan, operanda=bvx, operandb=bvy, pickle_size=139, sizeof=48) + check(BoolLessThan, operanda=bvx, operandb=bvy, pickle_size=131, sizeof=48) + check(BoolUnsignedLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=48) + check(BoolLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=48) + + check(BitvecOr, operanda=bvx, operandb=bvy, pickle_size=129, sizeof=56) + check(BitvecXor, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecAnd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecNot, operanda=bvx, pickle_size=112, sizeof=56) + check(BitvecNeg, operanda=bvx, pickle_size=112, sizeof=56) + check(BitvecAdd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecMul, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecSub, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecDiv, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecMod, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecUnsignedDiv, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) + check(BitvecRem, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitvecUnsignedRem, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) + + check(BitvecShiftLeft, operanda=bvx, operandb=bvy, pickle_size=136, sizeof=56) + check(BitvecShiftRight, operanda=bvx, operandb=bvy, pickle_size=137, sizeof=56) + check(BitvecArithmeticShiftLeft, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=56) + check(BitvecArithmeticShiftRight, operanda=bvx, operandb=bvy, pickle_size=147, sizeof=56) + + + check(BitvecZeroExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) + check(BitvecSignExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) + check(BitvecExtract, operand=bvx, offset=0, size=8, pickle_size=119, sizeof=64) + check(BitvecConcat, operands=(bvx, bvy), pickle_size=133, sizeof=56) + check(BitvecITE, condition=x, true_value=bvx, false_value=bvy, pickle_size=161, sizeof=56) a = ArrayVariable(index_size=32, value_size=32, length=324, name="name") - check(ArrayConstant, index_size=32, value_size=8, value=b"A", pickle_size=136, sizeof=72) - check(ArraySlice, array=a, offset=0, size=10 , pickle_size=136, sizeof=56) - check(ArraySelect, array=a, index=bvx, pickle_size=161, sizeof=64) - check(ArrayStore, array=a, index=bvx, value=bvy, pickle_size=188, sizeof=72) + check(ArrayConstant, index_size=32, value_size=8, value=b"A", pickle_size=136, sizeof=64) + check(ArraySlice, array=a, offset=0, size=10 , pickle_size=136, sizeof=48) + check(ArraySelect, array=a, index=bvx, pickle_size=161, sizeof=56) + check(ArrayStore, array=a, index=bvx, value=bvy, pickle_size=188, sizeof=80) def all_subclasses(cls): @@ -287,10 +289,12 @@ def all_subclasses(cls): all_types = all_subclasses(Expression) self.assertSetEqual(checked, all_types) + def test_Expression_BitvecOp(self): a = BitvecConstant(size=32,value=100) b = BitvecConstant(size=32,value=101) - + x = a+b + self.assertTrue(isinstance(x, Bitvec)) def test_Expression_BoolTaint(self): #Bool can not be instantiaated @@ -332,7 +336,6 @@ def test_Expression_Array(self): self.assertIn('red', z.taint) self.assertIn('blue', z.taint) - class ExpressionTestLoco(unittest.TestCase): _multiprocess_can_split_ = True From 94232b98fc2020fac3fd5e07ad8c553410526fa7 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 8 Sep 2020 12:52:57 -0300 Subject: [PATCH 080/126] blkn --- manticore/core/smtlib/constraints.py | 86 ++++----- manticore/core/smtlib/expression.py | 259 +++++++++++++++------------ manticore/core/smtlib/solver.py | 23 +-- manticore/core/smtlib/visitors.py | 120 ++++++------- 4 files changed, 264 insertions(+), 224 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 817e8e5f3..cfa37d52f 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -18,13 +18,7 @@ Variable, Constant, ) -from .visitors import ( - GetDeclarations, - TranslatorSmtlib, - get_variables, - simplify, - replace -) +from .visitors import GetDeclarations, TranslatorSmtlib, get_variables, simplify, replace from ...utils import config import logging @@ -38,14 +32,16 @@ class ConstraintException(SmtlibError): pass -class Model(): + +class Model: pass + class ConstraintSet: - """ Constraint Sets + """Constraint Sets - An object containing a set of constraints. Serves also as a factory for - new variables. + An object containing a set of constraints. Serves also as a factory for + new variables. """ def __init__(self): @@ -198,7 +194,7 @@ def to_string(self, replace_constants: bool = False) -> str: # if no variables then it is a constant if isinstance(constraint, Constant) and constraint.value == True: continue - #Translate one constraint + # Translate one constraint translator.visit(constraint) if replace_constants: @@ -278,18 +274,18 @@ def is_declared(self, expression_var) -> bool: return any(expression_var is x for x in self.get_declared_variables()) def migrate(self, expression, name_migration_map=None): - """ Migrate an expression created for a different constraint set to self. - Returns an expression that can be used with this constraintSet + """Migrate an expression created for a different constraint set to self. + Returns an expression that can be used with this constraintSet - All the foreign variables used in the expression are replaced by - variables of this constraint set. If the variable was replaced before - the replacement is taken from the provided migration map. + All the foreign variables used in the expression are replaced by + variables of this constraint set. If the variable was replaced before + the replacement is taken from the provided migration map. - The migration mapping is updated with new replacements. + The migration mapping is updated with new replacements. - :param expression: the potentially foreign expression - :param name_migration_map: mapping of already migrated variables. maps from string name of foreign variable to its currently existing migrated string name. this is updated during this migration. - :return: a migrated expression where all the variables are local. name_migration_map is updated + :param expression: the potentially foreign expression + :param name_migration_map: mapping of already migrated variables. maps from string name of foreign variable to its currently existing migrated string name. this is updated during this migration. + :return: a migrated expression where all the variables are local. name_migration_map is updated """ if name_migration_map is None: @@ -347,11 +343,11 @@ def migrate(self, expression, name_migration_map=None): return migrated_expression def new_bool(self, name=None, taint=frozenset(), avoid_collisions=False): - """ Declares a free symbolic boolean in the constraint store - :param name: try to assign name to internal variable representation, - if not unique, a numeric nonce will be appended - :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True - :return: a fresh BoolVariable + """Declares a free symbolic boolean in the constraint store + :param name: try to assign name to internal variable representation, + if not unique, a numeric nonce will be appended + :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True + :return: a fresh BoolVariable """ if name is None: name = "B" @@ -364,12 +360,12 @@ def new_bool(self, name=None, taint=frozenset(), avoid_collisions=False): return self._declare(var) def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False): - """ Declares a free symbolic bitvector in the constraint store - :param size: size in bits for the bitvector - :param name: try to assign name to internal variable representation, - if not unique, a numeric nonce will be appended - :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True - :return: a fresh BitvecVariable + """Declares a free symbolic bitvector in the constraint store + :param size: size in bits for the bitvector + :param name: try to assign name to internal variable representation, + if not unique, a numeric nonce will be appended + :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True + :return: a fresh BitvecVariable """ if size <= 0: raise ValueError(f"Bitvec size ({size}) can't be equal to or less than 0") @@ -393,15 +389,15 @@ def new_array( avoid_collisions=False, default=None, ): - """ Declares a free symbolic array of value_size long bitvectors in the constraint store. - :param index_size: size in bits for the array indexes one of [32, 64] - :param value_size: size in bits for the array values - :param name: try to assign name to internal variable representation, - if not unique, a numeric nonce will be appended - :param length: upper limit for indexes on this array (#FIXME) - :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True - :param default: default for not initialized values - :return: a fresh ArrayProxy + """Declares a free symbolic array of value_size long bitvectors in the constraint store. + :param index_size: size in bits for the array indexes one of [32, 64] + :param value_size: size in bits for the array values + :param name: try to assign name to internal variable representation, + if not unique, a numeric nonce will be appended + :param length: upper limit for indexes on this array (#FIXME) + :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True + :param default: default for not initialized values + :return: a fresh ArrayProxy """ if name is None: name = "A" @@ -412,6 +408,12 @@ def new_array( raise ValueError(f"Name {name} already used") var = self._declare( ArrayVariable( - index_size=index_size, length=length, value_size=value_size, name=name, taint=taint, default=default ) + index_size=index_size, + length=length, + value_size=value_size, + name=name, + taint=taint, + default=default, + ) ) return var diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index dbbf43c3e..fc9ac4fe0 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -29,49 +29,51 @@ class ExpressionError(Exception): """ Expression exception """ + pass class XSlotted(type): - """ Metaclass that will propagate slots on multi-inheritance classes - Every class should define __xslots__ (instead of __slots__) + """Metaclass that will propagate slots on multi-inheritance classes + Every class should define __xslots__ (instead of __slots__) - class Base(object, metaclass=XSlotted, abstract=True): - pass + class Base(object, metaclass=XSlotted, abstract=True): + pass - class A(Base, abstract=True): - __xslots__ = ('a',) - pass + class A(Base, abstract=True): + __xslots__ = ('a',) + pass - class B(Base, abstract=True): - __xslots__ = ('b',) - pass + class B(Base, abstract=True): + __xslots__ = ('b',) + pass - class C(A, B): - pass + class C(A, B): + pass - # Normal case / baseline - class X(object): - __slots__ = ('a', 'b') + # Normal case / baseline + class X(object): + __slots__ = ('a', 'b') - c = C() - c.a = 1 - c.b = 2 + c = C() + c.a = 1 + c.b = 2 - x = X() - x.a = 1 - x.b = 2 + x = X() + x.a = 1 + x.b = 2 - import sys - print (sys.getsizeof(c),sys.getsizeof(x)) #same value + import sys + print (sys.getsizeof(c),sys.getsizeof(x)) #same value """ + @staticmethod - def _remove_mod(attr:str) -> str: - """ xlots attrivutes could have modifficators after a # symbol - attribute#v means attribute is _volatile_ and should not be saved to storage + def _remove_mod(attr: str) -> str: + """xlots attrivutes could have modifficators after a # symbol + attribute#v means attribute is _volatile_ and should not be saved to storage """ - return attr.split('#')[0] + return attr.split("#")[0] def __new__(cls, clsname, bases, attrs, abstract=False): @@ -79,7 +81,7 @@ def __new__(cls, clsname, bases, attrs, abstract=False): # merge the xslots of all the bases with the one defined here for base in bases: xslots = xslots.union(getattr(base, "__xslots__", ())) - attrs["__xslots__"] : Tuple[str] = tuple(xslots) + attrs["__xslots__"]: Tuple[str] = tuple(xslots) if abstract: attrs["__slots__"] = () else: @@ -90,7 +92,8 @@ def __new__(cls, clsname, bases, attrs, abstract=False): class Expression(object, metaclass=XSlotted, abstract=True): """ Abstract taintable Expression. """ - __xslots__ : Tuple[str, ...] = ("_taint", ) + + __xslots__: Tuple[str, ...] = ("_taint",) def __init__(self, taint: Union[tuple, frozenset] = ()): """ @@ -101,8 +104,9 @@ def __init__(self, taint: Union[tuple, frozenset] = ()): super().__init__() def __repr__(self): - return "<{:s} at {:x}{:s}>".format(type(self).__name__, id(self), self._taint and "-T" or "") - + return "<{:s} at {:x}{:s}>".format( + type(self).__name__, id(self), self._taint and "-T" or "" + ) @property def is_tainted(self): @@ -131,10 +135,10 @@ def __setstate__(self, state): class Variable(Expression, abstract=True): """ Variable is an Expression that has a name """ - __xslots__:Tuple[str, ...] = ("_name",) + __xslots__: Tuple[str, ...] = ("_name",) def __init__(self, name: str, **kwargs): - """ Variable is an Expression that has a name + """Variable is an Expression that has a name :param name: The Variable name """ super().__init__(**kwargs) @@ -150,10 +154,11 @@ def __repr__(self): class Constant(Expression, abstract=True): """ Constants expressions have a concrete python value. """ - __xslots__:Tuple[str, ...] = ("_value",) + + __xslots__: Tuple[str, ...] = ("_value",) def __init__(self, value: Union[bool, int, bytes, List[int]], **kwargs): - """ A constant expression has a value + """A constant expression has a value :param value: The constant value """ @@ -167,20 +172,21 @@ def value(self): class Operation(Expression, abstract=True): """ Operation expressions contain operands which are also Expressions. """ - __xslots__:Tuple[str, ...] = ("_operands",) + + __xslots__: Tuple[str, ...] = ("_operands",) def __init__(self, operands: Tuple[Expression, ...], **kwargs): - """ An operation has operands + """An operation has operands :param operands: A tuple of expression operands """ self._operands = operands - taint = kwargs.get('taint') + taint = kwargs.get("taint") # If taint was not forced by a keyword argument, calculate default if taint is None: operands_taints = map(lambda x: x.taint, operands) taint = reduce(lambda x, y: x.union(y), operands_taints, frozenset()) - kwargs['taint'] = taint + kwargs["taint"] = taint super().__init__(**kwargs) @property @@ -192,6 +198,7 @@ def operands(self): # Booleans class Bool(Expression, abstract=True): """Bool expression represent symbolic value of truth""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -265,9 +272,11 @@ def __hash__(self): class BoolOperation(Bool, Operation, abstract=True): """ It's an operation that results in a Bool """ + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + class BoolNot(BoolOperation): def __init__(self, operand: Bool, **kwargs): super().__init__(operands=(operand,), **kwargs) @@ -295,10 +304,11 @@ def __init__(self, cond: Bool, true: Bool, false: Bool, **kwargs): class Bitvec(Expression, abstract=True): """ Bitvector expressions have a fixed bit size """ - __xslots__:Tuple[str, ...] = ("_size",) + + __xslots__: Tuple[str, ...] = ("_size",) def __init__(self, size: int, **kwargs): - """ This is bit vector expression + """This is bit vector expression :param size: number of buts used """ @@ -317,9 +327,7 @@ def mask(self): def signmask(self): return 1 << (self.size - 1) - def cast( - self, value: Union["Bitvec", str, int, bytes], **kwargs - ) -> "Bitvec": + def cast(self, value: Union["Bitvec", str, int, bytes], **kwargs) -> "Bitvec": """ Cast a value int a Bitvec """ if isinstance(value, Bitvec): if value.size != self.size: @@ -512,6 +520,7 @@ def Bool(self): class BitvecVariable(Bitvec, Variable): pass + class BitvecConstant(Bitvec, Constant): def __init__(self, size: int, value: int, **kwargs): """ A bitvector constant """ @@ -542,8 +551,10 @@ def __hash__(self): # need to overload because we defined an __eq__ return object.__hash__(self) + class BitvecOperation(Bitvec, Operation, abstract=True): """ Operations that result in a Bitvec """ + pass @@ -571,26 +582,32 @@ class BitvecUnsignedDiv(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecMod(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecRem(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecUnsignedRem(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecShiftLeft(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecShiftRight(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecArithmeticShiftLeft(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) @@ -600,6 +617,7 @@ class BitvecArithmeticShiftRight(BitvecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) + class BitvecAnd(BitvecOperation): def __init__(self, operanda, operandb, *args, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) @@ -630,10 +648,12 @@ class BoolLessThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) + class BoolLessOrEqualThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) + class BoolEqual(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): assert isinstance(operanda, Expression) @@ -646,6 +666,7 @@ def __bool__(self): return simplified.value raise NotImplementedError + class BoolGreaterThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) @@ -679,13 +700,14 @@ def __init__(self, operanda, operandb, **kwargs): class Array(Expression, abstract=True): - """ An Array expression is an unmutable mapping from bitvector to bitvector + """An Array expression is an unmutable mapping from bitvector to bitvector array.index_size is the number of bits used for addressing a value array.value_size is the number of bits used in the values array.length counts the valid indexes starting at 0. Accessing outside the bound is undefined """ + @property def index_size(self): """ The bit size of the index part. Must be overloaded by a more specific class""" @@ -711,14 +733,14 @@ def store(self, index, value): @property def default(self): - """ If defined, reading from an uninitialized index return the default value. + """If defined, reading from an uninitialized index return the default value. Otherwise, reading from an uninitialized index gives a symbol (normal Array behavior) """ raise NotImplementedError @property def written(self): - """ Returns the set of potentially symbolic indexes that were written in + """Returns the set of potentially symbolic indexes that were written in this array. Note that as you could overwrite an index so this could have more elements @@ -732,7 +754,7 @@ def is_known(self, index) -> Union[Bool, bool]: ########################################################################### ## following methods are implemented on top of the abstract methods ^ - def in_bounds(self, index:Union[Bitvec, int]) -> Union[Bool, bool]: + def in_bounds(self, index: Union[Bitvec, int]) -> Union[Bool, bool]: """ True if the index points inside the array (or array is unbounded)""" if self.length is not None: return (0 <= index) & (index < self.length) @@ -747,21 +769,27 @@ def get(self, index): return self.select(index) def cast(self, array) -> "Array": - """ Builds an Array from bytes or bytearray - FIXME: this assigns a random name to a new variable and does not use - a ConstraintSet as a Factory + """Builds an Array from bytes or bytearray + FIXME: this assigns a random name to a new variable and does not use + a ConstraintSet as a Factory """ logger.error("THis is creating a variable out of band FTAG4985732") if isinstance(array, Array): return array - arr = ArrayVariable(index_size=self.index_size, length=len(array), default=0, value_size=self.value_size, name="cast{}".format(uuid.uuid1())) + arr = ArrayVariable( + index_size=self.index_size, + length=len(array), + default=0, + value_size=self.value_size, + name="cast{}".format(uuid.uuid1()), + ) for pos, byte in enumerate(array): arr = arr.store(pos, byte) return arr def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: - """ Forgiving casting method that will translate compatible values into - a compliant BitVec for indexing""" + """Forgiving casting method that will translate compatible values into + a compliant BitVec for indexing""" if isinstance(index, int): return BitvecConstant(self.index_size, index) if not isinstance(index, Bitvec) or index.size != self.index_size: @@ -769,8 +797,8 @@ def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: return simplify(index) def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: - """ Forgiving casting method that will translate compatible values into - a compliant Bitvec to be used as a value""" + """Forgiving casting method that will translate compatible values into + a compliant Bitvec to be used as a value""" if not isinstance(value, (Bitvec, bytes, int)): raise TypeError if isinstance(value, Bitvec): @@ -784,8 +812,8 @@ def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: return BitvecConstant(self.value_size, value) def write(self, offset, buf): - """ Builds a new unmutable Array instance on top of current array by - writing buf at offset """ + """Builds a new unmutable Array instance on top of current array by + writing buf at offset""" array = self for i, value in enumerate(buf): array = array.store(offset + i, value) @@ -796,10 +824,10 @@ def read(self, offset, size): return ArraySlice(self, offset=offset, size=size) def __getitem__(self, index): - """ __getitem__ allows for pythonic access - A = ArrayVariable(index_size=32, value_size=8) - A[10] := a symbol representing the value under index 10 in array A - A[10:20] := a symbol representing a slice of array A + """__getitem__ allows for pythonic access + A = ArrayVariable(index_size=32, value_size=8) + A[10] := a symbol representing the value under index 10 in array A + A[10:20] := a symbol representing a slice of array A """ if isinstance(index, slice): start, stop, size = self._fix_slice(index) @@ -824,7 +852,7 @@ def _compare_buffers(a, b): return cond def __eq__(self, other): - """ If both arrays has the same elements they are equal. + """If both arrays has the same elements they are equal. The difference in taints are ignored.""" return self._compare_buffers(self, other) @@ -841,6 +869,7 @@ def _fix_slice(self, index: slice): size = stop - start if isinstance(size, Bitvec): from .visitors import simplify + size = simplify(size) else: size = BitvecConstant(self.index_size, size) @@ -850,11 +879,11 @@ def _fix_slice(self, index: slice): def _concatenate(self, array_a, array_b): new_arr = ArrayVariable( - index_size=self.index_size, - length=len(array_a) + len(array_b), - value_size=self.value_size, - name="concatenation", - ) + index_size=self.index_size, + length=len(array_a) + len(array_b), + value_size=self.value_size, + name="concatenation", + ) for index in range(len(array_a)): new_arr = new_arr.store(index, simplify(array_a[index])) @@ -906,12 +935,11 @@ def write_LE(self, address, value, size): class ArrayConstant(Array, Constant): - __xslots__: Tuple[str, ...] = ( - "_index_size", - "_value_size") + __xslots__: Tuple[str, ...] = ("_index_size", "_value_size") def __init__( - self, *, + self, + *, index_size: int, value_size: int, **kwargs, @@ -936,17 +964,25 @@ def select(self, index): """ ArrayConstant get """ index = self.cast_index(index) if isinstance(index, Constant): - return BitvecConstant(size=self.value_size, value=self.value[index.value], taint=self.taint) + return BitvecConstant( + size=self.value_size, value=self.value[index.value], taint=self.taint + ) # Index being symbolic generates a sybolic result ! - result = BitvecConstant(size=self.value_size, value=0, taint=('out_of_bounds')) + result = BitvecConstant(size=self.value_size, value=0, taint=("out_of_bounds")) for i, c in enumerate(self.value): - result = BitvecITE(index == i, BitvecConstant(size=self.value_size, value=c), result, taint=self.taint) + result = BitvecITE( + index == i, BitvecConstant(size=self.value_size, value=c), result, taint=self.taint + ) return result def read(self, offset, size): - assert (len(self.value[offset:offset + size]) == size) - return ArrayConstant(index_size=self.index_size, value_size=self.value_size, value=self.value[offset:offset+size]) + assert len(self.value[offset : offset + size]) == size + return ArrayConstant( + index_size=self.index_size, + value_size=self.value_size, + value=self.value[offset : offset + size], + ) class ArrayVariable(Array, Variable): @@ -961,12 +997,13 @@ class ArrayVariable(Array, Variable): Otherwise the array is unbounded. """ + __xslots__: Tuple[str, ...] = ( - "_index_size", - "_value_size", - "_length", - "_default", - ) + "_index_size", + "_value_size", + "_length", + "_default", + ) @property def length(self): @@ -1022,8 +1059,7 @@ def default(self): return self._default def get(self, index, default=None): - """ Gets an element from an empty Array. - """ + """Gets an element from an empty Array.""" index = self.cast_index(index) if default is None: default = self._default @@ -1053,10 +1089,13 @@ def underlying_variable(self): array = array.array return array + class ArrayOperation(Array, Operation, abstract=True): """ It's an operation that results in an Array""" + pass + def get_items(array): if isinstance(array, ArrayStore): yield from get_items_array_store(array) @@ -1066,10 +1105,11 @@ def get_items(array): yield from get_items_array_constant(array) return + def get_items_array_slice(array): assert isinstance(array, ArraySlice) for offset, value in get_items(array.array): - yield offset+array.offset, value + yield offset + array.offset, value def get_items_array_store(array): @@ -1079,15 +1119,18 @@ def get_items_array_store(array): array = array.array yield from get_items(array) + def get_items_array_constant(array): assert isinstance(array, ArrayConstant) for index, value in enumerate(array.value): yield index, value + def get_items_array_variable(array): assert isinstance(array, ArrayVariable) raise GeneratorExit + class ArrayStore(ArrayOperation): __xslots__: Tuple[str, ...] = ( "_written#v", @@ -1096,7 +1139,6 @@ class ArrayStore(ArrayOperation): "_default#v", ) - @property def concrete_cache(self): if self._concrete_cache is not None: @@ -1141,8 +1183,8 @@ def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): self._length = array.length self._default = array.default - #recreate and reuse cache - #if isinstance(index, Constant) and isinstance(array, ArrayStore) and array._concrete_cache is not None: + # recreate and reuse cache + # if isinstance(index, Constant) and isinstance(array, ArrayStore) and array._concrete_cache is not None: # self._concrete_cache = dict(array._concrete_cache) # self._concrete_cache[index.value] = value @@ -1167,7 +1209,6 @@ def index_size(self): def value_size(self): return self.value.size - def __hash__(self): return object.__hash__(self) @@ -1189,7 +1230,7 @@ def value(self): def select(self, index): - """ Gets an element from the Array. + """Gets an element from the Array. If the element was not previously the default is used. """ index = simplify(self.cast_index(index)) @@ -1197,9 +1238,7 @@ def select(self, index): # Emulate list[-1] has_length = self.length is not None if has_length: - index = simplify( - BitvecITE(index < 0, self.length + index, index) - ) + index = simplify(BitvecITE(index < 0, self.length + index, index)) if isinstance(index, Constant): if has_length and index.value >= self.length: @@ -1247,14 +1286,12 @@ def store(self, index, value): return new_array - class ArraySlice(ArrayOperation): - """ Provides a projection of an underlying array. - Lets you slice an array without copying it. - (It needs to be simplified out before translating it smtlib) + """Provides a projection of an underlying array. + Lets you slice an array without copying it. + (It needs to be simplified out before translating it smtlib) """ - def __init__(self, array: "Array", offset: int, size: int, **kwargs): if not isinstance(array, Array): raise ValueError("Array expected") @@ -1310,6 +1347,7 @@ def store(self, index, value): def default(self): return self.array.default + class MutableArray: """ Arrayproxy is a layer on top of an array that provides mutability and some @@ -1396,7 +1434,6 @@ def __setitem__(self, index, value): assert self._array is not None return self - def write_BE(self, address, value, size): self._array = self._array.write_BE(address, value, size) assert self._array is not None @@ -1412,7 +1449,7 @@ def write(self, offset, buf): return self def read(self, offset, size): - return MutableArray(self._array[offset: offset + size]) + return MutableArray(self._array[offset : offset + size]) def __eq__(self, other): return self.array == other @@ -1430,6 +1467,7 @@ def __radd__(self, other): other = other.array return MutableArray(other + self.array) + class ArraySelect(BitvecOperation): __xslots__ = BitvecOperation.__xslots__ @@ -1458,14 +1496,13 @@ def extend(self): return self.size - self.operands[0].size - class BitvecZeroExtend(BitvecOperation): def __init__(self, size: int, operand: Bitvec, *args, **kwargs): super().__init__(size=size, operands=(operand,), **kwargs) @property def extend(self): - return self.size - self.operands[0].size + return self.size - self.operands[0].size class BitvecExtract(BitvecOperation): @@ -1490,7 +1527,6 @@ def end(self): class BitvecConcat(BitvecOperation): - def __init__(self, operands: Tuple[Bitvec, ...], **kwargs): size = sum(x.size for x in operands) super().__init__(size=size, operands=operands, **kwargs) @@ -1507,7 +1543,9 @@ def __init__( **kwargs, ): - super().__init__(size=true_value.size, operands=(condition, true_value, false_value), **kwargs) + super().__init__( + size=true_value.size, operands=(condition, true_value, false_value), **kwargs + ) @property def condition(self): @@ -1525,13 +1563,13 @@ def false_value(self): # auxiliar functions maybe move to operators def issymbolic(value) -> bool: """ - Helper to determine whether an object is symbolic (e.g checking - if data read from memory is symbolic) + Helper to determine whether an object is symbolic (e.g checking + if data read from memory is symbolic) - :param object value: object to check - :return: whether `value` is symbolic - :rtype: bool - """ + :param object value: object to check + :return: whether `value` is symbolic + :rtype: bool + """ return isinstance(value, (Expression, MutableArray)) @@ -1594,4 +1632,3 @@ def taint_with(arg, *taints, value_size=256, index_size=256): from .visitors import simplify - diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 74a81972c..8150f3f42 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -168,7 +168,7 @@ def minmax(self, constraints, x, iters=10000): class SmtlibProc: def __init__(self, command: str, debug: bool = False): - """ Single smtlib interactive process + """Single smtlib interactive process :param command: the shell command to execute :param debug: log all messaging @@ -214,7 +214,7 @@ def __readline_and_count(self): assert self._proc.stdout buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error - #print (">",buf) + # print (">",buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -227,7 +227,7 @@ def send(self, cmd: str) -> None: :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ - #print ("<",cmd) + # print ("<",cmd) if self._debug: logger.debug(">%s", cmd) self._proc.stdout.flush() # type: ignore @@ -422,20 +422,23 @@ def get_model(self, constraints: ConstraintSet): value.append(variable_i) value = bytes(value) else: - #Only works if we know the max index of the arrray + # Only works if we know the max index of the arrray used_indexes = map(self.__getvalue_bv, variable.written) valued = {} for i in used_indexes: valued[i] = self.__getvalue_bv(variable[i]) + class A: def __init__(self, d, default): self._d = d self._default = default + def __getitem__(self, index): return self._d.get(index, self._default) - value = A(valued,variable.default) + + value = A(valued, variable.default) except Exception as e: - value = None #We failed to get the model from the solver + value = None # We failed to get the model from the solver model[variable.name] = value return model @@ -622,7 +625,7 @@ def _optimize_fancy(self, constraints: ConstraintSet, x: Bitvec, goal: str, max_ temp_cs.add(operation(X, aux)) self._reset(temp_cs.to_string()) - #self._assert(operation(X, aux)) + # self._assert(operation(X, aux)) self._smtlib.send("(%s %s)" % (goal, aux.name)) self._smtlib.send("(check-sat)") _status = self._smtlib.recv() @@ -635,13 +638,13 @@ def get_value(self, constraints: ConstraintSet, *expressions): Ask the solver for one possible result of given expressions using given set of constraints. """ - self._cache = getattr(self, '_cache', {}) + self._cache = getattr(self, "_cache", {}) model = self.get_model(constraints) ####################33 values = [] start = time.time() - #with constraints.related_to(*expressions) as temp_cs: + # with constraints.related_to(*expressions) as temp_cs: with constraints as temp_cs: for expression in expressions: bucket = self._cache.setdefault(hash(constraints), {}) @@ -707,8 +710,6 @@ def get_value(self, constraints: ConstraintSet, *expressions): if time.time() - start > consts.timeout: raise SolverError("Timeout") - - if len(expressions) == 1: return values[0] else: diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index c2f9ce460..ded5afd12 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -10,24 +10,26 @@ logger = logging.getLogger(__name__) + class VisitorException(Exception): pass + class Visitor: - """ Class/Type Visitor + """Class/Type Visitor - Inherit your class visitor from this one and get called on a different - visiting function for each type of expression. It will call the first - implemented method for the __mro__ class order. - For example for a BitvecAdd expression this will try - visit_BitvecAdd() if not defined(*) then it will try with - visit_BitvecOperation() if not defined(*) then it will try with - visit_Bitvec() if not defined(*) then it will try with - visit_Expression() + Inherit your class visitor from this one and get called on a different + visiting function for each type of expression. It will call the first + implemented method for the __mro__ class order. + For example for a BitvecAdd expression this will try + visit_BitvecAdd() if not defined(*) then it will try with + visit_BitvecOperation() if not defined(*) then it will try with + visit_Bitvec() if not defined(*) then it will try with + visit_Expression() - (*) Or it is defined and it returns None for the node. - You can overload the visiting method to react to different semantic - aspects of an Exrpession. + (*) Or it is defined and it returns None for the node. + You can overload the visiting method to react to different semantic + aspects of an Exrpession. """ @@ -120,15 +122,15 @@ def _method(self, expression, *operands): return value return self._rebuild(expression, operands) - def _changed(self, expression:Expression, operands): + def _changed(self, expression: Expression, operands): return any(x is not y for x, y in zip(expression.operands, operands)) - def _rebuild(self, expression:Operation, operands): - """ Default operation used when no visiting method was successful for - this expression. If he operands have changed this reubild the curren expression - with the new operands. + def _rebuild(self, expression: Operation, operands): + """Default operation used when no visiting method was successful for + this expression. If he operands have changed this reubild the curren expression + with the new operands. - Assumes the stack is used for Expresisons + Assumes the stack is used for Expresisons """ if self._changed(expression, operands): aux = copy.copy(expression) @@ -141,10 +143,10 @@ def _rebuild(self, expression:Operation, operands): class Translator(Visitor): - """ Simple visitor to translate an expression into something else - """ + """Simple visitor to translate an expression into something else""" + def _rebuild(self, expression, operands): - """ The stack holds the translation of the expression. + """The stack holds the translation of the expression. There is no default action :param expression: Current expression @@ -155,9 +157,10 @@ def _rebuild(self, expression, operands): class GetDeclarations(Translator): - """ Simple visitor to collect all variables in an expression or set of - expressions + """Simple visitor to collect all variables in an expression or set of + expressions """ + def _rebuild(self, expression, operands): return expression @@ -174,9 +177,10 @@ def result(self): class GetDepth(Translator): - """ Simple visitor to collect all variables in an expression or set of - expressions + """Simple visitor to collect all variables in an expression or set of + expressions """ + def _rebuild(self, expression, operands): return expression @@ -245,7 +249,6 @@ def visit_Constant(self, expression): self._print(expression.value) return True - def visit_Variable(self, expression): self._print(expression.name) return True @@ -395,7 +398,6 @@ def constant_folder(expression): class ArithmeticSimplifier(Visitor): - @staticmethod def _same_constant(a, b): return isinstance(a, Constant) and isinstance(b, Constant) and a.value == b.value or a is b @@ -463,9 +465,9 @@ def visit_BoolNot(self, expression, *operands): return operands[0].operands[0] def visit_BoolEqual(self, expression, *operands): - """ (EQ, ITE(cond, constant1, constant2), constant1) -> cond - (EQ, ITE(cond, constant1, constant2), constant2) -> NOT cond - (EQ (extract a, b, c) (extract a, b, c)) + """(EQ, ITE(cond, constant1, constant2), constant1) -> cond + (EQ, ITE(cond, constant1, constant2), constant2) -> NOT cond + (EQ (extract a, b, c) (extract a, b, c)) """ if isinstance(operands[0], BitvecITE) and isinstance(operands[1], Constant): if isinstance(operands[0].operands[1], Constant) and isinstance( @@ -523,9 +525,9 @@ def visit_BitvecITE(self, expression, *operands): return BitvecITE(*operands, taint=expression.taint) def visit_BitvecConcat(self, expression, *operands): - """ concat( extract(k1, 0, a), extract(sizeof(a)-k1, k1, a)) ==> a - concat( extract(k1, beg, a), extract(end, k1, a)) ==> extract(beg, end, a) - concat( x , extract(k1, beg, a), extract(end, k1, a), z) ==> concat( x , extract(k1, beg, a), extract(end, k1, a), z) + """concat( extract(k1, 0, a), extract(sizeof(a)-k1, k1, a)) ==> a + concat( extract(k1, beg, a), extract(end, k1, a)) ==> extract(beg, end, a) + concat( x , extract(k1, beg, a), extract(end, k1, a), z) ==> concat( x , extract(k1, beg, a), extract(end, k1, a), z) """ if len(operands) == 1: return operands[0] @@ -589,9 +591,9 @@ def visit_BitvecConcat(self, expression, *operands): return value def visit_BitvecExtract(self, expression, *operands): - """ extract(sizeof(a), 0)(a) ==> a - extract(16, 0)( concat(a,b,c,d) ) => concat(c, d) - extract(m,M)(and/or/xor a b ) => and/or/xor((extract(m,M) a) (extract(m,M) a) + """extract(sizeof(a), 0)(a) ==> a + extract(16, 0)( concat(a,b,c,d) ) => concat(c, d) + extract(m,M)(and/or/xor a b ) => and/or/xor((extract(m,M) a) (extract(m,M) a) """ op = operands[0] begining = expression.begining @@ -638,8 +640,8 @@ def visit_BitvecExtract(self, expression, *operands): ) def visit_BitvecAdd(self, expression, *operands): - """ a + 0 ==> a - 0 + a ==> a + """a + 0 ==> a + 0 + a ==> a """ left = operands[0] right = operands[1] @@ -651,9 +653,9 @@ def visit_BitvecAdd(self, expression, *operands): return right def visit_BitvecSub(self, expression, *operands): - """ a - 0 ==> 0 - (a + b) - b ==> a - (b + a) - b ==> a + """a - 0 ==> 0 + (a + b) - b ==> a + (b + a) - b ==> a """ left = operands[0] right = operands[1] @@ -676,10 +678,10 @@ def visit_BitvecSub(self, expression, *operands): ) def visit_BitvecOr(self, expression, *operands): - """ a | 0 => a - 0 | a => a - 0xffffffff & a => 0xffffffff - a & 0xffffffff => 0xffffffff + """a | 0 => a + 0 | a => a + 0xffffffff & a => 0xffffffff + a & 0xffffffff => 0xffffffff """ left = operands[0] @@ -698,11 +700,11 @@ def visit_BitvecOr(self, expression, *operands): return BitvecOr(right, left, taint=expression.taint) def visit_BitvecAnd(self, expression, *operands): - """ ct & x => x & ct move constants to the right - a & 0 => 0 remove zero - a & 0xffffffff => a remove full mask - (b & ct2) & ct => b & (ct&ct2) associative property - (a & (b | c) => a&b | a&c distribute over | + """ct & x => x & ct move constants to the right + a & 0 => 0 remove zero + a & 0xffffffff => a remove full mask + (b & ct2) & ct => b & (ct&ct2) associative property + (a & (b | c) => a&b | a&c distribute over | """ left = operands[0] right = operands[1] @@ -725,8 +727,8 @@ def visit_BitvecAnd(self, expression, *operands): return BitvecAnd(right, left, taint=expression.taint) def visit_BitvecShiftLeft(self, expression, *operands): - """ a << 0 => a remove zero - a << ct => 0 if ct > sizeof(a) remove big constant shift + """a << 0 => a remove zero + a << ct => 0 if ct > sizeof(a) remove big constant shift """ left = operands[0] right = operands[1] @@ -737,8 +739,8 @@ def visit_BitvecShiftLeft(self, expression, *operands): return left def visit_ArraySelect(self, expression, *operands): - """ ArraySelect (ArrayStore((ArrayStore(x0,v0) ...),xn, vn), x0) - -> v0 + """ArraySelect (ArrayStore((ArrayStore(x0,v0) ...),xn, vn), x0) + -> v0 """ return None arr, index = operands @@ -789,8 +791,8 @@ def arithmetic_simplify(expression): def to_constant(expression): """ - Iff the expression can be simplified to a Constant get the actual concrete value. - This discards/ignore any taint + Iff the expression can be simplified to a Constant get the actual concrete value. + This discards/ignore any taint """ if isinstance(expression, MutableArray): expression = expression.array @@ -819,8 +821,7 @@ def simplify(expression): class TranslatorSmtlib(Translator): - """ Simple visitor to translate an expression to its smtlib representation - """ + """Simple visitor to translate an expression to its smtlib representation""" unique = 0 @@ -832,7 +833,6 @@ def __init__(self, use_bindings=False, *args, **kw): self._bindings = [] self._variables = set() - def _add_binding(self, expression, smtlib): if not self.use_bindings or len(smtlib) <= 10: return smtlib @@ -932,7 +932,7 @@ def result(self): return output def declarations(self): - result = '' + result = "" for exp in self._variables: if isinstance(exp, Bitvec): result += f"(declare-fun {exp.name} () (_ BitVec {exp.size}))\n" From 570b1e4063001e6dfe8ffceebc9b2209c23322a3 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 29 Sep 2020 12:17:38 -0300 Subject: [PATCH 081/126] Asorted expression fixes and refactors --- manticore/core/smtlib/expression.py | 3 ++- manticore/platforms/evm.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index fc9ac4fe0..750e1ebc0 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1048,12 +1048,13 @@ def index_size(self): def value_size(self): return self._value_size + ''' @property def index_max(self): if self._length is None: return None return self._length - 1 - +''' @property def default(self): return self._default diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 8bf332d1c..6ad133b5b 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -3220,7 +3220,7 @@ def start_transaction( assert self._pending_transaction is None, "Already started tx" assert caller is not None if issymbolic(data ): - assert data.index_max is not None + assert data.length is not None assert data.value_size == 8 self._pending_transaction = PendingTransaction( From 7168829ac1ebb2ad8330fed18dc3847b54aced4d Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 12:16:03 -0300 Subject: [PATCH 082/126] blkn --- manticore/core/smtlib/expression.py | 5 +- manticore/core/state.py | 30 +-- manticore/ethereum/manticore.py | 286 ++++++++++++++-------------- manticore/platforms/evm.py | 100 ++++++---- 4 files changed, 226 insertions(+), 195 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 750e1ebc0..03ef816dc 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1048,13 +1048,14 @@ def index_size(self): def value_size(self): return self._value_size - ''' + """ @property def index_max(self): if self._length is None: return None return self._length - 1 -''' +""" + @property def default(self): return self._default diff --git a/manticore/core/state.py b/manticore/core/state.py index 03d8c218c..aec999826 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -22,8 +22,8 @@ def __init__(self, message, testcase=False): class AbandonState(TerminateState): - """ Exception returned for abandoned states when - execution is finished + """Exception returned for abandoned states when + execution is finished """ def __init__(self, message="Abandoned state"): @@ -31,12 +31,12 @@ def __init__(self, message="Abandoned state"): class Concretize(StateException): - """ Base class for all exceptions that trigger the concretization - of a symbolic expression + """Base class for all exceptions that trigger the concretization + of a symbolic expression - This will fork the state using a pre-set concretization policy - Optional `setstate` function set the state to the actual concretized value. - #Fixme Doc. + This will fork the state using a pre-set concretization policy + Optional `setstate` function set the state to the actual concretized value. + #Fixme Doc. """ @@ -57,8 +57,8 @@ def __init__(self, message, expression, setstate=None, policy=None, **kwargs): class SerializeState(Concretize): - """ Allows the user to save a copy of the current state somewhere on the - disk so that analysis can later be resumed from this point. + """Allows the user to save a copy of the current state somewhere on the + disk so that analysis can later be resumed from this point. """ def _setstate(self, state, _value): @@ -77,12 +77,12 @@ def __init__(self, filename, **kwargs): class ForkState(Concretize): - """ Specialized concretization class for Bool expressions. - It tries True and False as concrete solutions. / + """Specialized concretization class for Bool expressions. + It tries True and False as concrete solutions. / - Note: as setstate is None the concrete value is not written back - to the state. So the expression could still by symbolic(but constrained) - in forked states. + Note: as setstate is None the concrete value is not written back + to the state. So the expression could still by symbolic(but constrained) + in forked states. """ def __init__(self, message, expression: Bool, **kwargs): @@ -326,7 +326,7 @@ def _solver(self): def migrate_expression(self, expression): if isinstance(expression, MutableArray): - expression=expression.array + expression = expression.array if not issymbolic(expression): return expression migration_map = self.context.get("migration_map") diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index ad2c4cd3a..365078bb7 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -111,43 +111,43 @@ def calculate_coverage(runtime_bytecode, seen): class ManticoreEVM(ManticoreBase): - """ Manticore EVM manager - - Usage Ex:: - - from manticore.ethereum import ManticoreEVM, ABI - m = ManticoreEVM() - #And now make the contract account to analyze - source_code = ''' - pragma solidity ^0.4.15; - contract AnInt { - uint private i=0; - function set(uint value){ - i=value - } + """Manticore EVM manager + + Usage Ex:: + + from manticore.ethereum import ManticoreEVM, ABI + m = ManticoreEVM() + #And now make the contract account to analyze + source_code = ''' + pragma solidity ^0.4.15; + contract AnInt { + uint private i=0; + function set(uint value){ + i=value } - ''' - #Initialize user and contracts - user_account = m.create_account(balance=1000) - contract_account = m.solidity_create_contract(source_code, owner=user_account, balance=0) - contract_account.set(12345, value=100) - - m.finalize() + } + ''' + #Initialize user and contracts + user_account = m.create_account(balance=1000) + contract_account = m.solidity_create_contract(source_code, owner=user_account, balance=0) + contract_account.set(12345, value=100) + + m.finalize() """ def make_symbolic_buffer(self, size, name=None, avoid_collisions=False, default=None): - """ Creates a symbolic buffer of size bytes to be used in transactions. - You can operate on it normally and add constraints to manticore.constraints - via manticore.constrain(constraint_expression) - - Example use:: - - symbolic_data = m.make_symbolic_buffer(320) - m.constrain(symbolic_data[0] == 0x65) - m.transaction(caller=attacker_account, - address=contract_account, - data=symbolic_data, - value=100000 ) + """Creates a symbolic buffer of size bytes to be used in transactions. + You can operate on it normally and add constraints to manticore.constraints + via manticore.constrain(constraint_expression) + + Example use:: + + symbolic_data = m.make_symbolic_buffer(320) + m.constrain(symbolic_data[0] == 0x65) + m.transaction(caller=attacker_account, + address=contract_account, + data=symbolic_data, + value=100000 ) """ if name is None: name = "TXBUFFER" @@ -160,23 +160,23 @@ def make_symbolic_buffer(self, size, name=None, avoid_collisions=False, default= value_size=8, taint=frozenset(), avoid_collisions=avoid_collisions, - default=default + default=default, ) def make_symbolic_value(self, nbits=256, name=None): - """ Creates a symbolic value, normally a uint256, to be used in transactions. - You can operate on it normally and add constraints to manticore.constraints - via manticore.constrain(constraint_expression) + """Creates a symbolic value, normally a uint256, to be used in transactions. + You can operate on it normally and add constraints to manticore.constraints + via manticore.constrain(constraint_expression) - Example use:: + Example use:: - symbolic_value = m.make_symbolic_value() - m.constrain(symbolic_value > 100) - m.constrain(symbolic_value < 1000) - m.transaction(caller=attacker_account, - address=contract_account, - data=data, - value=symbolic_value ) + symbolic_value = m.make_symbolic_value() + m.constrain(symbolic_value > 100) + m.constrain(symbolic_value < 1000) + m.transaction(caller=attacker_account, + address=contract_account, + data=data, + value=symbolic_value ) """ avoid_collisions = False @@ -319,16 +319,16 @@ def _compile_through_crytic_compile(filename, contract_name, libraries, crytic_c @staticmethod def _compile(source_code, contract_name, libraries=None, crytic_compile_args=None): - """ Compile a Solidity contract, used internally - - :param source_code: solidity source - :type source_code: string (filename, directory, etherscan address) or a file handle - :param contract_name: a string with the name of the contract to analyze - :param libraries: an itemizable of pairs (library_name, address) - :param crytic_compile_args: crytic compile options (https://github.com/crytic/crytic-compile/wiki/Configuration) - :type crytic_compile_args: dict - :return: name, source_code, bytecode, srcmap, srcmap_runtime, hashes - :return: name, source_code, bytecode, runtime, srcmap, srcmap_runtime, hashes, abi, warnings + """Compile a Solidity contract, used internally + + :param source_code: solidity source + :type source_code: string (filename, directory, etherscan address) or a file handle + :param contract_name: a string with the name of the contract to analyze + :param libraries: an itemizable of pairs (library_name, address) + :param crytic_compile_args: crytic compile options (https://github.com/crytic/crytic-compile/wiki/Configuration) + :type crytic_compile_args: dict + :return: name, source_code, bytecode, srcmap, srcmap_runtime, hashes + :return: name, source_code, bytecode, runtime, srcmap, srcmap_runtime, hashes, abi, warnings """ crytic_compile_args = dict() if crytic_compile_args is None else crytic_compile_args @@ -486,7 +486,7 @@ def human_transactions(self, state_id=None): def make_symbolic_arguments(self, types): """ - Build a reasonable set of symbolic arguments matching the types list + Build a reasonable set of symbolic arguments matching the types list """ from . import abitypes @@ -539,24 +539,24 @@ def solidity_create_contract( gas=None, compile_args=None, ): - """ Creates a solidity contract and library dependencies - - :param source_code: solidity source code - :type source_code: string (filename, directory, etherscan address) or a file handle - :param owner: owner account (will be default caller in any transactions) - :type owner: int or EVMAccount - :param contract_name: Name of the contract to analyze (optional if there is a single one in the source code) - :type contract_name: str - :param balance: balance to be transferred on creation - :type balance: int or BitvecVariable - :param address: the address for the new contract (optional) - :type address: int or EVMAccount - :param tuple args: constructor arguments - :param compile_args: crytic compile options #FIXME(https://github.com/crytic/crytic-compile/wiki/Configuration) - :type compile_args: dict - :param gas: gas budget for each contract creation needed (may be more than one if several related contracts defined in the solidity source) - :type gas: int - :rtype: EVMAccount + """Creates a solidity contract and library dependencies + + :param source_code: solidity source code + :type source_code: string (filename, directory, etherscan address) or a file handle + :param owner: owner account (will be default caller in any transactions) + :type owner: int or EVMAccount + :param contract_name: Name of the contract to analyze (optional if there is a single one in the source code) + :type contract_name: str + :param balance: balance to be transferred on creation + :type balance: int or BitvecVariable + :param address: the address for the new contract (optional) + :type address: int or EVMAccount + :param tuple args: constructor arguments + :param compile_args: crytic compile options #FIXME(https://github.com/crytic/crytic-compile/wiki/Configuration) + :type compile_args: dict + :param gas: gas budget for each contract creation needed (may be more than one if several related contracts defined in the solidity source) + :type gas: int + :rtype: EVMAccount """ if compile_args is None: compile_args = dict() @@ -596,11 +596,13 @@ def solidity_create_contract( for state in self.ready_states: world = state.platform - if not state.can_be_true(Operators.UGE(world.get_balance(owner.address), balance)): - #if not SelectedSolver.instance().can_be_true( - # self.constraints, - # Operators.UGE(world.get_balance(owner.address), balance), - #): + if not state.can_be_true( + Operators.UGE(world.get_balance(owner.address), balance) + ): + # if not SelectedSolver.instance().can_be_true( + # self.constraints, + # Operators.UGE(world.get_balance(owner.address), balance), + # ): raise EthereumError( f"Can't create solidity contract with balance ({balance}) " f"because the owner account ({owner}) has insufficient balance." @@ -662,17 +664,17 @@ def get_nonce(self, address): return next(iter(nonces)) def create_contract(self, owner, balance=0, address=None, init=None, name=None, gas=None): - """ Creates a contract - - :param owner: owner account (will be default caller in any transactions) - :type owner: int or EVMAccount - :param balance: balance to be transferred on creation - :type balance: int or BitvecVariable - :param int address: the address for the new contract (optional) - :param str init: initializing evm bytecode and arguments - :param str name: a unique name for reference - :param gas: gas budget for the creation/initialization of the contract - :rtype: EVMAccount + """Creates a contract + + :param owner: owner account (will be default caller in any transactions) + :type owner: int or EVMAccount + :param balance: balance to be transferred on creation + :type balance: int or BitvecVariable + :param int address: the address for the new contract (optional) + :param str init: initializing evm bytecode and arguments + :param str name: a unique name for reference + :param gas: gas budget for the creation/initialization of the contract + :rtype: EVMAccount """ if not self.count_ready_states(): raise NoAliveStates @@ -748,18 +750,18 @@ def end_block(self): world.end_block() def transaction(self, caller, address, value, data, gas=None, price=1): - """ Issue a symbolic transaction in all running states - - :param caller: the address of the account sending the transaction - :type caller: int or EVMAccount - :param address: the address of the contract to call - :type address: int or EVMAccount - :param value: balance to be transfered on creation - :type value: int or BitvecVariable - :param data: initial data - :param gas: gas budget - :param price: gas unit price - :raises NoAliveStates: if there are no alive states to execute + """Issue a symbolic transaction in all running states + + :param caller: the address of the account sending the transaction + :type caller: int or EVMAccount + :param address: the address of the contract to call + :type address: int or EVMAccount + :param value: balance to be transfered on creation + :type value: int or BitvecVariable + :param data: initial data + :param gas: gas budget + :param price: gas unit price + :raises NoAliveStates: if there are no alive states to execute """ if isinstance(data, MutableArray): data = data.array @@ -768,16 +770,16 @@ def transaction(self, caller, address, value, data, gas=None, price=1): ) def create_account(self, balance=0, address=None, code=None, name=None, nonce=None): - """ Low level creates an account. This won't generate a transaction. - - :param balance: balance to be set on creation (optional) - :type balance: int or BitvecVariable - :param address: the address for the new account (optional) - :type address: int - :param code: the runtime code for the new account (None means normal account), str or bytes (optional) - :param nonce: force a specific nonce - :param name: a global account name eg. for use as reference in the reports (optional) - :return: an EVMAccount + """Low level creates an account. This won't generate a transaction. + + :param balance: balance to be set on creation (optional) + :type balance: int or BitvecVariable + :param address: the address for the new account (optional) + :type address: int + :param code: the runtime code for the new account (None means normal account), str or bytes (optional) + :param nonce: force a specific nonce + :param name: a global account name eg. for use as reference in the reports (optional) + :return: an EVMAccount """ # Need at least one state where to apply this if not self.count_ready_states(): @@ -871,17 +873,17 @@ def _migrate_tx_expressions(self, state, caller, address, value, data, gas, pric return caller, address, value, data, gas, price def _transaction(self, sort, caller, value=0, address=None, data=None, gas=None, price=1): - """ Initiates a transaction - - :param caller: caller account - :type caller: int or EVMAccount - :param int address: the address for the transaction (optional) - :param value: value to be transferred - :param price: the price of gas for this transaction. - :type value: int or BitvecVariable - :param str data: initializing evm bytecode and arguments or transaction call data - :param gas: gas budget for current transaction - :rtype: EVMAccount + """Initiates a transaction + + :param caller: caller account + :type caller: int or EVMAccount + :param int address: the address for the transaction (optional) + :param value: value to be transferred + :param price: the price of gas for this transaction. + :type value: int or BitvecVariable + :param str data: initializing evm bytecode and arguments or transaction call data + :param gas: gas budget for current transaction + :rtype: EVMAccount """ if gas is None: gas = consts.defaultgas @@ -988,13 +990,13 @@ def preconstraint_for_call_transaction( value: Optional[Union[int, Expression]] = None, contract_metadata: Optional[SolidityMetadata] = None, ): - """ Returns a constraint that excludes combinations of value and data that would cause an exception in the EVM - contract dispatcher. + """Returns a constraint that excludes combinations of value and data that would cause an exception in the EVM + contract dispatcher. - :param address: address of the contract to call - :param value: balance to be transferred (optional) - :param data: symbolic transaction data - :param contract_metadata: SolidityMetadata for the contract (optional) + :param address: address of the contract to call + :param value: balance to be transferred (optional) + :param data: symbolic transaction data + :param contract_metadata: SolidityMetadata for the contract (optional) """ if isinstance(address, EVMAccount): address = int(address) @@ -1262,8 +1264,8 @@ def _on_unsound_symbolication(self, state, func, data, result): result.append(value) def fix_unsound_symbolication_fake(self, state): - """ This method goes through all the applied symbolic functions and tries - to find a concrete matching set of pairs + """This method goes through all the applied symbolic functions and tries + to find a concrete matching set of pairs """ def make_cond(state, table): @@ -1281,8 +1283,8 @@ def make_cond(state, table): return state.can_be_true(True) def fix_unsound_symbolication_sound(self, state): - """ This method goes through all the applied symbolic functions and tries - to find a concrete matching set of pairs + """This method goes through all the applied symbolic functions and tries + to find a concrete matching set of pairs """ def concretize_known_pairs(state, symbolic_pairs, known_pairs): @@ -1299,7 +1301,7 @@ def concretize_known_pairs(state, symbolic_pairs, known_pairs): return def match(state, func, symbolic_pairs, concrete_pairs, start=None): - """ Tries to find a concrete match for the symbolic pairs. It uses + """Tries to find a concrete match for the symbolic pairs. It uses concrete_pairs (and potentially extends it with solved pairs) until a matching set of concrete pairs is found, or fail. @@ -1416,9 +1418,9 @@ def fix_unsound_symbolication(self, state): return state.context["soundcheck"] def _terminate_state_callback(self, state, e): - """ INTERNAL USE - Every time a state finishes executing the last transaction, we save it in - our private list + """INTERNAL USE + Every time a state finishes executing the last transaction, we save it in + our private list """ if isinstance(e, AbandonState): # do nothing @@ -1475,8 +1477,8 @@ def _did_evm_execute_instruction_callback(self, state, instruction, arguments, r ) def get_metadata(self, address) -> Optional[SolidityMetadata]: - """ Gets the solidity metadata for address. - This is available only if address is a contract created from solidity + """Gets the solidity metadata for address. + This is available only if address is a contract created from solidity """ return self.metadata.get(address) @@ -1841,9 +1843,9 @@ def worker_finalize(q): self.remove_all() def global_coverage(self, account): - """ Returns code coverage for the contract on `account_address`. - This sums up all the visited code lines from any of the explored - states. + """Returns code coverage for the contract on `account_address`. + This sums up all the visited code lines from any of the explored + states. """ account_address = int(account) runtime_bytecode = None diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 6ad133b5b..d52292a44 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -104,7 +104,9 @@ def globalfakesha3(data): description="Max calldata size to explore in each CALLDATACOPY. Iff size in a calldata related instruction are symbolic it will be constrained to be less than this constant. -1 means free(only use when gas is being tracked)", ) consts.add( - "ignore_balance", default=False, description="Do not try to solve symbolic balances", + "ignore_balance", + default=False, + description="Do not try to solve symbolic balances", ) @@ -181,7 +183,24 @@ def concretize(self, state, constrain=False): :param state: a manticore state :param bool constrain: If True, constrain expr to concretized value """ - conc_caller, conc_address, conc_value, conc_gas, conc_data, conc_return_data,conc_used_gas = state.solve_one_n(self.caller,self.address,self.value,self.gas,self.data,self.return_data,self.used_gas,constrain=constrain) + ( + conc_caller, + conc_address, + conc_value, + conc_gas, + conc_data, + conc_return_data, + conc_used_gas, + ) = state.solve_one_n( + self.caller, + self.address, + self.value, + self.gas, + self.data, + self.return_data, + self.used_gas, + constrain=constrain, + ) return Transaction( self.sort, conc_address, @@ -486,7 +505,7 @@ def __init__(self, result, data=None): if isinstance(data, MutableArray): data = data.array if not isinstance(data, (type(None), Array, bytes)): - raise EVMException("Invalid end transaction data type") + raise EVMException("Invalid end transaction data type") self.result = result self.data = data @@ -771,13 +790,15 @@ def extend_with_zeroes(b): # raise EVMException("Need code") self._constraints = constraints # Uninitialized values in memory are 0 by spec - self.memory = MutableArray(constraints.new_array( - index_size=256, - value_size=8, - name=f"EMPTY_MEMORY_{address:x}", - avoid_collisions=True, - default=0, - )) + self.memory = MutableArray( + constraints.new_array( + index_size=256, + value_size=8, + name=f"EMPTY_MEMORY_{address:x}", + avoid_collisions=True, + default=0, + ) + ) self.address = address self.caller = ( caller # address of the account that is directly responsible for this execution @@ -849,7 +870,7 @@ def constraints(self): @constraints.setter def constraints(self, constraints): self._constraints = constraints - #self.memory.constraints = constraints + # self.memory.constraints = constraints @property def gas(self): @@ -978,7 +999,7 @@ def disassemble(self): def PC(self): return self.pc - def _getcode(self, pc:int = 0, zerotail:bool = True): + def _getcode(self, pc: int = 0, zerotail: bool = True): bytecode = self.bytecode for pc_i in range(pc, len(bytecode)): c = bytecode[pc_i] @@ -990,7 +1011,6 @@ def _getcode(self, pc:int = 0, zerotail:bool = True): while True: yield 0 # STOP opcode - @property def instruction(self): """ @@ -1046,7 +1066,7 @@ def _push(self, value): if isinstance(value, int): value = value & TT256M1 - #value = simplify(value) + # value = simplify(value) if isinstance(value, Constant) and not value.taint: value = value.value @@ -1171,14 +1191,13 @@ def _push_results(self, instruction, result): assert result is None def _calculate_gas(self, *arguments): - start= time.time() + start = time.time() current = self.instruction implementation = getattr(self, f"{current.semantics}_gas", None) if implementation is None: return current.fee return current.fee + implementation(*arguments) - def _handler(self, *arguments): current = self.instruction implementation = getattr(self, current.semantics, None) @@ -1410,7 +1429,7 @@ def write_buffer(self, offset, data): def _load(self, offset, size=1): value = self.memory.read_BE(offset, size) - #value = simplify(value) + # value = simplify(value) if isinstance(value, Constant) and not value.taint: value = value.value self._publish("did_evm_read_memory", offset, value, size) @@ -1634,9 +1653,9 @@ def SHA3_gas(self, start, size): @concretized_args(size="ALL") def SHA3(self, start, size): """Compute Keccak-256 hash - If the size is symbolic the potential solutions will be sampled as - defined by the default policy and the analysis will be forked. - The `size` can be considered concrete in this handler. + If the size is symbolic the potential solutions will be sampled as + defined by the default policy and the analysis will be forked. + The `size` can be considered concrete in this handler. """ data = self.read_buffer(start, size) @@ -1686,7 +1705,12 @@ def CALLDATALOAD(self, offset): bytes = [] for i in range(32): try: - c=Operators.ITEBV(8, Operators.ULT(self.safe_add(offset, i), data_length), self.data[offset + i], 0, ) + c = Operators.ITEBV( + 8, + Operators.ULT(self.safe_add(offset, i), data_length), + self.data[offset + i], + 0, + ) c = simplify(c) except IndexError: # offset + i is concrete and outside data @@ -1695,8 +1719,8 @@ def CALLDATALOAD(self, offset): return Operators.CONCAT(256, *bytes) def _use_calldata(self, offset, size): - """ To improve reporting we maintain how much of the calldata is actually - used. CALLDATACOPY and CALLDATA LOAD update this limit accordingly """ + """To improve reporting we maintain how much of the calldata is actually + used. CALLDATACOPY and CALLDATA LOAD update this limit accordingly""" self._used_calldata_size = Operators.ITEBV( 256, size != 0, self._used_calldata_size + offset + size, self._used_calldata_size ) @@ -1746,7 +1770,6 @@ def CALLDATACOPY(self, mem_offset, data_offset, size): max_size = cap self.constraints.add(Operators.ULE(size, cap)) - for i in range(max_size): try: c1 = Operators.ITEBV( @@ -1784,7 +1807,6 @@ def CODECOPY(self, mem_offset, code_offset, size): copyfee = self.safe_mul(GCOPY, Operators.UDIV(self.safe_add(size, 31), 32)) self._consume(copyfee) - if issymbolic(size): max_size = SelectedSolver.instance().max(self.constraints, size) else: @@ -1816,7 +1838,10 @@ def CODECOPY(self, mem_offset, code_offset, size): self._store(mem_offset + i, value) - assert SelectedSolver.instance().must_be_true(self.constraints,self.memory[0:max_size] == self.bytecode[code_offset:code_offset + max_size]) + assert SelectedSolver.instance().must_be_true( + self.constraints, + self.memory[0:max_size] == self.bytecode[code_offset : code_offset + max_size], + ) self._publish("did_evm_read_code", self.address, code_offset, size) @@ -2148,7 +2173,9 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): keccak_init = self.world.symbolic_function(globalsha3, data) caller = msg.caller.read_BE(0, 20) salt = salt.read_BE(0, 32) - address = self.world.symbolic_function(b"\xff" + caller + salt + keccak_init) & ((1<<0x20)-1) + address = self.world.symbolic_function(b"\xff" + caller + salt + keccak_init) & ( + (1 << 0x20) - 1 + ) self.world.start_transaction( "CREATE", @@ -2173,7 +2200,6 @@ def CREATE2(self, value, offset, size): address = 0 return address - def CALL_gas(self, wanted_gas, address, value, in_offset, in_size, out_offset, out_size): """ Dynamic gas for CALL instruction. _arguably turing complete in itself_ """ GCALLVALUE = 9000 @@ -3138,13 +3164,15 @@ def create_account(self, address=None, balance=0, code=None, storage=None, nonce if storage is None: # Uninitialized values in a storage are 0 by spec - storage = MutableArray(self.constraints.new_array( - index_size=256, - value_size=256, - name=f"STORAGE_{address:x}", - avoid_collisions=True, - default=0, - )) + storage = MutableArray( + self.constraints.new_array( + index_size=256, + value_size=256, + name=f"STORAGE_{address:x}", + avoid_collisions=True, + default=0, + ) + ) else: if isinstance(storage, MutableArray): if storage.index_bits != 256 or storage.value_bits != 256: @@ -3219,7 +3247,7 @@ def start_transaction( """ assert self._pending_transaction is None, "Already started tx" assert caller is not None - if issymbolic(data ): + if issymbolic(data): assert data.length is not None assert data.value_size == 8 From d669e41ec93aa3f3f0376c6fef9cdd7a901beb2f Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 15:10:50 -0300 Subject: [PATCH 083/126] Silence CClimate --- manticore/core/smtlib/expression.py | 13 +++++-------- manticore/platforms/evm.py | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 03ef816dc..3391dacb2 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -19,11 +19,13 @@ from functools import reduce import uuid - import re import copy -from typing import Union, Optional, Dict, Tuple, List, AnyStr +from typing import Union, Optional, Tuple, List +def simplify(e): + from .visitors import simplify as visitor_simplify + return visitor_simplify(e) class ExpressionError(Exception): """ @@ -752,8 +754,7 @@ def is_known(self, index) -> Union[Bool, bool]: """ Returned Boolean Expression holds when the index was used""" raise NotImplementedError - ########################################################################### - ## following methods are implemented on top of the abstract methods ^ + # Following methods are implemented on top of the abstract methods ^ def in_bounds(self, index: Union[Bitvec, int]) -> Union[Bool, bool]: """ True if the index points inside the array (or array is unbounded)""" if self.length is not None: @@ -868,8 +869,6 @@ def _fix_slice(self, index: slice): stop = len(self) size = stop - start if isinstance(size, Bitvec): - from .visitors import simplify - size = simplify(size) else: size = BitvecConstant(self.index_size, size) @@ -1632,5 +1631,3 @@ def taint_with(arg, *taints, value_size=256, index_size=256): return arg - -from .visitors import simplify diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index d52292a44..35325f794 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -2594,7 +2594,7 @@ def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value else: tx_fee = GTRANSACTION # Simple transaction fee - ## This popcnt like thing is expensive when the bytecode or + # This popcnt like thing is expensive when the bytecode or # data has symbolic content zerocount = 0 From cce9dd472ddcbfce810748966b3ce2fd00a88397 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 15:34:31 -0300 Subject: [PATCH 084/126] Silence CClimate --- manticore/core/smtlib/expression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 3391dacb2..a07ab11f6 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -23,10 +23,12 @@ import copy from typing import Union, Optional, Tuple, List + def simplify(e): from .visitors import simplify as visitor_simplify return visitor_simplify(e) + class ExpressionError(Exception): """ Expression exception @@ -1630,4 +1632,3 @@ def taint_with(arg, *taints, value_size=256, index_size=256): arg._taint |= tainted_fset return arg - From 37446fc939eba6c64b8ca353daf32693c2abfa83 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 18:02:41 -0300 Subject: [PATCH 085/126] Silence black --- examples/evm/minimal.py | 2 +- manticore/core/smtlib/expression.py | 1 + manticore/core/state.py | 10 +-- manticore/native/memory.py | 7 +- tests/ethereum/test_general.py | 17 ++-- tests/other/test_smtlibv2.py | 124 ++++++++++++++-------------- 6 files changed, 83 insertions(+), 78 deletions(-) diff --git a/examples/evm/minimal.py b/examples/evm/minimal.py index ac1442c7d..60cd0413f 100644 --- a/examples/evm/minimal.py +++ b/examples/evm/minimal.py @@ -25,7 +25,7 @@ } """ -user_account = m.create_account(balance=10**10, name="user_account") +user_account = m.create_account(balance=10 ** 10, name="user_account") print("[+] Creating a user account", user_account.name_) contract_account = m.solidity_create_contract( diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index a07ab11f6..be7ebc5b9 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -26,6 +26,7 @@ def simplify(e): from .visitors import simplify as visitor_simplify + return visitor_simplify(e) diff --git a/manticore/core/state.py b/manticore/core/state.py index 6d0821a42..3c49518bd 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -271,12 +271,12 @@ def constraints(self, constraints): def _update_state_descriptor(self, descriptor: StateDescriptor, *args, **kwargs): """ - Called on execution_intermittent to update the descriptor for this state. This is intended for information - like the PC or instruction count, where updating after each instruction would be a waste of cycles. - This one updates the execution counts + Called on execution_intermittent to update the descriptor for this state. This is intended for information + like the PC or instruction count, where updating after each instruction would be a waste of cycles. + This one updates the execution counts - :param descriptor: StateDescriptor for this state - """ + :param descriptor: StateDescriptor for this state + """ descriptor.total_execs = self._total_exec descriptor.own_execs = self._own_exec diff --git a/manticore/native/memory.py b/manticore/native/memory.py index fb04cde78..d1a8f2710 100644 --- a/manticore/native/memory.py +++ b/manticore/native/memory.py @@ -347,7 +347,9 @@ def __init__( self._array = backing_array else: self._array = expression.MutableArray( - expression.ArrayVariable(index_size=index_bits, length=size, value_size=8, name=name) + expression.ArrayVariable( + index_size=index_bits, length=size, value_size=8, name=name + ) ) def __reduce__(self): @@ -1376,8 +1378,7 @@ class LazySMemory(SMemory): def __init__(self, constraints, *args, **kwargs): super(LazySMemory, self).__init__(constraints, *args, **kwargs) - self.backing_array = constraints.new_array( - index_size=self.memory_bit_size) + self.backing_array = constraints.new_array(index_size=self.memory_bit_size) self.backed_by_symbolic_store = set() def __reduce__(self): diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index e87c72626..a3370a562 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -1628,9 +1628,9 @@ def test_overloaded_functions_and_events(self): class EthSpecificTxIntructionTests(unittest.TestCase): def test_jmpdest_check(self): """ - This test that jumping to a JUMPDEST in the operand of a PUSH should - be treated as an INVALID instruction. - https://github.com/trailofbits/manticore/issues/1169 + This test that jumping to a JUMPDEST in the operand of a PUSH should + be treated as an INVALID instruction. + https://github.com/trailofbits/manticore/issues/1169 """ constraints = ConstraintSet() @@ -1665,8 +1665,8 @@ def test_jmpdest_check(self): def test_delegatecall_env(self): """ - This test that the delegatecalled environment is identicall to the caller - https://github.com/trailofbits/manticore/issues/1169 + This test that the delegatecalled environment is identicall to the caller + https://github.com/trailofbits/manticore/issues/1169 """ constraints = ConstraintSet() world = evm.EVMWorld(constraints) @@ -1742,7 +1742,12 @@ def test_delegatecall_env(self): self.assertEqual(world.get_balance(0x111111111111111111111111111111111111111), 0) self.assertEqual(world.get_balance(0x222222222222222222222222222222222222222), 10) from manticore.core.smtlib.visitors import translate_to_smtlib, simplify - print ( translate_to_smtlib(simplify(world.get_balance(0x333333333333333333333333333333333333333)))) + + print( + translate_to_smtlib( + simplify(world.get_balance(0x333333333333333333333333333333333333333)) + ) + ) self.assertEqual( world.get_balance(0x333333333333333333333333333333333333333), 100000000000000000000000 - 10, diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index ba90fca68..194104eea 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -38,8 +38,8 @@ def assertItemsEqual(self, a, b): self.assertEqual(sorted(a), sorted(b)) def test_xslotted(self): - """ Test that XSlotted multi inheritance classes uses same amount - of memory than a single class object with slots + """Test that XSlotted multi inheritance classes uses same amount + of memory than a single class object with slots """ class Base(object, metaclass=XSlotted, abstract=True): @@ -75,18 +75,22 @@ class X(object): self.assertEqual(sys.getsizeof(c), sys.getsizeof(x)) def test_Bitvec_ops(self): - a = BitvecVariable(size=32, name='BV') - b = BitvecVariable(size=32, name='BV1') - c = BitvecVariable(size=32, name='BV2') + a = BitvecVariable(size=32, name="BV") + b = BitvecVariable(size=32, name="BV1") + c = BitvecVariable(size=32, name="BV2") x = BitvecConstant(size=32, value=100, taint=("T",)) - z = ((b + 1) % b < a * x / c - 5) + z = (b + 1) % b < a * x / c - 5 self.assertSetEqual(z.taint, set(("T",))) - self.assertEqual(translate_to_smtlib(z), - "(bvslt (bvsmod (bvadd BV1 #x00000001) BV1) (bvsub (bvsdiv (bvmul BV #x00000064) BV2) #x00000005))") - z = ((1 + b) / b <= a - x * 5 + c) + self.assertEqual( + translate_to_smtlib(z), + "(bvslt (bvsmod (bvadd BV1 #x00000001) BV1) (bvsub (bvsdiv (bvmul BV #x00000064) BV2) #x00000005))", + ) + z = (1 + b) / b <= a - x * 5 + c self.assertSetEqual(z.taint, set(("T",))) - self.assertEqual(translate_to_smtlib(z), - "(bvsle (bvsdiv (bvadd #x00000001 BV1) BV1) (bvadd (bvsub BV (bvmul #x00000064 #x00000005)) BV2))") + self.assertEqual( + translate_to_smtlib(z), + "(bvsle (bvsdiv (bvadd #x00000001 BV1) BV1) (bvadd (bvsub BV (bvmul #x00000064 #x00000005)) BV2))", + ) def test_ConstantArrayBitvec(self): c = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") @@ -108,9 +112,10 @@ def test_ConstantArrayBitvec2(self): c[2] = 100 self.assertEqual(c[2], 100) - def test_ArrayDefault3(self): - c = MutableArray(ArrayVariable(index_size=32, value_size=8, length=5, default=0, name="ARR")) + c = MutableArray( + ArrayVariable(index_size=32, value_size=8, length=5, default=0, name="ARR") + ) self.assertEqual(c[1], 0) self.assertEqual(c[2], 0) self.assertEqual(c[3], 0) @@ -124,18 +129,18 @@ def test_ArrayDefault3(self): def test_ArrayDefault4(self): cs = ConstraintSet() a = MutableArray(cs.new_array(index_size=32, value_size=8, length=4, default=0, name="ARR")) - i = cs.new_bitvec(size = a.index_size) + i = cs.new_bitvec(size=a.index_size) SelectedSolver.instance().must_be_true(cs, 0 == a.default) SelectedSolver.instance().must_be_true(cs, a[i] == a.default) - cs.add(i==2) + cs.add(i == 2) SelectedSolver.instance().must_be_true(cs, 0 == a.default) SelectedSolver.instance().must_be_true(cs, a[i] == a.default) b = a[:] - i = cs.new_bitvec(size = a.index_size) + i = cs.new_bitvec(size=a.index_size) SelectedSolver.instance().must_be_true(cs, 0 == b.default) SelectedSolver.instance().must_be_true(cs, b[i] == b.default) - + a[1] = 10 a[2] = 20 a[3] = 30 @@ -146,7 +151,6 @@ def test_ArrayDefault4(self): SelectedSolver.instance().must_be_true(cs, a[2] == 20) SelectedSolver.instance().must_be_true(cs, a[3] == 30) # SelectedSolver.instance().must_be_true(cs, a[4] == 0) #undefined! - b = a[:] # b := 0 10 20 30 0 0 x x x x (x undefined) @@ -155,9 +159,6 @@ def test_ArrayDefault4(self): SelectedSolver.instance().must_be_true(cs, b[1] == 10) SelectedSolver.instance().must_be_true(cs, b[2] == 20) SelectedSolver.instance().must_be_true(cs, b[3] == 30) - - - def test_Expression(self): # Used to check if all Expression have test @@ -176,10 +177,10 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): not hasattr(x, "__dict__"), ) """ - #self.assertEqual(len(pickle_dumps(x)), pickle_size) + # self.assertEqual(len(pickle_dumps(x)), pickle_size) self.assertEqual(sys.getsizeof(x), sizeof) self.assertFalse(hasattr(x, "__dict__")) # slots! - self.assertTrue(hasattr(x, "_taint")) # taint! + self.assertTrue(hasattr(x, "_taint")) # taint! checked.add(ty) # Can not instantiate an Expression @@ -266,7 +267,6 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): check(BitvecArithmeticShiftLeft, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=56) check(BitvecArithmeticShiftRight, operanda=bvx, operandb=bvy, pickle_size=147, sizeof=56) - check(BitvecZeroExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) check(BitvecSignExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) check(BitvecExtract, operand=bvx, offset=0, size=8, pickle_size=119, sizeof=64) @@ -275,11 +275,10 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): a = ArrayVariable(index_size=32, value_size=32, length=324, name="name") check(ArrayConstant, index_size=32, value_size=8, value=b"A", pickle_size=136, sizeof=64) - check(ArraySlice, array=a, offset=0, size=10 , pickle_size=136, sizeof=48) + check(ArraySlice, array=a, offset=0, size=10, pickle_size=136, sizeof=48) check(ArraySelect, array=a, index=bvx, pickle_size=161, sizeof=56) check(ArrayStore, array=a, index=bvx, value=bvy, pickle_size=188, sizeof=80) - def all_subclasses(cls): return set((Expression,)).union( set(cls.__subclasses__()).union( @@ -291,50 +290,50 @@ def all_subclasses(cls): self.assertSetEqual(checked, all_types) def test_Expression_BitvecOp(self): - a = BitvecConstant(size=32,value=100) - b = BitvecConstant(size=32,value=101) - x = a+b + a = BitvecConstant(size=32, value=100) + b = BitvecConstant(size=32, value=101) + x = a + b self.assertTrue(isinstance(x, Bitvec)) def test_Expression_BoolTaint(self): - #Bool can not be instantiaated + # Bool can not be instantiaated self.assertRaises(Exception, Bool, ()) - x = BoolConstant(value=True, taint=('red',)) - y = BoolConstant(value=False, taint=('blue',)) - z = BoolOr(x,y) - self.assertIn('red', x.taint) - self.assertIn('blue', y.taint) - self.assertIn('red', z.taint) - self.assertIn('blue', z.taint) + x = BoolConstant(value=True, taint=("red",)) + y = BoolConstant(value=False, taint=("blue",)) + z = BoolOr(x, y) + self.assertIn("red", x.taint) + self.assertIn("blue", y.taint) + self.assertIn("red", z.taint) + self.assertIn("blue", z.taint) def test_Expression_BitvecTaint(self): - #Bool can not be instantiaated + # Bool can not be instantiaated self.assertRaises(Exception, Bitvec, ()) - x = BitvecConstant(size=32, value=123, taint=('red',)) - y = BitvecConstant(size=32, value=124, taint=('blue',)) - z = BoolGreaterOrEqualThan(x,y) - self.assertIn('red', x.taint) - self.assertIn('blue', y.taint) - self.assertIn('red', z.taint) - self.assertIn('blue', z.taint) - + x = BitvecConstant(size=32, value=123, taint=("red",)) + y = BitvecConstant(size=32, value=124, taint=("blue",)) + z = BoolGreaterOrEqualThan(x, y) + self.assertIn("red", x.taint) + self.assertIn("blue", y.taint) + self.assertIn("red", z.taint) + self.assertIn("blue", z.taint) def test_Expression_Array(self): - #Bool can not be instantiaated + # Bool can not be instantiaated self.assertRaises(Exception, Array, ()) a = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") - a[0] == ord('A') + a[0] == ord("A") + + x = BitvecConstant(size=32, value=123, taint=("red",)) + y = BitvecConstant(size=32, value=124, taint=("blue",)) + z = BoolGreaterOrEqualThan(x, y) + self.assertIn("red", x.taint) + self.assertIn("blue", y.taint) + self.assertIn("red", z.taint) + self.assertIn("blue", z.taint) - x = BitvecConstant(size=32, value=123, taint=('red',)) - y = BitvecConstant(size=32, value=124, taint=('blue',)) - z = BoolGreaterOrEqualThan(x,y) - self.assertIn('red', x.taint) - self.assertIn('blue', y.taint) - self.assertIn('red', z.taint) - self.assertIn('blue', z.taint) class ExpressionTestLoco(unittest.TestCase): _multiprocess_can_split_ = True @@ -344,7 +343,6 @@ def setUp(self): cs = ConstraintSet() self.assertTrue(self.solver.check(cs)) - def assertItemsEqual(self, a, b): # Required for Python3 compatibility self.assertEqual(sorted(a), sorted(b)) @@ -383,7 +381,6 @@ def test_signed_unsigned_LT_simple(self): self.assertTrue(self.solver.must_be_true(cs, lt)) - class ExpressionTest(unittest.TestCase): _multiprocess_can_split_ = False @@ -755,18 +752,19 @@ def testBasicArrayProxySymbIdx2(self): ) # get a concrete solution for index 1 (default 100) self.assertItemsEqual(solutions, (100, 2)) - def testBasicConstatArray(self): cs = ConstraintSet() - array1 = MutableArray(cs.new_array(index_size=32, value_size=32, length=10, name="array1", default=0)) - array2 = MutableArray(cs.new_array(index_size=32, value_size=32, length=10, name="array2", default=0)) + array1 = MutableArray( + cs.new_array(index_size=32, value_size=32, length=10, name="array1", default=0) + ) + array2 = MutableArray( + cs.new_array(index_size=32, value_size=32, length=10, name="array2", default=0) + ) array1[0:10] = range(10) self.assertTrue(array1[0] == 0) - #yeah right self.assertTrue(array1[0:10] == range(10)) + # yeah right self.assertTrue(array1[0:10] == range(10)) array_slice = array1[0:10] self.assertTrue(array_slice[0] == 0) - - def testBasicPickle(self): import pickle From bc81838a830cc0a99c584270bf2ec60bcbcf3e91 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 18:04:35 -0300 Subject: [PATCH 086/126] BitVec -> Bitvec --- manticore/native/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manticore/native/models.py b/manticore/native/models.py index 9e43a5845..114d2fad8 100644 --- a/manticore/native/models.py +++ b/manticore/native/models.py @@ -5,7 +5,7 @@ from .cpu.abstractcpu import Cpu, ConcretizeArgument from .state import State from ..core.smtlib import issymbolic, Bitvec -from ..core.smtlib.solver import SelectedSolver, issymbolic, BitVec +from ..core.smtlib.solver import SelectedSolver, issymbolic, Bitvec from ..core.smtlib.operators import ITEBV, ZEXTEND from ..core.state import Concretize from typing import Union @@ -77,7 +77,7 @@ def can_be_NULL(state, byte) -> bool: return byte == 0 -def _find_zero(cpu, state, ptr: Union[int, BitVec]) -> int: +def _find_zero(cpu, state, ptr: Union[int, Bitvec]) -> int: """ Helper for finding the closest NULL or, effectively NULL byte from a starting address. From f163497de0ff1c113b91de3cca49a5b8edf38462 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 18:27:40 -0300 Subject: [PATCH 087/126] blkn --- manticore/core/smtlib/expression.py | 22 +++++----------------- manticore/core/state.py | 6 +----- manticore/platforms/evm.py | 11 ++--------- 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index be7ebc5b9..975131e33 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -940,11 +940,7 @@ class ArrayConstant(Array, Constant): __xslots__: Tuple[str, ...] = ("_index_size", "_value_size") def __init__( - self, - *, - index_size: int, - value_size: int, - **kwargs, + self, *, index_size: int, value_size: int, **kwargs, ): self._index_size = index_size self._value_size = value_size @@ -1193,8 +1189,7 @@ def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): # self._concrete_cache[index.value] = value super().__init__( - operands=(array, index, value), - **kwargs, + operands=(array, index, value), **kwargs, ) @property @@ -1301,8 +1296,7 @@ def __init__(self, array: "Array", offset: int, size: int, **kwargs): raise ValueError("Array expected") super().__init__( - operands=(array, array.cast_index(offset), array.cast_index(size)), - **kwargs, + operands=(array, array.cast_index(offset), array.cast_index(size)), **kwargs, ) def __hash__(self): @@ -1342,9 +1336,7 @@ def select(self, index): def store(self, index, value): return ArraySlice( - self.array.store(index + self.offset, value), - offset=self.offset, - size=len(self), + self.array.store(index + self.offset, value), offset=self.offset, size=len(self), ) @property @@ -1540,11 +1532,7 @@ class BitvecITE(BitvecOperation): __xslots__ = BitvecOperation.__xslots__ def __init__( - self, - condition: Bool, - true_value: Bitvec, - false_value: Bitvec, - **kwargs, + self, condition: Bool, true_value: Bitvec, false_value: Bitvec, **kwargs, ): super().__init__( diff --git a/manticore/core/state.py b/manticore/core/state.py index 3c49518bd..ed0e9145f 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -323,11 +323,7 @@ def new_symbolic_buffer(self, nbytes, **options): avoid_collisions = True taint = options.get("taint", frozenset()) expr = self._constraints.new_array( - name=label, - length=nbytes, - value_size=8, - taint=taint, - avoid_collisions=avoid_collisions, + name=label, length=nbytes, value_size=8, taint=taint, avoid_collisions=avoid_collisions, ) self._input_symbols.append(expr) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 8b7c808be..0a37896a8 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -104,9 +104,7 @@ def globalfakesha3(data): description="Max calldata size to explore in each CALLDATACOPY. Iff size in a calldata related instruction are symbolic it will be constrained to be less than this constant. -1 means free(only use when gas is being tracked)", ) consts.add( - "ignore_balance", - default=False, - description="Do not try to solve symbolic balances", + "ignore_balance", default=False, description="Do not try to solve symbolic balances", ) @@ -2194,12 +2192,7 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): ) self.world.start_transaction( - "CREATE", - address, - data=data, - caller=self.address, - value=value, - gas=self.gas, + "CREATE", address, data=data, caller=self.address, value=value, gas=self.gas, ) raise StartTx() From c232d58c2a6842923c107dcb5ba2d8aeef7fa1cd Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 7 Oct 2020 23:39:48 -0300 Subject: [PATCH 088/126] Lint --- manticore/core/smtlib/expression.py | 25 ++++++++++++------------- manticore/platforms/evm.py | 11 ++++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 975131e33..1a0230114 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -24,9 +24,8 @@ from typing import Union, Optional, Tuple, List -def simplify(e): +def local_simplify(e): from .visitors import simplify as visitor_simplify - return visitor_simplify(e) @@ -666,7 +665,7 @@ def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) def __bool__(self): - simplified = simplify(self) + simplified = local_simplify(self) if isinstance(simplified, Constant): return simplified.value raise NotImplementedError @@ -777,7 +776,7 @@ def cast(self, array) -> "Array": FIXME: this assigns a random name to a new variable and does not use a ConstraintSet as a Factory """ - logger.error("THis is creating a variable out of band FTAG4985732") + # logger.error("THis is creating a variable out of band FTAG4985732") if isinstance(array, Array): return array arr = ArrayVariable( @@ -798,7 +797,7 @@ def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: return BitvecConstant(self.index_size, index) if not isinstance(index, Bitvec) or index.size != self.index_size: raise ExpressionError(f"Expected Bitvector of size {self.index_size}") - return simplify(index) + return local_simplify(index) def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: """Forgiving casting method that will translate compatible values into @@ -872,7 +871,7 @@ def _fix_slice(self, index: slice): stop = len(self) size = stop - start if isinstance(size, Bitvec): - size = simplify(size) + size = local_simplify(size) else: size = BitvecConstant(self.index_size, size) if not isinstance(size, BitvecConstant): @@ -888,9 +887,9 @@ def _concatenate(self, array_a, array_b): ) for index in range(len(array_a)): - new_arr = new_arr.store(index, simplify(array_a[index])) + new_arr = new_arr.store(index, local_simplify(array_a[index])) for index in range(len(array_b)): - new_arr = new_arr.store(index + len(array_a), simplify(array_b[index])) + new_arr = new_arr.store(index + len(array_a), local_simplify(array_b[index])) return new_arr def __add__(self, other): @@ -1072,7 +1071,7 @@ def select(self, index): def store(self, index, value): index = self.cast_index(index) - value = simplify(self.cast_value(value)) + value = local_simplify(self.cast_value(value)) return ArrayStore(array=self, index=index, value=value) @property @@ -1232,12 +1231,12 @@ def select(self, index): """Gets an element from the Array. If the element was not previously the default is used. """ - index = simplify(self.cast_index(index)) + index = local_simplify(self.cast_index(index)) # Emulate list[-1] has_length = self.length is not None if has_length: - index = simplify(BitvecITE(index < 0, self.length + index, index)) + index = local_simplify(BitvecITE(index < 0, self.length + index, index)) if isinstance(index, Constant): if has_length and index.value >= self.length: @@ -1279,7 +1278,7 @@ def select(self, index): return result def store(self, index, value): - index = simplify(self.cast_index(index)) + index = local_simplify(self.cast_index(index)) value = self.cast_value(value) new_array = ArrayStore(self, index, value) return new_array @@ -1332,7 +1331,7 @@ def select(self, index): length = self.length if length is not None and index.value >= length: raise IndexError - return self.array.select(simplify(index + self.offset)) + return self.array.select (local_simplify(index + self.offset)) def store(self, index, value): return ArraySlice( diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 0a37896a8..d340349c5 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -6,8 +6,7 @@ import io import copy import inspect -from functools import wraps -from typing import List, Set, Tuple, Union +from typing import List, Set, Tuple, Union, Dict from ..platforms.platform import * from ..core.smtlib import ( SelectedSolver, @@ -21,20 +20,20 @@ BitvecConstant, translate_to_smtlib, to_constant, - simplify, get_depth, issymbolic, get_taints, istainted, taint_with, + simplify ) from ..core.state import Concretize, TerminateState from ..utils.event import Eventful from ..utils.helpers import printable_bytes from ..utils import config -from ..core.smtlib.visitors import simplify from ..exceptions import EthereumError import pyevmasm as EVMAsm + import logging from collections import namedtuple import sha3 @@ -1030,7 +1029,7 @@ def get_instruction(self, pc: Union[Bitvec, int]): try: _decoding_cache = getattr(self, "_decoding_cache") except Exception: - self._decoding_cache = {} + self._decoding_cache: Dict[int, EVMAsm.Instruction] = {} _decoding_cache = self._decoding_cache if isinstance(pc, Constant): @@ -1039,6 +1038,8 @@ def get_instruction(self, pc: Union[Bitvec, int]): if pc in _decoding_cache: return _decoding_cache[pc] + if isinstance(pc, Bitvec): + raise EVMException("Trying to decode from symbolic pc") instruction = EVMAsm.disassemble_one(self._getcode(pc), pc=pc, fork=self.evmfork) _decoding_cache[pc] = instruction return instruction From 9200178643b05e7dba895ddad57abce532821875 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 8 Oct 2020 00:02:44 -0300 Subject: [PATCH 089/126] Lint --- manticore/core/smtlib/expression.py | 3 ++- manticore/platforms/evm.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 1a0230114..1eade2c9b 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -26,6 +26,7 @@ def local_simplify(e): from .visitors import simplify as visitor_simplify + return visitor_simplify(e) @@ -1331,7 +1332,7 @@ def select(self, index): length = self.length if length is not None and index.value >= length: raise IndexError - return self.array.select (local_simplify(index + self.offset)) + return self.array.select(local_simplify(index + self.offset)) def store(self, index, value): return ArraySlice( diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index d340349c5..1669d77bd 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -25,7 +25,7 @@ get_taints, istainted, taint_with, - simplify + simplify, ) from ..core.state import Concretize, TerminateState from ..utils.event import Eventful From ef844bd27de07eb077075116df68824be043db4b Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 8 Oct 2020 00:20:17 -0300 Subject: [PATCH 090/126] Fix debug assertion --- manticore/platforms/evm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 1669d77bd..266f89d8c 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -1855,7 +1855,7 @@ def CODECOPY(self, mem_offset, code_offset, size): assert SelectedSolver.instance().must_be_true( self.constraints, - self.memory[0:max_size] == self.bytecode[code_offset : code_offset + max_size], + self.memory[mem_offset:mem_offset+max_size] == self.bytecode[code_offset : code_offset + max_size], ) self._publish("did_evm_read_code", self.address, code_offset, size) From 79c0b66975393779fd1d0f12b5b2de42c6cccb51 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 8 Oct 2020 00:29:27 -0300 Subject: [PATCH 091/126] blkn --- manticore/platforms/evm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 266f89d8c..30d822867 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -1855,7 +1855,8 @@ def CODECOPY(self, mem_offset, code_offset, size): assert SelectedSolver.instance().must_be_true( self.constraints, - self.memory[mem_offset:mem_offset+max_size] == self.bytecode[code_offset : code_offset + max_size], + self.memory[mem_offset : mem_offset + max_size] + == self.bytecode[code_offset : code_offset + max_size], ) self._publish("did_evm_read_code", self.address, code_offset, size) From f17bd7f456cc3506334d752f275e8425eca836c7 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 15 Oct 2020 20:05:12 -0300 Subject: [PATCH 092/126] Apply suggestions from code review Co-authored-by: Eric Kilmer --- examples/evm/minimal.py | 2 +- manticore/core/smtlib/constraints.py | 1 - manticore/core/smtlib/expression.py | 9 +++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/evm/minimal.py b/examples/evm/minimal.py index 60cd0413f..4a4c939ec 100644 --- a/examples/evm/minimal.py +++ b/examples/evm/minimal.py @@ -31,7 +31,7 @@ contract_account = m.solidity_create_contract( source_code, owner=user_account, name="contract_account" ) -print("[+] Creating a contract account", contract_account) +print(f"[+] Creating a contract account {contract_account}") contract_account.named_func(1) print("[+] Now the symbolic values") diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index cfa37d52f..9d8161341 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -185,7 +185,6 @@ def to_string(self, replace_constants: bool = False) -> str: ): constant_bindings[expression.operands[0]] = expression.operands[1] - result = "" translator = TranslatorSmtlib(use_bindings=False) tuple(translator.visit_Variable(v) for v in variables) for constraint in constraints: diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 1eade2c9b..b0a614180 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -75,7 +75,7 @@ class X(object): @staticmethod def _remove_mod(attr: str) -> str: - """xlots attrivutes could have modifficators after a # symbol + """xslots attributes could have modifiers after a # symbol attribute#v means attribute is _volatile_ and should not be saved to storage """ return attr.split("#")[0] @@ -88,7 +88,7 @@ def __new__(cls, clsname, bases, attrs, abstract=False): xslots = xslots.union(getattr(base, "__xslots__", ())) attrs["__xslots__"]: Tuple[str] = tuple(xslots) if abstract: - attrs["__slots__"] = () + attrs["__slots__"] = tuple() else: attrs["__slots__"]: Tuple[str] = tuple(map(cls._remove_mod, attrs["__xslots__"])) @@ -127,10 +127,7 @@ def operands(self): return () def __getstate__(self): - state = {} - for attr in self.__slots__: - state[attr] = getattr(self, attr) - return state + return {attr: getattr(self, attr) for attr in self.__slots__} def __setstate__(self, state): for attr in self.__slots__: From b7a5bc0d0724fe3db22e5192af6a0fbf2e511149 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 15 Oct 2020 20:25:04 -0300 Subject: [PATCH 093/126] Apply suggestions from code review Co-authored-by: Eric Kilmer --- manticore/core/smtlib/expression.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index b0a614180..38c5da4d6 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -758,7 +758,7 @@ def is_known(self, index) -> Union[Bool, bool]: def in_bounds(self, index: Union[Bitvec, int]) -> Union[Bool, bool]: """ True if the index points inside the array (or array is unbounded)""" if self.length is not None: - return (0 <= index) & (index < self.length) + return 0 <= index < self.length return True def __len__(self): @@ -774,7 +774,6 @@ def cast(self, array) -> "Array": FIXME: this assigns a random name to a new variable and does not use a ConstraintSet as a Factory """ - # logger.error("THis is creating a variable out of band FTAG4985732") if isinstance(array, Array): return array arr = ArrayVariable( @@ -782,7 +781,7 @@ def cast(self, array) -> "Array": length=len(array), default=0, value_size=self.value_size, - name="cast{}".format(uuid.uuid1()), + name=f"cast{uuid.uuid1()}", ) for pos, byte in enumerate(array): arr = arr.store(pos, byte) @@ -963,7 +962,7 @@ def select(self, index): size=self.value_size, value=self.value[index.value], taint=self.taint ) - # Index being symbolic generates a sybolic result ! + # Index being symbolic generates a symbolic result ! result = BitvecConstant(size=self.value_size, value=0, taint=("out_of_bounds")) for i, c in enumerate(self.value): result = BitvecITE( @@ -988,7 +987,7 @@ class ArrayVariable(Array, Variable): If a default value is provided reading from an unused index will return the default. Otherwise each unused position in the array represents a free bitvector. - If an length maximun index is provided accessing over the max is undefined. + If a length maximum index is provided, accessing over the max is undefined. Otherwise the array is unbounded. """ @@ -1028,7 +1027,7 @@ def __init__( """ assert index_size in (32, 64, 256) assert value_size in (8, 16, 32, 64, 256) - assert length is None or length >= 0 and length < 2 ** index_size + assert length is None or 0 <= length < 2 ** index_size self._index_size = index_size self._length = length self._value_size = value_size @@ -1043,13 +1042,6 @@ def index_size(self): def value_size(self): return self._value_size - """ - @property - def index_max(self): - if self._length is None: - return None - return self._length - 1 -""" @property def default(self): @@ -1153,10 +1145,7 @@ def written(self): # Calculate only first time # This can have repeated and reused written indexes. if self._written is None: - written = set() - for offset, value in get_items(self): - written.add(offset) - self._written = written + self._written = {offset for offset, _ in get_items(self)} return self._written def is_known(self, index): @@ -1549,7 +1538,7 @@ def false_value(self): return self.operands[2] -# auxiliar functions maybe move to operators +# auxiliary functions. Maybe move to operators def issymbolic(value) -> bool: """ Helper to determine whether an object is symbolic (e.g checking From e57975e62248cb3fb4b3103c81b656de8e1c7963 Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 15 Oct 2020 20:27:19 -0300 Subject: [PATCH 094/126] review comments --- manticore/core/smtlib/constraints.py | 7 ++- manticore/core/smtlib/expression.py | 75 ++++++++-------------------- manticore/core/smtlib/solver.py | 1 - manticore/platforms/evm.py | 10 ++-- tests/ethereum/test_general.py | 7 --- 5 files changed, 30 insertions(+), 70 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index cfa37d52f..9167b54c4 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -178,16 +178,19 @@ def to_string(self, replace_constants: bool = False) -> str: if replace_constants: constant_bindings = {} for expression in constraints: + # FIXME this will not catch Constant == Variable + # Maybe we need a way to canonicalize the expressions somewhere if ( isinstance(expression, BoolEqual) and isinstance(expression.operands[0], Variable) - and not isinstance(expression.operands[1], (Variable, Constant)) + and isinstance(expression.operands[1], Constant) ): constant_bindings[expression.operands[0]] = expression.operands[1] result = "" translator = TranslatorSmtlib(use_bindings=False) - tuple(translator.visit_Variable(v) for v in variables) + for v in variables: + translator.visit_Variable(v) for constraint in constraints: if replace_constants: constraint = simplify(replace(constraint, constant_bindings)) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 1eade2c9b..f00ac4b8d 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -16,7 +16,7 @@ cs.add( condition2 ) """ - +from abc import ABC, abstractmethod from functools import reduce import uuid import re @@ -72,14 +72,6 @@ class X(object): print (sys.getsizeof(c),sys.getsizeof(x)) #same value """ - - @staticmethod - def _remove_mod(attr: str) -> str: - """xlots attrivutes could have modifficators after a # symbol - attribute#v means attribute is _volatile_ and should not be saved to storage - """ - return attr.split("#")[0] - def __new__(cls, clsname, bases, attrs, abstract=False): xslots = set(attrs.get("__xslots__", ())) @@ -90,17 +82,18 @@ def __new__(cls, clsname, bases, attrs, abstract=False): if abstract: attrs["__slots__"] = () else: - attrs["__slots__"]: Tuple[str] = tuple(map(cls._remove_mod, attrs["__xslots__"])) + attrs["__slots__"]: Tuple[str] = tuple( + map(lambda attr: attr.split('#', 1)[0], attrs["__xslots__"])) return super().__new__(cls, clsname, bases, attrs) -class Expression(object, metaclass=XSlotted, abstract=True): +class Expression(ABC, metaclass=XSlotted, abstract=True): """ Abstract taintable Expression. """ __xslots__: Tuple[str, ...] = ("_taint",) - def __init__(self, taint: Union[tuple, frozenset] = ()): + def __init__(self, *, taint: Union[tuple, frozenset] = ()): """ An abstract Unmutable Taintable Expression :param taint: A frozenzset @@ -142,7 +135,7 @@ class Variable(Expression, abstract=True): __xslots__: Tuple[str, ...] = ("_name",) - def __init__(self, name: str, **kwargs): + def __init__(self, *, name: str, **kwargs): """Variable is an Expression that has a name :param name: The Variable name """ @@ -162,7 +155,7 @@ class Constant(Expression, abstract=True): __xslots__: Tuple[str, ...] = ("_value",) - def __init__(self, value: Union[bool, int, bytes, List[int]], **kwargs): + def __init__(self, *, value: Union[bool, int, bytes, List[int]], **kwargs): """A constant expression has a value :param value: The constant value @@ -180,7 +173,7 @@ class Operation(Expression, abstract=True): __xslots__: Tuple[str, ...] = ("_operands",) - def __init__(self, operands: Tuple[Expression, ...], **kwargs): + def __init__(self, *, operands: Tuple[Expression, ...], **kwargs): """An operation has operands :param operands: A tuple of expression operands @@ -204,9 +197,6 @@ def operands(self): class Bool(Expression, abstract=True): """Bool expression represent symbolic value of truth""" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def cast(self, value: Union["Bool", int, bool], **kwargs) -> Union["BoolConstant", "Bool"]: """ Cast any type into a Bool or fail """ if isinstance(value, Bool): @@ -249,12 +239,11 @@ def __rxor__(self, other): return BoolXor(self.cast(other), self) def __bool__(self): - raise NotImplementedError + raise ExpressionError("You tried to use a Bool Expression as a boolean constant. Expressions could represent a set of concrete values.") class BoolVariable(Bool, Variable): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + pass class BoolConstant(Bool, Constant): @@ -281,6 +270,14 @@ class BoolOperation(Bool, Operation, abstract=True): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + def __bool__(self): + # FIXME: TODO: re-think is we want to be this forgiving every use of + # local_simplify looks hacky + simplified = local_simplify(self) + if isinstance(simplified, Constant): + return simplified.value + raise ExpressionError("BoolOperation can not be reduced to a constant") + class BoolNot(BoolOperation): def __init__(self, operand: Bool, **kwargs): @@ -559,7 +556,6 @@ def __hash__(self): class BitvecOperation(Bitvec, Operation, abstract=True): """ Operations that result in a Bitvec """ - pass @@ -665,12 +661,6 @@ def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): assert isinstance(operandb, Expression) super().__init__(operands=(operanda, operandb), **kwargs) - def __bool__(self): - simplified = local_simplify(self) - if isinstance(simplified, Constant): - return simplified.value - raise NotImplementedError - class BoolGreaterThan(BoolOperation): def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): @@ -703,7 +693,6 @@ def __init__(self, operanda, operandb, **kwargs): operands=(operanda, operandb), **kwargs ) - class Array(Expression, abstract=True): """An Array expression is an unmutable mapping from bitvector to bitvector @@ -714,9 +703,10 @@ class Array(Expression, abstract=True): """ @property + @abstractmethod def index_size(self): """ The bit size of the index part. Must be overloaded by a more specific class""" - raise NotImplementedError + ... @property def value_size(self): @@ -1253,31 +1243,6 @@ def select(self, index): # if a default is defined we need to check if the index was previously written return BitvecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default)) - # build a big ITE expression - array, offset, items = self, 0, [] - while not isinstance(array, ArrayVariable): - if isinstance(array, ArraySlice): - # jump over array slices - offset += array.offset - else: - assert isinstance(array, ArrayStore) - # The index written to underlaying Array are displaced when sliced - cond = index == (array.index - offset) - if isinstance(cond, Constant): - if cond.value == True: - items.insert(0, (cond, array.value)) - break - else: - array = array.array - continue - items.insert(0, (cond, array.value)) - array = array.array - - result = self.cast_value(default) - for cond_i, value_i in items: - result = BitvecITE(cond_i, value_i, result) - return result - def store(self, index, value): index = local_simplify(self.cast_index(index)) value = self.cast_value(value) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 8150f3f42..863df056c 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -833,5 +833,4 @@ def instance(cls): cls.choice = consts.solver SelectedSolver = {"cvc4": CVC4Solver, "yices": YicesSolver, "z3": Z3Solver}[cls.choice.name] - return YicesSolver.instance() return SelectedSolver.instance() diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 30d822867..beea916a3 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -2679,11 +2679,11 @@ def _open_transaction(self, sort, address, price, bytecode_or_data, caller, valu if sort not in {"CALL", "CREATE", "DELEGATECALL", "CALLCODE", "STATICCALL"}: raise EVMException(f"Transaction type '{sort}' not supported") - if caller not in self.accounts: - logger.info("Caller not in account") - raise EVMException( - f"Caller account {hex(caller)} does not exist; valid accounts: {list(map(hex, self.accounts))}" - ) + #if caller not in self.accounts: + # logger.info("Caller not in account") + # raise EVMException( + # f"Caller account {hex(caller)} does not exist; valid accounts: {list(map(hex, self.accounts))}" + # ) if sort == "CREATE": expected_address = self.new_address(sender=caller) diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index a3370a562..d94827afa 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -1741,13 +1741,6 @@ def test_delegatecall_env(self): # check balances self.assertEqual(world.get_balance(0x111111111111111111111111111111111111111), 0) self.assertEqual(world.get_balance(0x222222222222222222222222222222222222222), 10) - from manticore.core.smtlib.visitors import translate_to_smtlib, simplify - - print( - translate_to_smtlib( - simplify(world.get_balance(0x333333333333333333333333333333333333333)) - ) - ) self.assertEqual( world.get_balance(0x333333333333333333333333333333333333333), 100000000000000000000000 - 10, From a102dae9076103dbe4a7a21971846ccc370c16fb Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 15 Oct 2020 22:59:52 -0300 Subject: [PATCH 095/126] blkn --- manticore/platforms/evm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index beea916a3..532fcd70d 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -2679,7 +2679,7 @@ def _open_transaction(self, sort, address, price, bytecode_or_data, caller, valu if sort not in {"CALL", "CREATE", "DELEGATECALL", "CALLCODE", "STATICCALL"}: raise EVMException(f"Transaction type '{sort}' not supported") - #if caller not in self.accounts: + # if caller not in self.accounts: # logger.info("Caller not in account") # raise EVMException( # f"Caller account {hex(caller)} does not exist; valid accounts: {list(map(hex, self.accounts))}" From 87a68877155dbdd6d046fad60b91c6b856b2216f Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 16 Oct 2020 10:40:28 -0300 Subject: [PATCH 096/126] backpedal on ABC --- manticore/core/smtlib/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index ff522dc6b..8b547c832 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -90,7 +90,7 @@ def __new__(cls, clsname, bases, attrs, abstract=False): return super().__new__(cls, clsname, bases, attrs) -class Expression(ABC, metaclass=XSlotted, abstract=True): +class Expression(object, metaclass=XSlotted, abstract=True): """ Abstract taintable Expression. """ __xslots__: Tuple[str, ...] = ("_taint",) From 9123dcf1461cab282acfe5677c756934c2262bf4 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 16 Oct 2020 11:27:45 -0300 Subject: [PATCH 097/126] relax size test --- tests/other/test_smtlibv2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 194104eea..4c0ce5171 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -178,7 +178,9 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): ) """ # self.assertEqual(len(pickle_dumps(x)), pickle_size) - self.assertEqual(sys.getsizeof(x), sizeof) + #self.assertEqual(sys.getsizeof(x), sizeof) + #Te test numbers are taken from Python 3.8.5 older pythons use 8 more bytes sometimes + self.assertLessEqual(sys.getsizeof(x), sizeof+8) self.assertFalse(hasattr(x, "__dict__")) # slots! self.assertTrue(hasattr(x, "_taint")) # taint! checked.add(ty) From 0595ce03576c8c14d7652ec30e2279d876136742 Mon Sep 17 00:00:00 2001 From: feliam Date: Fri, 16 Oct 2020 11:28:30 -0300 Subject: [PATCH 098/126] lint --- tests/other/test_smtlibv2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 4c0ce5171..9c4f759f8 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -178,9 +178,9 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): ) """ # self.assertEqual(len(pickle_dumps(x)), pickle_size) - #self.assertEqual(sys.getsizeof(x), sizeof) - #Te test numbers are taken from Python 3.8.5 older pythons use 8 more bytes sometimes - self.assertLessEqual(sys.getsizeof(x), sizeof+8) + # self.assertEqual(sys.getsizeof(x), sizeof) + # The test numbers are taken from Python 3.8.5 older pythons use 8 more bytes sometimes + self.assertLessEqual(sys.getsizeof(x), sizeof + 8) self.assertFalse(hasattr(x, "__dict__")) # slots! self.assertTrue(hasattr(x, "_taint")) # taint! checked.add(ty) From d45e01b0157312862a5c30c20e9b5c39d9a6e7f2 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 19 Oct 2020 17:36:19 -0300 Subject: [PATCH 099/126] Fixed some tests --- manticore/core/smtlib/constraints.py | 4 +-- manticore/native/memory.py | 11 ++++---- manticore/platforms/linux.py | 4 +-- tests/native/test_cpu_manual.py | 40 +++++++++++++-------------- tests/native/test_manticore.py | 4 +-- tests/native/test_manticore_logger.py | 24 ++++++++++++++++ tests/native/test_register.py | 4 +-- tests/native/test_state.py | 10 +++---- 8 files changed, 63 insertions(+), 38 deletions(-) create mode 100644 tests/native/test_manticore_logger.py diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 3d89c58c6..4c82f2004 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -130,8 +130,8 @@ def related_to(self, *related_to) -> "ConstraintSet": Slices this ConstraintSet keeping only the related constraints. Two constraints are independient if they can be expressed full using a disjoint set of variables. - Todo: Research. constraints refering differen not overlapping parts of the same array - should be considered independient. + Todo: Research. constraints referring different not overlapping parts of the same array + should be considered independient. :param related_to: An expression :return: """ diff --git a/manticore/native/memory.py b/manticore/native/memory.py index d1a8f2710..b66367665 100644 --- a/manticore/native/memory.py +++ b/manticore/native/memory.py @@ -12,6 +12,7 @@ expression, issymbolic, Expression, + MutableArray ) from ..native.mappings import mmap, munmap from ..utils.helpers import interval_intersection @@ -359,7 +360,7 @@ def __reduce__(self): self.start, len(self), self._perms, - self._array.index_bits, + self._array.index_size, self._array, self._array.name, ), @@ -378,16 +379,16 @@ def split(self, address: int): return self, None assert self.start < address < self.end - index_bits, value_bits = self._array.index_bits, self._array.value_bits + index_bits, value_bits = self._array.index_size, self._array.value_size left_size, right_size = address - self.start, self.end - address left_name, right_name = ["{}_{:d}".format(self._array.name, i) for i in range(2)] head_arr = expression.MutableArray( - expression.ArrayVariable(index_bits, left_size, value_bits, name=left_name) + expression.ArrayVariable(index_size=index_bits, length=left_size, value_size=value_bits, name=left_name) ) tail_arr = expression.MutableArray( - expression.ArrayVariable(index_bits, right_size, value_bits, name=right_name) + expression.ArrayVariable(index_size=index_bits, length=right_size, value_size=value_bits, name=right_name) ) head = ArrayMap(self.start, left_size, self.perms, index_bits, head_arr, left_name) @@ -1378,7 +1379,7 @@ class LazySMemory(SMemory): def __init__(self, constraints, *args, **kwargs): super(LazySMemory, self).__init__(constraints, *args, **kwargs) - self.backing_array = constraints.new_array(index_size=self.memory_bit_size) + self.backing_array = MutableArray(constraints.new_array(index_size=self.memory_bit_size)) self.backed_by_symbolic_store = set() def __reduce__(self): diff --git a/manticore/platforms/linux.py b/manticore/platforms/linux.py index 6d5101714..7bf153b26 100644 --- a/manticore/platforms/linux.py +++ b/manticore/platforms/linux.py @@ -19,9 +19,9 @@ import os import random -from elftools.elf.descriptions import describe_symbol_type # Remove in favor of binary.py +from elftools.elf.descriptions import describe_symbol_type from elftools.elf.elffile import ELFFile from elftools.elf.sections import SymbolTableSection @@ -483,7 +483,7 @@ def __init__( # build the constraints array size = len(data) - self.array = constraints.new_array(name=self.name, length=size) + self.array = MutableArray(constraints.new_array(name=self.name, length=size)) symbols_cnt = 0 for i in range(size): diff --git a/tests/native/test_cpu_manual.py b/tests/native/test_cpu_manual.py index 4cc2595dd..0d706bc4a 100644 --- a/tests/native/test_cpu_manual.py +++ b/tests/native/test_cpu_manual.py @@ -497,13 +497,13 @@ def test_le_or(self): cpu.write_int(0x1000, 0x4142434445464748, 64) cpu.write_int(0x1000, cpu.read_int(0x1000, 32) | 0, 32) - addr1 = cs.new_Bitvec(64) + addr1 = cs.new_bitvec(64) cs.add(addr1 == 0x1004) cpu.write_int(addr1, 0x58, 8) self.assertEqual(cpu.read_int(0x1000, 32), 0x45464748) - addr1 = cs.new_Bitvec(64) + addr1 = cs.new_bitvec(64) cs.add(addr1 == 0x1000) cpu.write_int(addr1, 0x59, 8) @@ -592,7 +592,7 @@ def test_cache_003(self): for i in range(8): self.assertEqual(cpu.read_int(0x1008 + i, 8), ord("hgfedcba"[i])) - addr1 = cs.new_Bitvec(64) + addr1 = cs.new_bitvec(64) cs.add(addr1 == 0x1004) cpu.write_int(addr1, 0x58, 8) @@ -601,7 +601,7 @@ def test_cache_003(self): value = cpu.read_int(0x1004, 16) self.assertItemsEqual(solver.get_all_values(cs, value), [0x4358]) - addr2 = cs.new_Bitvec(64) + addr2 = cs.new_bitvec(64) cs.add(Operators.AND(addr2 >= 0x1000, addr2 <= 0x100C)) cpu.write_int(addr2, 0x5959, 16) @@ -728,11 +728,11 @@ def test_IDIV_symbolic(self): mem[code : code + 3] = "\xf7\x7d\xf4" cpu.EIP = code - cpu.EAX = cs.new_Bitvec(32, "EAX") + cpu.EAX = cs.new_bitvec(32, "EAX") cs.add(cpu.EAX == 116) - cpu.EBP = cs.new_Bitvec(32, "EBP") + cpu.EBP = cs.new_bitvec(32, "EBP") cs.add(cpu.EBP == stack + 0x700) - value = cs.new_Bitvec(32, "VALUE") + value = cs.new_bitvec(32, "VALUE") cpu.write_int(cpu.EBP - 0xC, value, 32) cs.add(value == 100) cpu.execute() @@ -765,11 +765,11 @@ def test_IDIV_grr001_symbolic(self): mem[code : code + 2] = "\xf7\xf9" cpu.EIP = code - cpu.EAX = cs.new_Bitvec(32, "EAX") + cpu.EAX = cs.new_bitvec(32, "EAX") cs.add(cpu.EAX == 0xFFFFFFFF) - cpu.EDX = cs.new_Bitvec(32, "EDX") + cpu.EDX = cs.new_bitvec(32, "EDX") cs.add(cpu.EDX == 0xFFFFFFFF) - cpu.ECX = cs.new_Bitvec(32, "ECX") + cpu.ECX = cs.new_bitvec(32, "ECX") cs.add(cpu.ECX == 0x32) cpu.execute() @@ -808,9 +808,9 @@ def test_ADC_001_symbolic(self): mem[code : code + 2] = "\x13\xf2" cpu.EIP = code - cpu.ESI = cs.new_Bitvec(32, "ESI") + cpu.ESI = cs.new_bitvec(32, "ESI") cs.add(cpu.ESI == 0) - cpu.EDX = cs.new_Bitvec(32, "EDX") + cpu.EDX = cs.new_bitvec(32, "EDX") cs.add(cpu.EDX == 0xFFFFFFFF) cpu.CF = cs.new_bool("CF") cs.add(cpu.CF) @@ -874,7 +874,7 @@ def test_CMPXCHG8B_symbolic(self): mem[code : code + 5] = "\xf0\x0f\xc7\x0f;" cpu.EIP = code - cpu.EDI = cs.new_Bitvec(32, "EDI") + cpu.EDI = cs.new_bitvec(32, "EDI") cs.add(Operators.OR(cpu.EDI == 0x2000, cpu.EDI == 0x2100, cpu.EDI == 0x2200)) self.assertEqual(sorted(solver.get_all_values(cs, cpu.EDI)), [0x2000, 0x2100, 0x2200]) self.assertEqual(cpu.read_int(0x2000, 64), 0) @@ -885,7 +885,7 @@ def test_CMPXCHG8B_symbolic(self): cpu.write_int(0x2100, 0x4142434445464748, 64) - cpu.EAX = cs.new_Bitvec(32, "EAX") + cpu.EAX = cs.new_bitvec(32, "EAX") cs.add(Operators.OR(cpu.EAX == 0x41424344, cpu.EAX == 0x0BADF00D, cpu.EAX == 0xF7F7F7F7)) cpu.EDX = 0x45464748 @@ -1060,7 +1060,7 @@ def test_SAR_1_symbolic(self): cs.add(cpu.AF == False) cpu.OF = cs.new_bool() cs.add(cpu.OF == False) - cpu.EAX = cs.new_Bitvec(32) + cpu.EAX = cs.new_bitvec(32) cs.add(cpu.EAX == 0xFFFFFFF7) done = False @@ -1138,7 +1138,7 @@ def test_SAR_2_symbolicsa(self): cs.add(cpu.AF == False) cpu.OF = cs.new_bool() cs.add(cpu.OF == False) - cpu.EAX = cs.new_Bitvec(32) + cpu.EAX = cs.new_bitvec(32) cs.add(cpu.EAX == 0xFFFFFFFF) done = False @@ -1192,15 +1192,15 @@ def test_SAR_3_symbolic(self): mem[0x804D601] = "\x78" mem[0x804D602] = "\x00" mem[0x804D603] = "\xff" - addr = cs.new_Bitvec(32) + addr = cs.new_bitvec(32) cs.add(addr == 0xFFFFB000) - value = cs.new_Bitvec(8) + value = cs.new_bitvec(8) cs.add(value == 0x8F) mem[addr] = value - cpu.EAX = cs.new_Bitvec(32) + cpu.EAX = cs.new_bitvec(32) cs.add(cpu.EAX == 0xFFFFB000) - cpu.CL = cs.new_Bitvec(8) + cpu.CL = cs.new_bitvec(8) cs.add(cpu.CL == 0xFF) cpu.EIP = 0x804D600 diff --git a/tests/native/test_manticore.py b/tests/native/test_manticore.py index 8217e45b7..fcf4e3bc6 100644 --- a/tests/native/test_manticore.py +++ b/tests/native/test_manticore.py @@ -19,7 +19,7 @@ def setUp(self): def test_profiling_data(self): p = Profiler() - set_verbosity(0) + set_verbosity(1) self.m.register_plugin(p) self.m.run() self.m.finalize() @@ -139,7 +139,7 @@ def test_integration_basic_stdin(self): class ManticoreLogger(unittest.TestCase): """Make sure we set the logging levels correctly""" - _multiprocess_can_split_ = True + _multiprocess_can_split_ = False def test_logging(self): set_verbosity(5) diff --git a/tests/native/test_manticore_logger.py b/tests/native/test_manticore_logger.py new file mode 100644 index 000000000..8592b8d77 --- /dev/null +++ b/tests/native/test_manticore_logger.py @@ -0,0 +1,24 @@ +import unittest +import os +import logging +import filecmp + +from manticore.native import Manticore +from manticore.utils.log import get_verbosity, set_verbosity + + +class ManticoreLogger(unittest.TestCase): + """Make sure we set the logging levels correctly""" + + _multiprocess_can_split_ = False + + def test_logging(self): + set_verbosity(5) + self.assertEqual(get_verbosity("manticore.native.cpu.abstractcpu"), logging.DEBUG) + self.assertEqual(get_verbosity("manticore.ethereum.abi"), logging.DEBUG) + + set_verbosity(1) + self.assertEqual(get_verbosity("manticore.native.cpu.abstractcpu"), logging.WARNING) + self.assertEqual(get_verbosity("manticore.ethereum.abi"), logging.INFO) + + set_verbosity(0) #this is global and does not work in concurrent envs \ No newline at end of file diff --git a/tests/native/test_register.py b/tests/native/test_register.py index b38cef3e5..821a672f5 100644 --- a/tests/native/test_register.py +++ b/tests/native/test_register.py @@ -1,6 +1,6 @@ import unittest -from manticore.core.smtlib import Bool, BitvecConstant +from manticore.core.smtlib import Bool, BoolVariable, BitvecConstant from manticore.native.cpu.register import Register @@ -47,7 +47,7 @@ def test_bool_write_nonflag(self): def test_Bool(self): r = Register(32) - b = Bool() + b = BoolVariable(name="B") r.write(b) self.assertIs(r.read(), b) diff --git a/tests/native/test_state.py b/tests/native/test_state.py index 1bd17ea9c..6bf0d782d 100644 --- a/tests/native/test_state.py +++ b/tests/native/test_state.py @@ -79,27 +79,27 @@ def setUp(self): def test_solve_one(self): val = 42 - expr = BitvecVariable(32, "tmp") + expr = BitvecVariable(size=32, name="tmp") self.state.constrain(expr == val) solved = self.state.solve_one(expr) self.assertEqual(solved, val) def test_solve_n(self): - expr = BitvecVariable(32, "tmp") + expr = BitvecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) solved = sorted(self.state.solve_n(expr, 2)) self.assertEqual(solved, [5, 6]) def test_solve_n2(self): - expr = BitvecVariable(32, "tmp") + expr = BitvecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 100) solved = self.state.solve_n(expr, 5) self.assertEqual(len(solved), 5) def test_solve_min_max(self): - expr = BitvecVariable(32, "tmp") + expr = BitvecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) self.assertEqual(self.state.solve_min(expr), 5) @@ -107,7 +107,7 @@ def test_solve_min_max(self): self.assertEqual(self.state.solve_minmax(expr), (5, 6)) def test_policy_one(self): - expr = BitvecVariable(32, "tmp") + expr = BitvecVariable(size=32, name="tmp") self.state.constrain(expr > 0) self.state.constrain(expr < 100) solved = self.state.concretize(expr, "ONE") From 2aa5b7063f07fc0d518568eaeb14d985242188c0 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 19 Oct 2020 17:51:17 -0300 Subject: [PATCH 100/126] blkn --- manticore/native/memory.py | 10 +++++++--- tests/native/test_manticore_logger.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/manticore/native/memory.py b/manticore/native/memory.py index b66367665..223974349 100644 --- a/manticore/native/memory.py +++ b/manticore/native/memory.py @@ -12,7 +12,7 @@ expression, issymbolic, Expression, - MutableArray + MutableArray, ) from ..native.mappings import mmap, munmap from ..utils.helpers import interval_intersection @@ -385,10 +385,14 @@ def split(self, address: int): left_name, right_name = ["{}_{:d}".format(self._array.name, i) for i in range(2)] head_arr = expression.MutableArray( - expression.ArrayVariable(index_size=index_bits, length=left_size, value_size=value_bits, name=left_name) + expression.ArrayVariable( + index_size=index_bits, length=left_size, value_size=value_bits, name=left_name + ) ) tail_arr = expression.MutableArray( - expression.ArrayVariable(index_size=index_bits, length=right_size, value_size=value_bits, name=right_name) + expression.ArrayVariable( + index_size=index_bits, length=right_size, value_size=value_bits, name=right_name + ) ) head = ArrayMap(self.start, left_size, self.perms, index_bits, head_arr, left_name) diff --git a/tests/native/test_manticore_logger.py b/tests/native/test_manticore_logger.py index 8592b8d77..635028093 100644 --- a/tests/native/test_manticore_logger.py +++ b/tests/native/test_manticore_logger.py @@ -21,4 +21,4 @@ def test_logging(self): self.assertEqual(get_verbosity("manticore.native.cpu.abstractcpu"), logging.WARNING) self.assertEqual(get_verbosity("manticore.ethereum.abi"), logging.INFO) - set_verbosity(0) #this is global and does not work in concurrent envs \ No newline at end of file + set_verbosity(0) # this is global and does not work in concurrent envs From 188054ce108d43bfd0f21602208daca690d7ed5a Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 20 Oct 2020 00:02:09 -0300 Subject: [PATCH 101/126] Disable wasm --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ff3ab10e..4228cbc87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - type: ["ethereum_truffle", "ethereum_bench", "examples", "ethereum", "ethereum_vm", "native", "wasm", "wasm_sym", "other"] + type: ["ethereum_truffle", "ethereum_bench", "examples", "ethereum", "ethereum_vm", "native", "other"] # "wasm", "wasm_sym", steps: - uses: actions/checkout@v1 - name: Set up Python 3.6 From ba5499bfd5e06d3e2ad0c4016102ac34d4831a93 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 20 Oct 2020 12:27:37 -0300 Subject: [PATCH 102/126] Apply suggestions from code review Co-authored-by: Eric Kilmer --- manticore/core/smtlib/visitors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index c521f6eff..f3e6818c9 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -128,10 +128,10 @@ def _changed(self, expression: Expression, operands): def _rebuild(self, expression: Operation, operands): """Default operation used when no visiting method was successful for - this expression. If he operands have changed this reubild the curren expression + this expression. If the operands have changed this rebuild the current expression with the new operands. - Assumes the stack is used for Expresisons + Assumes the stack is used for Expressions """ if self._changed(expression, operands): aux = copy.copy(expression) From 2fe9a064dd74dd8746bc7337ab0e39e9ad1ba7e7 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 20 Oct 2020 17:52:12 -0300 Subject: [PATCH 103/126] Fix tests --- manticore/core/smtlib/solver.py | 2 +- manticore/ethereum/detectors.py | 4 ++++ tests/native/test_driver.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 863df056c..2ecfbd9e5 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -567,7 +567,7 @@ def get_all_values( length=expression.length, value_size=expression.value_size, taint=expression.taint, - ).array + ) else: raise NotImplementedError( f"get_all_values only implemented for {type(expression)} expression type." diff --git a/manticore/ethereum/detectors.py b/manticore/ethereum/detectors.py index fd90e0ba2..ded85c1b2 100644 --- a/manticore/ethereum/detectors.py +++ b/manticore/ethereum/detectors.py @@ -632,6 +632,10 @@ class DetectDelegatecall(Detector): def _to_constant(self, expression): if isinstance(expression, Constant): return expression.value + else: + expression = simplify(expression) + if isinstance(expression, Constant): + return expression.value return expression def will_evm_execute_instruction_callback(self, state, instruction, arguments): diff --git a/tests/native/test_driver.py b/tests/native/test_driver.py index 3aa1d80b4..f4bac24ec 100644 --- a/tests/native/test_driver.py +++ b/tests/native/test_driver.py @@ -26,7 +26,7 @@ def testCreating(self): m.log_file = "/dev/null" def test_issymbolic(self): - v = BitvecVariable(32, "sym") + v = BitvecVariable(size=32, name="sym") self.assertTrue(issymbolic(v)) def test_issymbolic_neg(self): From 8d334284e2872577a7d9eda434e28fdada8a919a Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 1 Dec 2020 13:47:32 -0300 Subject: [PATCH 104/126] CI Test --- manticore/__main__.py | 6 +- manticore/core/manticore.py | 139 +++--- manticore/core/plugin.py | 12 +- manticore/core/smtlib/__init__.py | 2 +- manticore/core/smtlib/constraints.py | 83 +++- manticore/core/smtlib/expression.py | 381 +++++++++-------- manticore/core/smtlib/operators.py | 102 ++--- manticore/core/smtlib/solver.py | 37 +- manticore/core/smtlib/visitors.py | 404 +++++++++++------- manticore/core/state.py | 27 +- manticore/core/worker.py | 59 +-- manticore/ethereum/abi.py | 26 +- manticore/ethereum/abitypes.py | 40 +- manticore/ethereum/manticore.py | 25 +- manticore/ethereum/parsetab.py | 126 ++---- manticore/ethereum/plugins.py | 52 +-- manticore/ethereum/solidity.py | 4 +- manticore/ethereum/verifier.py | 12 +- manticore/native/cpu/abstractcpu.py | 22 +- manticore/native/cpu/arm.py | 6 +- manticore/native/cpu/bitwise.py | 48 +-- manticore/native/cpu/disasm.py | 3 +- manticore/native/cpu/register.py | 4 +- manticore/native/cpu/x86.py | 60 +-- manticore/native/manticore.py | 7 +- manticore/native/memory.py | 12 +- manticore/native/models.py | 18 +- manticore/native/state_merging.py | 4 +- manticore/platforms/decree.py | 151 ++++--- manticore/platforms/evm.py | 123 +++--- manticore/platforms/linux.py | 38 +- manticore/utils/emulate.py | 2 +- manticore/wasm/executor.py | 4 +- manticore/wasm/structure.py | 12 +- manticore/wasm/types.py | 10 +- .../contracts/simple_int_overflow.sol | 4 +- tests/ethereum/test_detectors.py | 8 +- tests/ethereum/test_general.py | 4 +- tests/ethereum/test_regressions.py | 44 +- tests/native/test_cpu_automatic.py | 1 + tests/native/test_cpu_manual.py | 22 +- tests/native/test_driver.py | 4 +- tests/native/test_integration_native.py | 10 +- tests/native/test_linux.py | 2 +- tests/native/test_models.py | 2 +- tests/native/test_register.py | 6 +- tests/native/test_state.py | 12 +- tests/other/test_smtlibv2.py | 271 ++++++++---- 48 files changed, 1343 insertions(+), 1108 deletions(-) diff --git a/manticore/__main__.py b/manticore/__main__.py index 12d553465..9eef5c569 100644 --- a/manticore/__main__.py +++ b/manticore/__main__.py @@ -26,6 +26,8 @@ def main() -> None: + import pdb + """ Dispatches execution into one of Manticore's engines: evm or native. """ @@ -209,7 +211,9 @@ def positive(value): ) eth_flags.add_argument( - "--limit-loops", action="store_true", help="Limit loops depth", + "--limit-loops", + action="store_true", + help="Limit loops depth", ) eth_flags.add_argument( diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index e3919f437..1650b66e3 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -136,7 +136,7 @@ def newFunction(self, *args, **kw): def at_running(func: Callable) -> Callable: # type: ignore """Allows the decorated method to run only when manticore is actively - exploring states + exploring states """ @functools.wraps(func) @@ -149,7 +149,7 @@ def newFunction(self, *args, **kw): def at_not_running(func: Callable) -> Callable: # type: ignore """Allows the decorated method to run only when manticore is NOT - exploring states + exploring states """ @functools.wraps(func) @@ -162,8 +162,7 @@ def newFunction(self, *args, **kw): return newFunction def only_from_main_script(func: Callable) -> Callable: # type: ignore - """Allows the decorated method to run only from the main manticore script - """ + """Allows the decorated method to run only from the main manticore script""" @functools.wraps(func) def newFunction(self, *args, **kw): @@ -379,14 +378,14 @@ def __init__( self._main_id = os.getpid(), threading.current_thread().ident def is_main(self): - """ True if called from the main process/script - Note: in "single" mode this is _most likely_ True """ + """True if called from the main process/script + Note: in "single" mode this is _most likely_ True""" return self._main_id == (os.getpid(), threading.current_thread().ident) @sync @only_from_main_script def take_snapshot(self): - """ Copy/Duplicate/backup all ready states and save it in a snapshot. + """Copy/Duplicate/backup all ready states and save it in a snapshot. If there is a snapshot already saved it will be overrwritten """ if self._snapshot is not None: @@ -401,8 +400,8 @@ def take_snapshot(self): @sync @only_from_main_script def goto_snapshot(self): - """ REMOVE current ready states and replace them with the saved states - in a snapshot """ + """REMOVE current ready states and replace them with the saved states + in a snapshot""" if not self._snapshot: raise ManticoreError("No snapshot to go to") self.clear_ready_states() @@ -530,32 +529,32 @@ def setstate(x, y): @staticmethod @deprecated("Use utils.log.set_verbosity instead.") def verbosity(level): - """ Sets global verbosity level. - This will activate different logging profiles globally depending - on the provided numeric value + """Sets global verbosity level. + This will activate different logging profiles globally depending + on the provided numeric value """ set_verbosity(level) # State storage @Eventful.will_did("save_state", can_raise=False) def _save(self, state, state_id=None) -> int: - """ Store or update a state in secondary storage under state_id. - Use a fresh id is None is provided. + """Store or update a state in secondary storage under state_id. + Use a fresh id is None is provided. - :param state: A manticore State - :param state_id: if not None force state_id (overwrite) - :type state_id: int or None - :returns: the state id used + :param state: A manticore State + :param state_id: if not None force state_id (overwrite) + :type state_id: int or None + :returns: the state id used """ state._id = self._workspace.save_state(state, state_id=state_id) return state.id @Eventful.will_did("load_state", can_raise=False) def _load(self, state_id: int) -> StateBase: - """ Load the state from the secondary storage + """Load the state from the secondary storage - :param state_id: a state id - :returns: the loaded state + :param state_id: a state id + :returns: the loaded state """ if not hasattr(self, "stcache"): self.stcache: weakref.WeakValueDictionary = weakref.WeakValueDictionary() @@ -570,9 +569,9 @@ def _load(self, state_id: int) -> StateBase: @Eventful.will_did("remove_state", can_raise=False) def _remove(self, state_id: int) -> int: - """ Remove a state from secondary storage + """Remove a state from secondary storage - :param state_id: a state id + :param state_id: a state id """ if not hasattr(self, "stcache"): self.stcache = weakref.WeakValueDictionary() @@ -584,14 +583,14 @@ def _remove(self, state_id: int) -> int: # Internal support for state lists def _put_state(self, state) -> int: - """ This enqueues the state for exploration. + """This enqueues the state for exploration. - Serialize and store the state with a fresh state_id. Then add it to - the shared READY states list + Serialize and store the state with a fresh state_id. Then add it to + the shared READY states list - +-------+ - State +----- >+ READY | - +-------+ + +-------+ + State +----- >+ READY | + +-------+ """ self._publish("will_enqueue_state", state, can_raise=False) @@ -646,11 +645,11 @@ def _get_state(self, wait=False) -> typing.Optional[StateBase]: @sync def _revive_state(self, state_id: int): - """ Send a state back to READY list + """Send a state back to READY list - +--------+ +------------------+ - | READY +<-------+ BUSY/TERMINATED | - +---+----+ +----------------+ + +--------+ +------------------+ + | READY +<-------+ BUSY/TERMINATED | + +---+----+ +----------------+ """ # Move from BUSY or TERMINATED to READY @@ -669,15 +668,15 @@ def _revive_state(self, state_id: int): @sync def _terminate_state(self, state_id: int, delete=False): - """ Send a BUSY state to the TERMINATED list or trash it if delete is True + """Send a BUSY state to the TERMINATED list or trash it if delete is True - +------+ +------------+ - | BUSY +------->+ TERMINATED | - +---+--+ +------------+ - | - v - ### - ### + +------+ +------------+ + | BUSY +------->+ TERMINATED | + +---+--+ +------------+ + | + v + ### + ### """ # wait for a state id to be added to the ready list and remove it @@ -698,15 +697,15 @@ def _terminate_state(self, state_id: int, delete=False): @sync def _kill_state(self, state_id: int, delete=False): - """ Send a BUSY state to the KILLED list or trash it if delete is True + """Send a BUSY state to the KILLED list or trash it if delete is True - +------+ +--------+ - | BUSY +------->+ KILLED | - +---+--+ +--------+ - | - v - ### - ### + +------+ +--------+ + | BUSY +------->+ KILLED | + +---+--+ +--------+ + | + v + ### + ### """ # wait for a state id to be added to the ready list and remove it @@ -727,12 +726,12 @@ def _kill_state(self, state_id: int, delete=False): @sync def kill_state(self, state: typing.Union[StateBase, int], delete: bool = False): - """ Kill a state. - A state is moved from any list to the kill list or fully - removed from secondary storage + """Kill a state. + A state is moved from any list to the kill list or fully + removed from secondary storage - :param state: a state - :param delete: if true remove the state from the secondary storage + :param state: a state + :param delete: if true remove the state from the secondary storage """ state_id = getattr(state, "id", state) @@ -814,10 +813,10 @@ def killed_states(self): @sync @at_not_running def _all_states(self): - """ Only allowed at not running. - (At running we can have states at busy) - Returns a tuple with all active state ids. - Notably the "killed" states are not included here. + """Only allowed at not running. + (At running we can have states at busy) + Returns a tuple with all active state ids. + Notably the "killed" states are not included here. """ return tuple(self._ready_states) + tuple(self._terminated_states) @@ -943,8 +942,8 @@ def register_plugin(self, plugin: Plugin): @at_not_running def unregister_plugin(self, plugin: typing.Union[str, Plugin]): - """ Removes a plugin from manticore. - No events should be sent to it after + """Removes a plugin from manticore. + No events should be sent to it after """ if isinstance(plugin, str): # Passed plugin.unique_name instead of value assert plugin in self.plugins, "Plugin instance not registered" @@ -968,10 +967,10 @@ def subscribe(self, name, callback): @property # type: ignore @at_not_running def context(self): - """ Convenient access to shared context. We maintain a local copy of the - share context during the time manticore is not running. - This local context is copied to the shared context when a run starts - and copied back when a run finishes + """Convenient access to shared context. We maintain a local copy of the + share context during the time manticore is not running. + This local context is copied to the shared context when a run starts + and copied back when a run finishes """ return self._shared_context @@ -1030,9 +1029,9 @@ def wait(self, condition): @sync def kill(self): - """ Attempt to cancel and kill all the workers. - Workers must terminate - RUNNING, STANDBY -> KILLED + """Attempt to cancel and kill all the workers. + Workers must terminate + RUNNING, STANDBY -> KILLED """ self._publish("will_terminate_execution", self._output) self._killed.value = True @@ -1065,8 +1064,8 @@ def workspace(self): @contextmanager def kill_timeout(self, timeout=None): - """ A convenient context manager that will kill a manticore run after - timeout seconds + """A convenient context manager that will kill a manticore run after + timeout seconds """ if timeout is None: timeout = consts.timeout @@ -1154,7 +1153,7 @@ def run(self): @at_not_running def remove_all(self): """ - Deletes all streams from storage and clean state lists + Deletes all streams from storage and clean state lists """ for state_id in self._all_states: self._remove(state_id) diff --git a/manticore/core/plugin.py b/manticore/core/plugin.py index 33de6b276..9d2dac38d 100644 --- a/manticore/core/plugin.py +++ b/manticore/core/plugin.py @@ -100,8 +100,8 @@ def on_unregister(self): pass def generate_testcase(self, state, testcase, message): - """ Called so the plugin can attach some results to the testcase if the - state needs it""" + """Called so the plugin can attach some results to the testcase if the + state needs it""" pass @@ -361,8 +361,8 @@ def did_execute_instruction_callback(self, state, pc, target_pc, instruction): logger.info("did_execute_instruction %r %r %r %r", state, pc, target_pc, instruction) def will_run_callback(self, state): - """ Called once at the beginning of the run. - state is the initial root state + """Called once at the beginning of the run. + state is the initial root state """ logger.info("will_run") @@ -638,7 +638,9 @@ def on_execution_intermittent_callback( state.id, ) update_cb( - context.setdefault(state.id, StateDescriptor(state_id=state.id)), *args, **kwargs, + context.setdefault(state.id, StateDescriptor(state_id=state.id)), + *args, + **kwargs, ) context[state.id].last_intermittent_update = datetime.now() diff --git a/manticore/core/smtlib/__init__.py b/manticore/core/smtlib/__init__.py index cc75d1150..0128c94f6 100644 --- a/manticore/core/smtlib/__init__.py +++ b/manticore/core/smtlib/__init__.py @@ -1,4 +1,4 @@ -from .expression import Expression, Bool, Bitvec, Array, BitvecConstant, issymbolic # noqa +from .expression import Expression, Bool, BitVec, Array, BitVecConstant, issymbolic # noqa from .constraints import ConstraintSet # noqa from .solver import * # noqa from . import operators as Operators # noqa diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 4c82f2004..64615d060 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -1,25 +1,32 @@ import itertools import sys import copy -from typing import Optional from ...utils.helpers import PickleSerializer from ...exceptions import SmtlibError from .expression import ( Expression, - BitvecVariable, + BitVecVariable, BoolVariable, ArrayVariable, Array, Bool, - Bitvec, + BitVec, BoolConstant, MutableArray, BoolEqual, Variable, Constant, ) -from .visitors import GetDeclarations, TranslatorSmtlib, get_variables, simplify, replace -from ...utils import config +from .visitors import ( + GetBindings, + GetDeclarations, + TranslatorSmtlib, + get_variables, + simplify, + replace, + CountExpressionUse, + translate_to_smtlib, +) import logging logger = logging.getLogger(__name__) @@ -93,6 +100,7 @@ def add(self, constraint) -> None: constraint = BoolConstant(constraint) assert isinstance(constraint, Bool) constraint = simplify(constraint) + # If self._child is not None this constraint set has been forked and a # a derived constraintset may be using this. So we can't add any more # constraints to this one. After the child constraintSet is deleted @@ -174,28 +182,65 @@ def related_to(self, *related_to) -> "ConstraintSet": return cs def to_string(self, replace_constants: bool = False) -> str: + replace_constants = True variables, constraints = self.get_declared_variables(), self.constraints + # rep_bindings = {} + # extra_variables, extra_constraints = [], [] + # counts = CountExpressionUse() + # for c in constraints: + # counts.visit(c) + # for exp, count in counts.counts.items(): + # if count > 1: + # if isinstance(exp, BitVec): + # new_var = BitVecVariable(size=exp.size, name=new_name) + # if isinstance(exp, Bool): + # new_var = BoolVariable(name=new_name) + # if isinstance(exp, Array): + # new_var = ArrayVariable(name=new_name, index_size=exp.index_size, value_size=exp.value_size) + # extra_constraints.append(new_var == exp) + # extra_variables.append(new_var) + # rep_bindings[exp] = new_var + if replace_constants: constant_bindings = {} for expression in constraints: - # FIXME this will not catch Constant == Variable - # Maybe we need a way to canonicalize the expressions somewhere if ( isinstance(expression, BoolEqual) and isinstance(expression.operands[0], Variable) and isinstance(expression.operands[1], Constant) ): constant_bindings[expression.operands[0]] = expression.operands[1] + if ( + isinstance(expression, BoolEqual) + and isinstance(expression.operands[1], Variable) + and isinstance(expression.operands[0], Constant) + ): + constant_bindings[expression.operands[1]] = expression.operands[0] - translator = TranslatorSmtlib(use_bindings=False) + translator = TranslatorSmtlib(use_bindings=True) + # gb = GetBindings() for v in variables: translator.visit_Variable(v) + # for v in extra_variables: + # translator.visit_Variable(v) + # if constraints: + # for constraint in constraints: + # gb.visit(constraint) for constraint in constraints: if replace_constants: constraint = simplify(replace(constraint, constant_bindings)) - # if no variables then it is a constant - if isinstance(constraint, Constant) and constraint.value == True: - continue + if ( + isinstance(constraint, BoolEqual) + and isinstance(constraint.operands[0], Variable) + and isinstance(constraint.operands[1], Variable) + and constraint.operands[1] in constant_bindings + ): + constraint = simplify(replace(constraint, constant_bindings)) + + # constraint = simplify(replace(constraint, rep_bindings)) + # if no variables then it is a constant + if isinstance(constraint, Constant) and constraint.value == True: + continue # Translate one constraint translator.visit(constraint) @@ -203,6 +248,14 @@ def to_string(self, replace_constants: bool = False) -> str: for k, v in constant_bindings.items(): translator.visit(k == v) + # for constraint in extra_constraints: + # if replace_constants: + # constraint = simplify(replace(constraint, constant_bindings)) + # if isinstance(constraint, Constant) and constraint.value == True: + # continue + # # Translate one constraint + # translator.visit(constraint) + return translator.smtlib() def _declare(self, var): @@ -321,7 +374,7 @@ def migrate(self, expression, name_migration_map=None): # Create and declare a new variable of given type if isinstance(foreign_var, Bool): new_var = self.new_bool(name=migrated_name) - elif isinstance(foreign_var, Bitvec): + elif isinstance(foreign_var, BitVec): new_var = self.new_bitvec(foreign_var.size, name=migrated_name) elif isinstance(foreign_var, Array): # Note that we are discarding the ArrayProxy encapsulation @@ -367,10 +420,10 @@ def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False) :param name: try to assign name to internal variable representation, if not unique, a numeric nonce will be appended :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True - :return: a fresh BitvecVariable + :return: a fresh BitVecVariable """ if size <= 0: - raise ValueError(f"Bitvec size ({size}) can't be equal to or less than 0") + raise ValueError(f"BitVec size ({size}) can't be equal to or less than 0") if name is None: name = "BV" avoid_collisions = True @@ -378,7 +431,7 @@ def new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False) name = self._make_unique_name(name) if not avoid_collisions and name in self._declarations: raise ValueError(f"Name {name} already used") - var = BitvecVariable(size=size, name=name, taint=taint) + var = BitVecVariable(size=size, name=name, taint=taint) return self._declare(var) def new_array( diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 8b547c832..09b05b0be 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1,7 +1,7 @@ """ Module for Symbolic Expression ConstraintSets are considered a factory for new symbolic variables of type: -BoolVariable, BitvecVariable and ArrayVariable. +BoolVariable, BitVecVariable and ArrayVariable. Normal python operators are overloaded in each class, complex expressions trees are built operating over expression variables and constants @@ -22,12 +22,16 @@ import re import copy from typing import Union, Optional, Tuple, List +from functools import lru_cache def local_simplify(e): from .visitors import simplify as visitor_simplify - return visitor_simplify(e) + x = visitor_simplify(e) + if isinstance(e, Bool): + assert isinstance(x, Bool) + return x class ExpressionError(Exception): @@ -75,7 +79,7 @@ class X(object): def __new__(cls, clsname, bases, attrs, abstract=False): - xslots = set(attrs.get("__xslots__", ())) + xslots = frozenset(attrs.get("__xslots__", ())) # merge the xslots of all the bases with the one defined here for base in bases: xslots = xslots.union(getattr(base, "__xslots__", ())) @@ -86,7 +90,26 @@ def __new__(cls, clsname, bases, attrs, abstract=False): attrs["__slots__"]: Tuple[str] = tuple( map(lambda attr: attr.split("#", 1)[0], attrs["__xslots__"]) ) - + """ + def h(self): + print(self.__class__, self.__slots__) + s = [] + for name in self.__slots__: + if name in ("_concrete_cache", "_written"): + continue + try: + print (name) + x = getattr(self, name) + hash(x) + s.append(name) + except Exception as e: + print ("AAAAAAAAAAAAAAAAAAAAERRR", name, e) + return hash((clsname, tuple(getattr(self, name) for name in s))) + + attrs["__hash__"] = h + """ + attrs["__hash__"] = object.__hash__ + # attrs["__hash__"] = lambda self : hash((clsname, tuple(getattr(self, name) for name in self.__slots__ if name not in ("_concrete_cache", "_written")))) return super().__new__(cls, clsname, bases, attrs) @@ -148,6 +171,12 @@ def name(self): def __repr__(self): return "<{:s}({:s}) at {:x}>".format(type(self).__name__, self.name, id(self)) + """ + def __eq__(self, other): + # Ignore the taint for eq comparison + return self._name==other._name and super().__eq__(other) + """ + class Constant(Expression, abstract=True): """ Constants expressions have a concrete python value. """ @@ -213,9 +242,6 @@ def __eq__(self, other): # will have its __hash__() implicitly set to None. return BoolEqual(self, self.cast(other)) - def __hash__(self): - return object.__hash__(self) - def __ne__(self, other): return BoolNot(self == self.cast(other)) @@ -237,7 +263,7 @@ def __ror__(self, other): def __rxor__(self, other): return BoolXor(self.cast(other), self) - def __bool__(self): + def __dbool__(self): raise ExpressionError( "You tried to use a Bool Expression as a boolean constant. Expressions could represent a set of concrete values." ) @@ -254,16 +280,6 @@ def __init__(self, value: bool, **kwargs): def __bool__(self): return self._value - def __eq__(self, other): - # A class that overrides __eq__() and does not define __hash__() - # will have its __hash__() implicitly set to None. - if self.taint: - return super().__eq__(other) - return self.value == other - - def __hash__(self): - return object.__hash__(self) - class BoolOperation(Bool, Operation, abstract=True): """ It's an operation that results in a Bool """ @@ -271,10 +287,10 @@ class BoolOperation(Bool, Operation, abstract=True): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def __bool__(self): + def __xbool__(self): # FIXME: TODO: re-think is we want to be this forgiving every use of # local_simplify looks hacky - simplified = local_simplify(self) + simplified = self # local_simplify(self) if isinstance(simplified, Constant): return simplified.value raise ExpressionError("BoolOperation can not be reduced to a constant") @@ -305,8 +321,8 @@ def __init__(self, cond: Bool, true: Bool, false: Bool, **kwargs): super().__init__(operands=(cond, true, false), **kwargs) -class Bitvec(Expression, abstract=True): - """ Bitvector expressions have a fixed bit size """ +class BitVec(Expression, abstract=True): + """ BitVector expressions have a fixed bit size """ __xslots__: Tuple[str, ...] = ("_size",) @@ -330,49 +346,49 @@ def mask(self): def signmask(self): return 1 << (self.size - 1) - def cast(self, value: Union["Bitvec", str, int, bytes], **kwargs) -> "Bitvec": - """ Cast a value int a Bitvec """ - if isinstance(value, Bitvec): + def cast(self, value: Union["BitVec", str, int, bytes], **kwargs) -> "BitVec": + """ Cast a value int a BitVec """ + if isinstance(value, BitVec): if value.size != self.size: - raise ExpressionError("Bitvector of unexpected size") + raise ExpressionError("BitVector of unexpected size") return value if isinstance(value, (str, bytes)) and len(value) == 1: value = ord(value) # Try to support not Integral types that can be casted to int value = int(value) & self.mask if not isinstance(value, int): - raise ExpressionError("Not cast-able to Bitvec") - return BitvecConstant(self.size, value, **kwargs) + raise ExpressionError("Not cast-able to BitVec") + return BitVecConstant(self.size, value, **kwargs) def __add__(self, other): - return BitvecAdd(self, self.cast(other)) + return BitVecAdd(self, self.cast(other)) def __sub__(self, other): - return BitvecSub(self, self.cast(other)) + return BitVecSub(self, self.cast(other)) def __mul__(self, other): - return BitvecMul(self, self.cast(other)) + return BitVecMul(self, self.cast(other)) def __mod__(self, other): - return BitvecMod(self, self.cast(other)) + return BitVecMod(self, self.cast(other)) # object.__divmod__(self, other) # object.__pow__(self, other[, modulo]) def __lshift__(self, other): - return BitvecShiftLeft(self, self.cast(other)) + return BitVecShiftLeft(self, self.cast(other)) def __rshift__(self, other): - return BitvecShiftRight(self, self.cast(other)) + return BitVecShiftRight(self, self.cast(other)) def __and__(self, other): - return BitvecAnd(self, self.cast(other)) + return BitVecAnd(self, self.cast(other)) def __xor__(self, other): - return BitvecXor(self, self.cast(other)) + return BitVecXor(self, self.cast(other)) def __or__(self, other): - return BitvecOr(self, self.cast(other)) + return BitVecOr(self, self.cast(other)) # The division operator (/) is implemented by these methods. The # __truediv__() method is used when __future__.division is in effect, @@ -381,10 +397,10 @@ def __or__(self, other): # TypeError will be raised instead. def __div__(self, other): - return BitvecDiv(self, self.cast(other)) + return BitVecDiv(self, self.cast(other)) def __truediv__(self, other): - return BitvecDiv(self, self.cast(other)) + return BitVecDiv(self, self.cast(other)) def __floordiv__(self, other): return self / other @@ -398,43 +414,43 @@ def __floordiv__(self, other): # y.__rsub__(x) is called if x.__sub__(y) returns NotImplemented. def __radd__(self, other): - return BitvecAdd(self.cast(other), self) + return BitVecAdd(self.cast(other), self) def __rsub__(self, other): - return BitvecSub(self.cast(other), self) + return BitVecSub(self.cast(other), self) def __rmul__(self, other): - return BitvecMul(self.cast(other), self) + return BitVecMul(self.cast(other), self) def __rmod__(self, other): - return BitvecMod(self.cast(other), self) + return BitVecMod(self.cast(other), self) def __rtruediv__(self, other): - return BitvecDiv(self.cast(other), self) + return BitVecDiv(self.cast(other), self) def __rdiv__(self, other): - return BitvecDiv(self.cast(other), self) + return BitVecDiv(self.cast(other), self) # object.__rdivmod__(self, other) # object.__rpow__(self, other) def __rlshift__(self, other): - return BitvecShiftLeft(self.cast(other), self) + return BitVecShiftLeft(self.cast(other), self) def __rrshift__(self, other): - return BitvecShiftRight(self.cast(other), self) + return BitVecShiftRight(self.cast(other), self) def __rand__(self, other): - return BitvecAnd(self.cast(other), self) + return BitVecAnd(self.cast(other), self) def __rxor__(self, other): - return BitvecXor(self.cast(other), self) + return BitVecXor(self.cast(other), self) def __ror__(self, other): - return BitvecOr(self.cast(other), self) + return BitVecOr(self.cast(other), self) def __invert__(self): - return BitvecXor(self, self.cast(self.mask)) + return BitVecXor(self, self.cast(self.mask)) # These are the so-called "rich comparison" methods, and are called # for comparison operators in preference to __cmp__() below. The @@ -458,10 +474,9 @@ def __eq__(self, other): # will have its __hash__() implicitly set to None. return BoolEqual(self, self.cast(other)) - def __hash__(self): - return object.__hash__(self) - def __ne__(self, other): + # A class that overrides __eq__() and does not define __hash__() + # will have its __hash__() implicitly set to None. return BoolNot(BoolEqual(self, self.cast(other))) def __gt__(self, other): @@ -471,7 +486,7 @@ def __ge__(self, other): return BoolGreaterOrEqualThan(self, self.cast(other)) def __neg__(self): - return BitvecNeg(self) + return BitVecNeg(self) # Unsigned comparisons def ugt(self, other): @@ -487,44 +502,44 @@ def ule(self, other): return BoolUnsignedLessOrEqualThan(self, self.cast(other)) def udiv(self, other): - return BitvecUnsignedDiv(self, self.cast(other)) + return BitVecUnsignedDiv(self, self.cast(other)) def rudiv(self, other): - return BitvecUnsignedDiv(self.cast(other), self) + return BitVecUnsignedDiv(self.cast(other), self) def sdiv(self, other): - return BitvecDiv(self, self.cast(other)) + return BitVecDiv(self, self.cast(other)) def rsdiv(self, other): - return BitvecDiv(self.cast(other), self) + return BitVecDiv(self.cast(other), self) def srem(self, other): - return BitvecRem(self, self.cast(other)) + return BitVecRem(self, self.cast(other)) def rsrem(self, other): - return BitvecRem(self.cast(other), self) + return BitVecRem(self.cast(other), self) def urem(self, other): - return BitvecUnsignedRem(self, self.cast(other)) + return BitVecUnsignedRem(self, self.cast(other)) def rurem(self, other): - return BitvecUnsignedRem(self.cast(other), self) + return BitVecUnsignedRem(self.cast(other), self) def sar(self, other): - return BitvecArithmeticShiftRight(self, self.cast(other)) + return BitVecArithmeticShiftRight(self, self.cast(other)) def sal(self, other): - return BitvecArithmeticShiftLeft(self, self.cast(other)) + return BitVecArithmeticShiftLeft(self, self.cast(other)) def Bool(self): return self != 0 -class BitvecVariable(Bitvec, Variable): +class BitVecVariable(BitVec, Variable): pass -class BitvecConstant(Bitvec, Constant): +class BitVecConstant(BitVec, Constant): def __init__(self, size: int, value: int, **kwargs): """ A bitvector constant """ value &= (1 << size) - 1 # Can not use self.mask yet @@ -545,142 +560,139 @@ def signed_value(self): return self._value def __eq__(self, other): - # If not tainted use the concrete value - if not self.taint: - return self.value == other - return super().__eq__(other) + # Ignore the taint for eq comparison + return self._value == other - def __hash__(self): - # need to overload because we defined an __eq__ - return object.__hash__(self) + def __repr__(self): + return f"BitVecConstant<{self.size}, {self.value}>" -class BitvecOperation(Bitvec, Operation, abstract=True): - """ Operations that result in a Bitvec """ +class BitVecOperation(BitVec, Operation, abstract=True): + """ Operations that result in a BitVec """ pass -class BitvecAdd(BitvecOperation): +class BitVecAdd(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecSub(BitvecOperation): +class BitVecSub(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecMul(BitvecOperation): +class BitVecMul(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecDiv(BitvecOperation): +class BitVecDiv(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecUnsignedDiv(BitvecOperation): +class BitVecUnsignedDiv(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecMod(BitvecOperation): +class BitVecMod(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecRem(BitvecOperation): +class BitVecRem(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecUnsignedRem(BitvecOperation): +class BitVecUnsignedRem(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecShiftLeft(BitvecOperation): +class BitVecShiftLeft(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecShiftRight(BitvecOperation): +class BitVecShiftRight(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecArithmeticShiftLeft(BitvecOperation): +class BitVecArithmeticShiftLeft(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecArithmeticShiftRight(BitvecOperation): +class BitVecArithmeticShiftRight(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecAnd(BitvecOperation): +class BitVecAnd(BitVecOperation): def __init__(self, operanda, operandb, *args, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecOr(BitvecOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, *args, **kwargs): +class BitVecOr(BitVecOperation): + def __init__(self, operanda: BitVec, operandb: BitVec, *args, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecXor(BitvecOperation): +class BitVecXor(BitVecOperation): def __init__(self, operanda, operandb, **kwargs): super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) -class BitvecNot(BitvecOperation): +class BitVecNot(BitVecOperation): def __init__(self, operanda, **kwargs): super().__init__(size=operanda.size, operands=(operanda,), **kwargs) -class BitvecNeg(BitvecOperation): +class BitVecNeg(BitVecOperation): def __init__(self, operanda, **kwargs): super().__init__(size=operanda.size, operands=(operanda,), **kwargs) # Comparing two bitvectors results in a Bool class BoolLessThan(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) class BoolLessOrEqualThan(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) class BoolEqual(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): assert isinstance(operanda, Expression) assert isinstance(operandb, Expression) super().__init__(operands=(operanda, operandb), **kwargs) class BoolGreaterThan(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) class BoolGreaterOrEqualThan(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, *args, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, *args, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) class BoolUnsignedLessThan(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) class BoolUnsignedLessOrEqualThan(BoolOperation): - def __init__(self, operanda: Bitvec, operandb: Bitvec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) @@ -751,7 +763,7 @@ def is_known(self, index) -> Union[Bool, bool]: raise NotImplementedError # Following methods are implemented on top of the abstract methods ^ - def in_bounds(self, index: Union[Bitvec, int]) -> Union[Bool, bool]: + def in_bounds(self, index: Union[BitVec, int]) -> Union[Bool, bool]: """ True if the index points inside the array (or array is unbounded)""" if self.length is not None: return 0 <= index < self.length @@ -761,10 +773,6 @@ def __len__(self): """ Number of values. """ return self.length - def get(self, index): - """ FIXME: Should this exist?""" - return self.select(index) - def cast(self, array) -> "Array": """Builds an Array from bytes or bytearray FIXME: this assigns a random name to a new variable and does not use @@ -783,21 +791,23 @@ def cast(self, array) -> "Array": arr = arr.store(pos, byte) return arr - def cast_index(self, index: Union[int, Bitvec]) -> Bitvec: + def cast_index(self, index: Union[int, BitVec]) -> BitVec: """Forgiving casting method that will translate compatible values into a compliant BitVec for indexing""" if isinstance(index, int): - return BitvecConstant(self.index_size, index) - if not isinstance(index, Bitvec) or index.size != self.index_size: - raise ExpressionError(f"Expected Bitvector of size {self.index_size}") + return BitVecConstant(self.index_size, index) + if not isinstance(index, BitVec) or index.size != self.index_size: + raise ExpressionError(f"Expected BitVector of size {self.index_size}") + if isinstance(index, Constant): + return index return local_simplify(index) - def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: + def cast_value(self, value: Union[BitVec, bytes, int]) -> BitVec: """Forgiving casting method that will translate compatible values into - a compliant Bitvec to be used as a value""" - if not isinstance(value, (Bitvec, bytes, int)): + a compliant BitVec to be used as a value""" + if not isinstance(value, (BitVec, bytes, int)): raise TypeError - if isinstance(value, Bitvec): + if isinstance(value, BitVec): if value.size != self.value_size: raise ValueError return value @@ -805,7 +815,7 @@ def cast_value(self, value: Union[Bitvec, bytes, int]) -> Bitvec: value = ord(value) if not isinstance(value, int): value = int(value) - return BitvecConstant(self.value_size, value) + return BitVecConstant(self.value_size, value) def write(self, offset, buf): """Builds a new unmutable Array instance on top of current array by @@ -838,10 +848,10 @@ def __iter__(self): @staticmethod def _compare_buffers(a, b): """ Builds an expression that represents equality between the two arrays.""" - if len(a) != len(b): + if a.length != b.length: return BoolConstant(False) cond = BoolConstant(True) - for i in range(len(a)): + for i in range(a.length): cond = BoolAnd(cond.cast(a[i] == b[i]), cond) if cond is BoolConstant(False): return BoolConstant(False) @@ -850,7 +860,7 @@ def _compare_buffers(a, b): def __eq__(self, other): """If both arrays has the same elements they are equal. The difference in taints are ignored.""" - return self._compare_buffers(self, other) + return self._compare_buffers(self, self.cast(other)) def __ne__(self, other): return BoolNot(self == other) @@ -863,11 +873,11 @@ def _fix_slice(self, index: slice): if stop is None: stop = len(self) size = stop - start - if isinstance(size, Bitvec): + if isinstance(size, BitVec): size = local_simplify(size) else: - size = BitvecConstant(self.index_size, size) - if not isinstance(size, BitvecConstant): + size = BitVecConstant(self.index_size, size) + if not isinstance(size, BitVecConstant): raise ExpressionError("Size could not be simplified to a constant in a slice operation") return start, stop, size.value @@ -891,39 +901,41 @@ def __add__(self, other): def __radd__(self, other): return self._concatenate(other, self) + @lru_cache(maxsize=128, typed=True) def read_BE(self, address, size): address = self.cast_index(address) bytes = [] for offset in range(size): - bytes.append(self.cast_value(self.get(address + offset))) - return BitvecConcat(operands=tuple(bytes)) + bytes.append(self.cast_value(self[address + offset])) + return BitVecConcat(operands=tuple(bytes)) + @lru_cache(maxsize=128, typed=True) def read_LE(self, address, size): address = self.cast_index(address) bytes = [] for offset in range(size): bytes.append(self.get(address + offset, self._default)) - return BitvecConcat(operands=reversed(bytes)) + return BitVecConcat(operands=reversed(bytes)) def write_BE(self, address, value, size): address = self.cast_index(address) - value = BitvecConstant(size=size * self.value_size, value=0).cast(value) + value = BitVecConstant(size=size * self.value_size, value=0).cast(value) array = self for offset in range(size): array = array.store( address + offset, - BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), + BitVecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), ) return array def write_LE(self, address, value, size): address = self.cast_index(address) - value = Bitvec(size * self.value_size).cast(value) + value = BitVec(size * self.value_size).cast(value) array = self for offset in reversed(range(size)): array = array.store( address + offset, - BitvecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), + BitVecExtract(value, (size - 1 - offset) * self.value_size, self.value_size), ) return array @@ -932,7 +944,11 @@ class ArrayConstant(Array, Constant): __xslots__: Tuple[str, ...] = ("_index_size", "_value_size") def __init__( - self, *, index_size: int, value_size: int, **kwargs, + self, + *, + index_size: int, + value_size: int, + **kwargs, ): self._index_size = index_size self._value_size = value_size @@ -951,18 +967,24 @@ def length(self): return len(self.value) def select(self, index): + """ ArrayConstant get """ + index = self.cast_index(index) + return self._select(index) + + @lru_cache(maxsize=128, typed=True) + def _select(self, index): """ ArrayConstant get """ index = self.cast_index(index) if isinstance(index, Constant): - return BitvecConstant( + return BitVecConstant( size=self.value_size, value=self.value[index.value], taint=self.taint ) # Index being symbolic generates a symbolic result ! - result = BitvecConstant(size=self.value_size, value=0, taint=("out_of_bounds")) + result = BitVecConstant(size=self.value_size, value=0, taint=("out_of_bounds")) for i, c in enumerate(self.value): - result = BitvecITE( - index == i, BitvecConstant(size=self.value_size, value=c), result, taint=self.taint + result = BitVecITE( + index == i, BitVecConstant(size=self.value_size, value=c), result, taint=self.taint ) return result @@ -999,9 +1021,6 @@ class ArrayVariable(Array, Variable): def length(self): return self._length - def __hash__(self): - return object.__hash__(self) - def __init__( self, index_size: int, @@ -1042,18 +1061,15 @@ def value_size(self): def default(self): return self._default - def get(self, index, default=None): + @lru_cache(maxsize=128, typed=True) + def select(self, index): """Gets an element from an empty Array.""" - index = self.cast_index(index) - if default is None: - default = self._default + default = self._default if default is not None: return default + index = self.cast_index(index) return ArraySelect(self, index) - def select(self, index): - return self.get(index) - def store(self, index, value): index = self.cast_index(index) value = local_simplify(self.cast_value(value)) @@ -1061,7 +1077,7 @@ def store(self, index, value): @property def written(self): - return set() + return frozenset() def is_known(self, index): return False @@ -1156,7 +1172,7 @@ def is_known(self, index): is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) return is_known_index - def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): + def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size self._written = None # Cache of the known indexes @@ -1170,7 +1186,8 @@ def __init__(self, array: Array, index: Bitvec, value: Bitvec, **kwargs): # self._concrete_cache[index.value] = value super().__init__( - operands=(array, index, value), **kwargs, + operands=(array, index, value), + **kwargs, ) @property @@ -1189,9 +1206,6 @@ def index_size(self): def value_size(self): return self.value.size - def __hash__(self): - return object.__hash__(self) - @property def array(self): return self.operands[0] @@ -1218,7 +1232,7 @@ def select(self, index): # Emulate list[-1] has_length = self.length is not None if has_length: - index = local_simplify(BitvecITE(index < 0, self.length + index, index)) + index = local_simplify(BitVecITE(index < 0, self.length + index, index)) if isinstance(index, Constant): if has_length and index.value >= self.length: @@ -1232,14 +1246,18 @@ def select(self, index): return ArraySelect(self, index) # if a default is defined we need to check if the index was previously written - return BitvecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default)) + return BitVecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default)) def store(self, index, value): - index = local_simplify(self.cast_index(index)) + casted = self.cast_index(index) + index = local_simplify(casted) value = self.cast_value(value) new_array = ArrayStore(self, index, value) return new_array + def __eq__(self, other): + return self._compare_buffers(self, self.cast(other)) + class ArraySlice(ArrayOperation): """Provides a projection of an underlying array. @@ -1252,12 +1270,10 @@ def __init__(self, array: "Array", offset: int, size: int, **kwargs): raise ValueError("Array expected") super().__init__( - operands=(array, array.cast_index(offset), array.cast_index(size)), **kwargs, + operands=(array, array.cast_index(offset), array.cast_index(size)), + **kwargs, ) - def __hash__(self): - return object.__hash__(self) - @property def array(self): return self.operands[0] @@ -1292,13 +1308,19 @@ def select(self, index): def store(self, index, value): return ArraySlice( - self.array.store(index + self.offset, value), offset=self.offset, size=len(self), + self.array.store(index + self.offset, value), + offset=self.offset, + size=len(self), ) @property def default(self): return self.array.default + def __eq__(self, other): + # Ignore the taint for eq comparison + return self._compare_buffers(self, self.cast(other)) + class MutableArray: """ @@ -1318,6 +1340,13 @@ def __init__(self, array: Array): self._array: Array = array + def __eq__(self, other): + """ Comparing the inner array of a MutableArray with other""" + return self.array == other + + def __hash__(self): + raise NotImplementedError() + @property def underlying_variable(self): return self._array.underlying_variable @@ -1403,9 +1432,6 @@ def write(self, offset, buf): def read(self, offset, size): return MutableArray(self._array[offset : offset + size]) - def __eq__(self, other): - return self.array == other - def __ne__(self, other): return BoolNot(self == other) @@ -1420,10 +1446,11 @@ def __radd__(self, other): return MutableArray(other + self.array) -class ArraySelect(BitvecOperation): - __xslots__ = BitvecOperation.__xslots__ +class ArraySelect(BitVecOperation): + __xslots__ = BitVecOperation.__xslots__ - def __init__(self, array: "Array", index: "Bitvec", *args, **kwargs): + def __init__(self, array: "Array", index: "BitVec", *args, **kwargs): + assert isinstance(array, Array) assert index.size == array.index_size super().__init__(size=array.value_size, operands=(array, index), **kwargs) @@ -1439,8 +1466,8 @@ def __repr__(self): return f"" -class BitvecSignExtend(BitvecOperation): - def __init__(self, operand: Bitvec, size: int, *args, **kwargs): +class BitVecSignExtend(BitVecOperation): + def __init__(self, operand: BitVec, size: int, *args, **kwargs): super().__init__(size=size, operands=(operand,), **kwargs) @property @@ -1448,8 +1475,8 @@ def extend(self): return self.size - self.operands[0].size -class BitvecZeroExtend(BitvecOperation): - def __init__(self, size: int, operand: Bitvec, *args, **kwargs): +class BitVecZeroExtend(BitVecOperation): + def __init__(self, size: int, operand: BitVec, *args, **kwargs): super().__init__(size=size, operands=(operand,), **kwargs) @property @@ -1457,10 +1484,10 @@ def extend(self): return self.size - self.operands[0].size -class BitvecExtract(BitvecOperation): +class BitVecExtract(BitVecOperation): __xslots__ = ("_offset",) - def __init__(self, operand: "Bitvec", offset: int, size: int, *args, **kwargs): + def __init__(self, operand: "BitVec", offset: int, size: int, *args, **kwargs): assert offset >= 0 and offset + size <= operand.size super().__init__(size=size, operands=(operand,), **kwargs) self._offset = offset @@ -1478,17 +1505,21 @@ def end(self): return self.begining + self.size - 1 -class BitvecConcat(BitvecOperation): - def __init__(self, operands: Tuple[Bitvec, ...], **kwargs): +class BitVecConcat(BitVecOperation): + def __init__(self, operands: Tuple[BitVec, ...], **kwargs): size = sum(x.size for x in operands) super().__init__(size=size, operands=operands, **kwargs) -class BitvecITE(BitvecOperation): - __xslots__ = BitvecOperation.__xslots__ +class BitVecITE(BitVecOperation): + __xslots__ = BitVecOperation.__xslots__ def __init__( - self, condition: Bool, true_value: Bitvec, false_value: Bitvec, **kwargs, + self, + condition: Bool, + true_value: BitVec, + false_value: BitVec, + **kwargs, ): super().__init__( @@ -1567,7 +1598,7 @@ def taint_with(arg, *taints, value_size=256, index_size=256): tainted_fset = frozenset(tuple(taints)) if not issymbolic(arg): if isinstance(arg, int): - arg = BitvecConstant(value_size, arg) + arg = BitVecConstant(value_size, arg) arg._taint = tainted_fset else: raise ValueError("type not supported") diff --git a/manticore/core/smtlib/operators.py b/manticore/core/smtlib/operators.py index b30a4f5b3..5f381e23d 100644 --- a/manticore/core/smtlib/operators.py +++ b/manticore/core/smtlib/operators.py @@ -1,12 +1,12 @@ from .expression import ( - Bitvec, - BitvecExtract, - BitvecSignExtend, - BitvecZeroExtend, - BitvecConstant, - BitvecConcat, + BitVec, + BitVecExtract, + BitVecSignExtend, + BitVecZeroExtend, + BitVecConstant, + BitVecConcat, Bool, - BitvecITE, + BitVecITE, BoolConstant, BoolITE, ) @@ -15,11 +15,11 @@ def ORD(s): - if isinstance(s, Bitvec): + if isinstance(s, BitVec): if s.size == 8: return s else: - return BitvecExtract(s, 0, 8) + return BitVecExtract(s, 0, 8) elif isinstance(s, int): return s & 0xFF else: @@ -27,11 +27,11 @@ def ORD(s): def CHR(s): - if isinstance(s, Bitvec): + if isinstance(s, BitVec): if s.size == 8: return s else: - return BitvecExtract(s, 0, 8) + return BitVecExtract(s, 0, 8) elif isinstance(s, int): return bytes([s & 0xFF]) else: @@ -66,15 +66,15 @@ def OR(a, b, *others): if isinstance(b, Bool): return b | a result = a | b - if isinstance(result, (Bitvec, int)): + if isinstance(result, (BitVec, int)): result = ITE(result != 0, True, False) return result def UGT(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.ugt(b) - if isinstance(b, Bitvec): + if isinstance(b, BitVec): return b.ult(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -85,9 +85,9 @@ def UGT(a, b): def UGE(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.uge(b) - if isinstance(b, Bitvec): + if isinstance(b, BitVec): return b.ule(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -98,9 +98,9 @@ def UGE(a, b): def ULT(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.ult(b) - if isinstance(b, Bitvec): + if isinstance(b, BitVec): return b.ugt(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -111,9 +111,9 @@ def ULT(a, b): def ULE(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.ule(b) - if isinstance(b, Bitvec): + if isinstance(b, BitVec): return b.uge(a) if a < 0: a = a & 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF @@ -124,12 +124,12 @@ def ULE(a, b): def EXTRACT(x, offset, size): - if isinstance(x, Bitvec) and isinstance(offset, Bitvec): - return BitvecExtract(x >> offset, 0, size) - elif isinstance(x, Bitvec): + if isinstance(x, BitVec) and isinstance(offset, BitVec): + return BitVecExtract(x >> offset, 0, size) + elif isinstance(x, BitVec): if offset == 0 and size == x.size: return x - return BitvecExtract(x, offset, size) + return BitVecExtract(x, offset, size) else: return (x >> offset) & ((1 << size) - 1) @@ -140,15 +140,15 @@ def SEXTEND(x, size_src, size_dest): x -= 1 << size_src return x & ((1 << size_dest) - 1) assert x.size == size_src - return BitvecSignExtend(x, size_dest) + return BitVecSignExtend(x, size_dest) def ZEXTEND(x, size): if isinstance(x, int): return x & ((1 << size) - 1) - assert isinstance(x, Bitvec) and size - x.size >= 0 + assert isinstance(x, BitVec) and size - x.size >= 0 if size - x.size > 0: - return BitvecZeroExtend(size, x) + return BitVecZeroExtend(size, x) else: return x @@ -160,10 +160,10 @@ def CONCAT(total_size, *args): def cast(x): if isinstance(x, int): - return BitvecConstant(arg_size, x) + return BitVecConstant(arg_size, x) return x - return BitvecConcat(operands=tuple(map(cast, args))) + return BitVecConcat(operands=tuple(map(cast, args))) else: return args[0] else: @@ -174,8 +174,8 @@ def cast(x): def ITE(cond, true_value, false_value): - assert isinstance(true_value, (Bool, bool, Bitvec, int)) - assert isinstance(false_value, (Bool, bool, Bitvec, int)) + assert isinstance(true_value, (Bool, bool, BitVec, int)) + assert isinstance(false_value, (Bool, bool, BitVec, int)) assert isinstance(cond, (Bool, bool)) if isinstance(cond, bool): if cond: @@ -193,14 +193,14 @@ def ITE(cond, true_value, false_value): def ITEBV(size, cond, true_value, false_value): - if isinstance(cond, Bitvec): + if isinstance(cond, BitVec): cond = cond.Bool() if isinstance(cond, int): cond = cond != 0 assert isinstance(cond, (Bool, bool)) - assert isinstance(true_value, (Bitvec, int)) - assert isinstance(false_value, (Bitvec, int)) + assert isinstance(true_value, (BitVec, int)) + assert isinstance(false_value, (BitVec, int)) assert isinstance(size, int) if isinstance(cond, BoolConstant) and not cond.taint: @@ -213,42 +213,42 @@ def ITEBV(size, cond, true_value, false_value): return false_value if isinstance(true_value, int): - true_value = BitvecConstant(size, true_value) + true_value = BitVecConstant(size, true_value) if isinstance(false_value, int): - false_value = BitvecConstant(size, false_value) - return BitvecITE(cond, true_value, false_value) + false_value = BitVecConstant(size, false_value) + return BitVecITE(cond, true_value, false_value) def UDIV(dividend, divisor): - if isinstance(dividend, Bitvec): + if isinstance(dividend, BitVec): return dividend.udiv(divisor) - elif isinstance(divisor, Bitvec): + elif isinstance(divisor, BitVec): return divisor.rudiv(dividend) assert dividend >= 0 or divisor > 0 # unsigned-es return dividend // divisor def SDIV(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.sdiv(b) - elif isinstance(b, Bitvec): + elif isinstance(b, BitVec): return b.rsdiv(a) return int(math.trunc(float(a) / float(b))) def SMOD(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.smod(b) - elif isinstance(b, Bitvec): + elif isinstance(b, BitVec): return b.rsmod(a) return int(math.fmod(a, b)) def SREM(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.srem(b) - elif isinstance(b, Bitvec): + elif isinstance(b, BitVec): return b.rsrem(a) elif isinstance(a, int) and isinstance(b, int): return a - int(a / b) * b @@ -256,22 +256,22 @@ def SREM(a, b): def UREM(a, b): - if isinstance(a, Bitvec): + if isinstance(a, BitVec): return a.urem(b) - elif isinstance(b, Bitvec): + elif isinstance(b, BitVec): return b.rurem(a) return a % b def SAR(size, a, b): assert isinstance(size, int) - if isinstance(b, Bitvec) and b.size != size: + if isinstance(b, BitVec) and b.size != size: b = ZEXTEND(b, size) - if isinstance(a, Bitvec): + if isinstance(a, BitVec): assert size == a.size return a.sar(b) - elif isinstance(b, Bitvec): - return BitvecConstant(size, a).sar(b) + elif isinstance(b, BitVec): + return BitVecConstant(size, a).sar(b) else: tempDest = a tempCount = b diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 2ecfbd9e5..7842ee1b5 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -133,16 +133,16 @@ def get_value(self, constraints, expression): """Ask the solver for one possible result of given expression using given set of constraints.""" raise SolverException("Abstract method not implemented") - def max(self, constraints, X: Bitvec, M=10000): + def max(self, constraints, X: BitVec, M=10000): """ Iteratively finds the maximum value for a symbol within given constraints. :param X: a symbol or expression :param M: maximum number of iterations allowed """ - assert isinstance(X, Bitvec) + assert isinstance(X, BitVec) return self.optimize(constraints, X, "maximize", M) - def min(self, constraints, X: Bitvec, M=10000): + def min(self, constraints, X: BitVec, M=10000): """ Iteratively finds the minimum value for a symbol within given constraints. @@ -150,7 +150,7 @@ def min(self, constraints, X: Bitvec, M=10000): :param X: a symbol or expression :param M: maximum number of iterations allowed """ - assert isinstance(X, Bitvec) + assert isinstance(X, BitVec) return self.optimize(constraints, X, "minimize", M) def minmax(self, constraints, x, iters=10000): @@ -223,7 +223,7 @@ def __readline_and_count(self): def send(self, cmd: str) -> None: """ - Send a string to the solver. + Send a string to the solver.s :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ @@ -245,6 +245,7 @@ def recv(self) -> str: buf = "".join(bufl).strip() + # print (">",buf) if self._debug: logger.debug("<%s", buf) @@ -381,11 +382,11 @@ def _getvalue(self, expression) -> Union[int, bool, bytes]: else: if isinstance(expression, BoolVariable): return self.__getvalue_bool(expression.name) - elif isinstance(expression, BitvecVariable): + elif isinstance(expression, BitVecVariable): return self.__getvalue_bv(expression.name) raise NotImplementedError( - f"_getvalue only implemented for Bool, Bitvec and Array. Got {type(expression)}" + f"_getvalue only implemented for Bool, BitVec and Array. Got {type(expression)}" ) # push pop @@ -408,7 +409,7 @@ def get_model(self, constraints: ConstraintSet): value = None if isinstance(variable, BoolVariable): value = self.__getvalue_bool(variable.name) - elif isinstance(variable, BitvecVariable): + elif isinstance(variable, BitVecVariable): value = self.__getvalue_bv(variable.name) elif isinstance(variable, Array): try: @@ -460,7 +461,7 @@ def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool] return self._is_sat() # get-all-values min max minmax - def _optimize_generic(self, constraints: ConstraintSet, x: Bitvec, goal: str, max_iter=10000): + def _optimize_generic(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): """ Iteratively finds the maximum or minimum value for the operation (Normally Operators.UGT or Operators.ULT) @@ -560,11 +561,12 @@ def get_all_values( with constraints as temp_cs: if isinstance(expression, Bool): var = temp_cs.new_bool() - elif isinstance(expression, Bitvec): + elif isinstance(expression, BitVec): var = temp_cs.new_bitvec(expression.size) elif isinstance(expression, Array): var = temp_cs.new_array( length=expression.length, + index_size=expression.index_size, value_size=expression.value_size, taint=expression.taint, ) @@ -572,7 +574,6 @@ def get_all_values( raise NotImplementedError( f"get_all_values only implemented for {type(expression)} expression type." ) - temp_cs.add(var == expression) self._reset(temp_cs.to_string()) result = [] @@ -595,7 +596,7 @@ def get_all_values( logger.info("Timeout searching for all solutions") return list(result) raise SolverError("Timeout") - # Sometimes adding a new contraint after a check-sat eats all the mem + # Sometimes adding a new constraint after a check-sat eats all the mem if self._multiple_check: self._smtlib.send(f"(assert {translate_to_smtlib(var != value)})") else: @@ -603,7 +604,7 @@ def get_all_values( self._reset(temp_cs.to_string()) return list(result) - def _optimize_fancy(self, constraints: ConstraintSet, x: Bitvec, goal: str, max_iter=10000): + def _optimize_fancy(self, constraints: ConstraintSet, x: BitVec, goal: str, max_iter=10000): """ Iteratively finds the maximum or minimum value for the operation (Normally Operators.UGT or Operators.ULT) @@ -641,7 +642,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): self._cache = getattr(self, "_cache", {}) model = self.get_model(constraints) - ####################33 + #################### values = [] start = time.time() # with constraints.related_to(*expressions) as temp_cs: @@ -660,10 +661,10 @@ def get_value(self, constraints: ConstraintSet, *expressions): if not issymbolic(expression): values.append(expression) continue - assert isinstance(expression, (Bool, Bitvec, Array)) + assert isinstance(expression, (Bool, BitVec, Array)) if isinstance(expression, Bool): var = temp_cs.new_bool() - elif isinstance(expression, Bitvec): + elif isinstance(expression, BitVec): var = temp_cs.new_bitvec(expression.size) elif isinstance(expression, Array): var = [] @@ -704,7 +705,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): if isinstance(expression, Bool): values.append(self.__getvalue_bool(var.name)) - if isinstance(expression, Bitvec): + if isinstance(expression, BitVec): values.append(self.__getvalue_bv(var.name)) bucket[hash(expression)] = values[-1] if time.time() - start > consts.timeout: @@ -762,7 +763,7 @@ def __autoconfig(self): # Certain version of Z3 fails to handle multiple check-sat # https://gist.github.com/feliam/0f125c00cb99ef05a6939a08c4578902 - multiple_check = self.version < Version(4, 8, 7) + multiple_check = False # self.version < Version(4, 8, 7) return init, support_minmax, support_reset, multiple_check def _solver_version(self) -> Version: diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index f3e6818c9..9328e3a54 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -1,3 +1,5 @@ +from typing import Optional, Dict + from ...utils.helpers import CacheDict from ...exceptions import SmtlibError from .expression import * @@ -6,7 +8,6 @@ import logging import operator import math -import threading from decimal import Decimal logger = logging.getLogger(__name__) @@ -22,10 +23,11 @@ class Visitor: Inherit your class visitor from this one and get called on a different visiting function for each type of expression. It will call the first implemented method for the __mro__ class order. - For example for a BitvecAdd expression this will try - visit_BitvecAdd() if not defined(*) then it will try with - visit_BitvecOperation() if not defined(*) then it will try with - visit_Bitvec() if not defined(*) then it will try with + For example for a BitVecAdd expression this will try + visit_BitVecAdd() if not defined(*) then it will try with + visit_BitVecOperation() if not defined(*) then it will try with + visit_BitVec() if not defined(*) then it will try with + visit_Operation() if not defined(*) then it will try with visit_Expression() (*) Or it is defined and it returns None for the node. @@ -70,8 +72,7 @@ def visit(self, node, use_fixed_point=False): """ cache = self._cache visited = set() - local_stack = [] - local_stack.append(node) # initially the stack contains only the visiting node + local_stack = [node] # initially the stack contains only the visiting node while local_stack: node = local_stack.pop() if node in cache: # If seen we do not really need to visit this @@ -124,8 +125,10 @@ def _method(self, expression, *operands): return self._rebuild(expression, operands) def _changed(self, expression: Expression, operands): - return any(x is not y for x, y in zip(expression.operands, operands)) + changed = any(x is not y for x, y in zip(expression.operands, operands)) + return changed + @lru_cache(maxsize=32, typed=True) def _rebuild(self, expression: Operation, operands): """Default operation used when no visiting method was successful for this expression. If the operands have changed this rebuild the current expression @@ -201,16 +204,18 @@ def get_depth(exp): return visitor.result +import io + + class PrettyPrinter(Visitor): def __init__(self, depth=None, **kwargs): super().__init__(**kwargs) - self.output = "" + self.output = io.StringIO() self.indent = 0 self.depth = depth def _print(self, s, e=None): - self.output += " " * self.indent + str(s) # + '(%016x)'%hash(e) - self.output += "\n" + self.output.write(" " * self.indent + str(s) + "\n") # + '(%016x)'%hash(e) def visit(self, expression): """ @@ -232,7 +237,7 @@ def visit_Operation(self, expression, *operands): self.indent -= 2 return True - def visit_BitvecExtract(self, expression): + def visit_BitVecExtract(self, expression): self._print( expression.__class__.__name__ + "{%d:%d}" % (expression.begining, expression.end), expression, @@ -256,7 +261,7 @@ def visit_Variable(self, expression): @property def result(self): - return self.output + return self.output.getvalue() def pretty_print(expression, **kwargs): @@ -270,17 +275,17 @@ def pretty_print(expression, **kwargs): class ConstantFolderSimplifier(Visitor): operations = { - BitvecMod: operator.__mod__, - BitvecAdd: operator.__add__, - BitvecSub: operator.__sub__, - BitvecMul: operator.__mul__, - BitvecShiftLeft: operator.__lshift__, - BitvecShiftRight: operator.__rshift__, - BitvecAnd: operator.__and__, - BitvecOr: operator.__or__, - BitvecXor: operator.__xor__, - BitvecNot: operator.__not__, - BitvecNeg: operator.__invert__, + BitVecMod: operator.__mod__, + BitVecAdd: operator.__add__, + BitVecSub: operator.__sub__, + BitVecMul: operator.__mul__, + BitVecShiftLeft: operator.__lshift__, + BitVecShiftRight: operator.__rshift__, + BitVecAnd: operator.__and__, + BitVecOr: operator.__or__, + BitVecXor: operator.__xor__, + BitVecNot: operator.__not__, + BitVecNeg: operator.__invert__, BoolAnd: operator.__and__, BoolEqual: operator.__eq__, BoolOr: operator.__or__, @@ -291,7 +296,7 @@ class ConstantFolderSimplifier(Visitor): BoolUnsignedGreaterOrEqualThan: operator.__ge__, } - def visit_BitvecUnsignedDiv(self, expression, *operands) -> Optional[BitvecConstant]: + def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].value b = operands[1].value @@ -299,7 +304,7 @@ def visit_BitvecUnsignedDiv(self, expression, *operands) -> Optional[BitvecConst ret = 0 else: ret = math.trunc(Decimal(a) / Decimal(b)) - return BitvecConstant(expression.size, ret, taint=expression.taint) + return BitVecConstant(expression.size, ret, taint=expression.taint) return None def visit_BoolLessThan(self, expression, *operands) -> Optional[BoolConstant]: @@ -330,7 +335,7 @@ def visit_BoolGreaterOrEqual(self, expression, *operands) -> Optional[BoolConsta return BoolConstant(a >= b, taint=expression.taint) return None - def visit_BitvecDiv(self, expression, *operands) -> Optional[BitvecConstant]: + def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask mask = operands[0].mask @@ -344,26 +349,26 @@ def visit_BitvecDiv(self, expression, *operands) -> Optional[BitvecConstant]: result = 0 else: result = math.trunc(Decimal(numeral) / Decimal(dividend)) - return BitvecConstant(expression.size, result, taint=expression.taint) + return BitVecConstant(expression.size, result, taint=expression.taint) return None - def visit_BitvecConcat(self, expression, *operands): + def visit_BitVecConcat(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): result = 0 for o in operands: result <<= o.size result |= o.value - return BitvecConstant(expression.size, result, taint=expression.taint) + return BitVecConstant(expression.size, result, taint=expression.taint) - def visit_BitvecZeroExtend(self, expression, *operands): + def visit_BitVecZeroExtend(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): - return BitvecConstant(expression.size, operands[0].value, taint=expression.taint) + return BitVecConstant(expression.size, operands[0].value, taint=expression.taint) - def visit_BitvecSignExtend(self, expression, *operands): + def visit_BitVecSignExtend(self, expression, *operands): if expression.extend == 0: return operands[0] - def visit_BitvecExtract(self, expression, *operands): + def visit_BitVecExtract(self, expression, *operands): if all(isinstance(o, Constant) for o in operands): value = operands[0].value begining = expression.begining @@ -371,7 +376,7 @@ def visit_BitvecExtract(self, expression, *operands): value = value >> begining mask = (1 << (end - begining)) - 1 value = value & mask - return BitvecConstant(expression.size, value, taint=expression.taint) + return BitVecConstant(expression.size, value, taint=expression.taint) def visit_BoolAnd(self, expression, a, b): if isinstance(a, Constant) and a.value == True: @@ -384,8 +389,8 @@ def visit_Operation(self, expression, *operands): operation = self.operations.get(type(expression), None) if operation is not None and all(isinstance(o, Constant) for o in operands): value = operation(*(x.value for x in operands)) - if isinstance(expression, Bitvec): - return BitvecConstant(expression.size, value, taint=expression.taint) + if isinstance(expression, BitVec): + return BitVecConstant(expression.size, value, taint=expression.taint) else: isinstance(expression, Bool) return BoolConstant(value, taint=expression.taint) @@ -410,9 +415,9 @@ def visit_Operation(self, expression, *operands): expression = constant_folder(expression) return expression - def visit_BitvecZeroExtend(self, expression, *operands): + def visit_BitVecZeroExtend(self, expression, *operands): if self._changed(expression, operands): - return BitvecZeroExtend(expression.size, *operands, taint=expression.taint) + return BitVecZeroExtend(expression.size, *operands, taint=expression.taint) else: return expression @@ -435,10 +440,10 @@ def visit_BoolAnd(self, expression, *operands): operand_1_1 = operand_1.operands[1] if ( - isinstance(operand_0_0, BitvecExtract) - and isinstance(operand_0_1, BitvecExtract) - and isinstance(operand_1_0, BitvecExtract) - and isinstance(operand_1_1, BitvecExtract) + isinstance(operand_0_0, BitVecExtract) + and isinstance(operand_0_1, BitVecExtract) + and isinstance(operand_1_0, BitVecExtract) + and isinstance(operand_1_1, BitVecExtract) ): if ( @@ -457,7 +462,7 @@ def visit_BoolAnd(self, expression, *operands): value1 = operand_0_1.value beg = min(operand_0_0.begining, operand_1_0.begining) end = max(operand_0_0.end, operand_1_0.end) - return BitvecExtract(value0, beg, end - beg + 1) == BitvecExtract( + return BitVecExtract(value0, beg, end - beg + 1) == BitVecExtract( value1, beg, end - beg + 1 ) @@ -470,7 +475,7 @@ def visit_BoolEqual(self, expression, *operands): (EQ, ITE(cond, constant1, constant2), constant2) -> NOT cond (EQ (extract a, b, c) (extract a, b, c)) """ - if isinstance(operands[0], BitvecITE) and isinstance(operands[1], Constant): + if isinstance(operands[0], BitVecITE) and isinstance(operands[1], Constant): if isinstance(operands[0].operands[1], Constant) and isinstance( operands[0].operands[2], Constant ): @@ -487,7 +492,7 @@ def visit_BoolEqual(self, expression, *operands): if operands[0] is operands[1]: return BoolConstant(True, taint=expression.taint) - if isinstance(operands[0], BitvecExtract) and isinstance(operands[1], BitvecExtract): + if isinstance(operands[0], BitVecExtract) and isinstance(operands[1], BitVecExtract): if ( operands[0].value is operands[1].value and operands[0].end == operands[1].end @@ -510,7 +515,7 @@ def visit_BoolOr(self, expression, a, b): if a is b: return a - def visit_BitvecITE(self, expression, *operands): + def visit_BitVecITE(self, expression, *operands): if isinstance(operands[0], Constant): if operands[0].value: result = operands[1] @@ -523,9 +528,9 @@ def visit_BitvecITE(self, expression, *operands): return result if self._changed(expression, operands): - return BitvecITE(*operands, taint=expression.taint) + return BitVecITE(*operands, taint=expression.taint) - def visit_BitvecConcat(self, expression, *operands): + def visit_BitVecConcat(self, expression, *operands): """concat( extract(k1, 0, a), extract(sizeof(a)-k1, k1, a)) ==> a concat( extract(k1, beg, a), extract(end, k1, a)) ==> extract(beg, end, a) concat( x , extract(k1, beg, a), extract(end, k1, a), z) ==> concat( x , extract(k1, beg, a), extract(end, k1, a), z) @@ -537,12 +542,12 @@ def visit_BitvecConcat(self, expression, *operands): last_o = None new_operands = [] for o in operands: - if isinstance(o, BitvecExtract): + if isinstance(o, BitVecExtract): if last_o is None: last_o = o else: if last_o.value is o.value and last_o.begining == o.end + 1: - last_o = BitvecExtract( + last_o = BitVecExtract( o.value, o.begining, last_o.end - o.begining + 1, taint=expression.taint ) changed = True @@ -557,15 +562,15 @@ def visit_BitvecConcat(self, expression, *operands): if last_o is not None: new_operands.append(last_o) if changed: - return BitvecConcat(operands=tuple(new_operands)) + return BitVecConcat(operands=tuple(new_operands)) op = operands[0] value = None end = None begining = None for o in operands: - # If found a non BitvecExtract, do not apply - if not isinstance(o, BitvecExtract): + # If found a non BitVecExtract, do not apply + if not isinstance(o, BitVecExtract): value = None break # Set the value for the first item @@ -587,11 +592,11 @@ def visit_BitvecConcat(self, expression, *operands): if value is not None: if end + 1 != value.size or begining != 0: - return BitvecExtract(value, begining, end - begining + 1, taint=expression.taint) + return BitVecExtract(value, begining, end - begining + 1, taint=expression.taint) return value - def visit_BitvecExtract(self, expression, *operands): + def visit_BitVecExtract(self, expression, *operands): """extract(sizeof(a), 0)(a) ==> a extract(16, 0)( concat(a,b,c,d) ) => concat(c, d) extract(m,M)(and/or/xor a b ) => and/or/xor((extract(m,M) a) (extract(m,M) a) @@ -603,14 +608,14 @@ def visit_BitvecExtract(self, expression, *operands): # extract(sizeof(a), 0)(a) ==> a if begining == 0 and end + 1 == op.size: return op - elif isinstance(op, BitvecExtract): - return BitvecExtract(op.value, op.begining + begining, size, taint=expression.taint) - elif isinstance(op, BitvecConcat): + elif isinstance(op, BitVecExtract): + return BitVecExtract(op.value, op.begining + begining, size, taint=expression.taint) + elif isinstance(op, BitVecConcat): new_operands = [] for item in reversed(op.operands): if size == 0: assert expression.size == sum([x.size for x in new_operands]) - return BitvecConcat( + return BitVecConcat( operands=tuple(reversed(new_operands)), taint=expression.taint ) @@ -623,62 +628,62 @@ def visit_BitvecExtract(self, expression, *operands): size = 0 else: if size <= item.size - begining: - new_operands.append(BitvecExtract(item, begining, size)) + new_operands.append(BitVecExtract(item, begining, size)) size = 0 else: - new_operands.append(BitvecExtract(item, begining, item.size - begining)) + new_operands.append(BitVecExtract(item, begining, item.size - begining)) size -= item.size - begining begining = 0 - elif isinstance(op, BitvecConstant): - return BitvecConstant(size, (op.value >> begining) & ~(1 << size)) + elif isinstance(op, BitVecConstant): + return BitVecConstant(size, (op.value >> begining) & ~(1 << size)) - if isinstance(op, (BitvecAnd, BitvecOr, BitvecXor)): + if isinstance(op, (BitVecAnd, BitVecOr, BitVecXor)): bitoperand_a, bitoperand_b = op.operands return op.__class__( - BitvecExtract(bitoperand_a, begining, expression.size), - BitvecExtract(bitoperand_b, begining, expression.size), + BitVecExtract(bitoperand_a, begining, expression.size), + BitVecExtract(bitoperand_b, begining, expression.size), taint=expression.taint, ) - def visit_BitvecAdd(self, expression, *operands): + def visit_BitVecAdd(self, expression, *operands): """a + 0 ==> a 0 + a ==> a """ left = operands[0] right = operands[1] - if isinstance(right, BitvecConstant): + if isinstance(right, BitVecConstant): if right.value == 0: return left - if isinstance(left, BitvecConstant): + if isinstance(left, BitVecConstant): if left.value == 0: return right - def visit_BitvecSub(self, expression, *operands): + def visit_BitVecSub(self, expression, *operands): """a - 0 ==> 0 (a + b) - b ==> a (b + a) - b ==> a """ left = operands[0] right = operands[1] - if isinstance(left, BitvecAdd): + if isinstance(left, BitVecAdd): if self._same_constant(left.operands[0], right): return left.operands[1] elif self._same_constant(left.operands[1], right): return left.operands[0] - elif isinstance(left, BitvecSub) and isinstance(right, Constant): + elif isinstance(left, BitVecSub) and isinstance(right, Constant): subleft = left.operands[0] subright = left.operands[1] if isinstance(subright, Constant): - return BitvecSub( + return BitVecSub( subleft, - BitvecConstant( + BitVecConstant( subleft.size, subright.value + right.value, taint=subright.taint | right.taint, ), ) - def visit_BitvecOr(self, expression, *operands): + def visit_BitVecOr(self, expression, *operands): """a | 0 => a 0 | a => a 0xffffffff & a => 0xffffffff @@ -687,20 +692,20 @@ def visit_BitvecOr(self, expression, *operands): """ left = operands[0] right = operands[1] - if isinstance(right, BitvecConstant): + if isinstance(right, BitVecConstant): if right.value == 0: return left elif right.value == left.mask: return right - elif isinstance(left, BitvecOr): + elif isinstance(left, BitVecOr): left_left = left.operands[0] left_right = left.operands[1] if isinstance(right, Constant): - return BitvecOr(left_left, (left_right | right), taint=expression.taint) - elif isinstance(left, BitvecConstant): - return BitvecOr(right, left, taint=expression.taint) + return BitVecOr(left_left, (left_right | right), taint=expression.taint) + elif isinstance(left, BitVecConstant): + return BitVecOr(right, left, taint=expression.taint) - def visit_BitvecAnd(self, expression, *operands): + def visit_BitVecAnd(self, expression, *operands): """ct & x => x & ct move constants to the right a & 0 => 0 remove zero a & 0xffffffff => a remove full mask @@ -709,31 +714,31 @@ def visit_BitvecAnd(self, expression, *operands): """ left = operands[0] right = operands[1] - if isinstance(right, BitvecConstant): + if isinstance(right, BitVecConstant): if right.value == 0: return right elif right.value == right.mask: return left - elif isinstance(left, BitvecAnd): + elif isinstance(left, BitVecAnd): left_left = left.operands[0] left_right = left.operands[1] if isinstance(right, Constant): - return BitvecAnd(left_left, left_right & right, taint=expression.taint) - elif isinstance(left, BitvecOr): + return BitVecAnd(left_left, left_right & right, taint=expression.taint) + elif isinstance(left, BitVecOr): left_left = left.operands[0] left_right = left.operands[1] - return BitvecOr(right & left_left, right & left_right, taint=expression.taint) + return BitVecOr(right & left_left, right & left_right, taint=expression.taint) - elif isinstance(left, BitvecConstant): - return BitvecAnd(right, left, taint=expression.taint) + elif isinstance(left, BitVecConstant): + return BitVecAnd(right, left, taint=expression.taint) - def visit_BitvecShiftLeft(self, expression, *operands): + def visit_BitVecShiftLeft(self, expression, *operands): """a << 0 => a remove zero a << ct => 0 if ct > sizeof(a) remove big constant shift """ left = operands[0] right = operands[1] - if isinstance(right, BitvecConstant): + if isinstance(right, BitVecConstant): if right.value == 0: return left elif right.value >= right.size: @@ -743,27 +748,26 @@ def visit_ArraySelect(self, expression, *operands): """ArraySelect (ArrayStore((ArrayStore(x0,v0) ...),xn, vn), x0) -> v0 """ - return None arr, index = operands if isinstance(arr, ArrayVariable): - return self._visit_Operation(expression, *operands) + return None - if isinstance(index, BitvecConstant): + if isinstance(index, BitVecConstant): ival = index.value # props are slow and using them in tight loops should be avoided, esp when they offer no additional validation # arr._operands[1] = arr.index, arr._operands[0] = arr.array while ( isinstance(arr, ArrayStore) - and isinstance(arr._operands[1], BitvecConstant) + and isinstance(arr._operands[1], BitVecConstant) and arr._operands[1]._value != ival ): arr = arr._operands[0] # arr.array if ( - isinstance(index, BitvecConstant) + isinstance(index, BitVecConstant) and isinstance(arr, ArrayStore) - and isinstance(arr.index, BitvecConstant) + and isinstance(arr.index, BitVecConstant) and arr.index.value == index.value ): if arr.value is not None: @@ -773,7 +777,6 @@ def visit_ArraySelect(self, expression, *operands): out = arr.select(index) if out is not None: return arr.select(index) - return self._visit_operation(expression, *operands) def visit_Expression(self, expression, *operands): assert len(operands) == 0 @@ -818,37 +821,61 @@ def to_constant(expression): @lru_cache(maxsize=128, typed=True) def simplify(expression): - return arithmetic_simplify(expression) + simplified = arithmetic_simplify(expression) + + def t(x): + try: + return translate_to_smtlib(x) + except: + return str(x) + + return simplified + + +class CountExpressionUse(Visitor): + def _rebuild(self, expression, operands): + return expression + + def __init__(self, *args, counts: Optional[Dict[Expression, int]] = None, **kw): + super().__init__(*args, **kw) + if counts is not None: + self.counts = counts + else: + self.counts = {} + + def visit_Expression(self, expression, *args): + if not isinstance(expression, (Variable, Constant)): + try: + self.counts[expression] += 1 + except KeyError: + self.counts[expression] = 1 class TranslatorSmtlib(Translator): """Simple visitor to translate an expression to its smtlib representation""" - unique = 0 - unique_lock = threading.Lock() - - def __init__(self, use_bindings=False, *args, **kw): + def __init__(self, use_bindings=True, *args, **kw): assert "bindings" not in kw super().__init__(*args, **kw) - self.use_bindings = use_bindings - self._bindings_cache = {} - self._bindings = [] + self._unique = 0 + self.use_bindings = False # use_bindings + self._bindings_cache_exp = {} + self._bindings = {} self._variables = set() def _add_binding(self, expression, smtlib): - if not self.use_bindings or len(smtlib) <= 10: + if not self.use_bindings: return smtlib - if smtlib in self._bindings_cache: - return self._bindings_cache[smtlib] - - with TranslatorSmtlib.unique_lock: - TranslatorSmtlib.unique += 1 - name = "a_%d" % TranslatorSmtlib.unique - - self._bindings.append((name, expression, smtlib)) + if isinstance(expression, (Constant, Variable)): + return smtlib + if expression in self._bindings_cache_exp: + return self._bindings_cache_exp[expression] + self._unique += 1 + name = "!a_%d!" % self._unique - self._bindings_cache[expression] = name + self._bindings[name] = expression, smtlib + self._bindings_cache_exp[expression] = name return name @property @@ -862,23 +889,23 @@ def bindings(self): BoolOr: "or", BoolXor: "xor", BoolITE: "ite", - BitvecAdd: "bvadd", - BitvecSub: "bvsub", - BitvecMul: "bvmul", - BitvecDiv: "bvsdiv", - BitvecUnsignedDiv: "bvudiv", - BitvecMod: "bvsmod", - BitvecRem: "bvsrem", - BitvecUnsignedRem: "bvurem", - BitvecShiftLeft: "bvshl", - BitvecShiftRight: "bvlshr", - BitvecArithmeticShiftLeft: "bvashl", - BitvecArithmeticShiftRight: "bvashr", - BitvecAnd: "bvand", - BitvecOr: "bvor", - BitvecXor: "bvxor", - BitvecNot: "bvnot", - BitvecNeg: "bvneg", + BitVecAdd: "bvadd", + BitVecSub: "bvsub", + BitVecMul: "bvmul", + BitVecDiv: "bvsdiv", + BitVecUnsignedDiv: "bvudiv", + BitVecMod: "bvsmod", + BitVecRem: "bvsrem", + BitVecUnsignedRem: "bvurem", + BitVecShiftLeft: "bvshl", + BitVecShiftRight: "bvlshr", + BitVecArithmeticShiftLeft: "bvashl", + BitVecArithmeticShiftRight: "bvashr", + BitVecAnd: "bvand", + BitVecOr: "bvor", + BitVecXor: "bvxor", + BitVecNot: "bvnot", + BitVecNeg: "bvneg", BoolLessThan: "bvslt", BoolLessOrEqualThan: "bvsle", BoolGreaterThan: "bvsgt", @@ -887,17 +914,17 @@ def bindings(self): BoolUnsignedLessOrEqualThan: "bvule", BoolUnsignedGreaterThan: "bvugt", BoolUnsignedGreaterOrEqualThan: "bvuge", - BitvecSignExtend: "(_ sign_extend %d)", - BitvecZeroExtend: "(_ zero_extend %d)", - BitvecExtract: "(_ extract %d %d)", - BitvecConcat: "concat", - BitvecITE: "ite", + BitVecSignExtend: "(_ sign_extend %d)", + BitVecZeroExtend: "(_ zero_extend %d)", + BitVecExtract: "(_ extract %d %d)", + BitVecConcat: "concat", + BitVecITE: "ite", ArrayStore: "store", ArraySelect: "select", } - def visit_BitvecConstant(self, expression): - assert isinstance(expression, BitvecConstant) + def visit_BitVecConstant(self, expression): + assert isinstance(expression, BitVecConstant) if expression.size == 1: return "#" + bin(expression.value & expression.mask)[1:] else: @@ -912,32 +939,61 @@ def visit_Variable(self, expression): def visit_ArraySelect(self, expression, *operands): array_smt, index_smt = operands - if isinstance(expression.array, ArrayStore): - array_smt = self._add_binding(expression.array, array_smt) - return "(select %s %s)" % (array_smt, index_smt) + array_smt = self._add_binding(expression.array, array_smt) + return f"(select {array_smt} {index_smt})" def visit_Operation(self, expression, *operands): operation = self.translation_table[type(expression)] - if isinstance(expression, (BitvecSignExtend, BitvecZeroExtend)): + if isinstance(expression, (BitVecSignExtend, BitVecZeroExtend)): operation = operation % expression.extend - elif isinstance(expression, BitvecExtract): + elif isinstance(expression, BitVecExtract): operation = operation % (expression.end, expression.begining) operands = [self._add_binding(*x) for x in zip(expression.operands, operands)] - return "(%s %s)" % (operation, " ".join(operands)) + return f"({operation} {' '.join(operands)})" @property def result(self): - output = super().result - if self.use_bindings: - for name, expr, smtlib in reversed(self._bindings): - output = "( let ((%s %s)) %s )" % (name, smtlib, output) + return self.apply_bindings(self._stack[-1]) + + def apply_bindings(self, main_smtlib_str): + # Python program to print topological sorting of a DAG + from toposort import toposort_flatten as toposort + + G = {} + for name, (expr, smtlib) in self._bindings.items(): + variables = {v.name for v in get_variables(expr)} + variables.update(re.findall(r"!a_\d+!", smtlib)) + + for v_name in variables: + G.setdefault(name, set()).add(v_name) + if not variables: + G[name] = set() + + output = "" + # Build let statement + for name in reversed(toposort(G)): + if name not in self._bindings: + continue + expr, smtlib = self._bindings[name] + + # FIXME: too much string manipulation. Search occurrences in the Expression realm + if main_smtlib_str.count(name) + output.count(name) == 1: + main_smtlib_str = main_smtlib_str.replace(name, smtlib) + output = output.replace(name, smtlib) + else: + output = f"({name} {smtlib}) {output}" + + if output: + output = f"(let ({output}) {main_smtlib_str})" + else: + output = main_smtlib_str return output def declarations(self): result = "" for exp in self._variables: - if isinstance(exp, Bitvec): + if isinstance(exp, BitVec): result += f"(declare-fun {exp.name} () (_ BitVec {exp.size}))\n" elif isinstance(exp, Bool): result += f"(declare-fun {exp.name} () Bool)\n" @@ -950,15 +1006,14 @@ def declarations(self): def smtlib(self): result = self.declarations() for constraint_str in self._stack: - if constraint_str != "true": - result += f"(assert {constraint_str})\n" + result += f"(assert {self.apply_bindings(constraint_str)})\n" return result -def translate_to_smtlib(expression, **kwargs): +def translate_to_smtlib(expression, use_bindings=True, **kwargs): if isinstance(expression, MutableArray): expression = expression.array - translator = TranslatorSmtlib(**kwargs) + translator = TranslatorSmtlib(use_bindings=use_bindings, **kwargs) translator.visit(expression) return translator.result @@ -1003,7 +1058,7 @@ def __init__(self, target_index, **kwargs): self.stores = [] def visit_ArrayStore(self, exp, target, where, what): - if not isinstance(what, BitvecConstant): + if not isinstance(what, BitVecConstant): raise self.ExpressionNotSimple if where.value == self._target_index: @@ -1017,10 +1072,41 @@ def simplify_array_select(array_exp): return simplifier.stores +@lru_cache(maxsize=128, typed=True) +def _get_variables(expression): + visitor = GetDeclarations() + visitor.visit(expression) + return visitor.result + + def get_variables(expression): if isinstance(expression, MutableArray): expression = expression.array + return _get_variables(expression) - visitor = GetDeclarations() - visitor.visit(expression) - return visitor.result + +class GetBindings(Visitor): + """Simple visitor to collect all variables in an expression or set of + expressions + """ + + def _rebuild(self, expression, operands): + return expression + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.expressions = {} + + def visit_Operation(self, expression, *operands): + try: + self.expressions[expression] += 1 + import pdb + + pdb.set_trace() + print("A TWO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 222222222222222222222222222222222") + except KeyError as e: + self.expressions[expression] = 1 + + @property + def result(self): + return self.expressions diff --git a/manticore/core/state.py b/manticore/core/state.py index ed0e9145f..c15b3a90e 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,7 +1,7 @@ import copy import logging -from .smtlib import solver, Bool, issymbolic, BitvecConstant, MutableArray +from .smtlib import solver, Bool, issymbolic, BitVecConstant, MutableArray from ..utils.event import Eventful from ..utils.helpers import PickleSerializer from ..utils import config @@ -76,7 +76,7 @@ def _setstate(self, state, _value): def __init__(self, filename, **kwargs): super().__init__( f"Saving state to {filename}", - BitvecConstant(32, 0), + BitVecConstant(32, 0), setstate=self._setstate, policy="ONE", **kwargs, @@ -323,7 +323,11 @@ def new_symbolic_buffer(self, nbytes, **options): avoid_collisions = True taint = options.get("taint", frozenset()) expr = self._constraints.new_array( - name=label, length=nbytes, value_size=8, taint=taint, avoid_collisions=avoid_collisions, + name=label, + length=nbytes, + value_size=8, + taint=taint, + avoid_collisions=avoid_collisions, ) self._input_symbols.append(expr) @@ -459,19 +463,10 @@ def solve_one_n(self, *exprs, constrain=False): :return: Concrete value or a tuple of concrete values :rtype: int """ - values = [] - for expr in exprs: - if not issymbolic(expr): - values.append(expr) - else: - expr = self.migrate_expression(expr) - value = self._solver.get_value(self._constraints, expr) - if constrain: - self.constrain(expr == value) - # Include forgiveness here - if isinstance(value, bytearray): - value = bytes(value) - values.append(value) + expressions = [self.migrate_expression(e) for e in exprs] + values = self._solver.get_value(self._constraints, *expressions) + if len(expressions) == 1: + values = (values,) return values def solve_n(self, expr, nsolves): diff --git a/manticore/core/worker.py b/manticore/core/worker.py index 31270bdbc..1e9d74012 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -21,30 +21,30 @@ class Worker: """ - A Manticore Worker. - This will run forever potentially in a different process. Normally it - will be spawned at Manticore constructor and will stay alive until killed. - A Worker can be in 3 phases: STANDBY, RUNNING, KILLED. And will react to - different events: start, stop, kill. - The events are transmitted via 2 conditional variable: m._killed and - m._started. - - .. code-block:: none - - STANDBY: Waiting for the start event - RUNNING: Exploring and spawning states until no more READY states or - the cancel event is received - KIlLED: This is the end. No more manticoring in this worker process - - +---------+ +---------+ - +--->+ STANDBY +<--->+ RUNNING | - +-+-------+ +-------+-+ - | | - | +--------+ | - +----->+ KILLED <-----+ - +----+---+ - | - # + A Manticore Worker. + This will run forever potentially in a different process. Normally it + will be spawned at Manticore constructor and will stay alive until killed. + A Worker can be in 3 phases: STANDBY, RUNNING, KILLED. And will react to + different events: start, stop, kill. + The events are transmitted via 2 conditional variable: m._killed and + m._started. + + .. code-block:: none + + STANDBY: Waiting for the start event + RUNNING: Exploring and spawning states until no more READY states or + the cancel event is received + KIlLED: This is the end. No more manticoring in this worker process + + +---------+ +---------+ + +--->+ STANDBY +<--->+ RUNNING | + +-+-------+ +-------+-+ + | | + | +--------+ | + +----->+ KILLED <-----+ + +----+---+ + | + # """ def __init__(self, *, id, manticore, single=False): @@ -186,9 +186,9 @@ def run(self, *args): class WorkerSingle(Worker): - """ A single worker that will run in the current process and current thread. - As this will not provide any concurrency is normally only used for - profiling underlying arch emulation and debugging.""" + """A single worker that will run in the current process and current thread. + As this will not provide any concurrency is normally only used for + profiling underlying arch emulation and debugging.""" def __init__(self, *args, **kwargs): super().__init__(*args, single=True, **kwargs) @@ -246,7 +246,10 @@ def start(self, target: typing.Optional[typing.Callable] = None): thread as an argument. """ logger.debug( - "Starting Daemon %d. (Pid %d Tid %d).", self.id, os.getpid(), threading.get_ident(), + "Starting Daemon %d. (Pid %d Tid %d).", + self.id, + os.getpid(), + threading.get_ident(), ) self._t = threading.Thread(target=self.run if target is None else target, args=(self,)) diff --git a/manticore/ethereum/abi.py b/manticore/ethereum/abi.py index 112a2c9d9..3db593007 100644 --- a/manticore/ethereum/abi.py +++ b/manticore/ethereum/abi.py @@ -9,7 +9,7 @@ from ..core.smtlib import ( Array, Operators, - Bitvec, + BitVec, ArrayVariable, MutableArray, to_constant, @@ -22,10 +22,10 @@ class ABI: """ - This class contains methods to handle the ABI. - The Application Binary Interface is the standard way to interact with - contracts in the Ethereum ecosystem, both from outside the blockchain - and for contract-to-contract interaction. + This class contains methods to handle the ABI. + The Application Binary Interface is the standard way to interact with + contracts in the Ethereum ecosystem, both from outside the blockchain + and for contract-to-contract interaction. """ @@ -271,18 +271,18 @@ def _deserialize(ty, buf: typing.Union[bytearray, bytes, Array], offset=0): @staticmethod def _serialize_uint(value, size=32, padding=0): """ - Translates a python integral or a Bitvec into a 32 byte string, MSB first + Translates a python integral or a BitVec into a 32 byte string, MSB first """ if size <= 0 or size > 32: raise ValueError from .account import EVMAccount # because of circular import - if not isinstance(value, (int, Bitvec, EVMAccount)): + if not isinstance(value, (int, BitVec, EVMAccount)): raise ValueError if issymbolic(value): # Help mypy out. Can remove this by teaching it how issymbolic works - assert isinstance(value, Bitvec) + assert isinstance(value, BitVec) # FIXME This temporary array variable should be obtained from a specific constraint store buffer = ArrayVariable( index_size=256, length=32, value_size=8, name="temp{}".format(uuid.uuid1()) @@ -290,7 +290,7 @@ def _serialize_uint(value, size=32, padding=0): if value.size <= size * 8: value = Operators.ZEXTEND(value, size * 8) else: - # automatically truncate, e.g. if they passed a Bitvec(256) for an `address` argument (160 bits) + # automatically truncate, e.g. if they passed a BitVec(256) for an `address` argument (160 bits) value = Operators.EXTRACT(value, 0, size * 8) buffer = buffer.write_BE(padding, value, size) else: @@ -305,17 +305,17 @@ def _serialize_uint(value, size=32, padding=0): return buffer @staticmethod - def _serialize_int(value: typing.Union[int, Bitvec], size=32, padding=0): + def _serialize_int(value: typing.Union[int, BitVec], size=32, padding=0): """ - Translates a signed python integral or a Bitvec into a 32 byte string, MSB first + Translates a signed python integral or a BitVec into a 32 byte string, MSB first """ if size <= 0 or size > 32: raise ValueError - if not isinstance(value, (int, Bitvec)): + if not isinstance(value, (int, BitVec)): raise ValueError if issymbolic(value): # Help mypy out. Can remove this by teaching it how issymbolic works - assert isinstance(value, Bitvec) + assert isinstance(value, BitVec) buf = ArrayVariable( index_size=256, length=32, value_size=8, name="temp{}".format(uuid.uuid1()) ) diff --git a/manticore/ethereum/abitypes.py b/manticore/ethereum/abitypes.py index 95f7f9511..c6f2edda1 100644 --- a/manticore/ethereum/abitypes.py +++ b/manticore/ethereum/abitypes.py @@ -151,20 +151,20 @@ def t_error(t): # parser def p_basic_type(p): """ - T : UINTN - T : UINT - T : INTN - T : INT - T : ADDRESS - T : BOOL - T : FIXEDMN - T : UFIXEDMN - T : FIXED - T : UFIXED - T : BYTESM - T : FUNCTION - T : BYTES - T : STRING + T : UINTN + T : UINT + T : INTN + T : INT + T : ADDRESS + T : BOOL + T : FIXEDMN + T : UFIXEDMN + T : FIXED + T : UFIXED + T : BYTESM + T : FUNCTION + T : BYTES + T : STRING """ p[0] = p[1] @@ -172,35 +172,35 @@ def p_basic_type(p): def p_type_list_one(p): """ - TL : T + TL : T """ p[0] = (p[1],) def p_type_list(p): """ - TL : T COMMA TL + TL : T COMMA TL """ p[0] = (p[1],) + p[3] def p_tuple(p): """ - T : LPAREN TL RPAREN + T : LPAREN TL RPAREN """ p[0] = ("tuple", p[2]) def p_tuple_empty(p): """ - T : LPAREN RPAREN + T : LPAREN RPAREN """ p[0] = ("tuple", ()) def p_dynamic_type(p): """ - T : T LBRAKET RBRAKET + T : T LBRAKET RBRAKET """ reps = None base_type = p[1] @@ -209,7 +209,7 @@ def p_dynamic_type(p): def p_dynamic_fixed_type(p): """ - T : T LBRAKET NUMBER RBRAKET + T : T LBRAKET NUMBER RBRAKET """ reps = int(p[3]) base_type = p[1] diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 439dc2595..5f5896c05 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -18,7 +18,7 @@ ConstraintSet, Array, MutableArray, - Bitvec, + BitVec, Operators, BoolConstant, Expression, @@ -120,7 +120,7 @@ class ManticoreEVM(ManticoreBase): _published_events = {"solve"} def make_symbolic_buffer(self, size, name=None, avoid_collisions=False, default=None): - """ Creates a symbolic buffer of size bytes to be used in transactions. + """Creates a symbolic buffer of size bytes to be used in transactions. You can operate on it normally and add constraints to manticore.constraints via manticore.constrain(constraint_expression) @@ -175,7 +175,7 @@ def make_symbolic_address(self, *accounts, name=None, select="both"): :param name: Name of the symbolic variable. Defaults to 'TXADDR' and later to 'TXADDR_' :param select: Whether to select contracts or normal accounts. Not implemented for now. - :return: Symbolic address in form of a BitvecVariable. + :return: Symbolic address in form of a BitVecVariable. """ if select not in ("both", "normal", "contract"): raise EthereumError("Wrong selection type") @@ -538,7 +538,7 @@ def solidity_create_contract( :param contract_name: Name of the contract to analyze (optional if there is a single one in the source code) :type contract_name: str :param balance: balance to be transferred on creation - :type balance: int or BitvecVariable + :type balance: int or BitVecVariable :param address: the address for the new contract (optional) :type address: int or EVMAccount :param tuple args: constructor arguments @@ -670,7 +670,7 @@ def create_contract(self, owner, balance=0, address=None, init=None, name=None, :param owner: owner account (will be default caller in any transactions) :type owner: int or EVMAccount :param balance: balance to be transferred on creation - :type balance: int or BitvecVariable + :type balance: int or BitVecVariable :param int address: the address for the new contract (optional) :param str init: initializing evm bytecode and arguments :param str name: a unique name for reference @@ -758,7 +758,7 @@ def transaction(self, caller, address, value, data, gas=None, price=1): :param address: the address of the contract to call :type address: int or EVMAccount :param value: balance to be transfered on creation - :type value: int or BitvecVariable + :type value: int or BitVecVariable :param data: initial data :param gas: gas budget :param price: gas unit price @@ -774,7 +774,7 @@ def create_account(self, balance=0, address=None, code=None, name=None, nonce=No """Low level creates an account. This won't generate a transaction. :param balance: balance to be set on creation (optional) - :type balance: int or BitvecVariable + :type balance: int or BitVecVariable :param address: the address for the new account (optional) :type address: int :param code: the runtime code for the new account (None means normal account), str or bytes (optional) @@ -881,7 +881,7 @@ def _transaction(self, sort, caller, value=0, address=None, data=None, gas=None, :param int address: the address for the transaction (optional) :param value: value to be transferred :param price: the price of gas for this transaction. - :type value: int or BitvecVariable + :type value: int or BitVecVariable :param str data: initializing evm bytecode and arguments or transaction call data :param gas: gas budget for current transaction :rtype: EVMAccount @@ -902,13 +902,13 @@ def _transaction(self, sort, caller, value=0, address=None, data=None, gas=None, raise TypeError("code bad type") # Check types - if not isinstance(caller, (int, Bitvec)): + if not isinstance(caller, (int, BitVec)): raise TypeError("Caller invalid type") - if not isinstance(value, (int, Bitvec)): + if not isinstance(value, (int, BitVec)): raise TypeError("Value invalid type") - if not isinstance(address, (int, Bitvec)): + if not isinstance(address, (int, BitVec)): raise TypeError("address invalid type") if not isinstance(price, int): @@ -1087,7 +1087,8 @@ def multi_tx_analysis( logger.info("Starting symbolic transaction: %d", tx_no) # run_symbolic_tx - symbolic_data = self.make_symbolic_buffer(320, default=0) + symbolic_data = self.make_symbolic_buffer(320, default=None) + symbolic_data = MutableArray(symbolic_data) if tx_send_ether: value = self.make_symbolic_value() else: diff --git a/manticore/ethereum/parsetab.py b/manticore/ethereum/parsetab.py index d71b9d2e7..3ef2495e9 100644 --- a/manticore/ethereum/parsetab.py +++ b/manticore/ethereum/parsetab.py @@ -1,108 +1,50 @@ + # parsetab.py # This file is automatically generated. Do not edit. # pylint: disable=W,C,R -_tabversion = "3.10" - -_lr_method = "LALR" +_tabversion = '3.10' -_lr_signature = "ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n " +_lr_method = 'LALR' -_lr_action_items = { - "UINTN": ([0, 16, 24], [2, 2, 2]), - "UINT": ([0, 16, 24], [3, 3, 3]), - "INTN": ([0, 16, 24], [4, 4, 4]), - "INT": ([0, 16, 24], [5, 5, 5]), - "ADDRESS": ([0, 16, 24], [6, 6, 6]), - "BOOL": ([0, 16, 24], [7, 7, 7]), - "FIXEDMN": ([0, 16, 24], [8, 8, 8]), - "UFIXEDMN": ([0, 16, 24], [9, 9, 9]), - "FIXED": ([0, 16, 24], [10, 10, 10]), - "UFIXED": ([0, 16, 24], [11, 11, 11]), - "BYTESM": ([0, 16, 24], [12, 12, 12]), - "FUNCTION": ([0, 16, 24], [13, 13, 13]), - "BYTES": ([0, 16, 24], [14, 14, 14]), - "STRING": ([0, 16, 24], [15, 15, 15]), - "LPAREN": ([0, 16, 24], [16, 16, 16]), - "$end": ( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 21, 23, 25], - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, -19, -17, -20], - ), - "LBRAKET": ( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 25], - [17, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, 17, -19, -17, -20], - ), - "COMMA": ( - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 25], - [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, 24, -19, -17, -20], - ), - "RPAREN": ( - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 23, 25, 26], - [ - -1, - -2, - -3, - -4, - -5, - -6, - -7, - -8, - -9, - -10, - -11, - -12, - -13, - -14, - 19, - 23, - -18, - -15, - -19, - -17, - -20, - -16, - ], - ), - "RBRAKET": ([17, 22], [21, 25]), - "NUMBER": ([17], [22]), -} +_lr_signature = 'ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n ' + +_lr_action_items = {'UINTN':([0,16,24,],[2,2,2,]),'UINT':([0,16,24,],[3,3,3,]),'INTN':([0,16,24,],[4,4,4,]),'INT':([0,16,24,],[5,5,5,]),'ADDRESS':([0,16,24,],[6,6,6,]),'BOOL':([0,16,24,],[7,7,7,]),'FIXEDMN':([0,16,24,],[8,8,8,]),'UFIXEDMN':([0,16,24,],[9,9,9,]),'FIXED':([0,16,24,],[10,10,10,]),'UFIXED':([0,16,24,],[11,11,11,]),'BYTESM':([0,16,24,],[12,12,12,]),'FUNCTION':([0,16,24,],[13,13,13,]),'BYTES':([0,16,24,],[14,14,14,]),'STRING':([0,16,24,],[15,15,15,]),'LPAREN':([0,16,24,],[16,16,16,]),'$end':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,21,23,25,],[0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,-19,-17,-20,]),'LBRAKET':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[17,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,17,-19,-17,-20,]),'COMMA':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,24,-19,-17,-20,]),'RPAREN':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,23,25,26,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,19,23,-18,-15,-19,-17,-20,-16,]),'RBRAKET':([17,22,],[21,25,]),'NUMBER':([17,],[22,]),} _lr_action = {} for _k, _v in _lr_action_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_action: - _lr_action[_x] = {} - _lr_action[_x][_k] = _y + for _x,_y in zip(_v[0],_v[1]): + if not _x in _lr_action: _lr_action[_x] = {} + _lr_action[_x][_k] = _y del _lr_action_items -_lr_goto_items = {"T": ([0, 16, 24], [1, 20, 20]), "TL": ([16, 24], [18, 26])} +_lr_goto_items = {'T':([0,16,24,],[1,20,20,]),'TL':([16,24,],[18,26,]),} _lr_goto = {} for _k, _v in _lr_goto_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_goto: - _lr_goto[_x] = {} - _lr_goto[_x][_k] = _y + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_goto: _lr_goto[_x] = {} + _lr_goto[_x][_k] = _y del _lr_goto_items _lr_productions = [ - ("S' -> T", "S'", 1, None, None, None), - ("T -> UINTN", "T", 1, "p_basic_type", "abitypes.py", 154), - ("T -> UINT", "T", 1, "p_basic_type", "abitypes.py", 155), - ("T -> INTN", "T", 1, "p_basic_type", "abitypes.py", 156), - ("T -> INT", "T", 1, "p_basic_type", "abitypes.py", 157), - ("T -> ADDRESS", "T", 1, "p_basic_type", "abitypes.py", 158), - ("T -> BOOL", "T", 1, "p_basic_type", "abitypes.py", 159), - ("T -> FIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 160), - ("T -> UFIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 161), - ("T -> FIXED", "T", 1, "p_basic_type", "abitypes.py", 162), - ("T -> UFIXED", "T", 1, "p_basic_type", "abitypes.py", 163), - ("T -> BYTESM", "T", 1, "p_basic_type", "abitypes.py", 164), - ("T -> FUNCTION", "T", 1, "p_basic_type", "abitypes.py", 165), - ("T -> BYTES", "T", 1, "p_basic_type", "abitypes.py", 166), - ("T -> STRING", "T", 1, "p_basic_type", "abitypes.py", 167), - ("TL -> T", "TL", 1, "p_type_list_one", "abitypes.py", 175), - ("TL -> T COMMA TL", "TL", 3, "p_type_list", "abitypes.py", 182), - ("T -> LPAREN TL RPAREN", "T", 3, "p_tuple", "abitypes.py", 189), - ("T -> LPAREN RPAREN", "T", 2, "p_tuple_empty", "abitypes.py", 196), - ("T -> T LBRAKET RBRAKET", "T", 3, "p_dynamic_type", "abitypes.py", 203), - ("T -> T LBRAKET NUMBER RBRAKET", "T", 4, "p_dynamic_fixed_type", "abitypes.py", 212), + ("S' -> T","S'",1,None,None,None), + ('T -> UINTN','T',1,'p_basic_type','abitypes.py',154), + ('T -> UINT','T',1,'p_basic_type','abitypes.py',155), + ('T -> INTN','T',1,'p_basic_type','abitypes.py',156), + ('T -> INT','T',1,'p_basic_type','abitypes.py',157), + ('T -> ADDRESS','T',1,'p_basic_type','abitypes.py',158), + ('T -> BOOL','T',1,'p_basic_type','abitypes.py',159), + ('T -> FIXEDMN','T',1,'p_basic_type','abitypes.py',160), + ('T -> UFIXEDMN','T',1,'p_basic_type','abitypes.py',161), + ('T -> FIXED','T',1,'p_basic_type','abitypes.py',162), + ('T -> UFIXED','T',1,'p_basic_type','abitypes.py',163), + ('T -> BYTESM','T',1,'p_basic_type','abitypes.py',164), + ('T -> FUNCTION','T',1,'p_basic_type','abitypes.py',165), + ('T -> BYTES','T',1,'p_basic_type','abitypes.py',166), + ('T -> STRING','T',1,'p_basic_type','abitypes.py',167), + ('TL -> T','TL',1,'p_type_list_one','abitypes.py',175), + ('TL -> T COMMA TL','TL',3,'p_type_list','abitypes.py',182), + ('T -> LPAREN TL RPAREN','T',3,'p_tuple','abitypes.py',189), + ('T -> LPAREN RPAREN','T',2,'p_tuple_empty','abitypes.py',196), + ('T -> T LBRAKET RBRAKET','T',3,'p_dynamic_type','abitypes.py',203), + ('T -> T LBRAKET NUMBER RBRAKET','T',4,'p_dynamic_fixed_type','abitypes.py',212), ] diff --git a/manticore/ethereum/plugins.py b/manticore/ethereum/plugins.py index ee3a2fb40..dad5a53bb 100644 --- a/manticore/ethereum/plugins.py +++ b/manticore/ethereum/plugins.py @@ -17,21 +17,21 @@ def __init__( self, regexp=r".*", mutability="both", depth="both", fallback=False, include=True, **kwargs ): """ - Constrain input based on function metadata. Include or avoid functions - selected by the specified criteria. + Constrain input based on function metadata. Include or avoid functions + selected by the specified criteria. - Examples: - #Do not explore any human transactions that end up calling a constant function - no_human_constant = FilterFunctions(depth='human', mutability='constant', include=False) + Examples: + #Do not explore any human transactions that end up calling a constant function + no_human_constant = FilterFunctions(depth='human', mutability='constant', include=False) - #At human tx depth only accept synthetic check functions - only_tests = FilterFunctions(regexp=r'mcore_.*', depth='human', include=False) + #At human tx depth only accept synthetic check functions + only_tests = FilterFunctions(regexp=r'mcore_.*', depth='human', include=False) - :param regexp: a regular expression over the name of the function '.*' will match all functions - :param mutability: mutable, constant or both will match functions declared in the abi to be of such class - :param depth: match functions in internal transactions, in human initiated transactions or in both types - :param fallback: if True include the fallback function. Hash will be 00000000 for it - :param include: if False exclude the selected functions, if True include them + :param regexp: a regular expression over the name of the function '.*' will match all functions + :param mutability: mutable, constant or both will match functions declared in the abi to be of such class + :param depth: match functions in internal transactions, in human initiated transactions or in both types + :param fallback: if True include the fallback function. Hash will be 00000000 for it + :param include: if False exclude the selected functions, if True include them """ super().__init__(**kwargs) depth = depth.lower() @@ -155,27 +155,27 @@ def will_evm_execute_instruction_callback(self, state, instruction, arguments): class KeepOnlyIfStorageChanges(Plugin): - """ This plugin discards all transactions that results in states where - the underlying EVM storage did not change or in other words, - there were no writes to it. + """This plugin discards all transactions that results in states where + the underlying EVM storage did not change or in other words, + there were no writes to it. - This allows to speed-up EVM engine exploration as we don't - explore states that have the same storage (contract data). + This allows to speed-up EVM engine exploration as we don't + explore states that have the same storage (contract data). - However, keep in mind that if the (contract) code relies on - account balance and the balance is not a symbolic value - it might be that a certain state will not be covered by the - execution when this plugin is used. + However, keep in mind that if the (contract) code relies on + account balance and the balance is not a symbolic value + it might be that a certain state will not be covered by the + execution when this plugin is used. """ def did_open_transaction_callback(self, state, tx, *args): - """ We need a stack. Each tx (internal or not) starts with a "False" flag - denoting that it did not write anything to the storage + """We need a stack. Each tx (internal or not) starts with a "False" flag + denoting that it did not write anything to the storage """ state.context["written"].append(False) def did_close_transaction_callback(self, state, tx, *args): - """ When a tx (internal or not) is closed a value is popped out from the + """When a tx (internal or not) is closed a value is popped out from the flag stack. Depending on the result if the storage is not rolled back the next flag in the stack is updated. Not that if the a tx is reverted the changes it may have done on the storage will not affect the final result. @@ -195,7 +195,7 @@ def did_close_transaction_callback(self, state, tx, *args): state.context["written"][-1] = state.context["written"][-1] or flag def did_evm_write_storage_callback(self, state, *args): - """ Turn on the corresponding flag is the storage has been modified. + """Turn on the corresponding flag is the storage has been modified. Note: subject to change if the current transaction is reverted""" state.context["written"][-1] = True @@ -206,7 +206,7 @@ def will_run_callback(self, *args): def did_run_callback(self): """When human tx/run just ended remove the states that have not changed - the storage""" + the storage""" with self.manticore.locked_context("ethereum.saved_states", list) as saved_states: # Normally the ready states are consumed and forked, eth save the # states that finished ok in a special context list. This list will diff --git a/manticore/ethereum/solidity.py b/manticore/ethereum/solidity.py index f177d05fe..38dd241ed 100644 --- a/manticore/ethereum/solidity.py +++ b/manticore/ethereum/solidity.py @@ -157,8 +157,8 @@ def init_bytecode(self): return self._without_metadata(self._init_bytecode) def get_source_for(self, asm_offset, runtime=True): - """ Solidity source code snippet related to `asm_pos` evm bytecode offset. - If runtime is False, initialization bytecode source map is used + """Solidity source code snippet related to `asm_pos` evm bytecode offset. + If runtime is False, initialization bytecode source map is used """ srcmap = self.get_srcmap(runtime) diff --git a/manticore/ethereum/verifier.py b/manticore/ethereum/verifier.py index d1637f7b9..c4ce7a075 100644 --- a/manticore/ethereum/verifier.py +++ b/manticore/ethereum/verifier.py @@ -58,7 +58,7 @@ def manticore_verifier( outputspace_url=None, timeout=100, ): - """ Verify solidity properties + """Verify solidity properties The results are dumped to stdout and to the workspace folder. $manticore-verifier property.sol --contract TestToken --smt.solver yices --maxt 4 @@ -366,7 +366,11 @@ def main(): cryticparser.init(parser) parser.add_argument( - "source_code", type=str, nargs="*", default=[], help="Contract source code", + "source_code", + type=str, + nargs="*", + default=[], + help="Contract source code", ) parser.add_argument( "-v", action="count", default=0, help="Specify verbosity level from -v to -vvvv" @@ -387,7 +391,9 @@ def main(): help="Show program version information", ) parser.add_argument( - "--propconfig", type=str, help="Solidity property accounts config file (.yml)", + "--propconfig", + type=str, + help="Solidity property accounts config file (.yml)", ) eth_flags = parser.add_argument_group("Ethereum flags") diff --git a/manticore/native/cpu/abstractcpu.py b/manticore/native/cpu/abstractcpu.py index c533215f3..6e847937c 100644 --- a/manticore/native/cpu/abstractcpu.py +++ b/manticore/native/cpu/abstractcpu.py @@ -44,7 +44,7 @@ class CpuException(Exception): class DecodeException(CpuException): """ - Raised when trying to decode an unknown or invalid instruction """ + Raised when trying to decode an unknown or invalid instruction""" def __init__(self, pc, bytes): super().__init__("Error decoding instruction @ 0x{:x}".format(pc)) @@ -90,7 +90,11 @@ class ConcretizeRegister(CpuException): """ def __init__( - self, cpu: "Cpu", reg_name: str, message: Optional[str] = None, policy: str = "MINMAX", + self, + cpu: "Cpu", + reg_name: str, + message: Optional[str] = None, + policy: str = "MINMAX", ): self.message = message if message else f"Concretizing {reg_name}" @@ -177,11 +181,11 @@ def __getattr__(self, name): @property def type(self): - """ This property encapsulates the operand type. - It may be one of the following: - register - memory - immediate + """This property encapsulates the operand type. + It may be one of the following: + register + memory + immediate """ raise NotImplementedError @@ -664,7 +668,7 @@ def write_int(self, where, expression, size=None, force=False): :param int where: address to write to :param expr: value to write - :type expr: int or Bitvec + :type expr: int or BitVec :param size: bit size of `expr` :param force: whether to ignore memory permissions """ @@ -722,7 +726,7 @@ def read_int(self, where, size=None, force=False): :param int where: address to read from :param size: number of bits to read :return: the value read - :rtype: int or Bitvec + :rtype: int or BitVec :param force: whether to ignore memory permissions """ if size is None: diff --git a/manticore/native/cpu/arm.py b/manticore/native/cpu/arm.py index 10c6c611d..6d87aea84 100644 --- a/manticore/native/cpu/arm.py +++ b/manticore/native/cpu/arm.py @@ -10,7 +10,7 @@ from .abstractcpu import instruction as abstract_instruction from .bitwise import * from .register import Register -from ...core.smtlib import Operators, BitvecConstant, issymbolic +from ...core.smtlib import Operators, BitVecConstant, issymbolic from typing import NamedTuple @@ -405,7 +405,7 @@ def _read_APSR(self): def make_apsr_flag(flag_expr, offset): """Helper for constructing an expression for the APSR register""" return Operators.ITEBV( - 32, flag_expr, BitvecConstant(32, 1 << offset), BitvecConstant(32, 0) + 32, flag_expr, BitVecConstant(32, 1 << offset), BitVecConstant(32, 0) ) apsr = 0 @@ -719,7 +719,7 @@ def stack_push(self, data, nbytes=None): nbytes = nbytes or self.address_bit_size // 8 self.SP -= nbytes self.write_int(self.SP, data, nbytes * 8) - elif isinstance(data, Bitvec): + elif isinstance(data, BitVec): self.SP -= data.size // 8 self.write_int(self.SP, data, data.size) elif isinstance(data, str): diff --git a/manticore/native/cpu/bitwise.py b/manticore/native/cpu/bitwise.py index 35702306d..d00dba1d5 100644 --- a/manticore/native/cpu/bitwise.py +++ b/manticore/native/cpu/bitwise.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from ...core.smtlib import Operators -from ...core.smtlib.expression import Bitvec +from ...core.smtlib.expression import BitVec def Mask(width): @@ -19,7 +19,7 @@ def Bit(value, idx): Extract `idx` bit from `value`. :param value: Source value from which to extract. - :type value: int or long or Bitvec + :type value: int or long or BitVec :param idx: Bit index :return: int or long or BitVex """ @@ -31,15 +31,15 @@ def GetNBits(value, nbits): Get the first `nbits` from `value`. :param value: Source value from which to extract - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int nbits: How many bits to extract :return: Low `nbits` bits of `value`. - :rtype int or long or Bitvec + :rtype int or long or BitVec """ # NOP if sizes are the same if isinstance(value, int): return Operators.EXTRACT(value, 0, nbits) - elif isinstance(value, Bitvec): + elif isinstance(value, BitVec): if value.size < nbits: return Operators.ZEXTEND(value, nbits) else: @@ -52,10 +52,10 @@ def SInt(value, width): representation. :param value: The value to convert. - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int width: The width of the bitstring to consider :return: The converted value - :rtype int or long or Bitvec + :rtype int or long or BitVec """ return Operators.ITEBV( width, @@ -70,10 +70,10 @@ def UInt(value, width): Return integer value of `value` as a bitstring of `width` width. :param value: The value to convert. - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int width: The width of the bitstring to consider :return: The integer value - :rtype int or long or Bitvec + :rtype int or long or BitVec """ return GetNBits(value, width) @@ -83,7 +83,7 @@ def LSL_C(value, amount, width, with_carry=True): The ARM LSL_C (logical left shift with carry) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value and the carry result @@ -107,11 +107,11 @@ def LSL(value, amount, width): The ARM LSL (logical left shift) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value - :rtype int or Bitvec + :rtype int or BitVec """ if isinstance(amount, int) and amount == 0: return value @@ -125,7 +125,7 @@ def LSR_C(value, amount, width, with_carry=True): The ARM LSR_C (logical shift right with carry) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value and carry result @@ -146,11 +146,11 @@ def LSR(value, amount, width): The ARM LSR (logical shift right) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value - :rtype int or Bitvec + :rtype int or BitVec """ if isinstance(amount, int) and amount == 0: return value @@ -163,7 +163,7 @@ def ASR_C(value, amount, width, with_carry=True): The ARM ASR_C (arithmetic shift right with carry) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value and carry result @@ -190,11 +190,11 @@ def ASR(value, amount, width): The ARM ASR (arithmetic shift right) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to shift it. :param int width: Width of the value :return: Resultant value - :rtype int or Bitvec + :rtype int or BitVec """ if isinstance(amount, int) and amount == 0: return value @@ -208,7 +208,7 @@ def ROR_C(value, amount, width, with_carry=True): The ARM ROR_C (rotate right with carry) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value and carry result @@ -234,11 +234,11 @@ def ROR(value, amount, width): The ARM ROR (rotate right) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value - :rtype int or Bitvec + :rtype int or BitVec """ if isinstance(amount, int) and amount == 0: return value @@ -251,7 +251,7 @@ def RRX_C(value, carry, width, with_carry=True): The ARM RRX (rotate right with extend and with carry) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value and carry result @@ -270,11 +270,11 @@ def RRX(value, carry, width): The ARM RRX (rotate right with extend) operation. :param value: Value to shift - :type value: int or long or Bitvec + :type value: int or long or BitVec :param int amount: How many bits to rotate it. :param int width: Width of the value :return: Resultant value - :rtype int or Bitvec + :rtype int or BitVec """ result = RRX_C(value, carry, width, with_carry=False) return result diff --git a/manticore/native/cpu/disasm.py b/manticore/native/cpu/disasm.py index cf62d3be9..462385e4e 100644 --- a/manticore/native/cpu/disasm.py +++ b/manticore/native/cpu/disasm.py @@ -4,8 +4,7 @@ class Instruction: - """Capstone-like instruction to be used internally - """ + """Capstone-like instruction to be used internally""" @property @abstractmethod diff --git a/manticore/native/cpu/register.py b/manticore/native/cpu/register.py index 150de95dc..8cac2ff4d 100644 --- a/manticore/native/cpu/register.py +++ b/manticore/native/cpu/register.py @@ -1,4 +1,4 @@ -from ...core.smtlib import Operators, Bitvec, Bool +from ...core.smtlib import Operators, BitVec, Bool class Register: @@ -20,7 +20,7 @@ def read(self): def write(self, val): if isinstance(val, (Bool, bool)): self.value = val - elif isinstance(val, Bitvec): + elif isinstance(val, BitVec): self.value = val.Bool() if self.is_flag() else val elif isinstance(val, int): self.value = Operators.EXTRACT(val, 0, self.width) diff --git a/manticore/native/cpu/x86.py b/manticore/native/cpu/x86.py index 3aacfa665..c43b95f7b 100644 --- a/manticore/native/cpu/x86.py +++ b/manticore/native/cpu/x86.py @@ -20,7 +20,7 @@ ) -from ...core.smtlib import Operators, Bitvec, Bool, BitvecConstant, operator, visitors, issymbolic +from ...core.smtlib import Operators, BitVec, Bool, BitVecConstant, operator, visitors, issymbolic from ..memory import Memory, ConcretizeMemory from functools import reduce @@ -585,7 +585,7 @@ def _set_bv(self, register_id, register_size, offset, size, reset, value): # if (value & ~((1<= bitoffset.size addr = bitbase.address() @@ -4691,14 +4691,14 @@ def PAUSE(cpu): @instruction def ANDN(cpu, dest, src1, src2): """Performs a bitwise logical AND of inverted second operand (the first source operand) - with the third operand (the second source operand). The result is stored in the first - operand (destination operand). + with the third operand (the second source operand). The result is stored in the first + operand (destination operand). - DEST <- (NOT SRC1) bitwiseAND SRC2; - SF <- DEST[OperandSize -1]; - ZF <- (DEST = 0); - Flags Affected - SF and ZF are updated based on result. OF and CF flags are cleared. AF and PF flags are undefined. + DEST <- (NOT SRC1) bitwiseAND SRC2; + SF <- DEST[OperandSize -1]; + ZF <- (DEST = 0); + Flags Affected + SF and ZF are updated based on result. OF and CF flags are cleared. AF and PF flags are undefined. """ value = ~src1.read() & src2.read() dest.write(value) @@ -4965,7 +4965,7 @@ def PSHUFW(cpu, op0, op1, op3): :param op0: destination operand. :param op1: source operand. :param op3: order operand. - """ + """ size = op0.size arg0 = op0.read() arg1 = op1.read() @@ -5019,7 +5019,7 @@ def PSHUFD(cpu, op0, op1, op3): :param op0: destination operand. :param op1: source operand. :param op3: order operand. - """ + """ size = op0.size arg0 = op0.read() arg1 = op1.read() @@ -5804,12 +5804,12 @@ def VORPS(cpu, dest, src, src2): @instruction def PTEST(cpu, dest, src): - """ PTEST - PTEST set the ZF flag if all bits in the result are 0 of the bitwise AND - of the first source operand (first operand) and the second source operand - (second operand). Also this sets the CF flag if all bits in the result - are 0 of the bitwise AND of the second source operand (second operand) - and the logical NOT of the destination operand. + """PTEST + PTEST set the ZF flag if all bits in the result are 0 of the bitwise AND + of the first source operand (first operand) and the second source operand + (second operand). Also this sets the CF flag if all bits in the result + are 0 of the bitwise AND of the second source operand (second operand) + and the logical NOT of the destination operand. """ cpu.OF = False cpu.AF = False @@ -6094,9 +6094,9 @@ def PEXTRW(cpu, dest, src, count): @instruction def PALIGNR(cpu, dest, src, offset): """ALIGNR concatenates the destination operand (the first operand) and the source - operand (the second operand) into an intermediate composite, shifts the composite - at byte granularity to the right by a constant immediate, and extracts the right- - aligned result into the destination.""" + operand (the second operand) into an intermediate composite, shifts the composite + at byte granularity to the right by a constant immediate, and extracts the right- + aligned result into the destination.""" dest.write( Operators.EXTRACT( Operators.CONCAT(dest.size * 2, dest.read(), src.read()), @@ -6107,7 +6107,7 @@ def PALIGNR(cpu, dest, src, offset): @instruction def PSLLDQ(cpu, dest, src): - """ Packed Shift Left Logical Double Quadword + """Packed Shift Left Logical Double Quadword Shifts the destination operand (first operand) to the left by the number of bytes specified in the count operand (second operand). The empty low-order bytes are cleared (set to all 0s). If the value specified by the count @@ -6240,7 +6240,7 @@ def VPSHUFB(cpu, op0, op1, op3): :param op0: destination operand. :param op1: source operand. :param op3: order operand. - """ + """ size = op0.size arg0 = op0.read() arg1 = op1.read() diff --git a/manticore/native/manticore.py b/manticore/native/manticore.py index e4333f5cb..b2dacf04f 100644 --- a/manticore/native/manticore.py +++ b/manticore/native/manticore.py @@ -192,11 +192,8 @@ def _assertions_callback(self, state, pc, instruction): # (It may dereference pointers) assertion = parse(program, state.cpu.read_int, state.cpu.read_register) if not state.can_be_true(assertion): - logger.info(str(state.cpu)) - logger.info( - "Assertion %x -> {%s} does not hold. Aborting state.", state.cpu.pc, program - ) - raise TerminateState() + message = f"Assertion {state.cpu.PC:x} -> {program:s} does not hold. Aborting state." + raise TerminateState(message=message) # Everything is good add it. state.constraints.add(assertion) diff --git a/manticore/native/memory.py b/manticore/native/memory.py index 223974349..2417db6cc 100644 --- a/manticore/native/memory.py +++ b/manticore/native/memory.py @@ -7,8 +7,8 @@ arithmetic_simplify, SelectedSolver, TooManySolutions, - Bitvec, - BitvecConstant, + BitVec, + BitVecConstant, expression, issymbolic, Expression, @@ -1138,7 +1138,7 @@ def __init__(self, constraints: ConstraintSet, symbols=None, *args, **kwargs): to their condition and value. The condition of a symbolic chunk can be concrete (True/False) or symbolic. The value should - always be symbolic (e.g. a BitvecVariable). + always be symbolic (e.g. a BitVecVariable). """ super().__init__(*args, **kwargs) assert isinstance(constraints, ConstraintSet) @@ -1161,11 +1161,11 @@ def constraints(self, constraints): self._constraints = constraints def _get_size(self, size): - if isinstance(size, Bitvec): + if isinstance(size, BitVec): size = arithmetic_simplify(size) else: - size = BitvecConstant(self.memory_bit_size, size) - assert isinstance(size, BitvecConstant) + size = BitVecConstant(self.memory_bit_size, size) + assert isinstance(size, BitVecConstant) return size.value def munmap(self, start, size): diff --git a/manticore/native/models.py b/manticore/native/models.py index 114d2fad8..d50920277 100644 --- a/manticore/native/models.py +++ b/manticore/native/models.py @@ -4,8 +4,8 @@ from .cpu.abstractcpu import Cpu, ConcretizeArgument from .state import State -from ..core.smtlib import issymbolic, Bitvec -from ..core.smtlib.solver import SelectedSolver, issymbolic, Bitvec +from ..core.smtlib import issymbolic, BitVec +from ..core.smtlib.solver import SelectedSolver, issymbolic, BitVec from ..core.smtlib.operators import ITEBV, ZEXTEND from ..core.state import Concretize from typing import Union @@ -77,7 +77,7 @@ def can_be_NULL(state, byte) -> bool: return byte == 0 -def _find_zero(cpu, state, ptr: Union[int, Bitvec]) -> int: +def _find_zero(cpu, state, ptr: Union[int, BitVec]) -> int: """ Helper for finding the closest NULL or, effectively NULL byte from a starting address. @@ -97,7 +97,7 @@ def _find_zero(cpu, state, ptr: Union[int, Bitvec]) -> int: return offset -def strcmp(state: State, s1: Union[int, Bitvec], s2: Union[int, Bitvec]): +def strcmp(state: State, s1: Union[int, BitVec], s2: Union[int, BitVec]): """ strcmp symbolic model. @@ -153,7 +153,7 @@ def strcmp(state: State, s1: Union[int, Bitvec], s2: Union[int, Bitvec]): return ret -def strlen_exact(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: +def strlen_exact(state: State, s: Union[int, BitVec]) -> Union[int, BitVec]: """ strlen symbolic model @@ -191,7 +191,7 @@ def strlen_exact(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: return offset -def strlen_approx(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: +def strlen_approx(state: State, s: Union[int, BitVec]) -> Union[int, BitVec]: """ strlen symbolic model @@ -220,7 +220,7 @@ def strlen_approx(state: State, s: Union[int, Bitvec]) -> Union[int, Bitvec]: return ret -def strcpy(state: State, dst: Union[int, Bitvec], src: Union[int, Bitvec]) -> Union[int, Bitvec]: +def strcpy(state: State, dst: Union[int, BitVec], src: Union[int, BitVec]) -> Union[int, BitVec]: """ strcpy symbolic model @@ -270,8 +270,8 @@ def strcpy(state: State, dst: Union[int, Bitvec], src: Union[int, Bitvec]) -> Un def strncpy( - state: State, dst: Union[int, Bitvec], src: Union[int, Bitvec], n: Union[int, Bitvec] -) -> Union[int, Bitvec]: + state: State, dst: Union[int, BitVec], src: Union[int, BitVec], n: Union[int, BitVec] +) -> Union[int, BitVec]: """ strncpy symbolic model diff --git a/manticore/native/state_merging.py b/manticore/native/state_merging.py index 0f3230439..332f95b73 100644 --- a/manticore/native/state_merging.py +++ b/manticore/native/state_merging.py @@ -1,4 +1,4 @@ -from ..core.smtlib import SelectedSolver, ConstraintSet, Operators, issymbolic, Bitvec +from ..core.smtlib import SelectedSolver, ConstraintSet, Operators, issymbolic, BitVec def compare_sockets(cs, socket1, socket2): @@ -188,7 +188,7 @@ def merge_cpu(cpu1, cpu2, state, exp1, merged_constraint): for reg in cpu1.canonical_registers: val1 = cpu1.read_register(reg) val2 = cpu2.read_register(reg) - if isinstance(val1, Bitvec) and isinstance(val2, Bitvec): + if isinstance(val1, BitVec) and isinstance(val2, BitVec): assert val1.size == val2.size if issymbolic(val1) or issymbolic(val2) or val1 != val2: val1_migrated = merged_constraint.migrate(val1) diff --git a/manticore/platforms/decree.py b/manticore/platforms/decree.py index 130b4ce7f..be36986a1 100644 --- a/manticore/platforms/decree.py +++ b/manticore/platforms/decree.py @@ -405,34 +405,34 @@ def _is_open(self, fd): return fd >= 0 and fd < len(self.files) and self.files[fd] is not None def sys_allocate(self, cpu, length, isX, addr): - """ allocate - allocate virtual memory - - The allocate system call creates a new allocation in the virtual address - space of the calling process. The length argument specifies the length of - the allocation in bytes which will be rounded up to the hardware page size. - - The kernel chooses the address at which to create the allocation; the - address of the new allocation is returned in *addr as the result of the call. - - All newly allocated memory is readable and writeable. In addition, the - is_X argument is a boolean that allows newly allocated memory to be marked - as executable (non-zero) or non-executable (zero). - - The allocate function is invoked through system call number 5. - - :param cpu: current CPU - :param length: the length of the allocation in bytes - :param isX: boolean that allows newly allocated memory to be marked as executable - :param addr: the address of the new allocation is returned in *addr - - :return: On success, allocate returns zero and a pointer to the allocated area - is returned in *addr. Otherwise, an error code is returned - and *addr is undefined. - EINVAL length is zero. - EINVAL length is too large. - EFAULT addr points to an invalid address. - ENOMEM No memory is available or the process' maximum number of allocations - would have been exceeded. + """allocate - allocate virtual memory + + The allocate system call creates a new allocation in the virtual address + space of the calling process. The length argument specifies the length of + the allocation in bytes which will be rounded up to the hardware page size. + + The kernel chooses the address at which to create the allocation; the + address of the new allocation is returned in *addr as the result of the call. + + All newly allocated memory is readable and writeable. In addition, the + is_X argument is a boolean that allows newly allocated memory to be marked + as executable (non-zero) or non-executable (zero). + + The allocate function is invoked through system call number 5. + + :param cpu: current CPU + :param length: the length of the allocation in bytes + :param isX: boolean that allows newly allocated memory to be marked as executable + :param addr: the address of the new allocation is returned in *addr + + :return: On success, allocate returns zero and a pointer to the allocated area + is returned in *addr. Otherwise, an error code is returned + and *addr is undefined. + EINVAL length is zero. + EINVAL length is too large. + EFAULT addr points to an invalid address. + ENOMEM No memory is available or the process' maximum number of allocations + would have been exceeded. """ # TODO: check 4 bytes from addr if addr not in cpu.memory: @@ -451,20 +451,20 @@ def sys_allocate(self, cpu, length, isX, addr): return 0 def sys_random(self, cpu, buf, count, rnd_bytes): - """ random - fill a buffer with random data + """random - fill a buffer with random data - The random system call populates the buffer referenced by buf with up to - count bytes of random data. If count is zero, random returns 0 and optionally - sets *rx_bytes to zero. If count is greater than SSIZE_MAX, the result is unspecified. + The random system call populates the buffer referenced by buf with up to + count bytes of random data. If count is zero, random returns 0 and optionally + sets *rx_bytes to zero. If count is greater than SSIZE_MAX, the result is unspecified. - :param cpu: current CPU - :param buf: a memory buffer - :param count: max number of bytes to receive - :param rnd_bytes: if valid, points to the actual number of random bytes + :param cpu: current CPU + :param buf: a memory buffer + :param count: max number of bytes to receive + :param rnd_bytes: if valid, points to the actual number of random bytes - :return: 0 On success - EINVAL count is invalid. - EFAULT buf or rnd_bytes points to an invalid address. + :return: 0 On success + EINVAL count is invalid. + EFAULT buf or rnd_bytes points to an invalid address. """ ret = 0 @@ -496,20 +496,20 @@ def sys_random(self, cpu, buf, count, rnd_bytes): return ret def sys_receive(self, cpu, fd, buf, count, rx_bytes): - """ receive - receive bytes from a file descriptor - - The receive system call reads up to count bytes from file descriptor fd to the - buffer pointed to by buf. If count is zero, receive returns 0 and optionally - sets *rx_bytes to zero. - - :param cpu: current CPU. - :param fd: a valid file descriptor - :param buf: a memory buffer - :param count: max number of bytes to receive - :param rx_bytes: if valid, points to the actual number of bytes received - :return: 0 Success - EBADF fd is not a valid file descriptor or is not open - EFAULT buf or rx_bytes points to an invalid address. + """receive - receive bytes from a file descriptor + + The receive system call reads up to count bytes from file descriptor fd to the + buffer pointed to by buf. If count is zero, receive returns 0 and optionally + sets *rx_bytes to zero. + + :param cpu: current CPU. + :param fd: a valid file descriptor + :param buf: a memory buffer + :param count: max number of bytes to receive + :param rx_bytes: if valid, points to the actual number of bytes received + :return: 0 Success + EBADF fd is not a valid file descriptor or is not open + EFAULT buf or rx_bytes points to an invalid address. """ data = "" if count != 0: @@ -554,19 +554,19 @@ def sys_receive(self, cpu, fd, buf, count, rx_bytes): return 0 def sys_transmit(self, cpu, fd, buf, count, tx_bytes): - """ transmit - send bytes through a file descriptor - The transmit system call writes up to count bytes from the buffer pointed - to by buf to the file descriptor fd. If count is zero, transmit returns 0 - and optionally sets *tx_bytes to zero. - - :param cpu current CPU - :param fd a valid file descriptor - :param buf a memory buffer - :param count number of bytes to send - :param tx_bytes if valid, points to the actual number of bytes transmitted - :return: 0 Success - EBADF fd is not a valid file descriptor or is not open. - EFAULT buf or tx_bytes points to an invalid address. + """transmit - send bytes through a file descriptor + The transmit system call writes up to count bytes from the buffer pointed + to by buf to the file descriptor fd. If count is zero, transmit returns 0 + and optionally sets *tx_bytes to zero. + + :param cpu current CPU + :param fd a valid file descriptor + :param buf a memory buffer + :param count number of bytes to send + :param tx_bytes if valid, points to the actual number of bytes transmitted + :return: 0 Success + EBADF fd is not a valid file descriptor or is not open. + EFAULT buf or tx_bytes points to an invalid address. """ data = [] if count != 0: @@ -632,7 +632,7 @@ def sys_terminate(self, cpu, error_code): return error_code def sys_deallocate(self, cpu, addr, size): - """ deallocate - remove allocations + """deallocate - remove allocations The deallocate system call deletes the allocations for the specified address range, and causes further references to the addresses within the range to generate invalid memory accesses. The region is also @@ -676,8 +676,7 @@ def sys_deallocate(self, cpu, addr, size): return 0 def sys_fdwait(self, cpu, nfds, readfds, writefds, timeout, readyfds): - """ fdwait - wait for file descriptors to become ready - """ + """fdwait - wait for file descriptors to become ready""" logger.debug( "FDWAIT(%d, 0x%08x, 0x%08x, 0x%08x, 0x%08x)" % (nfds, readfds, writefds, timeout, readyfds) @@ -794,10 +793,10 @@ def int80(self, cpu): cpu.EAX = func(*args[: nargs - 1]) def sched(self): - """ Yield CPU. - This will choose another process from the RUNNNIG list and change - current running process. May give the same cpu if only one running - process. + """Yield CPU. + This will choose another process from the RUNNNIG list and change + current running process. May give the same cpu if only one running + process. """ if len(self.procs) > 1: logger.info("SCHED:") @@ -825,9 +824,9 @@ def sched(self): self._current = next def wait(self, readfds, writefds, timeout): - """ Wait for filedescriptors or timeout. - Adds the current process to the corresponding waiting list and - yields the cpu to another running process. + """Wait for filedescriptors or timeout. + Adds the current process to the corresponding waiting list and + yields the cpu to another running process. """ logger.info("WAIT:") logger.info( diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 532fcd70d..cf9a6e25a 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -10,14 +10,14 @@ from ..platforms.platform import * from ..core.smtlib import ( SelectedSolver, - Bitvec, + BitVec, Array, MutableArray, Operators, Constant, ArrayVariable, ArrayStore, - BitvecConstant, + BitVecConstant, translate_to_smtlib, to_constant, get_depth, @@ -103,7 +103,9 @@ def globalfakesha3(data): description="Max calldata size to explore in each CALLDATACOPY. Iff size in a calldata related instruction are symbolic it will be constrained to be less than this constant. -1 means free(only use when gas is being tracked)", ) consts.add( - "ignore_balance", default=False, description="Do not try to solve symbolic balances", + "ignore_balance", + default=False, + description="Do not try to solve symbolic balances", ) @@ -127,7 +129,7 @@ def globalfakesha3(data): def ceil32(x): size = 256 - if isinstance(x, Bitvec): + if isinstance(x, BitVec): size = x.size return Operators.ITEBV(size, Operators.UREM(x, 32) == 0, x, x + 32 - Operators.UREM(x, 32)) @@ -392,7 +394,7 @@ def used_gas(self): def set_result(self, result, return_data=None, used_gas=None): if getattr(self, "result", None) is not None: raise EVMException("Transaction result already set") - if not isinstance(used_gas, (int, Bitvec, type(None))): + if not isinstance(used_gas, (int, BitVec, type(None))): raise EVMException("Invalid used gas in Transaction") if result not in {None, "TXERROR", "REVERT", "RETURN", "THROW", "STOP", "SELFDESTRUCT"}: raise EVMException("Invalid transaction result") @@ -832,7 +834,9 @@ def extend_with_zeroes(b): self._failed = False def fail_if(self, failed): - self._failed = Operators.OR(self._failed, failed) + old_failed = self.constraints.new_bool() + self.constraints.add(old_failed == self._failed) + self._failed = Operators.OR(old_failed, failed) def is_failed(self): if isinstance(self._failed, bool): @@ -846,7 +850,7 @@ def setstate(state, value): state.platform._failed = value raise Concretize( - "Transaction failed", expression=self._failed, setstate=lambda a, b: None, policy="ALL" + "Transaction failed", expression=self._failed, setstate=setstate, policy="ALL" ) @property @@ -1016,7 +1020,7 @@ def instruction(self): """ return self.get_instruction(pc=self.pc) - def get_instruction(self, pc: Union[Bitvec, int]): + def get_instruction(self, pc: Union[BitVec, int]): """ Current instruction pointed by self.pc """ @@ -1038,7 +1042,7 @@ def get_instruction(self, pc: Union[Bitvec, int]): if pc in _decoding_cache: return _decoding_cache[pc] - if isinstance(pc, Bitvec): + if isinstance(pc, BitVec): raise EVMException("Trying to decode from symbolic pc") instruction = EVMAsm.disassemble_one(self._getcode(pc), pc=pc, fork=self.evmfork) _decoding_cache[pc] = instruction @@ -1059,7 +1063,7 @@ def _push(self, value): ITEM2 sp-> {empty} """ - assert isinstance(value, int) or isinstance(value, Bitvec) and value.size == 256 + assert isinstance(value, int) or isinstance(value, BitVec) and value.size == 256 if len(self.stack) >= 1024: raise StackOverflow() @@ -1085,20 +1089,15 @@ def _pop(self): return self.stack.pop() def _consume(self, fee): + if consts.oog == "ignore": + return # Check type and bitvec size if isinstance(fee, int): if fee > (1 << 512) - 1: raise ValueError - elif isinstance(fee, Bitvec): + elif isinstance(fee, BitVec): if fee.size != 512: raise ValueError("Fees should be 512 bit long") - # This configuration variable allows the user to control and perhaps relax the gas calculation - # pedantic: gas is faithfully accounted and checked at instruction level. State may get forked in OOG/NoOOG - # complete: gas is faithfully accounted and checked at basic blocks limits. State may get forked in OOG/NoOOG - # concrete: Concretize gas. If the fee to be consumed gets to be symbolic. Choose some potential values and fork on those. - # optimistic: Try not to OOG. If it may be enough gas we ignore the OOG case. A constraint is added to assert the gas is enough and the OOG state is ignored. - # pesimistic: OOG soon. If it may NOT be enough gas we ignore the normal case. A constraint is added to assert the gas is NOT enough and the other state is ignored. - # ignore: Ignore gas. Do not account for it. Do not OOG. oog_condition = simplify(Operators.ULT(self._gas, fee)) self.fail_if(oog_condition) @@ -1191,6 +1190,9 @@ def _push_results(self, instruction, result): assert result is None def _calculate_gas(self, *arguments): + if consts.oog == "ignore": + return 0 + start = time.time() current = self.instruction implementation = getattr(self, f"{current.semantics}_gas", None) @@ -1329,12 +1331,14 @@ def execute(self): def setstate(state, value): if taints: - state.platform.current_vm.pc = BitvecConstant(256, value, taint=taints) + state.platform.current_vm.pc = BitVecConstant(256, value, taint=taints) else: state.platform.current_vm.pc = value raise Concretize("Symbolic PC", expression=expression, setstate=setstate, policy="ALL") try: + if getattr(getattr(self, self.instruction.semantics, None), "_pos", None) is None: + print(self) self._check_jmpdest() last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() result = self._handler(*arguments) @@ -1570,7 +1574,7 @@ def EXP(self, base, exponent): :param base: exponential base, concretized with sampled values :param exponent: exponent value, concretized with sampled values - :return: Bitvec* EXP result + :return: BitVec* EXP result """ if exponent == 0: return 1 @@ -2194,7 +2198,12 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): ) self.world.start_transaction( - "CREATE", address, data=data, caller=self.address, value=value, gas=self.gas, + "CREATE", + address, + data=data, + caller=self.address, + value=value, + gas=self.gas, ) raise StartTx() @@ -2421,9 +2430,12 @@ def __str__(self): sp = 0 for i in list(reversed(self.stack))[:10]: argname = args.get(sp, "") - r = "" if issymbolic(i): - r = "{:>12s} {:66s}".format(argname, repr(i)) + r = "{:>12s} {:66s} {:s}".format( + argname, + "[%x-%x]" % SelectedSolver.instance().minmax(self.constraints, i), + str(i.taint), + ) else: r = "{:>12s} 0x{:064x}".format(argname, i) sp += 1 @@ -2448,6 +2460,12 @@ def __str__(self): else: result.append(f"Gas: {gas}") + """ + vals = SelectedSolver.instance().get_all_values(self.constraints, + self.data[0:4], maxcnt=2, + silent=True) + result.append(f"Data: {vals}") + """ return "\n".join(hex(self.address) + ": " + x for x in result) @@ -2600,6 +2618,10 @@ def evmfork(self): return self._fork def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value): + return 0 + if consts.oog == "ignore": + return 0 + GTXCREATE = ( 32000 # Paid by all contract creating transactions after the Homestead transition. ) @@ -2611,23 +2633,29 @@ def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value else: tx_fee = GTRANSACTION # Simple transaction fee + # This is INSANE TDO FIXME # This popcnt like thing is expensive when the bytecode or # data has symbolic content - zerocount = 0 - nonzerocount = 0 - for index in range(len(bytecode_or_data)): - try: - c = bytecode_or_data.get(index) - except AttributeError: - c = bytecode_or_data[index] + size = 1 + while 2 ** size < len(bytecode_or_data): + size += 1 + if size > 512: + raise Exception("hahaha") - zerocount += Operators.ITEBV(512, c == 0, 1, 0) - nonzerocount += Operators.ITEBV(512, c == 0, 0, 1) + zerocount = 0 + len_bytecode_or_data = len(bytecode_or_data) + for index in range(len_bytecode_or_data): + c = bytecode_or_data[index] + zerocount += Operators.ITEBV(2 ** size, c == 0, 1, 0) + nonzerocount = len_bytecode_or_data - zerocount tx_fee += zerocount * GTXDATAZERO tx_fee += nonzerocount * GTXDATANONZERO - return simplify(tx_fee) + + x = self.constraints.new_bitvec(size=512) + self.constraints.add(x == simplify(Operators.ZEXTEND(tx_fee, 512))) + return x def _make_vm_for_tx(self, tx): if tx.sort == "CREATE": @@ -2892,9 +2920,9 @@ def get_storage_data(self, storage_address, offset): :param storage_address: an account address :param offset: the storage slot to use. - :type offset: int or Bitvec + :type offset: int or BitVec :return: the value - :rtype: int or Bitvec + :rtype: int or BitVec """ return self._world_state[storage_address]["storage"].select(offset) @@ -2904,9 +2932,9 @@ def set_storage_data(self, storage_address, offset, value): :param storage_address: an account address :param offset: the storage slot to use. - :type offset: int or Bitvec + :type offset: int or BitVec :param value: the value to write - :type value: int or Bitvec + :type value: int or BitVec """ self._world_state[storage_address]["storage"][offset] = value @@ -2969,7 +2997,7 @@ def increase_nonce(self, address): return new_nonce def set_balance(self, address, value): - if isinstance(value, Bitvec): + if isinstance(value, BitVec): value = Operators.ZEXTEND(value, 512) self._world_state[int(address)]["balance"] = simplify(value) @@ -2988,17 +3016,17 @@ def account_exists(self, address): ) def add_to_balance(self, address, value): - if isinstance(value, Bitvec): + if isinstance(value, BitVec): value = Operators.ZEXTEND(value, 512) self._world_state[address]["balance"] += value def sub_from_balance(self, address, value): - if isinstance(value, Bitvec): + if isinstance(value, BitVec): value = Operators.ZEXTEND(value, 512) self._world_state[address]["balance"] -= value def send_funds(self, sender, recipient, value): - if isinstance(value, Bitvec): + if isinstance(value, BitVec): value = Operators.ZEXTEND(value, 512) self._world_state[sender]["balance"] -= value self._world_state[recipient]["balance"] += value @@ -3367,7 +3395,7 @@ def _pending_transaction_failed(self): sort, address, price, data, caller, value, gas, failed = self._pending_transaction # Initially the failed flag is not set. For now we need the caller to be - # concrete so the caller balance is easy to get. Initialize falied here + # concrete so the caller balance is easy to get. Initialize failed here if failed is None: # Check depth failed = self.depth >= 1024 @@ -3376,7 +3404,7 @@ def _pending_transaction_failed(self): aux_src_balance = Operators.ZEXTEND(self.get_balance(caller), 512) aux_value = Operators.ZEXTEND(value, 512) enough_balance = Operators.UGE(aux_src_balance, aux_value) - if self.depth == 0: + if self.depth == 0: # the tx_fee is taken at depth 0 # take the gas from the balance aux_price = Operators.ZEXTEND(price, 512) aux_gas = Operators.ZEXTEND(gas, 512) @@ -3388,6 +3416,7 @@ def _pending_transaction_failed(self): failed = Operators.NOT(enough_balance) self._pending_transaction = sort, address, price, data, caller, value, gas, failed + # ok now failed exists ans it is initialized. Concretize or fork. if issymbolic(failed): # optimistic/pesimistic is inverted as the expresion represents fail policy = {"optimistic": "PESSIMISTIC", "pessimistic": "OPTIMISTIC"}.get( @@ -3414,14 +3443,6 @@ def set_failed(state, solution): policy=policy, ) - if self.depth != 0: - price = 0 - aux_price = Operators.ZEXTEND(price, 512) - aux_gas = Operators.ZEXTEND(gas, 512) - tx_fee = Operators.ITEBV(512, self.depth == 0, aux_price * aux_gas, 0) - aux_src_balance = Operators.ZEXTEND(self.get_balance(caller), 512) - aux_value = Operators.ZEXTEND(value, 512) - enough_balance = Operators.UGE(aux_src_balance, aux_value + tx_fee) return failed def _process_pending_transaction(self): diff --git a/manticore/platforms/linux.py b/manticore/platforms/linux.py index 7bf153b26..10ae0332f 100644 --- a/manticore/platforms/linux.py +++ b/manticore/platforms/linux.py @@ -1758,17 +1758,17 @@ def sys_read(self, fd: int, buf: int, count: int) -> int: return len(data) def sys_write(self, fd: int, buf, count) -> int: - """ write - send bytes through a file descriptor - The write system call writes up to count bytes from the buffer pointed - to by buf to the file descriptor fd. If count is zero, write returns 0 - and optionally sets *tx_bytes to zero. - - :param fd a valid file descriptor - :param buf a memory buffer - :param count number of bytes to send - :return: 0 Success - EBADF fd is not a valid file descriptor or is not open. - EFAULT buf or tx_bytes points to an invalid address. + """write - send bytes through a file descriptor + The write system call writes up to count bytes from the buffer pointed + to by buf to the file descriptor fd. If count is zero, write returns 0 + and optionally sets *tx_bytes to zero. + + :param fd a valid file descriptor + :param buf a memory buffer + :param count number of bytes to send + :return: 0 Success + EBADF fd is not a valid file descriptor or is not open. + EFAULT buf or tx_bytes points to an invalid address. """ data: bytes = bytes() cpu = self.current @@ -2773,10 +2773,10 @@ def sys_gettimeofday(self, tv, tz) -> int: return 0 def sched(self) -> None: - """ Yield CPU. - This will choose another process from the running list and change - current running process. May give the same cpu if only one running - process. + """Yield CPU. + This will choose another process from the running list and change + current running process. May give the same cpu if only one running + process. """ if len(self.procs) > 1: logger.debug("SCHED:") @@ -2803,9 +2803,9 @@ def sched(self) -> None: self._current = next_running_idx def wait(self, readfds, writefds, timeout) -> None: - """ Wait for file descriptors or timeout. - Adds the current process in the correspondent waiting list and - yield the cpu to another running process. + """Wait for file descriptors or timeout. + Adds the current process in the correspondent waiting list and + yield the cpu to another running process. """ logger.debug("WAIT:") logger.debug( @@ -2849,7 +2849,7 @@ def awake(self, procid) -> None: self._current = procid def connections(self, fd: int) -> Optional[int]: - """ File descriptors are connected to each other like pipes, except + """File descriptors are connected to each other like pipes, except for 0, 1, and 2. If you write to FD(N) for N >=3, then that comes out from FD(N+1) and vice-versa """ diff --git a/manticore/utils/emulate.py b/manticore/utils/emulate.py index 9a77b8b84..aad3e20cd 100644 --- a/manticore/utils/emulate.py +++ b/manticore/utils/emulate.py @@ -184,7 +184,7 @@ def protect_memory_callback(self, start, size, perms): self._emu.mem_protect(start, size, convert_permissions(perms)) def get_unicorn_pc(self): - """ Get the program counter from Unicorn regardless of architecture. + """Get the program counter from Unicorn regardless of architecture. Legacy method, since this module only works on x86.""" if self._cpu.arch == CS_ARCH_ARM: return self._emu.reg_read(UC_ARM_REG_R15) diff --git a/manticore/wasm/executor.py b/manticore/wasm/executor.py index 4aae85710..4bab43dff 100644 --- a/manticore/wasm/executor.py +++ b/manticore/wasm/executor.py @@ -21,7 +21,7 @@ Value_t, ZeroDivisionTrap, ) -from ..core.smtlib import Operators, Bitvec, issymbolic +from ..core.smtlib import Operators, BitVec, issymbolic from ..utils.event import Eventful from decimal import Decimal, InvalidOperation @@ -284,7 +284,7 @@ def select(self, store, stack): c = stack.pop() v2 = stack.pop() v1 = stack.pop() - assert isinstance(c, (I32, Bitvec)), f"{type(c)} is not I32" + assert isinstance(c, (I32, BitVec)), f"{type(c)} is not I32" if not issymbolic(v2) and not issymbolic(v1): assert type(v2) == type(v1), f"{type(v2)} is not the same as {type(v1)}" diff --git a/manticore/wasm/structure.py b/manticore/wasm/structure.py index 623e7fa83..f10f5b8bd 100644 --- a/manticore/wasm/structure.py +++ b/manticore/wasm/structure.py @@ -44,7 +44,7 @@ ZeroDivisionTrap, ) from .state import State -from ..core.smtlib import Bitvec, Bool, issymbolic, Operators, Expression +from ..core.smtlib import BitVec, Bool, issymbolic, Operators, Expression from ..core.state import Concretize from ..utils.event import Eventful from ..utils import config @@ -1021,7 +1021,7 @@ def invoke_by_name(self, name: str, stack, store, argv): passing argv :param name: Name of the function to look for - :param argv: Arguments to pass to the function. Can be Bitvecs or Values + :param argv: Arguments to pass to the function. Can be BitVecs or Values """ for export in self.exports: if export.name == name and isinstance(export.value, FuncAddr): @@ -1040,7 +1040,7 @@ def invoke(self, stack: "Stack", funcaddr: FuncAddr, store: Store, argv: typing. https://www.w3.org/TR/wasm-core-1/#invocation%E2%91%A1 :param funcaddr: Address (in Store) of the function to call - :param argv: Arguments to pass to the function. Can be Bitvecs or Values + :param argv: Arguments to pass to the function. Can be BitVecs or Values """ assert funcaddr in range(len(store.funcs)) funcinst = store.funcs[funcaddr] @@ -1795,15 +1795,15 @@ def empty(self) -> bool: def has_type_on_top(self, t: typing.Union[type, typing.Tuple[type, ...]], n: int): """ - *Asserts* that the stack has at least n values of type t or type Bitvec on the top + *Asserts* that the stack has at least n values of type t or type BitVec on the top - :param t: type of value to look for (Bitvec is always included as an option) + :param t: type of value to look for (BitVec is always included as an option) :param n: Number of values to check :return: True """ for i in range(1, n + 1): assert isinstance( - self.data[i * -1], (t, Bitvec) + self.data[i * -1], (t, BitVec) ), f"{type(self.data[i * -1])} is not an {t}!" return True diff --git a/manticore/wasm/types.py b/manticore/wasm/types.py index 47e6425be..fdecc8034 100644 --- a/manticore/wasm/types.py +++ b/manticore/wasm/types.py @@ -1,6 +1,6 @@ import typing from dataclasses import dataclass -from ..core.smtlib import issymbolic, Bitvec +from ..core.smtlib import issymbolic, BitVec from ctypes import * import wasm import struct @@ -165,9 +165,9 @@ def cast(cls, other): ValType = type #: https://www.w3.org/TR/wasm-core-1/#syntax-valtype -Value_t = (I32, I64, F32, F64, Bitvec) -# Value = typing.TypeVar('Value', I32, I64, F32, F64, Bitvec) #: https://www.w3.org/TR/wasm-core-1/#syntax-val -Value = typing.Union[I32, I64, F32, F64, Bitvec] #: https://www.w3.org/TR/wasm-core-1/#syntax-val +Value_t = (I32, I64, F32, F64, BitVec) +# Value = typing.TypeVar('Value', I32, I64, F32, F64, BitVec) #: https://www.w3.org/TR/wasm-core-1/#syntax-val +Value = typing.Union[I32, I64, F32, F64, BitVec] #: https://www.w3.org/TR/wasm-core-1/#syntax-val class Name(str): @@ -450,7 +450,7 @@ def __init__(self, depth: int, ty: type, message: str, expression, policy=None, :param depth: Index in the stack (should typically be negative) :param ty: The type to cast the :param message: Debug message describing the reason for concretization - :param expression: The expression to concretize, either a Value or a Bitvec + :param expression: The expression to concretize, either a Value or a BitVec """ def setstate(state, value): diff --git a/tests/ethereum/contracts/simple_int_overflow.sol b/tests/ethereum/contracts/simple_int_overflow.sol index 419c2a1c6..c7567a9d5 100644 --- a/tests/ethereum/contracts/simple_int_overflow.sol +++ b/tests/ethereum/contracts/simple_int_overflow.sol @@ -1,7 +1,7 @@ -pragma solidity ^0.4.15; +pragma solidity ^0.5.3; contract Overflow { - uint private sellerBalance=0; + uint private sellerBalance=10; function add(uint value) public { sellerBalance += value; // complicated math with possible overflow diff --git a/tests/ethereum/test_detectors.py b/tests/ethereum/test_detectors.py index b0431bf22..af5d368ff 100644 --- a/tests/ethereum/test_detectors.py +++ b/tests/ethereum/test_detectors.py @@ -29,8 +29,6 @@ from typing import Tuple, Type -consts = config.get_group("core") -consts.mprocessing = consts.mprocessing.single THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -47,6 +45,8 @@ class EthDetectorTest(unittest.TestCase): def setUp(self): self.mevm = ManticoreEVM() + consts = config.get_group("core") + consts.mprocessing = consts.mprocessing.single self.mevm.register_plugin(KeepOnlyIfStorageChanges()) log.set_verbosity(0) self.worksp = self.mevm.workspace @@ -63,7 +63,7 @@ def _test(self, name: str, should_find, use_ctor_sym_arg=False): dir = os.path.join(THIS_DIR, "contracts", "detectors") filepath = os.path.join(dir, f"{name}.sol") - + print (filepath) if use_ctor_sym_arg: ctor_arg: Tuple = (mevm.make_symbolic_value(),) else: @@ -71,7 +71,7 @@ def _test(self, name: str, should_find, use_ctor_sym_arg=False): self.mevm.register_detector(self.DETECTOR_CLASS()) - with self.mevm.kill_timeout(240): + with self.mevm.kill_timeout(48000): mevm.multi_tx_analysis( filepath, contract_name="DetectThis", diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index d94827afa..6406af9a1 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -16,7 +16,7 @@ from manticore.core.plugin import Plugin from manticore.core.smtlib import ConstraintSet, operators from manticore.core.smtlib import SelectedSolver -from manticore.core.smtlib.expression import BitvecVariable +from manticore.core.smtlib.expression import BitVecVariable from manticore.core.smtlib.visitors import to_constant from manticore.core.state import TerminateState from manticore.ethereum import ( @@ -1384,7 +1384,7 @@ def will_evm_execute_instruction_callback(self, state, i, *args, **kwargs): class EthHelpersTest(unittest.TestCase): def setUp(self): - self.bv = BitvecVariable(size=256, name="A") + self.bv = BitVecVariable(size=256, name="A") def test_concretizer(self): policy = "SOME_NONSTANDARD_POLICY" diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index 798bad2be..f55ff457a 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -253,17 +253,17 @@ def test_705(self): def test_addmod(self): """ - (declare-fun BV () (_ Bitvec 256)) - (declare-fun BV_2 () (_ Bitvec 256)) - (declare-fun BV_1 () (_ Bitvec 256)) - (declare-fun a_1 () (_ Bitvec 256))(assert (= a_1 (bvmul BV BV_1))) - (declare-fun a_2 () (_ Bitvec 512))(assert (= a_2 ((_ zero_extend 256) BV))) - (declare-fun a_3 () (_ Bitvec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) - (declare-fun a_4 () (_ Bitvec 512))(assert (= a_4 (bvmul a_2 a_3))) - (declare-fun a_5 () (_ Bitvec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) - (declare-fun a_6 () (_ Bitvec 512))(assert (= a_6 (bvsmod a_4 a_5))) - (declare-fun a_7 () (_ Bitvec 256))(assert (= a_7 ((_ extract 255 0) a_6))) - (declare-fun a_8 () (_ Bitvec 256))(assert (= a_8 (bvsmod a_1 BV_2))) + (declare-fun BV () (_ BitVec 256)) + (declare-fun BV_2 () (_ BitVec 256)) + (declare-fun BV_1 () (_ BitVec 256)) + (declare-fun a_1 () (_ BitVec 256))(assert (= a_1 (bvmul BV BV_1))) + (declare-fun a_2 () (_ BitVec 512))(assert (= a_2 ((_ zero_extend 256) BV))) + (declare-fun a_3 () (_ BitVec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) + (declare-fun a_4 () (_ BitVec 512))(assert (= a_4 (bvmul a_2 a_3))) + (declare-fun a_5 () (_ BitVec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) + (declare-fun a_6 () (_ BitVec 512))(assert (= a_6 (bvsmod a_4 a_5))) + (declare-fun a_7 () (_ BitVec 256))(assert (= a_7 ((_ extract 255 0) a_6))) + (declare-fun a_8 () (_ BitVec 256))(assert (= a_8 (bvsmod a_1 BV_2))) (declare-fun a_9 () Bool)(assert (= a_9 (= a_7 a_8))) (assert (not a_9)) @@ -310,17 +310,17 @@ def test_addmod(self): def test_mulmod(self): """ - (declare-fun BV () (_ Bitvec 256)) - (declare-fun BV_2 () (_ Bitvec 256)) - (declare-fun BV_1 () (_ Bitvec 256)) - (declare-fun a_1 () (_ Bitvec 256))(assert (= a_1 (bvmul BV BV_1))) - (declare-fun a_2 () (_ Bitvec 512))(assert (= a_2 ((_ zero_extend 256) BV))) - (declare-fun a_3 () (_ Bitvec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) - (declare-fun a_4 () (_ Bitvec 512))(assert (= a_4 (bvmul a_2 a_3))) - (declare-fun a_5 () (_ Bitvec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) - (declare-fun a_6 () (_ Bitvec 512))(assert (= a_6 (bvsmod a_4 a_5))) - (declare-fun a_7 () (_ Bitvec 256))(assert (= a_7 ((_ extract 255 0) a_6))) - (declare-fun a_8 () (_ Bitvec 256))(assert (= a_8 (bvsmod a_1 BV_2))) + (declare-fun BV () (_ BitVec 256)) + (declare-fun BV_2 () (_ BitVec 256)) + (declare-fun BV_1 () (_ BitVec 256)) + (declare-fun a_1 () (_ BitVec 256))(assert (= a_1 (bvmul BV BV_1))) + (declare-fun a_2 () (_ BitVec 512))(assert (= a_2 ((_ zero_extend 256) BV))) + (declare-fun a_3 () (_ BitVec 512))(assert (= a_3 ((_ zero_extend 256) BV_1))) + (declare-fun a_4 () (_ BitVec 512))(assert (= a_4 (bvmul a_2 a_3))) + (declare-fun a_5 () (_ BitVec 512))(assert (= a_5 ((_ zero_extend 256) BV_2))) + (declare-fun a_6 () (_ BitVec 512))(assert (= a_6 (bvsmod a_4 a_5))) + (declare-fun a_7 () (_ BitVec 256))(assert (= a_7 ((_ extract 255 0) a_6))) + (declare-fun a_8 () (_ BitVec 256))(assert (= a_8 (bvsmod a_1 BV_2))) (declare-fun a_9 () Bool)(assert (= a_9 (= a_7 a_8))) (assert (not a_9)) diff --git a/tests/native/test_cpu_automatic.py b/tests/native/test_cpu_automatic.py index 2ac594fe1..0d2290f49 100644 --- a/tests/native/test_cpu_automatic.py +++ b/tests/native/test_cpu_automatic.py @@ -12674,6 +12674,7 @@ def test_ADD_1_symbolic(self): with cs as temp_cs: temp_cs.add(condition) + print (temp_cs) self.assertTrue(solver.check(temp_cs)) with cs as temp_cs: temp_cs.add(condition == False) diff --git a/tests/native/test_cpu_manual.py b/tests/native/test_cpu_manual.py index 0d706bc4a..f7c772623 100644 --- a/tests/native/test_cpu_manual.py +++ b/tests/native/test_cpu_manual.py @@ -6,7 +6,7 @@ from manticore.native.cpu.abstractcpu import ConcretizeRegister from manticore.native.cpu.x86 import AMD64Cpu from manticore.native.memory import * -from manticore.core.smtlib import BitvecOr, operator, Bool +from manticore.core.smtlib import BitVecOr, operator, Bool from manticore.core.smtlib.solver import SelectedSolver from functools import reduce @@ -309,7 +309,7 @@ def _construct_flag_bitfield(self, flags): return reduce(operator.or_, (self._flags[f] for f in flags)) def _construct_sym_flag_bitfield(self, flags): - return reduce(operator.or_, (BitvecConstant(32, self._flags[f]) for f in flags)) + return reduce(operator.or_, (BitVecConstant(32, self._flags[f]) for f in flags)) def test_set_eflags(self) -> None: cpu = I386Cpu(Memory32()) @@ -351,15 +351,15 @@ def check_flag(obj, flag): check_flag(cpu.ZF, "ZF") def test_get_sym_eflags(self): - def flatten_ors(x: BitvecOr) -> List: + def flatten_ors(x: BitVecOr) -> List: """ - Retrieve all nodes of a BitvecOr expression tree + Retrieve all nodes of a BitVecOr expression tree """ - assert isinstance(x, BitvecOr) - if any(isinstance(op, BitvecOr) for op in x.operands): + assert isinstance(x, BitVecOr) + if any(isinstance(op, BitVecOr) for op in x.operands): ret: List = [] for op in x.operands: - if isinstance(op, BitvecOr): + if isinstance(op, BitVecOr): ret += flatten_ors(op) else: ret.append(op) @@ -371,14 +371,14 @@ def flatten_ors(x: BitvecOr) -> List: cpu.CF = 1 cpu.AF = 1 - a = BitvecConstant(32, 1) != 0 - b = BitvecConstant(32, 0) != 0 + a = BitVecConstant(32, 1) != 0 + b = BitVecConstant(32, 0) != 0 cpu.ZF = a cpu.SF = b flags = flatten_ors(cpu.EFLAGS) - self.assertTrue(isinstance(cpu.EFLAGS, BitvecOr)) + self.assertTrue(isinstance(cpu.EFLAGS, BitVecOr)) self.assertEqual(len(flags), 8) self.assertEqual(cpu.CF, 1) @@ -1292,7 +1292,7 @@ def test_symbolic_instruction(self): code = mem.mmap(0x1000, 0x1000, "rwx") stack = mem.mmap(0xF000, 0x1000, "rw") - mem[code] = BitvecConstant(8, 0x90) + mem[code] = BitVecConstant(8, 0x90) cpu.EIP = code cpu.EAX = 116 cpu.EBP = stack + 0x700 diff --git a/tests/native/test_driver.py b/tests/native/test_driver.py index f4bac24ec..a22eb2f44 100644 --- a/tests/native/test_driver.py +++ b/tests/native/test_driver.py @@ -5,7 +5,7 @@ import tempfile from manticore import issymbolic -from manticore.core.smtlib import BitvecVariable +from manticore.core.smtlib import BitVecVariable from manticore.native import Manticore @@ -26,7 +26,7 @@ def testCreating(self): m.log_file = "/dev/null" def test_issymbolic(self): - v = BitvecVariable(size=32, name="sym") + v = BitVecVariable(size=32, name="sym") self.assertTrue(issymbolic(v)) def test_issymbolic_neg(self): diff --git a/tests/native/test_integration_native.py b/tests/native/test_integration_native.py index 2821c559d..317891693 100644 --- a/tests/native/test_integration_native.py +++ b/tests/native/test_integration_native.py @@ -59,8 +59,7 @@ def test_timeout(self) -> None: workspace = os.path.join(self.test_dir, "workspace") t = time.time() with open(os.path.join(os.pardir, self.test_dir, "output.log"), "w") as output: - subprocess.check_call( - [ + cmd = [ "coverage", "run", # PYTHON_BIN, "-m", @@ -73,9 +72,9 @@ def test_timeout(self) -> None: "4", filename, "+++++++++", - ], - stdout=output, - ) + ] + + subprocess.check_call(cmd, stdout=output) self.assertTrue(time.time() - t < 20) @@ -129,6 +128,7 @@ def _test_arguments_assertions_aux( cmd += ["--assertions", assertions] cmd += [filename, "+++++++++"] + output = subprocess.check_output(cmd).splitlines() # self.assertIn(b'm.c.manticore:INFO: Verbosity set to 1.', output[0]) diff --git a/tests/native/test_linux.py b/tests/native/test_linux.py index e1c41b12d..9f9bd3df9 100644 --- a/tests/native/test_linux.py +++ b/tests/native/test_linux.py @@ -10,7 +10,7 @@ from manticore.native.cpu.abstractcpu import ConcretizeRegister from manticore.core.smtlib.solver import Z3Solver -from manticore.core.smtlib import BitvecVariable, issymbolic +from manticore.core.smtlib import BitVecVariable, issymbolic from manticore.native import Manticore from manticore.platforms import linux, linux_syscalls from manticore.utils.helpers import pickle_dumps diff --git a/tests/native/test_models.py b/tests/native/test_models.py index 0bbd584fa..2b41cafbf 100644 --- a/tests/native/test_models.py +++ b/tests/native/test_models.py @@ -11,7 +11,7 @@ Z3Solver, issymbolic, ArraySelect, - BitvecITE, + BitVecITE, ) from manticore.native.state import State from manticore.platforms import linux diff --git a/tests/native/test_register.py b/tests/native/test_register.py index 821a672f5..0663e811b 100644 --- a/tests/native/test_register.py +++ b/tests/native/test_register.py @@ -1,6 +1,6 @@ import unittest -from manticore.core.smtlib import Bool, BoolVariable, BitvecConstant +from manticore.core.smtlib import Bool, BoolVariable, BitVecConstant from manticore.native.cpu.register import Register @@ -53,14 +53,14 @@ def test_Bool(self): def test_bitvec_flag(self): r = Register(1) - b = BitvecConstant(32, 0) + b = BitVecConstant(32, 0) r.write(b) # __nonzero__ (==) currently unimplemented for Bool self.assertTrue(isinstance(r.read(), Bool)) def test_bitvec(self): r = Register(32) - b = BitvecConstant(32, 0) + b = BitVecConstant(32, 0) r.write(b) self.assertIs(r.read(), b) diff --git a/tests/native/test_state.py b/tests/native/test_state.py index 6bf0d782d..c4f2dc3a6 100644 --- a/tests/native/test_state.py +++ b/tests/native/test_state.py @@ -7,7 +7,7 @@ from manticore.utils.event import Eventful from manticore.platforms import linux from manticore.native.state import State -from manticore.core.smtlib import BitvecVariable, ConstraintSet +from manticore.core.smtlib import BitVecVariable, ConstraintSet from manticore.native import Manticore from manticore.native.plugins import Merger from manticore.core.plugin import Plugin @@ -79,27 +79,27 @@ def setUp(self): def test_solve_one(self): val = 42 - expr = BitvecVariable(size=32, name="tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr == val) solved = self.state.solve_one(expr) self.assertEqual(solved, val) def test_solve_n(self): - expr = BitvecVariable(size=32, name="tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) solved = sorted(self.state.solve_n(expr, 2)) self.assertEqual(solved, [5, 6]) def test_solve_n2(self): - expr = BitvecVariable(size=32, name="tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 100) solved = self.state.solve_n(expr, 5) self.assertEqual(len(solved), 5) def test_solve_min_max(self): - expr = BitvecVariable(size=32, name="tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 4) self.state.constrain(expr < 7) self.assertEqual(self.state.solve_min(expr), 5) @@ -107,7 +107,7 @@ def test_solve_min_max(self): self.assertEqual(self.state.solve_minmax(expr), (5, 6)) def test_policy_one(self): - expr = BitvecVariable(size=32, name="tmp") + expr = BitVecVariable(size=32, name="tmp") self.state.constrain(expr > 0) self.state.constrain(expr < 100) solved = self.state.concretize(expr, "ONE") diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 9c4f759f8..f736b7f2a 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -13,7 +13,7 @@ arithmetic_simplify, constant_folder, replace, - BitvecConstant, + BitVecConstant, ) from manticore.core.smtlib.solver import Z3Solver, YicesSolver, CVC4Solver, SelectedSolver from manticore.core.smtlib.expression import * @@ -74,11 +74,11 @@ class X(object): self.assertEqual(sys.getsizeof(c), sys.getsizeof(x)) - def test_Bitvec_ops(self): - a = BitvecVariable(size=32, name="BV") - b = BitvecVariable(size=32, name="BV1") - c = BitvecVariable(size=32, name="BV2") - x = BitvecConstant(size=32, value=100, taint=("T",)) + def test_BitVec_ops(self): + a = BitVecVariable(size=32, name="BV") + b = BitVecVariable(size=32, name="BV1") + c = BitVecVariable(size=32, name="BV2") + x = BitVecConstant(size=32, value=100, taint=("T",)) z = (b + 1) % b < a * x / c - 5 self.assertSetEqual(z.taint, set(("T",))) self.assertEqual( @@ -92,16 +92,16 @@ def test_Bitvec_ops(self): "(bvsle (bvsdiv (bvadd #x00000001 BV1) BV1) (bvadd (bvsub BV (bvmul #x00000064 #x00000005)) BV2))", ) - def test_ConstantArrayBitvec(self): + def test_ConstantArrayBitVec(self): c = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") self.assertEqual(c[0], ord("A")) self.assertEqual(c[1], ord("B")) self.assertEqual(c[2], ord("C")) self.assertEqual(c[3], ord("D")) self.assertEqual(c[4], ord("E")) - self.assertRaises(IndexError, c.get, 5) + self.assertRaises(IndexError, c.__getitem__, 5) - def test_ConstantArrayBitvec2(self): + def test_ConstantArrayBitVec2(self): c = MutableArray(ArrayVariable(index_size=32, value_size=8, length=5, name="ARR")) c[1] = 10 c[2] = 20 @@ -192,9 +192,9 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): Variable, Operation, BoolOperation, - BitvecOperation, + BitVecOperation, ArrayOperation, - Bitvec, + BitVec, Bool, Array, ): @@ -203,7 +203,7 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): self.assertTrue(ty.__doc__, ty) checked.add(ty) - check(BitvecVariable, size=32, name="name", pickle_size=111, sizeof=56) + check(BitVecVariable, size=32, name="name", pickle_size=111, sizeof=56) check(BoolVariable, name="name", pickle_size=99, sizeof=48) check( ArrayVariable, @@ -214,7 +214,7 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): pickle_size=156, sizeof=80, ) - check(BitvecConstant, size=32, value=10, pickle_size=107, sizeof=56) + check(BitVecConstant, size=32, value=10, pickle_size=107, sizeof=56) check(BoolConstant, value=False, pickle_size=94, sizeof=48) # TODO! But you can instantiate an ArraConstant @@ -236,8 +236,8 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): check(BoolNot, operand=x, pickle_size=137, sizeof=48) check(BoolITE, cond=z, true=x, false=y, pickle_size=130, sizeof=48) - bvx = BitvecVariable(size=32, name="bvx") - bvy = BitvecVariable(size=32, name="bvy") + bvx = BitVecVariable(size=32, name="bvx") + bvy = BitVecVariable(size=32, name="bvy") check(BoolUnsignedGreaterThan, operanda=bvx, operandb=bvy, pickle_size=142, sizeof=48) check(BoolGreaterThan, operanda=bvx, operandb=bvy, pickle_size=134, sizeof=48) @@ -250,30 +250,30 @@ def check(ty, pickle_size=None, sizeof=None, **kwargs): check(BoolUnsignedLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=48) check(BoolLessOrEqualThan, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=48) - check(BitvecOr, operanda=bvx, operandb=bvy, pickle_size=129, sizeof=56) - check(BitvecXor, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecAnd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecNot, operanda=bvx, pickle_size=112, sizeof=56) - check(BitvecNeg, operanda=bvx, pickle_size=112, sizeof=56) - check(BitvecAdd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecMul, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecSub, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecDiv, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecMod, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecUnsignedDiv, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) - check(BitvecRem, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) - check(BitvecUnsignedRem, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) - - check(BitvecShiftLeft, operanda=bvx, operandb=bvy, pickle_size=136, sizeof=56) - check(BitvecShiftRight, operanda=bvx, operandb=bvy, pickle_size=137, sizeof=56) - check(BitvecArithmeticShiftLeft, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=56) - check(BitvecArithmeticShiftRight, operanda=bvx, operandb=bvy, pickle_size=147, sizeof=56) - - check(BitvecZeroExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) - check(BitvecSignExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) - check(BitvecExtract, operand=bvx, offset=0, size=8, pickle_size=119, sizeof=64) - check(BitvecConcat, operands=(bvx, bvy), pickle_size=133, sizeof=56) - check(BitvecITE, condition=x, true_value=bvx, false_value=bvy, pickle_size=161, sizeof=56) + check(BitVecOr, operanda=bvx, operandb=bvy, pickle_size=129, sizeof=56) + check(BitVecXor, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecAnd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecNot, operanda=bvx, pickle_size=112, sizeof=56) + check(BitVecNeg, operanda=bvx, pickle_size=112, sizeof=56) + check(BitVecAdd, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecMul, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecSub, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecDiv, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecMod, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecUnsignedDiv, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) + check(BitVecRem, operanda=bvx, operandb=bvy, pickle_size=130, sizeof=56) + check(BitVecUnsignedRem, operanda=bvx, operandb=bvy, pickle_size=138, sizeof=56) + + check(BitVecShiftLeft, operanda=bvx, operandb=bvy, pickle_size=136, sizeof=56) + check(BitVecShiftRight, operanda=bvx, operandb=bvy, pickle_size=137, sizeof=56) + check(BitVecArithmeticShiftLeft, operanda=bvx, operandb=bvy, pickle_size=146, sizeof=56) + check(BitVecArithmeticShiftRight, operanda=bvx, operandb=bvy, pickle_size=147, sizeof=56) + + check(BitVecZeroExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) + check(BitVecSignExtend, operand=bvx, size=122, pickle_size=119, sizeof=56) + check(BitVecExtract, operand=bvx, offset=0, size=8, pickle_size=119, sizeof=64) + check(BitVecConcat, operands=(bvx, bvy), pickle_size=133, sizeof=56) + check(BitVecITE, condition=x, true_value=bvx, false_value=bvy, pickle_size=161, sizeof=56) a = ArrayVariable(index_size=32, value_size=32, length=324, name="name") check(ArrayConstant, index_size=32, value_size=8, value=b"A", pickle_size=136, sizeof=64) @@ -291,11 +291,11 @@ def all_subclasses(cls): all_types = all_subclasses(Expression) self.assertSetEqual(checked, all_types) - def test_Expression_BitvecOp(self): - a = BitvecConstant(size=32, value=100) - b = BitvecConstant(size=32, value=101) + def test_Expression_BitVecOp(self): + a = BitVecConstant(size=32, value=100) + b = BitVecConstant(size=32, value=101) x = a + b - self.assertTrue(isinstance(x, Bitvec)) + self.assertTrue(isinstance(x, BitVec)) def test_Expression_BoolTaint(self): # Bool can not be instantiaated @@ -309,12 +309,12 @@ def test_Expression_BoolTaint(self): self.assertIn("red", z.taint) self.assertIn("blue", z.taint) - def test_Expression_BitvecTaint(self): + def test_Expression_BitVecTaint(self): # Bool can not be instantiaated - self.assertRaises(Exception, Bitvec, ()) + self.assertRaises(Exception, BitVec, ()) - x = BitvecConstant(size=32, value=123, taint=("red",)) - y = BitvecConstant(size=32, value=124, taint=("blue",)) + x = BitVecConstant(size=32, value=123, taint=("red",)) + y = BitVecConstant(size=32, value=124, taint=("blue",)) z = BoolGreaterOrEqualThan(x, y) self.assertIn("red", x.taint) self.assertIn("blue", y.taint) @@ -328,8 +328,8 @@ def test_Expression_Array(self): a = ArrayConstant(index_size=32, value_size=8, value=b"ABCDE") a[0] == ord("A") - x = BitvecConstant(size=32, value=123, taint=("red",)) - y = BitvecConstant(size=32, value=124, taint=("blue",)) + x = BitVecConstant(size=32, value=123, taint=("red",)) + y = BitVecConstant(size=32, value=124, taint=("blue",)) z = BoolGreaterOrEqualThan(x, y) self.assertIn("red", x.taint) self.assertIn("blue", y.taint) @@ -399,9 +399,9 @@ def tearDown(self): def test_no_variable_expression_can_be_true(self): """ Tests if solver.can_be_true is correct when the expression has no nodes that subclass - from Variable (e.g. BitvecConstant) + from Variable (e.g. BitVecConstant) """ - x = BitvecConstant(32, 10) + x = BitVecConstant(32, 10) cs = ConstraintSet() self.assertFalse(self.solver.can_be_true(cs, x == False)) @@ -409,12 +409,12 @@ def test_constant_bitvec(self): """ Tests if higher bits are masked out """ - x = BitvecConstant(32, 0xFF00000000) + x = BitVecConstant(32, 0xFF00000000) self.assertTrue(x.value == 0) def testBasicAST_001(self): """ Can't build abstract classes """ - a = BitvecConstant(32, 100) + a = BitVecConstant(32, 100) self.assertRaises(Exception, Expression, ()) self.assertRaises(Exception, Constant, 123) @@ -423,28 +423,28 @@ def testBasicAST_001(self): def testBasicOperation(self): """ Add """ - a = BitvecConstant(size=32, value=100) - b = BitvecVariable(size=32, name="VAR") + a = BitVecConstant(size=32, value=100) + b = BitVecVariable(size=32, name="VAR") c = a + b - self.assertIsInstance(c, BitvecAdd) + self.assertIsInstance(c, BitVecAdd) self.assertIsInstance(c, Operation) self.assertIsInstance(c, Expression) def testBasicTaint(self): - a = BitvecConstant(32, 100, taint=("SOURCE1",)) - b = BitvecConstant(32, 200, taint=("SOURCE2",)) + a = BitVecConstant(32, 100, taint=("SOURCE1",)) + b = BitVecConstant(32, 200, taint=("SOURCE2",)) c = a + b - self.assertIsInstance(c, BitvecAdd) + self.assertIsInstance(c, BitVecAdd) self.assertIsInstance(c, Operation) self.assertIsInstance(c, Expression) self.assertTrue("SOURCE1" in c.taint) self.assertTrue("SOURCE2" in c.taint) def testBasicITETaint(self): - a = BitvecConstant(32, 100, taint=("SOURCE1",)) - b = BitvecConstant(32, 200, taint=("SOURCE2",)) - c = BitvecConstant(32, 300, taint=("SOURCE3",)) - d = BitvecConstant(32, 400, taint=("SOURCE4",)) + a = BitVecConstant(32, 100, taint=("SOURCE1",)) + b = BitVecConstant(32, 200, taint=("SOURCE2",)) + c = BitVecConstant(32, 300, taint=("SOURCE3",)) + d = BitVecConstant(32, 400, taint=("SOURCE4",)) x = Operators.ITEBV(32, a > b, c, d) self.assertTrue("SOURCE1" in x.taint) self.assertTrue("SOURCE2" in x.taint) @@ -456,12 +456,12 @@ def test_cs_new_bitvec_invalid_size(self): with self.assertRaises(ValueError) as e: cs.new_bitvec(size=0) - self.assertEqual(str(e.exception), "Bitvec size (0) can't be equal to or less than 0") + self.assertEqual(str(e.exception), "BitVec size (0) can't be equal to or less than 0") with self.assertRaises(ValueError) as e: cs.new_bitvec(size=-23) - self.assertEqual(str(e.exception), "Bitvec size (-23) can't be equal to or less than 0") + self.assertEqual(str(e.exception), "BitVec size (-23) can't be equal to or less than 0") def testBasicConstraints(self): cs = ConstraintSet() @@ -542,6 +542,72 @@ def testBasicArray(self): temp_cs.add(key == 1002) self.assertTrue(self.solver.check(temp_cs)) + def testBasicArraySelectCache(self): + cs = ConstraintSet() + # make array of 32->8 bits + array = cs.new_array(32, value_size=256) + # make free 32bit bitvector + key = cs.new_bitvec(32) + + expr1 = array[key] + expr2 = array[key] + self.assertTrue(hash(expr1) == hash(expr2)) + self.assertTrue(expr1 is expr2) + self.assertTrue(simplify(expr1) is simplify(expr2)) + d = {} + d[expr1] = 1 + d[expr2] = 2 + self.assertEqual(d[expr2], 2) + + ''' + expr3 = expr1 + expr2 + expr4 = expr1 + expr2 + self.assertTrue(hash(expr3) == hash(expr4)) + + b1 = expr3 == expr4 + b2 = expr3 == expr4 + d = {} + d[expr3] = 3 + d[expr4] = 4 + + key1 = cs.new_bitvec(32) + key2 = cs.new_bitvec(32) + + expr1 = array[key1] + expr2 = array[key2] + self.assertTrue(expr1 is not expr2) + self.assertTrue(simplify(expr1) is not simplify(expr2)) + + + expr1 = array[1] + expr2 = array[1] + self.assertTrue(expr1 is expr2) + self.assertTrue(hash(expr1) == hash(expr2)) + self.assertTrue(simplify(expr1) is simplify(expr2)) + + expr1 = array[1] + expr2 = array[2] + self.assertTrue(expr1 is not expr2) + self.assertTrue(hash(expr1) != hash(expr2)) + self.assertTrue(simplify(expr1) is not simplify(expr2)) + + expr1 = cs.new_bitvec(size=256) + expr2 = cs.new_bitvec(size=32) + self.assertFalse(hash(expr1) == hash(expr2)) + + expr1 = cs.new_bitvec(size=256) + expr2 = copy.copy(expr1) + self.assertTrue(hash(expr1) == hash(expr2)) + + expr1 = BitVecConstant(size=32, value=10) + expr2 = BitVecConstant(size=32, value=10) + self.assertTrue(hash(expr1) == hash(expr2)) + + expr1 = BitVecConstant(size=32, value=10) + expr2 = BitVecConstant(size=256, value=10) + self.assertTrue(hash(expr1) != hash(expr2)) + ''' + def testBasicArray256(self): cs = ConstraintSet() # make array of 32->8 bits @@ -553,7 +619,6 @@ def testBasicArray256(self): cs.add(array[key] == 11111111111111111111111111111111111111111111) # let's restrict key to be greater than 1000 cs.add(key.ugt(1000)) - with cs as temp_cs: # 1001 position of array can be 111...111 temp_cs.add(array[1001] == 11111111111111111111111111111111111111111111) @@ -672,6 +737,8 @@ def testBasicArrayConcatSlice(self): self.assertEqual(len(array), len(hw)) self.assertTrue(self.solver.must_be_true(cs, array == hw)) self.assertEqual(len(array.read(0, 12)), 12) + x = array.read(0, 12) == hw + self.assertTrue(self.solver.must_be_true(cs, array.read(0, 12) == hw)) cs.add(array.read(6, 6) == hw[6:12]) self.assertTrue(self.solver.must_be_true(cs, array.read(6, 6) == hw[6:12])) @@ -733,6 +800,7 @@ def testBasicArrayProxySymbIdx(self): # It should not be another solution for index self.assertFalse(self.solver.check(cs)) + def testBasicArrayProxySymbIdx2(self): cs = ConstraintSet() array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=100)) @@ -754,6 +822,7 @@ def testBasicArrayProxySymbIdx2(self): ) # get a concrete solution for index 1 (default 100) self.assertItemsEqual(solutions, (100, 2)) + def testBasicConstatArray(self): cs = ConstraintSet() array1 = MutableArray( @@ -785,7 +854,7 @@ def testBasicPickle(self): cs = pickle.loads(pickle_dumps(cs)) self.assertTrue(self.solver.check(cs)) - def testBitvector_add(self): + def testBitVector_add(self): cs = ConstraintSet() a = cs.new_bitvec(32) b = cs.new_bitvec(32) @@ -796,7 +865,7 @@ def testBitvector_add(self): self.assertTrue(self.solver.check(cs)) self.assertEqual(self.solver.get_value(cs, c), 11) - def testBitvector_add1(self): + def testBitVector_add1(self): cs = ConstraintSet() a = cs.new_bitvec(32) b = cs.new_bitvec(32) @@ -806,7 +875,7 @@ def testBitvector_add1(self): self.assertEqual(self.solver.check(cs), True) self.assertEqual(self.solver.get_value(cs, c), 11) - def testBitvector_add2(self): + def testBitVector_add2(self): cs = ConstraintSet() a = cs.new_bitvec(32) b = cs.new_bitvec(32) @@ -815,7 +884,7 @@ def testBitvector_add2(self): self.assertTrue(self.solver.check(cs)) self.assertEqual(self.solver.get_value(cs, a), 1) - def testBitvector_max(self): + def testBitVector_max(self): cs = ConstraintSet() a = cs.new_bitvec(32) cs.add(a <= 200) @@ -834,15 +903,15 @@ def testBitvector_max(self): self.assertEqual(self.solver.minmax(cs, a), (100, 200)) consts.optimize = True - def testBitvector_max_noop(self): + def testBitVector_max_noop(self): from manticore import config consts = config.get_group("smt") consts.optimize = False - self.testBitvector_max() + self.testBitVector_max() consts.optimize = True - def testBitvector_max1(self): + def testBitVector_max1(self): cs = ConstraintSet() a = cs.new_bitvec(32) cs.add(a < 200) @@ -850,12 +919,12 @@ def testBitvector_max1(self): self.assertTrue(self.solver.check(cs)) self.assertEqual(self.solver.minmax(cs, a), (101, 199)) - def testBitvector_max1_noop(self): + def testBitVector_max1_noop(self): from manticore import config consts = config.get_group("smt") consts.optimize = False - self.testBitvector_max1() + self.testBitVector_max1() consts.optimize = True def testBool_nonzero(self): @@ -883,12 +952,17 @@ def test_visitors(self): aux = arr[a + Operators.ZEXTEND(arr[a], 32)] self.assertEqual(get_depth(aux), 9) - self.maxDiff = 1500 + self.maxDiff = 5500 + print(translate_to_smtlib(aux)) self.assertEqual( - translate_to_smtlib(aux), + translate_to_smtlib(aux, use_bindings=False), "(select (store (store (store MEM #x00000000 #x61) #x00000001 #x62) #x00000003 (select (store (store MEM #x00000000 #x61) #x00000001 #x62) (bvadd VAR #x00000001))) (bvadd VAR ((_ zero_extend 24) (select (store (store (store MEM #x00000000 #x61) #x00000001 #x62) #x00000003 (select (store (store MEM #x00000000 #x61) #x00000001 #x62) (bvadd VAR #x00000001))) VAR))))", ) + self.assertEqual( + translate_to_smtlib(aux, use_bindings=True), + "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62)) (!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001)))) ) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR)))))") + values = arr[0:2] self.assertEqual(len(values), 2) self.assertItemsEqual(solver.get_all_values(cs, values[0]), [ord("a")]) @@ -901,15 +975,15 @@ def test_visitors(self): self.assertItemsEqual(solver.get_all_values(cs, values[1]), [ord("c")]) self.assertItemsEqual(solver.get_all_values(cs, values[2]), [ord("d")]) self.assertEqual( - pretty_print(aux, depth=2), "ArraySelect\n ArrayStore\n ...\n BitvecAdd\n ...\n" + pretty_print(aux, depth=2), "ArraySelect\n ArrayStore\n ...\n BitVecAdd\n ...\n" ) self.assertEqual( - pretty_print(Operators.EXTRACT(a, 0, 8), depth=1), "BitvecExtract{0:7}\n ...\n" + pretty_print(Operators.EXTRACT(a, 0, 8), depth=1), "BitVecExtract{0:7}\n ...\n" ) self.assertEqual(pretty_print(a, depth=2), "VAR\n") - x = BitvecConstant(32, 100, taint=("important",)) - y = BitvecConstant(32, 200, taint=("stuff",)) + x = BitVecConstant(32, 100, taint=("important",)) + y = BitVecConstant(32, 200, taint=("stuff",)) z = constant_folder(x + y) self.assertItemsEqual(z.taint, ("important", "stuff")) self.assertEqual(z.value, 300) @@ -979,18 +1053,18 @@ def test_arithmetic_simplify_extract(self): def test_arithmetic_simplify_udiv(self): cs = ConstraintSet() a = cs.new_bitvec(32, name="VARA") - b = a + Operators.UDIV(BitvecConstant(32, 0), BitvecConstant(32, 2)) + b = a + Operators.UDIV(BitVecConstant(32, 0), BitVecConstant(32, 2)) self.assertEqual(translate_to_smtlib(b), "(bvadd VARA (bvudiv #x00000000 #x00000002))") self.assertEqual(translate_to_smtlib(simplify(b)), "VARA") - c = a + Operators.UDIV(BitvecConstant(32, 2), BitvecConstant(32, 2)) + c = a + Operators.UDIV(BitVecConstant(32, 2), BitVecConstant(32, 2)) self.assertEqual(translate_to_smtlib(c), "(bvadd VARA (bvudiv #x00000002 #x00000002))") self.assertEqual(translate_to_smtlib(simplify(c)), "(bvadd VARA #x00000001)") def test_constant_folding_udiv(self): cs = ConstraintSet() - x = BitvecConstant(32, 0xFFFFFFFF, taint=("important",)) - y = BitvecConstant(32, 2, taint=("stuff",)) + x = BitVecConstant(32, 0xFFFFFFFF, taint=("important",)) + y = BitVecConstant(32, 2, taint=("stuff",)) z = constant_folder(x.udiv(y)) self.assertItemsEqual(z.taint, ("important", "stuff")) self.assertEqual(z.value, 0x7FFFFFFF) @@ -1006,9 +1080,9 @@ def test_simplify_OR(self): def testBasicReplace(self): """ Add """ - a = BitvecConstant(size=32, value=100) - b1 = BitvecVariable(size=32, name="VAR1") - b2 = BitvecVariable(size=32, name="VAR2") + a = BitVecConstant(size=32, value=100) + b1 = BitVecVariable(size=32, name="VAR1") + b2 = BitVecVariable(size=32, name="VAR2") c = a + b1 @@ -1280,16 +1354,33 @@ def test_UDIV(self): b = cs.new_bitvec(8) c = cs.new_bitvec(8) d = cs.new_bitvec(8) - cs.add(b == 0x86) # 134 cs.add(c == 0x11) # 17 cs.add(a == Operators.UDIV(b, c)) cs.add(d == b.udiv(c)) cs.add(a == d) - self.assertTrue(solver.check(cs)) self.assertEqual(solver.get_value(cs, a), 7) + + solver = Z3Solver.instance() + cs1 = ConstraintSet() + + a1 = cs1.new_bitvec(8) + b1 = cs1.new_bitvec(8) + c1 = cs1.new_bitvec(8) + d1 = cs1.new_bitvec(8) + self.assertTrue(solver.check(cs1)) + + cs1.add(b1 == 0x86) # -122 + cs1.add(c1 == 0x11) # 17 + cs1.add(a1 == Operators.SREM(b1, c1)) + cs1.add(d1 == b1.srem(c1)) + cs1.add(a1 == d1) + + self.assertTrue(solver.check(cs1)) + self.assertEqual(solver.get_value(cs1, a1), -3 & 0xFF) + def test_SDIV(self): solver = Z3Solver.instance() cs = ConstraintSet() From ed116261630266ad222cfa3a9eb3e1aa49556119 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Mon, 11 Jan 2021 14:56:43 -0500 Subject: [PATCH 105/126] Format with black 19.10b0 --- manticore/__main__.py | 4 +- manticore/core/plugin.py | 4 +- manticore/core/smtlib/expression.py | 22 +--- manticore/core/state.py | 6 +- manticore/core/worker.py | 5 +- manticore/ethereum/parsetab.py | 129 +++++++++++++++++------- manticore/ethereum/verifier.py | 10 +- manticore/native/cpu/abstractcpu.py | 6 +- manticore/platforms/evm.py | 13 +-- tests/ethereum/test_detectors.py | 2 +- tests/native/test_cpu_automatic.py | 2 +- tests/native/test_integration_native.py | 26 ++--- tests/other/test_smtlibv2.py | 14 ++- 13 files changed, 131 insertions(+), 112 deletions(-) diff --git a/manticore/__main__.py b/manticore/__main__.py index 9eef5c569..28be259d3 100644 --- a/manticore/__main__.py +++ b/manticore/__main__.py @@ -211,9 +211,7 @@ def positive(value): ) eth_flags.add_argument( - "--limit-loops", - action="store_true", - help="Limit loops depth", + "--limit-loops", action="store_true", help="Limit loops depth", ) eth_flags.add_argument( diff --git a/manticore/core/plugin.py b/manticore/core/plugin.py index 9d2dac38d..a409f25c8 100644 --- a/manticore/core/plugin.py +++ b/manticore/core/plugin.py @@ -638,9 +638,7 @@ def on_execution_intermittent_callback( state.id, ) update_cb( - context.setdefault(state.id, StateDescriptor(state_id=state.id)), - *args, - **kwargs, + context.setdefault(state.id, StateDescriptor(state_id=state.id)), *args, **kwargs, ) context[state.id].last_intermittent_update = datetime.now() diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 09b05b0be..becc96f11 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -944,11 +944,7 @@ class ArrayConstant(Array, Constant): __xslots__: Tuple[str, ...] = ("_index_size", "_value_size") def __init__( - self, - *, - index_size: int, - value_size: int, - **kwargs, + self, *, index_size: int, value_size: int, **kwargs, ): self._index_size = index_size self._value_size = value_size @@ -1186,8 +1182,7 @@ def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): # self._concrete_cache[index.value] = value super().__init__( - operands=(array, index, value), - **kwargs, + operands=(array, index, value), **kwargs, ) @property @@ -1270,8 +1265,7 @@ def __init__(self, array: "Array", offset: int, size: int, **kwargs): raise ValueError("Array expected") super().__init__( - operands=(array, array.cast_index(offset), array.cast_index(size)), - **kwargs, + operands=(array, array.cast_index(offset), array.cast_index(size)), **kwargs, ) @property @@ -1308,9 +1302,7 @@ def select(self, index): def store(self, index, value): return ArraySlice( - self.array.store(index + self.offset, value), - offset=self.offset, - size=len(self), + self.array.store(index + self.offset, value), offset=self.offset, size=len(self), ) @property @@ -1515,11 +1507,7 @@ class BitVecITE(BitVecOperation): __xslots__ = BitVecOperation.__xslots__ def __init__( - self, - condition: Bool, - true_value: BitVec, - false_value: BitVec, - **kwargs, + self, condition: Bool, true_value: BitVec, false_value: BitVec, **kwargs, ): super().__init__( diff --git a/manticore/core/state.py b/manticore/core/state.py index c15b3a90e..5e62eb566 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -323,11 +323,7 @@ def new_symbolic_buffer(self, nbytes, **options): avoid_collisions = True taint = options.get("taint", frozenset()) expr = self._constraints.new_array( - name=label, - length=nbytes, - value_size=8, - taint=taint, - avoid_collisions=avoid_collisions, + name=label, length=nbytes, value_size=8, taint=taint, avoid_collisions=avoid_collisions, ) self._input_symbols.append(expr) diff --git a/manticore/core/worker.py b/manticore/core/worker.py index 1e9d74012..b518d89a8 100644 --- a/manticore/core/worker.py +++ b/manticore/core/worker.py @@ -246,10 +246,7 @@ def start(self, target: typing.Optional[typing.Callable] = None): thread as an argument. """ logger.debug( - "Starting Daemon %d. (Pid %d Tid %d).", - self.id, - os.getpid(), - threading.get_ident(), + "Starting Daemon %d. (Pid %d Tid %d).", self.id, os.getpid(), threading.get_ident(), ) self._t = threading.Thread(target=self.run if target is None else target, args=(self,)) diff --git a/manticore/ethereum/parsetab.py b/manticore/ethereum/parsetab.py index 3ef2495e9..7570d6d8f 100644 --- a/manticore/ethereum/parsetab.py +++ b/manticore/ethereum/parsetab.py @@ -1,50 +1,111 @@ - # parsetab.py # This file is automatically generated. Do not edit. # pylint: disable=W,C,R -_tabversion = '3.10' +_tabversion = "3.10" + +_lr_method = "LALR" -_lr_method = 'LALR' +_lr_signature = "ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n " -_lr_signature = 'ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n ' - -_lr_action_items = {'UINTN':([0,16,24,],[2,2,2,]),'UINT':([0,16,24,],[3,3,3,]),'INTN':([0,16,24,],[4,4,4,]),'INT':([0,16,24,],[5,5,5,]),'ADDRESS':([0,16,24,],[6,6,6,]),'BOOL':([0,16,24,],[7,7,7,]),'FIXEDMN':([0,16,24,],[8,8,8,]),'UFIXEDMN':([0,16,24,],[9,9,9,]),'FIXED':([0,16,24,],[10,10,10,]),'UFIXED':([0,16,24,],[11,11,11,]),'BYTESM':([0,16,24,],[12,12,12,]),'FUNCTION':([0,16,24,],[13,13,13,]),'BYTES':([0,16,24,],[14,14,14,]),'STRING':([0,16,24,],[15,15,15,]),'LPAREN':([0,16,24,],[16,16,16,]),'$end':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,21,23,25,],[0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,-19,-17,-20,]),'LBRAKET':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[17,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,17,-19,-17,-20,]),'COMMA':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,24,-19,-17,-20,]),'RPAREN':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,23,25,26,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,19,23,-18,-15,-19,-17,-20,-16,]),'RBRAKET':([17,22,],[21,25,]),'NUMBER':([17,],[22,]),} +_lr_action_items = { + "UINTN": ([0, 16, 24,], [2, 2, 2,]), + "UINT": ([0, 16, 24,], [3, 3, 3,]), + "INTN": ([0, 16, 24,], [4, 4, 4,]), + "INT": ([0, 16, 24,], [5, 5, 5,]), + "ADDRESS": ([0, 16, 24,], [6, 6, 6,]), + "BOOL": ([0, 16, 24,], [7, 7, 7,]), + "FIXEDMN": ([0, 16, 24,], [8, 8, 8,]), + "UFIXEDMN": ([0, 16, 24,], [9, 9, 9,]), + "FIXED": ([0, 16, 24,], [10, 10, 10,]), + "UFIXED": ([0, 16, 24,], [11, 11, 11,]), + "BYTESM": ([0, 16, 24,], [12, 12, 12,]), + "FUNCTION": ([0, 16, 24,], [13, 13, 13,]), + "BYTES": ([0, 16, 24,], [14, 14, 14,]), + "STRING": ([0, 16, 24,], [15, 15, 15,]), + "LPAREN": ([0, 16, 24,], [16, 16, 16,]), + "$end": ( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 21, 23, 25,], + [0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, -19, -17, -20,], + ), + "LBRAKET": ( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 25,], + [17, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, 17, -19, -17, -20,], + ), + "COMMA": ( + [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 25,], + [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, 24, -19, -17, -20,], + ), + "RPAREN": ( + [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 23, 25, 26,], + [ + -1, + -2, + -3, + -4, + -5, + -6, + -7, + -8, + -9, + -10, + -11, + -12, + -13, + -14, + 19, + 23, + -18, + -15, + -19, + -17, + -20, + -16, + ], + ), + "RBRAKET": ([17, 22,], [21, 25,]), + "NUMBER": ([17,], [22,]), +} _lr_action = {} for _k, _v in _lr_action_items.items(): - for _x,_y in zip(_v[0],_v[1]): - if not _x in _lr_action: _lr_action[_x] = {} - _lr_action[_x][_k] = _y + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_action: + _lr_action[_x] = {} + _lr_action[_x][_k] = _y del _lr_action_items -_lr_goto_items = {'T':([0,16,24,],[1,20,20,]),'TL':([16,24,],[18,26,]),} +_lr_goto_items = { + "T": ([0, 16, 24,], [1, 20, 20,]), + "TL": ([16, 24,], [18, 26,]), +} _lr_goto = {} for _k, _v in _lr_goto_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_goto: _lr_goto[_x] = {} - _lr_goto[_x][_k] = _y + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_goto: + _lr_goto[_x] = {} + _lr_goto[_x][_k] = _y del _lr_goto_items _lr_productions = [ - ("S' -> T","S'",1,None,None,None), - ('T -> UINTN','T',1,'p_basic_type','abitypes.py',154), - ('T -> UINT','T',1,'p_basic_type','abitypes.py',155), - ('T -> INTN','T',1,'p_basic_type','abitypes.py',156), - ('T -> INT','T',1,'p_basic_type','abitypes.py',157), - ('T -> ADDRESS','T',1,'p_basic_type','abitypes.py',158), - ('T -> BOOL','T',1,'p_basic_type','abitypes.py',159), - ('T -> FIXEDMN','T',1,'p_basic_type','abitypes.py',160), - ('T -> UFIXEDMN','T',1,'p_basic_type','abitypes.py',161), - ('T -> FIXED','T',1,'p_basic_type','abitypes.py',162), - ('T -> UFIXED','T',1,'p_basic_type','abitypes.py',163), - ('T -> BYTESM','T',1,'p_basic_type','abitypes.py',164), - ('T -> FUNCTION','T',1,'p_basic_type','abitypes.py',165), - ('T -> BYTES','T',1,'p_basic_type','abitypes.py',166), - ('T -> STRING','T',1,'p_basic_type','abitypes.py',167), - ('TL -> T','TL',1,'p_type_list_one','abitypes.py',175), - ('TL -> T COMMA TL','TL',3,'p_type_list','abitypes.py',182), - ('T -> LPAREN TL RPAREN','T',3,'p_tuple','abitypes.py',189), - ('T -> LPAREN RPAREN','T',2,'p_tuple_empty','abitypes.py',196), - ('T -> T LBRAKET RBRAKET','T',3,'p_dynamic_type','abitypes.py',203), - ('T -> T LBRAKET NUMBER RBRAKET','T',4,'p_dynamic_fixed_type','abitypes.py',212), + ("S' -> T", "S'", 1, None, None, None), + ("T -> UINTN", "T", 1, "p_basic_type", "abitypes.py", 154), + ("T -> UINT", "T", 1, "p_basic_type", "abitypes.py", 155), + ("T -> INTN", "T", 1, "p_basic_type", "abitypes.py", 156), + ("T -> INT", "T", 1, "p_basic_type", "abitypes.py", 157), + ("T -> ADDRESS", "T", 1, "p_basic_type", "abitypes.py", 158), + ("T -> BOOL", "T", 1, "p_basic_type", "abitypes.py", 159), + ("T -> FIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 160), + ("T -> UFIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 161), + ("T -> FIXED", "T", 1, "p_basic_type", "abitypes.py", 162), + ("T -> UFIXED", "T", 1, "p_basic_type", "abitypes.py", 163), + ("T -> BYTESM", "T", 1, "p_basic_type", "abitypes.py", 164), + ("T -> FUNCTION", "T", 1, "p_basic_type", "abitypes.py", 165), + ("T -> BYTES", "T", 1, "p_basic_type", "abitypes.py", 166), + ("T -> STRING", "T", 1, "p_basic_type", "abitypes.py", 167), + ("TL -> T", "TL", 1, "p_type_list_one", "abitypes.py", 175), + ("TL -> T COMMA TL", "TL", 3, "p_type_list", "abitypes.py", 182), + ("T -> LPAREN TL RPAREN", "T", 3, "p_tuple", "abitypes.py", 189), + ("T -> LPAREN RPAREN", "T", 2, "p_tuple_empty", "abitypes.py", 196), + ("T -> T LBRAKET RBRAKET", "T", 3, "p_dynamic_type", "abitypes.py", 203), + ("T -> T LBRAKET NUMBER RBRAKET", "T", 4, "p_dynamic_fixed_type", "abitypes.py", 212), ] diff --git a/manticore/ethereum/verifier.py b/manticore/ethereum/verifier.py index c4ce7a075..47efac4a9 100644 --- a/manticore/ethereum/verifier.py +++ b/manticore/ethereum/verifier.py @@ -366,11 +366,7 @@ def main(): cryticparser.init(parser) parser.add_argument( - "source_code", - type=str, - nargs="*", - default=[], - help="Contract source code", + "source_code", type=str, nargs="*", default=[], help="Contract source code", ) parser.add_argument( "-v", action="count", default=0, help="Specify verbosity level from -v to -vvvv" @@ -391,9 +387,7 @@ def main(): help="Show program version information", ) parser.add_argument( - "--propconfig", - type=str, - help="Solidity property accounts config file (.yml)", + "--propconfig", type=str, help="Solidity property accounts config file (.yml)", ) eth_flags = parser.add_argument_group("Ethereum flags") diff --git a/manticore/native/cpu/abstractcpu.py b/manticore/native/cpu/abstractcpu.py index 6e847937c..f1b86b572 100644 --- a/manticore/native/cpu/abstractcpu.py +++ b/manticore/native/cpu/abstractcpu.py @@ -90,11 +90,7 @@ class ConcretizeRegister(CpuException): """ def __init__( - self, - cpu: "Cpu", - reg_name: str, - message: Optional[str] = None, - policy: str = "MINMAX", + self, cpu: "Cpu", reg_name: str, message: Optional[str] = None, policy: str = "MINMAX", ): self.message = message if message else f"Concretizing {reg_name}" diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index cf9a6e25a..4b5ac91db 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -103,9 +103,7 @@ def globalfakesha3(data): description="Max calldata size to explore in each CALLDATACOPY. Iff size in a calldata related instruction are symbolic it will be constrained to be less than this constant. -1 means free(only use when gas is being tracked)", ) consts.add( - "ignore_balance", - default=False, - description="Do not try to solve symbolic balances", + "ignore_balance", default=False, description="Do not try to solve symbolic balances", ) @@ -1338,7 +1336,7 @@ def setstate(state, value): raise Concretize("Symbolic PC", expression=expression, setstate=setstate, policy="ALL") try: if getattr(getattr(self, self.instruction.semantics, None), "_pos", None) is None: - print(self) + print(self) self._check_jmpdest() last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() result = self._handler(*arguments) @@ -2198,12 +2196,7 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): ) self.world.start_transaction( - "CREATE", - address, - data=data, - caller=self.address, - value=value, - gas=self.gas, + "CREATE", address, data=data, caller=self.address, value=value, gas=self.gas, ) raise StartTx() diff --git a/tests/ethereum/test_detectors.py b/tests/ethereum/test_detectors.py index af5d368ff..9ca957c8e 100644 --- a/tests/ethereum/test_detectors.py +++ b/tests/ethereum/test_detectors.py @@ -63,7 +63,7 @@ def _test(self, name: str, should_find, use_ctor_sym_arg=False): dir = os.path.join(THIS_DIR, "contracts", "detectors") filepath = os.path.join(dir, f"{name}.sol") - print (filepath) + print(filepath) if use_ctor_sym_arg: ctor_arg: Tuple = (mevm.make_symbolic_value(),) else: diff --git a/tests/native/test_cpu_automatic.py b/tests/native/test_cpu_automatic.py index 0d2290f49..e1754bdf9 100644 --- a/tests/native/test_cpu_automatic.py +++ b/tests/native/test_cpu_automatic.py @@ -12674,7 +12674,7 @@ def test_ADD_1_symbolic(self): with cs as temp_cs: temp_cs.add(condition) - print (temp_cs) + print(temp_cs) self.assertTrue(solver.check(temp_cs)) with cs as temp_cs: temp_cs.add(condition == False) diff --git a/tests/native/test_integration_native.py b/tests/native/test_integration_native.py index 317891693..cd34ee9a4 100644 --- a/tests/native/test_integration_native.py +++ b/tests/native/test_integration_native.py @@ -60,19 +60,19 @@ def test_timeout(self) -> None: t = time.time() with open(os.path.join(os.pardir, self.test_dir, "output.log"), "w") as output: cmd = [ - "coverage", - "run", # PYTHON_BIN, - "-m", - "manticore", - "--workspace", - workspace, - "--core.timeout", - "1", - "--core.procs", - "4", - filename, - "+++++++++", - ] + "coverage", + "run", # PYTHON_BIN, + "-m", + "manticore", + "--workspace", + workspace, + "--core.timeout", + "1", + "--core.procs", + "4", + filename, + "+++++++++", + ] subprocess.check_call(cmd, stdout=output) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index f736b7f2a..2bc527049 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -559,7 +559,7 @@ def testBasicArraySelectCache(self): d[expr2] = 2 self.assertEqual(d[expr2], 2) - ''' + """ expr3 = expr1 + expr2 expr4 = expr1 + expr2 self.assertTrue(hash(expr3) == hash(expr4)) @@ -606,7 +606,7 @@ def testBasicArraySelectCache(self): expr1 = BitVecConstant(size=32, value=10) expr2 = BitVecConstant(size=256, value=10) self.assertTrue(hash(expr1) != hash(expr2)) - ''' + """ def testBasicArray256(self): cs = ConstraintSet() @@ -738,7 +738,7 @@ def testBasicArrayConcatSlice(self): self.assertTrue(self.solver.must_be_true(cs, array == hw)) self.assertEqual(len(array.read(0, 12)), 12) x = array.read(0, 12) == hw - + self.assertTrue(self.solver.must_be_true(cs, array.read(0, 12) == hw)) cs.add(array.read(6, 6) == hw[6:12]) self.assertTrue(self.solver.must_be_true(cs, array.read(6, 6) == hw[6:12])) @@ -800,7 +800,6 @@ def testBasicArrayProxySymbIdx(self): # It should not be another solution for index self.assertFalse(self.solver.check(cs)) - def testBasicArrayProxySymbIdx2(self): cs = ConstraintSet() array = MutableArray(cs.new_array(index_size=32, value_size=32, name="array", default=100)) @@ -822,7 +821,6 @@ def testBasicArrayProxySymbIdx2(self): ) # get a concrete solution for index 1 (default 100) self.assertItemsEqual(solutions, (100, 2)) - def testBasicConstatArray(self): cs = ConstraintSet() array1 = MutableArray( @@ -960,8 +958,9 @@ def test_visitors(self): ) self.assertEqual( - translate_to_smtlib(aux, use_bindings=True), - "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62)) (!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001)))) ) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR)))))") + translate_to_smtlib(aux, use_bindings=True), + "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62)) (!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001)))) ) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR)))))", + ) values = arr[0:2] self.assertEqual(len(values), 2) @@ -1362,7 +1361,6 @@ def test_UDIV(self): self.assertTrue(solver.check(cs)) self.assertEqual(solver.get_value(cs, a), 7) - solver = Z3Solver.instance() cs1 = ConstraintSet() From 5ffe40a8d814c523b7a2b9b48b1ca4fac0c0a59d Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 10:27:56 -0500 Subject: [PATCH 106/126] Add toposort dependency --- mypy.ini | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/mypy.ini b/mypy.ini index 7f7279360..5d9ea804d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -34,6 +34,9 @@ ignore_missing_imports = True [mypy-rlp.*] ignore_missing_imports = True +[mypy-toposort.*] +ignore_missing_imports = True + [mypy-prettytable.*] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 0bc38d427..2e083b58a 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ def rtd_dependent_deps(): "wasm", "dataclasses; python_version < '3.7'", "pyevmasm>=0.2.3", + "toposort", ] + rtd_dependent_deps(), extras_require=extra_require, From 6a9f392fb49cb9ae1e2f4d5eff9631454a1aaab2 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 10:28:24 -0500 Subject: [PATCH 107/126] Fix x86 NEG instruction Python implementation --- manticore/native/cpu/x86.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/native/cpu/x86.py b/manticore/native/cpu/x86.py index c43b95f7b..5076bc15a 100644 --- a/manticore/native/cpu/x86.py +++ b/manticore/native/cpu/x86.py @@ -2079,7 +2079,7 @@ def NEG(cpu, dest): :param dest: destination operand. """ source = dest.read() - res = dest.write(-source) + res = dest.write((~source) + 1) cpu._calculate_logic_flags(dest.size, res) cpu.CF = source != 0 cpu.AF = (res & 0x0F) != 0x00 From 1cb97beda07a89edef21e9213eba461ddbcb4e0f Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 10:28:53 -0500 Subject: [PATCH 108/126] Fix BitVecUnsignedDiv division by zero --- manticore/core/smtlib/visitors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 9328e3a54..cb3c55562 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -300,7 +300,7 @@ def visit_BitVecUnsignedDiv(self, expression, *operands) -> Optional[BitVecConst if all(isinstance(o, Constant) for o in operands): a = operands[0].value b = operands[1].value - if a == 0: + if b == 0: ret = 0 else: ret = math.trunc(Decimal(a) / Decimal(b)) From c220799072322e887532fba71601d3be5827bc07 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 10:52:26 -0500 Subject: [PATCH 109/126] Update setup-python GHA --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4228cbc87..eae7b6aa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python 3.6 - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: '3.6' - name: Lint if: github.event_name == 'pull_request' env: @@ -46,9 +46,9 @@ jobs: steps: - uses: actions/checkout@v1 - name: Set up Python 3.6 - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: '3.6' - name: Install NPM uses: actions/setup-node@v1 with: @@ -98,9 +98,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python 3.6 - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: '3.6' - name: Build Dist run: | python3 -m pip install wheel From 5eea790e06aea520631d712a78cb996dbbe0b85b Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 11:40:52 -0500 Subject: [PATCH 110/126] Update mypy to 0.790 and add examples directory to checking Fixes some issues in the examples directory that can be found by running mypy as follows: mypy --check-untyped-defs examples/**/*.py --- examples/evm/asm_to_smtlib.py | 12 +++++---- examples/evm/coverage.py | 3 +-- examples/evm/mappingchallenge.py | 28 +++++++++++---------- examples/evm/minimal-json.py | 4 ++- examples/evm/use_def.py | 4 +++ examples/linux/binaries/concrete_solve.py | 6 ++--- examples/linux/binaries/symbolic_solve.py | 2 +- examples/script/concolic.py | 15 +++++------ examples/script/introduce_symbolic_bytes.py | 2 +- mypy.ini | 4 ++- setup.py | 2 +- 11 files changed, 46 insertions(+), 36 deletions(-) diff --git a/examples/evm/asm_to_smtlib.py b/examples/evm/asm_to_smtlib.py index c74913dce..c49df8cd9 100644 --- a/examples/evm/asm_to_smtlib.py +++ b/examples/evm/asm_to_smtlib.py @@ -7,7 +7,9 @@ from manticore.utils import log # log.set_verbosity(9) -config.out_of_gas = 1 +# FIXME: What's the equivalent? +consts = config.get_group("evm") +consts["oog"] = "complete" def printi(instruction): @@ -45,8 +47,8 @@ def printi(instruction): data = constraints.new_array(index_size=256, name="array") -class callbacks: - initial_stack = [] +class Callbacks: + initial_stack: List[BitVec] = [] def will_execute_instruction(self, pc, instr): for i in range(len(evm.stack), instr.pops): @@ -112,7 +114,7 @@ def send_funds(self, address, recipient, value): value = constraints.new_bitvec(256, name="value") world = DummyWorld(constraints) -callbacks = callbacks() +callbacks = Callbacks() # evm = world.current_vm evm = EVM(constraints, 0x41424344454647484950, data, caller, value, code, world=world, gas=1000000) @@ -138,5 +140,5 @@ def send_funds(self, address, recipient, value): print(constraints) print( - f"PC: {translate_to_smtlib(evm.pc)} {solver.get_all_values(constraints, evm.pc, maxcnt=3, silent=True)}" + f"PC: {translate_to_smtlib(evm.pc)} {SMTLIBSolver.get_all_values(constraints, evm.pc, maxcnt=3, silent=True)}" ) diff --git a/examples/evm/coverage.py b/examples/evm/coverage.py index 9704dcbd7..e2240471f 100644 --- a/examples/evm/coverage.py +++ b/examples/evm/coverage.py @@ -8,9 +8,8 @@ user_account = m.create_account(balance=1000) -bytecode = m.compile(source_code) # Initialize contract -contract_account = m.create_contract(owner=user_account, balance=0, init=bytecode) +contract_account = m.solidity_create_contract(source_code, owner=user_account, balance=0) m.transaction( caller=user_account, diff --git a/examples/evm/mappingchallenge.py b/examples/evm/mappingchallenge.py index 3de1fc785..b454b5192 100644 --- a/examples/evm/mappingchallenge.py +++ b/examples/evm/mappingchallenge.py @@ -30,22 +30,24 @@ class StopAtDepth(Detector): """ This just aborts explorations that are too deep """ def will_run_callback(self, *args): - with self.manticore.locked_context("seen_rep", dict) as reps: - reps.clear() + if self.manticore: + with self.manticore.locked_context("seen_rep", dict) as reps: + reps.clear() def will_decode_instruction_callback(self, state, pc): world = state.platform - with self.manticore.locked_context("seen_rep", dict) as reps: - item = ( - world.current_transaction.sort == "CREATE", - world.current_transaction.address, - pc, - ) - if not item in reps: - reps[item] = 0 - reps[item] += 1 - if reps[item] > 2: - state.abandon() + if self.manticore: + with self.manticore.locked_context("seen_rep", dict) as reps: + item = ( + world.current_transaction.sort == "CREATE", + world.current_transaction.address, + pc, + ) + if not item in reps: + reps[item] = 0 + reps[item] += 1 + if reps[item] > 2: + state.abandon() m.register_plugin(StopAtDepth()) diff --git a/examples/evm/minimal-json.py b/examples/evm/minimal-json.py index 2792ce8f7..639605cd6 100644 --- a/examples/evm/minimal-json.py +++ b/examples/evm/minimal-json.py @@ -1047,7 +1047,9 @@ user_account = m.create_account(balance=1000, name="user_account") print("[+] Creating a user account", user_account.name_) -contract_account = m.json_create_contract(truffle_json, owner=user_account, name="contract_account") +contract_account = m.solidity_create_contract( + truffle_json, owner=user_account, name="contract_account" +) print("[+] Creating a contract account", contract_account.name_) contract_account.sendCoin(1, 1) diff --git a/examples/evm/use_def.py b/examples/evm/use_def.py index 7fafb8938..8ecb23c58 100644 --- a/examples/evm/use_def.py +++ b/examples/evm/use_def.py @@ -41,6 +41,8 @@ class EVMUseDef(Plugin): def did_evm_write_storage_callback(self, state, address, offset, value): m = self.manticore + if not m: + return world = state.platform tx = world.all_transactions[-1] md = m.get_metadata(tx.address) @@ -55,6 +57,8 @@ def did_evm_write_storage_callback(self, state, address, offset, value): def did_evm_read_storage_callback(self, state, address, offset, value): m = self.manticore + if not m: + return world = state.platform tx = world.all_transactions[-1] md = m.get_metadata(tx.address) diff --git a/examples/linux/binaries/concrete_solve.py b/examples/linux/binaries/concrete_solve.py index f98767e0e..da95a8d46 100644 --- a/examples/linux/binaries/concrete_solve.py +++ b/examples/linux/binaries/concrete_solve.py @@ -1,4 +1,4 @@ -from manticore import Manticore +from manticore.native import Manticore def fixme(): @@ -6,11 +6,9 @@ def fixme(): # Let's initialize the manticore control object -m = Manticore("multiple-styles") - # First, let's give it some fake data for the input. Anything the same size as # the real flag should work fine! -m.concrete_data = "infiltrate miami!" +m = Manticore("multiple-styles", concrete_start="infiltrate miami!") # Now we're going to want to execute a few different hooks and share data, so # let's use the m.context dict to keep our solution in diff --git a/examples/linux/binaries/symbolic_solve.py b/examples/linux/binaries/symbolic_solve.py index 747364b2d..8de19f81b 100644 --- a/examples/linux/binaries/symbolic_solve.py +++ b/examples/linux/binaries/symbolic_solve.py @@ -1,4 +1,4 @@ -from manticore import Manticore +from manticore.native import Manticore def fixme(): diff --git a/examples/script/concolic.py b/examples/script/concolic.py index 37eff5cff..fa0a117fa 100755 --- a/examples/script/concolic.py +++ b/examples/script/concolic.py @@ -16,6 +16,7 @@ import queue import struct import itertools +from typing import Any from manticore import set_verbosity from manticore.native import Manticore @@ -66,21 +67,21 @@ def flip(constraint): """ flips a constraint (Equal) - (Equal (BitvecITE Cond IfC ElseC) IfC) + (Equal (BitVecITE Cond IfC ElseC) IfC) -> - (Equal (BitvecITE Cond IfC ElseC) ElseC) + (Equal (BitVecITE Cond IfC ElseC) ElseC) """ equal = copy.copy(constraint) assert len(equal.operands) == 2 # assume they are the equal -> ite form that we produce on standard branches ite, forcepc = equal.operands - if not (isinstance(ite, BitvecITE) and isinstance(forcepc, BitvecConstant)): + if not (isinstance(ite, BitVecITE) and isinstance(forcepc, BitVecConstant)): return constraint - assert isinstance(ite, BitvecITE) and isinstance(forcepc, BitvecConstant) + assert isinstance(ite, BitVecITE) and isinstance(forcepc, BitVecConstant) assert len(ite.operands) == 3 cond, iifpc, eelsepc = ite.operands - assert isinstance(iifpc, BitvecConstant) and isinstance(eelsepc, BitvecConstant) + assert isinstance(iifpc, BitVecConstant) and isinstance(eelsepc, BitVecConstant) equal._operands = (equal.operands[0], eelsepc if forcepc.value == iifpc.value else iifpc) @@ -234,7 +235,7 @@ def constraints_are_sat(cons): def get_new_constrs_for_queue(oldcons, newcons): - ret = [] + ret: List[Any] = [] # i'm pretty sure its correct to assume newcons is a superset of oldcons @@ -299,7 +300,7 @@ def concrete_input_to_constraints(ci, prev=None): def main(): - q = queue.Queue() + q: queue.Queue = queue.Queue() # todo randomly generated concrete start stdin = ints2inp(0, 5, 0) diff --git a/examples/script/introduce_symbolic_bytes.py b/examples/script/introduce_symbolic_bytes.py index 0ad3a187c..7d12ea1cd 100755 --- a/examples/script/introduce_symbolic_bytes.py +++ b/examples/script/introduce_symbolic_bytes.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys +import capstone from manticore import issymbolic from manticore.native import Manticore @@ -64,7 +65,6 @@ def introduce_sym(state): state.cpu.write_int(state.cpu.RBP - 0xC, val, 32) def has_tainted_operands(operands, taint_id): - # type: (list[manticore.core.cpu.abstractcpu.Operand], object) -> bool for operand in operands: op = operand.read() if issymbolic(op) and taint_id in op.taint: diff --git a/mypy.ini b/mypy.ini index 5d9ea804d..322afe080 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,8 @@ [mypy] python_version = 3.6 -files = manticore, tests +files = manticore, tests, examples/**/*.py +# TODO: LOTS OF ERRORS +# check_untyped_defs = True # Generated file [mypy-manticore.ethereum.parsetab] diff --git a/setup.py b/setup.py index 2e083b58a..0499d2f24 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def rtd_dependent_deps(): # (we need to know how to import a given native dependency so we can check if native dependencies are installed) native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc2"] -lint_deps = ["black==19.10b0", "mypy==0.770"] +lint_deps = ["black==19.10b0", "mypy==0.790"] auto_test_deps = ["py-evm"] From f44d570acf4a42757d5954a7b77bd4a5e459cf98 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 12:16:17 -0500 Subject: [PATCH 111/126] debug: No fast-fail for matrix --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eae7b6aa7..4667ecad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: tests: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: type: ["ethereum_truffle", "ethereum_bench", "examples", "ethereum", "ethereum_vm", "native", "other"] # "wasm", "wasm_sym", steps: From 2fa2561091e8ca7a36586ba0892af811f14275a9 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 12 Jan 2021 14:03:09 -0500 Subject: [PATCH 112/126] Fix coveralls uploading --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4667ecad0..1e56cc640 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: ./run_tests.sh - name: Coveralls Parallel run: | - coveralls + coveralls --service=github env: COVERALLS_PARALLEL: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -88,7 +88,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Coveralls Finished - uses: coverallsapp/github-action@v1.1.1 + uses: coverallsapp/github-action@v1.1.2 with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel-finished: true From 2605e46ee7cf4e7330d0d24de6930b23677b01a2 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 13 Jan 2021 18:19:27 -0300 Subject: [PATCH 113/126] merge --- manticore/core/manticore.py | 1 - manticore/core/state.py | 31 ++++---- manticore/core/workspace.py | 16 ++-- manticore/ethereum/manticore.py | 2 +- manticore/ethereum/parsetab.py | 129 +++++++++----------------------- manticore/native/manticore.py | 2 +- manticore/native/state.py | 3 + manticore/platforms/decree.py | 4 +- manticore/platforms/evm.py | 25 ++----- manticore/platforms/linux.py | 4 +- manticore/platforms/platform.py | 21 +++++- manticore/platforms/wasm.py | 4 +- 12 files changed, 93 insertions(+), 149 deletions(-) diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index 1650b66e3..4064545aa 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -59,7 +59,6 @@ description="The seed to use when randomly selecting states", ) - class ManticoreBase(Eventful): _published_events = {"solve"} diff --git a/manticore/core/state.py b/manticore/core/state.py index 5e62eb566..0610a95de 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,7 +1,8 @@ import copy import logging +from typing import Optional -from .smtlib import solver, Bool, issymbolic, BitVecConstant, MutableArray +from .smtlib import Bool, issymbolic, BitVecConstant, MutableArray from ..utils.event import Eventful from ..utils.helpers import PickleSerializer from ..utils import config @@ -174,14 +175,14 @@ class StateBase(Eventful): _published_events = {"execution_intermittent"} - def __init__(self, constraints, platform, **kwargs): + def __init__(self, *, constraints: "Constraints", platform: "Platform", manticore: Optional["ManticoreBase"] = None, **kwargs): super().__init__(**kwargs) + self._manticore = manticore self._platform = platform self._constraints = constraints - self._platform.constraints = constraints - self._input_symbols = list() self._child = None self._context = dict() + self._terminated_by = None self._solver = EventSolver() self._total_exec = 0 @@ -189,15 +190,15 @@ def __init__(self, constraints, platform, **kwargs): # 33 # Events are lost in serialization and fork !! self.forward_events_from(self._solver) - self.forward_events_from(platform) + platform.set_state(self) def __getstate__(self): state = super().__getstate__() state["platform"] = self._platform state["constraints"] = self._constraints - state["input_symbols"] = self._input_symbols state["child"] = self._child state["context"] = self._context + state["terminated_by"] = self._terminated_by state["exec_counter"] = self._total_exec return state @@ -206,9 +207,10 @@ def __setstate__(self, state): super().__setstate__(state) self._platform = state["platform"] self._constraints = state["constraints"] - self._input_symbols = state["input_symbols"] self._child = state["child"] self._context = state["context"] + self._manticore = None + self._terminated_by = state["terminated_by"] self._total_exec = state["exec_counter"] self._own_exec = 0 @@ -216,7 +218,7 @@ def __setstate__(self, state): # 33 # Events are lost in serialization and fork !! self.forward_events_from(self._solver) - self.forward_events_from(self._platform) + self.platform.set_state(self) @property def id(self): @@ -229,15 +231,16 @@ def __repr__(self): # This need to change. this is the center of ALL the problems. re. CoW def __enter__(self): assert self._child is None - self._platform.constraints = None - new_state = self.__class__(self._constraints.__enter__(), self._platform) - self.platform.constraints = new_state.constraints - new_state._input_symbols = list(self._input_symbols) + self._platform._constraints = None + new_state = self.__class__(constraints=self._constraints.__enter__(), platform=self._platform, manticore=self._manticore) + #Keep the same constraint + self.platform._constraints = new_state.constraints + #backup copy of the context new_state._context = copy.copy(self._context) new_state._id = None + new_state._total_exec = self._total_exec self.copy_eventful_state(new_state) - self._child = new_state assert new_state.platform.constraints is new_state.constraints @@ -246,7 +249,6 @@ def __enter__(self): def __exit__(self, ty, value, traceback): self._constraints.__exit__(ty, value, traceback) self._child = None - self.platform.constraints = self.constraints @property def input_symbols(self): @@ -267,7 +269,6 @@ def constraints(self): @constraints.setter def constraints(self, constraints): self._constraints = constraints - self.platform.constraints = constraints def _update_state_descriptor(self, descriptor: StateDescriptor, *args, **kwargs): """ diff --git a/manticore/core/workspace.py b/manticore/core/workspace.py index a4a711ae3..b2215ee2c 100644 --- a/manticore/core/workspace.py +++ b/manticore/core/workspace.py @@ -602,10 +602,17 @@ def save_testcase(self, state: StateBase, testcase: Testcase, message: str = "") # The workspaces should override `save_testcase` method # # Below is native-only + def save_input_symbols(testcase, state: StateBase): + with testcase.open_stream("input") as f: + for symbol in state.input_symbols: + # TODO - constrain=False here, so the extra migration shouldn't cause problems, right? + buf = state.solve_one(symbol) + f.write(f"{symbol.name}: {buf!r}\n") + self.save_summary(testcase, state, message) self.save_trace(testcase, state) self.save_constraints(testcase, state) - self.save_input_symbols(testcase, state) + save_input_symbols(testcase, state) for stream_name, data in state.platform.generate_workspace_files().items(): with testcase.open_stream(stream_name, binary=True) as stream: @@ -655,10 +662,3 @@ def save_constraints(testcase, state: StateBase): with testcase.open_stream("smt") as f: f.write(str(state.constraints)) - @staticmethod - def save_input_symbols(testcase, state: StateBase): - with testcase.open_stream("input") as f: - for symbol in state.input_symbols: - # TODO - constrain=False here, so the extra migration shouldn't cause problems, right? - buf = state.solve_one(symbol) - f.write(f"{symbol.name}: {buf!r}\n") diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 5f5896c05..1e0136ed1 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -390,7 +390,7 @@ def __init__(self, plugins=None, **kwargs): constraints = ConstraintSet() # make the ethereum world state world = evm.EVMWorld(constraints) - initial_state = State(constraints, world) + initial_state = State(constraints=constraints, platform=world, maticore=self) super().__init__(initial_state, **kwargs) if plugins is not None: for p in plugins: diff --git a/manticore/ethereum/parsetab.py b/manticore/ethereum/parsetab.py index 7570d6d8f..3ef2495e9 100644 --- a/manticore/ethereum/parsetab.py +++ b/manticore/ethereum/parsetab.py @@ -1,111 +1,50 @@ + # parsetab.py # This file is automatically generated. Do not edit. # pylint: disable=W,C,R -_tabversion = "3.10" - -_lr_method = "LALR" +_tabversion = '3.10' -_lr_signature = "ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n " +_lr_method = 'LALR' -_lr_action_items = { - "UINTN": ([0, 16, 24,], [2, 2, 2,]), - "UINT": ([0, 16, 24,], [3, 3, 3,]), - "INTN": ([0, 16, 24,], [4, 4, 4,]), - "INT": ([0, 16, 24,], [5, 5, 5,]), - "ADDRESS": ([0, 16, 24,], [6, 6, 6,]), - "BOOL": ([0, 16, 24,], [7, 7, 7,]), - "FIXEDMN": ([0, 16, 24,], [8, 8, 8,]), - "UFIXEDMN": ([0, 16, 24,], [9, 9, 9,]), - "FIXED": ([0, 16, 24,], [10, 10, 10,]), - "UFIXED": ([0, 16, 24,], [11, 11, 11,]), - "BYTESM": ([0, 16, 24,], [12, 12, 12,]), - "FUNCTION": ([0, 16, 24,], [13, 13, 13,]), - "BYTES": ([0, 16, 24,], [14, 14, 14,]), - "STRING": ([0, 16, 24,], [15, 15, 15,]), - "LPAREN": ([0, 16, 24,], [16, 16, 16,]), - "$end": ( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 21, 23, 25,], - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, -19, -17, -20,], - ), - "LBRAKET": ( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 25,], - [17, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, 17, -19, -17, -20,], - ), - "COMMA": ( - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 19, 20, 21, 23, 25,], - [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -18, 24, -19, -17, -20,], - ), - "RPAREN": ( - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 23, 25, 26,], - [ - -1, - -2, - -3, - -4, - -5, - -6, - -7, - -8, - -9, - -10, - -11, - -12, - -13, - -14, - 19, - 23, - -18, - -15, - -19, - -17, - -20, - -16, - ], - ), - "RBRAKET": ([17, 22,], [21, 25,]), - "NUMBER": ([17,], [22,]), -} +_lr_signature = 'ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n ' + +_lr_action_items = {'UINTN':([0,16,24,],[2,2,2,]),'UINT':([0,16,24,],[3,3,3,]),'INTN':([0,16,24,],[4,4,4,]),'INT':([0,16,24,],[5,5,5,]),'ADDRESS':([0,16,24,],[6,6,6,]),'BOOL':([0,16,24,],[7,7,7,]),'FIXEDMN':([0,16,24,],[8,8,8,]),'UFIXEDMN':([0,16,24,],[9,9,9,]),'FIXED':([0,16,24,],[10,10,10,]),'UFIXED':([0,16,24,],[11,11,11,]),'BYTESM':([0,16,24,],[12,12,12,]),'FUNCTION':([0,16,24,],[13,13,13,]),'BYTES':([0,16,24,],[14,14,14,]),'STRING':([0,16,24,],[15,15,15,]),'LPAREN':([0,16,24,],[16,16,16,]),'$end':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,21,23,25,],[0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,-19,-17,-20,]),'LBRAKET':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[17,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,17,-19,-17,-20,]),'COMMA':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,24,-19,-17,-20,]),'RPAREN':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,23,25,26,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,19,23,-18,-15,-19,-17,-20,-16,]),'RBRAKET':([17,22,],[21,25,]),'NUMBER':([17,],[22,]),} _lr_action = {} for _k, _v in _lr_action_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_action: - _lr_action[_x] = {} - _lr_action[_x][_k] = _y + for _x,_y in zip(_v[0],_v[1]): + if not _x in _lr_action: _lr_action[_x] = {} + _lr_action[_x][_k] = _y del _lr_action_items -_lr_goto_items = { - "T": ([0, 16, 24,], [1, 20, 20,]), - "TL": ([16, 24,], [18, 26,]), -} +_lr_goto_items = {'T':([0,16,24,],[1,20,20,]),'TL':([16,24,],[18,26,]),} _lr_goto = {} for _k, _v in _lr_goto_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_goto: - _lr_goto[_x] = {} - _lr_goto[_x][_k] = _y + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_goto: _lr_goto[_x] = {} + _lr_goto[_x][_k] = _y del _lr_goto_items _lr_productions = [ - ("S' -> T", "S'", 1, None, None, None), - ("T -> UINTN", "T", 1, "p_basic_type", "abitypes.py", 154), - ("T -> UINT", "T", 1, "p_basic_type", "abitypes.py", 155), - ("T -> INTN", "T", 1, "p_basic_type", "abitypes.py", 156), - ("T -> INT", "T", 1, "p_basic_type", "abitypes.py", 157), - ("T -> ADDRESS", "T", 1, "p_basic_type", "abitypes.py", 158), - ("T -> BOOL", "T", 1, "p_basic_type", "abitypes.py", 159), - ("T -> FIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 160), - ("T -> UFIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 161), - ("T -> FIXED", "T", 1, "p_basic_type", "abitypes.py", 162), - ("T -> UFIXED", "T", 1, "p_basic_type", "abitypes.py", 163), - ("T -> BYTESM", "T", 1, "p_basic_type", "abitypes.py", 164), - ("T -> FUNCTION", "T", 1, "p_basic_type", "abitypes.py", 165), - ("T -> BYTES", "T", 1, "p_basic_type", "abitypes.py", 166), - ("T -> STRING", "T", 1, "p_basic_type", "abitypes.py", 167), - ("TL -> T", "TL", 1, "p_type_list_one", "abitypes.py", 175), - ("TL -> T COMMA TL", "TL", 3, "p_type_list", "abitypes.py", 182), - ("T -> LPAREN TL RPAREN", "T", 3, "p_tuple", "abitypes.py", 189), - ("T -> LPAREN RPAREN", "T", 2, "p_tuple_empty", "abitypes.py", 196), - ("T -> T LBRAKET RBRAKET", "T", 3, "p_dynamic_type", "abitypes.py", 203), - ("T -> T LBRAKET NUMBER RBRAKET", "T", 4, "p_dynamic_fixed_type", "abitypes.py", 212), + ("S' -> T","S'",1,None,None,None), + ('T -> UINTN','T',1,'p_basic_type','abitypes.py',154), + ('T -> UINT','T',1,'p_basic_type','abitypes.py',155), + ('T -> INTN','T',1,'p_basic_type','abitypes.py',156), + ('T -> INT','T',1,'p_basic_type','abitypes.py',157), + ('T -> ADDRESS','T',1,'p_basic_type','abitypes.py',158), + ('T -> BOOL','T',1,'p_basic_type','abitypes.py',159), + ('T -> FIXEDMN','T',1,'p_basic_type','abitypes.py',160), + ('T -> UFIXEDMN','T',1,'p_basic_type','abitypes.py',161), + ('T -> FIXED','T',1,'p_basic_type','abitypes.py',162), + ('T -> UFIXED','T',1,'p_basic_type','abitypes.py',163), + ('T -> BYTESM','T',1,'p_basic_type','abitypes.py',164), + ('T -> FUNCTION','T',1,'p_basic_type','abitypes.py',165), + ('T -> BYTES','T',1,'p_basic_type','abitypes.py',166), + ('T -> STRING','T',1,'p_basic_type','abitypes.py',167), + ('TL -> T','TL',1,'p_type_list_one','abitypes.py',175), + ('TL -> T COMMA TL','TL',3,'p_type_list','abitypes.py',182), + ('T -> LPAREN TL RPAREN','T',3,'p_tuple','abitypes.py',189), + ('T -> LPAREN RPAREN','T',2,'p_tuple_empty','abitypes.py',196), + ('T -> T LBRAKET RBRAKET','T',3,'p_dynamic_type','abitypes.py',203), + ('T -> T LBRAKET NUMBER RBRAKET','T',4,'p_dynamic_fixed_type','abitypes.py',212), ] diff --git a/manticore/native/manticore.py b/manticore/native/manticore.py index b2dacf04f..aa457407c 100644 --- a/manticore/native/manticore.py +++ b/manticore/native/manticore.py @@ -422,7 +422,7 @@ def _make_linux( # TODO: use argv as arguments for function platform.set_entry(entry_pc) - initial_state = State(constraints, platform) + initial_state = State(constraints=constraints, platform=platform) if concrete_start != "": logger.info("Starting with concrete input: %s", concrete_start) diff --git a/manticore/native/state.py b/manticore/native/state.py index 33606c034..905460816 100644 --- a/manticore/native/state.py +++ b/manticore/native/state.py @@ -20,17 +20,20 @@ class CheckpointData(NamedTuple): class State(StateBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self._input_symbols = list() self._hooks: Dict[Optional[int], Set[HookCallback]] = {} self._after_hooks: Dict[Optional[int], Set[HookCallback]] = {} def __getstate__(self) -> Dict[str, Any]: state = super().__getstate__() + state["input_symbols"] = self._input_symbols state["hooks"] = self._hooks state["after_hooks"] = self._after_hooks return state def __setstate__(self, state: Dict[str, Any]) -> None: super().__setstate__(state) + self._input_symbols = state["input_symbols"] self._hooks = state["hooks"] self._after_hooks = state["after_hooks"] self._resub_hooks() diff --git a/manticore/platforms/decree.py b/manticore/platforms/decree.py index be36986a1..d5057b4fe 100644 --- a/manticore/platforms/decree.py +++ b/manticore/platforms/decree.py @@ -7,7 +7,7 @@ from ..core.smtlib import * from ..core.state import TerminateState from ..binary import CGCElf -from ..platforms.platform import Platform +from ..platforms.platform import NativePlatform import logging import random @@ -76,7 +76,7 @@ def _transmit(self, buf): return len(buf) -class Decree(Platform): +class Decree(NativePlatform): """ A simple Decree Operating System. This class emulates the most common Decree system calls diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 4b5ac91db..fa418bba7 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -708,7 +708,6 @@ def pos(self, pos): def __init__( self, - constraints, address, data, caller, @@ -732,6 +731,7 @@ def __init__( :param gas: gas budget for this transaction """ super().__init__(**kwargs) + constraints = world.constraints if data is not None and not issymbolic(data): data_size = len(data) data_symbolic = constraints.new_array( @@ -2482,10 +2482,9 @@ class EVMWorld(Platform): "solve", } - def __init__(self, constraints, fork=DEFAULT_FORK, **kwargs): - super().__init__(path="NOPATH", **kwargs) + def __init__(self, fork=DEFAULT_FORK, **kwargs): + super().__init__(**kwargs) self._world_state = {} - self._constraints = constraints self._callstack: List[ Tuple[Transaction, List[EVMLog], Set[int], Union[bytearray, MutableArray], EVM] ] = [] @@ -2502,7 +2501,6 @@ def __getstate__(self): state["_pending_transaction"] = self._pending_transaction state["_logs"] = self._logs state["_world_state"] = self._world_state - state["_constraints"] = self._constraints state["_callstack"] = self._callstack state["_deleted_accounts"] = self._deleted_accounts state["_transactions"] = self._transactions @@ -2513,7 +2511,6 @@ def __getstate__(self): def __setstate__(self, state): super().__setstate__(state) - self._constraints = state["_constraints"] self._pending_transaction = state["_pending_transaction"] self._world_state = state["_world_state"] self._deleted_accounts = state["_deleted_accounts"] @@ -2596,16 +2593,6 @@ def __str__(self): def logs(self): return self._logs - @property - def constraints(self): - return self._constraints - - @constraints.setter - def constraints(self, constraints): - self._constraints = constraints - if self.current_vm: - self.current_vm.constraints = constraints - @property def evmfork(self): return self._fork @@ -2672,7 +2659,7 @@ def _make_vm_for_tx(self, tx): gas = tx.gas - vm = EVM(self._constraints, address, data, caller, value, bytecode, world=self, gas=gas) + vm = EVM(address, data, caller, value, bytecode, world=self, gas=gas) if self.depth == 0: # Only at human level we need to debit the tx_fee from the gas # In case of an internal tx the CALL-like instruction will @@ -2752,9 +2739,9 @@ def _open_transaction(self, sort, address, price, bytecode_or_data, caller, valu def _close_transaction(self, result, data=None, rollback=False): self._publish("will_close_transaction", self._callstack[-1][0]) tx, logs, deleted_accounts, account_storage, vm = self._callstack.pop() - assert self.constraints == vm.constraints + # Keep constraints gathered in the last vm - self.constraints = vm.constraints + self._state.constraints = vm.constraints # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-211.md if data is not None and self.current_vm is not None: diff --git a/manticore/platforms/linux.py b/manticore/platforms/linux.py index 10ae0332f..64573fc43 100644 --- a/manticore/platforms/linux.py +++ b/manticore/platforms/linux.py @@ -43,7 +43,7 @@ InvalidMemoryAccess, ) from ..native.state import State -from ..platforms.platform import Platform, SyscallNotImplemented, unimplemented +from ..platforms.platform import NativePlatform, SyscallNotImplemented, unimplemented from typing import cast, Any, Deque, Dict, IO, Iterable, List, Optional, Set, Tuple, Union, Callable @@ -819,7 +819,7 @@ def setstate(state: State, value): return ret -class Linux(Platform): +class Linux(NativePlatform): """ A simple Linux Operating System Platform. This class emulates the most common Linux system calls diff --git a/manticore/platforms/platform.py b/manticore/platforms/platform.py index 9be3edcb3..ba736a144 100644 --- a/manticore/platforms/platform.py +++ b/manticore/platforms/platform.py @@ -1,4 +1,5 @@ import logging +from typing import Optional from functools import wraps from typing import Any, Callable, TypeVar @@ -50,11 +51,18 @@ class Platform(Eventful): _published_events = {"solve"} - def __init__(self, path, **kwargs): + + def __init__(self, *, state: Optional["StateBase"] = None, **kwargs): + self._state = state super().__init__(**kwargs) - def invoke_model(self, model, prefix_args=None): - self._function_abi.invoke(model, prefix_args) + def set_state(self, state: "StateBase"): + self._state = state + state.forward_events_from(self) + + @property + def constraints(self): + return self._state._constraints def __setstate__(self, state): super().__setstate__(state) @@ -63,5 +71,12 @@ def __getstate__(self): state = super().__getstate__() return state +class NativePlatform(Platform): + def __init__(self, path, **kwargs): + super().__init__(**kwargs) + + def invoke_model(self, model, prefix_args=None): + self._function_abi.invoke(model, prefix_args) + def generate_workspace_files(self): return {} diff --git a/manticore/platforms/wasm.py b/manticore/platforms/wasm.py index 8b47dfaee..9665fc9ff 100644 --- a/manticore/platforms/wasm.py +++ b/manticore/platforms/wasm.py @@ -1,4 +1,4 @@ -from .platform import Platform +from .platform import NativePlatform from ..wasm.structure import ( ModuleInstance, Store, @@ -33,7 +33,7 @@ def stub(arity, _state, *args): return [0 for _ in range(arity)] # TODO: Return symbolic values -class WASMWorld(Platform): +class WASMWorld(NativePlatform): """Manages global environment for a WASM state. Analagous to EVMWorld.""" def __init__(self, filename, name="self", **kwargs): From 487e5f48eb9f212e20bd32274bf97408295f70a4 Mon Sep 17 00:00:00 2001 From: feliam Date: Wed, 13 Jan 2021 18:37:57 -0300 Subject: [PATCH 114/126] merge --- manticore/core/smtlib/constraints.py | 2 +- manticore/core/smtlib/expression.py | 142 ++++++++++-------- manticore/core/smtlib/solver.py | 7 +- manticore/core/smtlib/visitors.py | 59 ++++---- manticore/core/state.py | 6 + manticore/ethereum/manticore.py | 2 + manticore/platforms/evm.py | 33 ++-- .../contracts/simple_int_overflow.sol | 2 +- tests/ethereum/test_general.py | 28 +++- tests/other/test_smtlibv2.py | 24 +-- 10 files changed, 174 insertions(+), 131 deletions(-) diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index 64615d060..c80ad4fbe 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -97,7 +97,7 @@ def add(self, constraint) -> None: :param constraint: The constraint to add to the set. """ if isinstance(constraint, bool): - constraint = BoolConstant(constraint) + constraint = BoolConstant(value=constraint) assert isinstance(constraint, Bool) constraint = simplify(constraint) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index becc96f11..e0c1075c7 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -21,7 +21,7 @@ import uuid import re import copy -from typing import Union, Optional, Tuple, List +from typing import overload, Union, Optional, Tuple, List, FrozenSet from functools import lru_cache @@ -118,12 +118,12 @@ class Expression(object, metaclass=XSlotted, abstract=True): __xslots__: Tuple[str, ...] = ("_taint",) - def __init__(self, *, taint: Union[tuple, frozenset] = ()): + def __init__(self, *, taint: FrozenSet[str] = frozenset()): """ An abstract Unmutable Taintable Expression - :param taint: A frozenzset + :param taint: A frozenzset of taints. Normally strings """ - self._taint = frozenset(taint) + self._taint = taint super().__init__() def __repr__(self): @@ -132,15 +132,15 @@ def __repr__(self): ) @property - def is_tainted(self): - return len(self._taint) != 0 + def is_tainted(self) -> bool: + return bool(self._taint) @property - def taint(self): + def taint(self) -> FrozenSet[str]: return self._taint @property - def operands(self): + def operands(self) -> Tuple["Expression"]: """ Hack so we can use any Expression as a node """ return () @@ -165,7 +165,7 @@ def __init__(self, *, name: str, **kwargs): self._name = name @property - def name(self): + def name(self) -> str: return self._name def __repr__(self): @@ -229,7 +229,7 @@ def cast(self, value: Union["Bool", int, bool], **kwargs) -> Union["BoolConstant """ Cast any type into a Bool or fail """ if isinstance(value, Bool): return value - return BoolConstant(bool(value), **kwargs) + return BoolConstant(value=bool(value), **kwargs) def __cmp__(self, *args): raise NotImplementedError("CMP for Bool") @@ -263,7 +263,7 @@ def __ror__(self, other): def __rxor__(self, other): return BoolXor(self.cast(other), self) - def __dbool__(self): + def __bool__(self): raise ExpressionError( "You tried to use a Bool Expression as a boolean constant. Expressions could represent a set of concrete values." ) @@ -274,26 +274,26 @@ class BoolVariable(Bool, Variable): class BoolConstant(Bool, Constant): - def __init__(self, value: bool, **kwargs): + def __init__(self, *, value: bool, **kwargs): super().__init__(value=bool(value), **kwargs) - def __bool__(self): + def __bool__(self) -> bool: return self._value class BoolOperation(Bool, Operation, abstract=True): """ It's an operation that results in a Bool """ + pass + #def __init__(self, *args, **kwargs): + # super().__init__(*args, **kwargs) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def __xbool__(self): - # FIXME: TODO: re-think is we want to be this forgiving every use of - # local_simplify looks hacky - simplified = self # local_simplify(self) - if isinstance(simplified, Constant): - return simplified.value - raise ExpressionError("BoolOperation can not be reduced to a constant") + #def __xbool__(self): + # # FIXME: TODO: re-think is we want to be this forgiving every use of + # # local_simplify looks hacky + # simplified = self # local_simplify(self) + # if isinstance(simplified, Constant): + # return simplified.value + # raise ExpressionError("BoolOperation can not be reduced to a constant") class BoolNot(BoolOperation): @@ -335,15 +335,15 @@ def __init__(self, size: int, **kwargs): self._size = size @property - def size(self): + def size(self) -> int: return self._size @property - def mask(self): + def mask(self) -> int: return (1 << self.size) - 1 @property - def signmask(self): + def signmask(self) -> int: return 1 << (self.size - 1) def cast(self, value: Union["BitVec", str, int, bytes], **kwargs) -> "BitVec": @@ -545,14 +545,14 @@ def __init__(self, size: int, value: int, **kwargs): value &= (1 << size) - 1 # Can not use self.mask yet super().__init__(size=size, value=value, **kwargs) - def __bool__(self): - return self.value != 0 + def __bool__(self) -> bool: + return bool(self.value) - def __int__(self): + def __int__(self) -> int: return self.value @property - def signed_value(self): + def signed_value(self) -> int: """ Gives signed python int representation """ if self._value & self.signmask: return self._value - (1 << self.size) @@ -560,6 +560,9 @@ def signed_value(self): return self._value def __eq__(self, other): + """ If tainted keep a tainted symbolic value""" + if self.taint: + return BoolEqual(self, self.cast(other)) # Ignore the taint for eq comparison return self._value == other @@ -569,12 +572,13 @@ def __repr__(self): class BitVecOperation(BitVec, Operation, abstract=True): """ Operations that result in a BitVec """ - - pass + def __init__(self, *, operands:Tuple[BitVec, ...], **kwargs): + super().__init__(operands=operands, **kwargs) class BitVecAdd(BitVecOperation): - def __init__(self, operanda, operandb, **kwargs): + def __init__(self, operanda:BitVec, operandb:BitVec, **kwargs): + assert operanda.size == operandb.size super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) @@ -670,9 +674,17 @@ def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): class BoolEqual(BoolOperation): + @overload def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): - assert isinstance(operanda, Expression) - assert isinstance(operandb, Expression) + ... + @overload + def __init__(self, operanda: Bool, operandb: Bool, **kwargs): + ... + @overload + def __init__(self, operanda: "Array", operandb: "Array", **kwargs): + ... + + def __init__(self, operanda, operandb, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) @@ -707,7 +719,7 @@ def __init__(self, operanda, operandb, **kwargs): operands=(operanda, operandb), **kwargs ) - +Array="Array" class Array(Expression, abstract=True): """An Array expression is an unmutable mapping from bitvector to bitvector @@ -719,30 +731,30 @@ class Array(Expression, abstract=True): @property @abstractmethod - def index_size(self): + def index_size(self) -> int: """ The bit size of the index part. Must be overloaded by a more specific class""" ... @property - def value_size(self): + def value_size(self) -> int: """ The bit size of the value part. Must be overloaded by a more specific class""" raise NotImplementedError @property - def length(self): + def length(self) -> int: """ Number of defined items. Must be overloaded by a more specific class""" raise NotImplementedError - def select(self, index): + def select(self, index) -> Union[BitVec, int]: """ Gets a bitvector element from the Array que la""" raise NotImplementedError - def store(self, index, value): + def store(self, index, value) -> "Array": """ Create a new array that contains the updated value""" raise NotImplementedError @property - def default(self): + def default(self) -> Optional[Union[BitVec, int]]: """If defined, reading from an uninitialized index return the default value. Otherwise, reading from an uninitialized index gives a symbol (normal Array behavior) """ @@ -817,7 +829,7 @@ def cast_value(self, value: Union[BitVec, bytes, int]) -> BitVec: value = int(value) return BitVecConstant(self.value_size, value) - def write(self, offset, buf): + def write(self, offset: Union[BitVec, int], buf:Union["Array", bytes]) -> "Array": """Builds a new unmutable Array instance on top of current array by writing buf at offset""" array = self @@ -825,10 +837,16 @@ def write(self, offset, buf): array = array.store(offset + i, value) return array - def read(self, offset, size): + def read(self, offset:Union[BitVec, int], size: int) -> "Array": """ A projection of the current array. """ return ArraySlice(self, offset=offset, size=size) + @overload + def __getitem__(self, index: Union[BitVec, int]) -> Union[BitVec, int]: + ... + @overload + def __getitem__(self, index: slice) -> "Array": + ... def __getitem__(self, index): """__getitem__ allows for pythonic access A = ArrayVariable(index_size=32, value_size=8) @@ -846,15 +864,15 @@ def __iter__(self): yield self[i] @staticmethod - def _compare_buffers(a, b): + def _compare_buffers(a:"Array", b:"Array") -> Union[Bool, bool]: """ Builds an expression that represents equality between the two arrays.""" if a.length != b.length: - return BoolConstant(False) - cond = BoolConstant(True) + return BoolConstant(value=False) + cond = BoolConstant(value=True) for i in range(a.length): cond = BoolAnd(cond.cast(a[i] == b[i]), cond) - if cond is BoolConstant(False): - return BoolConstant(False) + if cond is BoolConstant(value=False): + return BoolConstant(value=False) return cond def __eq__(self, other): @@ -881,14 +899,14 @@ def _fix_slice(self, index: slice): raise ExpressionError("Size could not be simplified to a constant in a slice operation") return start, stop, size.value - def _concatenate(self, array_a, array_b): + def _concatenate(self, array_a:"Array", array_b:"Array") -> "Array": + """Build a new array from the concatenation of the operands""" new_arr = ArrayVariable( index_size=self.index_size, length=len(array_a) + len(array_b), value_size=self.value_size, name="concatenation", ) - for index in range(len(array_a)): new_arr = new_arr.store(index, local_simplify(array_a[index])) for index in range(len(array_b)): @@ -902,7 +920,7 @@ def __radd__(self, other): return self._concatenate(other, self) @lru_cache(maxsize=128, typed=True) - def read_BE(self, address, size): + def read_BE(self, address: Union[int, BitVec], size:int) -> Union[BitVec, int]: address = self.cast_index(address) bytes = [] for offset in range(size): @@ -910,14 +928,14 @@ def read_BE(self, address, size): return BitVecConcat(operands=tuple(bytes)) @lru_cache(maxsize=128, typed=True) - def read_LE(self, address, size): + def read_LE(self, address: Union[int, BitVec], size:int) -> Union[BitVec, int]: address = self.cast_index(address) bytes = [] for offset in range(size): bytes.append(self.get(address + offset, self._default)) return BitVecConcat(operands=reversed(bytes)) - def write_BE(self, address, value, size): + def write_BE(self, address:Union[int, BitVec], value:Union[int, BitVec], size:int) -> Array: address = self.cast_index(address) value = BitVecConstant(size=size * self.value_size, value=0).cast(value) array = self @@ -928,7 +946,7 @@ def write_BE(self, address, value, size): ) return array - def write_LE(self, address, value, size): + def write_LE(self, address:Union[int, BitVec], value:Union[int, BitVec], size:int) -> Array: address = self.cast_index(address) value = BitVec(size * self.value_size).cast(value) array = self @@ -951,15 +969,15 @@ def __init__( super().__init__(**kwargs) @property - def index_size(self): + def index_size(self) -> int: return self._index_size @property - def value_size(self): + def value_size(self) -> int: return self._value_size @property - def length(self): + def length(self) -> int: return len(self.value) def select(self, index): @@ -1018,7 +1036,7 @@ def length(self): return self._length def __init__( - self, + self, *, index_size: int, value_size: int, length: Optional[int] = None, @@ -1157,14 +1175,14 @@ def written(self): def is_known(self, index): if isinstance(index, Constant) and index.value in self.concrete_cache: - return BoolConstant(True) + return BoolConstant(value=True) - is_known_index = BoolConstant(False) + is_known_index = BoolConstant(value=False) written = self.written for known_index in written: if isinstance(index, Constant) and isinstance(known_index, Constant): if known_index.value == index.value: - return BoolConstant(True) + return BoolConstant(value=True) is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) return is_known_index diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 7842ee1b5..361007762 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -645,7 +645,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): #################### values = [] start = time.time() - # with constraints.related_to(*expressions) as temp_cs: + #with constraints.related_to(*expressions) as temp_cs: with constraints as temp_cs: for expression in expressions: bucket = self._cache.setdefault(hash(constraints), {}) @@ -673,7 +673,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): expression_i = expression[i] if issymbolic(expression_i): subvar = temp_cs.new_bitvec(expression.value_size) - temp_cs.add(subvar == simplify(expression[i])) + temp_cs.add(subvar == expression[i]) var.append(subvar) else: var.append(expression_i) @@ -687,11 +687,10 @@ def get_value(self, constraints: ConstraintSet, *expressions): result.append(self.__getvalue_bv(var[i].name)) else: result.append(var[i]) - values.append(bytes(result)) bucket[hash(expression)] = values[-1] if time.time() - start > consts.timeout: - raise SolverError("Timeout") + raise SolverError(f"Timeout {expressions}") continue temp_cs.add(var == expression) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index cb3c55562..19d559d2f 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -311,21 +311,21 @@ def visit_BoolLessThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value - return BoolConstant(a < b, taint=expression.taint) + return BoolConstant(value=a < b, taint=expression.taint) return None def visit_BoolLessOrEqual(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value - return BoolConstant(a <= b, taint=expression.taint) + return BoolConstant(value=a <= b, taint=expression.taint) return None def visit_BoolGreaterThan(self, expression, *operands) -> Optional[BoolConstant]: if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value - return BoolConstant(a > b, taint=expression.taint) + return BoolConstant(value=a > b, taint=expression.taint) return None def visit_BoolGreaterOrEqual(self, expression, *operands) -> Optional[BoolConstant]: @@ -336,6 +336,7 @@ def visit_BoolGreaterOrEqual(self, expression, *operands) -> Optional[BoolConsta return None def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: + return None if all(isinstance(o, Constant) for o in operands): signmask = operands[0].signmask mask = operands[0].mask @@ -393,7 +394,7 @@ def visit_Operation(self, expression, *operands): return BitVecConstant(expression.size, value, taint=expression.taint) else: isinstance(expression, Bool) - return BoolConstant(value, taint=expression.taint) + return BoolConstant(value=value, taint=expression.taint) @lru_cache(maxsize=128, typed=True) @@ -490,7 +491,7 @@ def visit_BoolEqual(self, expression, *operands): return BoolNot(operands[0].operands[0], taint=expression.taint) if operands[0] is operands[1]: - return BoolConstant(True, taint=expression.taint) + return BoolConstant(value=True, taint=expression.taint) if isinstance(operands[0], BitVecExtract) and isinstance(operands[1], BitVecExtract): if ( @@ -499,7 +500,7 @@ def visit_BoolEqual(self, expression, *operands): and operands[0].begining == operands[1].begining ): - return BoolConstant(True, taint=expression.taint) + return BoolConstant(value=True, taint=expression.taint) def visit_BoolOr(self, expression, a, b): if isinstance(a, Constant): @@ -645,6 +646,21 @@ def visit_BitVecExtract(self, expression, *operands): taint=expression.taint, ) + def visit_BitVecMul(self, expression, *operands): + left = operands[0] + right = operands[1] + if isinstance(right, BitVecConstant): + if right.value == 1: + return left + if right.value == 0: + return right + if isinstance(left, BitVecConstant): + if left.value == 1: + return right + if left.value == 0: + return left + + def visit_BitVecAdd(self, expression, *operands): """a + 0 ==> a 0 + a ==> a @@ -783,9 +799,10 @@ def visit_Expression(self, expression, *operands): assert not isinstance(expression, Operation) return expression - +import time @lru_cache(maxsize=128, typed=True) def arithmetic_simplify(expression): + start = time.time() if not isinstance(expression, Expression): return expression simp = ArithmeticSimplifier() @@ -819,17 +836,7 @@ def to_constant(expression): return value -@lru_cache(maxsize=128, typed=True) -def simplify(expression): - simplified = arithmetic_simplify(expression) - - def t(x): - try: - return translate_to_smtlib(x) - except: - return str(x) - - return simplified +simplify = arithmetic_simplify class CountExpressionUse(Visitor): @@ -858,7 +865,7 @@ def __init__(self, use_bindings=True, *args, **kw): assert "bindings" not in kw super().__init__(*args, **kw) self._unique = 0 - self.use_bindings = False # use_bindings + self.use_bindings = use_bindings self._bindings_cache_exp = {} self._bindings = {} self._variables = set() @@ -1009,13 +1016,17 @@ def smtlib(self): result += f"(assert {self.apply_bindings(constraint_str)})\n" return result +@lru_cache(maxsize=128, typed=True) +def _translate_to_smtlib(expression, use_bindings=True, **kwargs): + translator = TranslatorSmtlib(use_bindings=use_bindings, **kwargs) + translator.visit(expression) + return translator.result def translate_to_smtlib(expression, use_bindings=True, **kwargs): if isinstance(expression, MutableArray): expression = expression.array - translator = TranslatorSmtlib(use_bindings=use_bindings, **kwargs) - translator.visit(expression) - return translator.result + result = _translate_to_smtlib(expression, use_bindings=use_bindings, **kwargs) + return result class Replace(Visitor): @@ -1100,10 +1111,6 @@ def __init__(self, **kwargs): def visit_Operation(self, expression, *operands): try: self.expressions[expression] += 1 - import pdb - - pdb.set_trace() - print("A TWO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1 222222222222222222222222222222222") except KeyError as e: self.expressions[expression] = 1 diff --git a/manticore/core/state.py b/manticore/core/state.py index 0610a95de..f182096a3 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -180,6 +180,10 @@ def __init__(self, *, constraints: "Constraints", platform: "Platform", manticor self._manticore = manticore self._platform = platform self._constraints = constraints + self._platform._constraints = constraints + self._platform.state=self + self._input_symbols = list() + self._child = None self._context = dict() @@ -206,6 +210,8 @@ def __getstate__(self): def __setstate__(self, state): super().__setstate__(state) self._platform = state["platform"] + self._platform.state=self + self._constraints = state["constraints"] self._child = state["child"] self._context = state["context"] diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 1e0136ed1..f3aeda7f6 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -1407,6 +1407,7 @@ def match(state, func, symbolic_pairs, concrete_pairs, start=None): return state.can_be_true(True) def fix_unsound_symbolication(self, state): + logger.info(f"Starting unsound symbolication search for {state.id}") soundcheck = state.context.get("soundcheck", None) if soundcheck is not None: return soundcheck @@ -1416,6 +1417,7 @@ def fix_unsound_symbolication(self, state): state.context["soundcheck"] = self.fix_unsound_symbolication_fake(state) else: state.context["soundcheck"] = True + logger.info(f"Done unsound symbolication search for {state.id}") return state.context["soundcheck"] def _terminate_state_callback(self, state, e): diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index fa418bba7..46b6ca68a 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -71,7 +71,7 @@ def globalfakesha3(data): "oog", default="complete", description=( - "Default behavior for symbolic gas." + "Default behavior for symbolic out of gas exception." "pedantic: Fully faithful. Test at every instruction. Forks." "complete: Mostly faithful. Test at BB limit. Forks." "concrete: Incomplete. Concretize gas to MIN/MAX values. Forks." @@ -133,7 +133,8 @@ def ceil32(x): def to_signed(i): - return Operators.ITEBV(256, i < TT255, i, i - TT256) + i &= ((1<<256)-1) + return Operators.ITEBV(256, Operators.EXTRACT(i, 255, 1) == 0, i, -((1 << 256) - i)) class Transaction: @@ -832,14 +833,11 @@ def extend_with_zeroes(b): self._failed = False def fail_if(self, failed): - old_failed = self.constraints.new_bool() - self.constraints.add(old_failed == self._failed) - self._failed = Operators.OR(old_failed, failed) + self._failed = Operators.OR(self._failed, failed) def is_failed(self): if isinstance(self._failed, bool): return self._failed - self._failed = simplify(self._failed) if isinstance(self._failed, Constant): return self._failed.value @@ -1335,10 +1333,9 @@ def setstate(state, value): raise Concretize("Symbolic PC", expression=expression, setstate=setstate, policy="ALL") try: - if getattr(getattr(self, self.instruction.semantics, None), "_pos", None) is None: - print(self) self._check_jmpdest() last_pc, last_gas, instruction, arguments, fee, allocated = self._checkpoint() + result = self._handler(*arguments) self._advance(result) except ConcretizeGas as ex: @@ -2424,11 +2421,7 @@ def __str__(self): for i in list(reversed(self.stack))[:10]: argname = args.get(sp, "") if issymbolic(i): - r = "{:>12s} {:66s} {:s}".format( - argname, - "[%x-%x]" % SelectedSolver.instance().minmax(self.constraints, i), - str(i.taint), - ) + r = "{:>12s}".format(argname) else: r = "{:>12s} 0x{:064x}".format(argname, i) sp += 1 @@ -2598,8 +2591,8 @@ def evmfork(self): return self._fork def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value): - return 0 - if consts.oog == "ignore": + return 21000 + if consts.tx_fee == "ignore": return 0 GTXCREATE = ( @@ -2613,7 +2606,7 @@ def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value else: tx_fee = GTRANSACTION # Simple transaction fee - # This is INSANE TDO FIXME + # This is INSANE TODO FIXME # This popcnt like thing is expensive when the bytecode or # data has symbolic content @@ -2633,9 +2626,7 @@ def _transaction_fee(self, sort, address, price, bytecode_or_data, caller, value tx_fee += zerocount * GTXDATAZERO tx_fee += nonzerocount * GTXDATANONZERO - x = self.constraints.new_bitvec(size=512) - self.constraints.add(x == simplify(Operators.ZEXTEND(tx_fee, 512))) - return x + return Operators.ZEXTEND(tx_fee, 512) def _make_vm_for_tx(self, tx): if tx.sort == "CREATE": @@ -2984,7 +2975,7 @@ def set_balance(self, address, value): def get_balance(self, address): if address not in self._world_state: return 0 - return Operators.EXTRACT(self._world_state[address]["balance"], 0, 256) + return simplify(Operators.EXTRACT(self._world_state[address]["balance"], 0, 256)) def account_exists(self, address): if address not in self._world_state: @@ -3385,7 +3376,7 @@ def _pending_transaction_failed(self): aux_value = Operators.ZEXTEND(value, 512) enough_balance = Operators.UGE(aux_src_balance, aux_value) if self.depth == 0: # the tx_fee is taken at depth 0 - # take the gas from the balance + # take the gas from t"he balance aux_price = Operators.ZEXTEND(price, 512) aux_gas = Operators.ZEXTEND(gas, 512) aux_fee = aux_price * aux_gas diff --git a/tests/ethereum/contracts/simple_int_overflow.sol b/tests/ethereum/contracts/simple_int_overflow.sol index c7567a9d5..8460a0018 100644 --- a/tests/ethereum/contracts/simple_int_overflow.sol +++ b/tests/ethereum/contracts/simple_int_overflow.sol @@ -1,4 +1,4 @@ -pragma solidity ^0.5.3; +pragma solidity ^0.4.24; contract Overflow { uint private sellerBalance=10; diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index 6406af9a1..d364c6b76 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -454,11 +454,12 @@ def test_SDIVS3(self): constraints, world, vm = self._make() xx = constraints.new_bitvec(256, name="x") yy = constraints.new_bitvec(256, name="y") - constraints.add(xx == 0x20) - constraints.add(yy == -1) + x, y = 0x20, -1 + constraints.add(xx == x) + constraints.add(yy == y) result = vm.SDIV(xx, yy) self.assertListEqual( - list(map(evm.to_signed, solver.get_all_values(constraints, result))), [-0x20] + list(map(evm.to_signed, solver.get_all_values(constraints, result))), [vm.SDIV(x, y)] ) def test_SDIVSx(self): @@ -466,14 +467,32 @@ def test_SDIVSx(self): constraints, world, vm = self._make() xx = constraints.new_bitvec(256, name="x") yy = constraints.new_bitvec(256, name="y") + zz = constraints.new_bitvec(256, name="z") constraints.add(xx == x) constraints.add(yy == y) result = vm.SDIV(xx, yy) + constraints.add(zz == result) + self.assertListEqual( + list(map(evm.to_signed, solver.get_all_values(constraints, zz))), [vm.SDIV(x, y)] + ) self.assertListEqual( list(map(evm.to_signed, solver.get_all_values(constraints, result))), [vm.SDIV(x, y)] ) + def test_to_sig(self): + self.assertEqual(evm.to_signed(-1), -1) + self.assertEqual(evm.to_signed(1), 1) + self.assertEqual(evm.to_signed(0), 0) + self.assertEqual(evm.to_signed(-2), -2) + self.assertEqual(evm.to_signed(2), 2) + self.assertEqual(evm.to_signed(0), 0) + self.assertEqual(evm.to_signed(0x8000000000000000000000000000000000000000000000000000000000000001), -0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff) + self.assertEqual(evm.to_signed(0x8000000000000000000000000000000000000000000000000000000000000002), -0x7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe) + self.assertEqual(evm.to_signed(0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff), 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff) + self.assertEqual(evm.to_signed(0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff), -1) + self.assertEqual(evm.to_signed( 0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe), -2) + class EthTests(unittest.TestCase): def setUp(self): @@ -555,6 +574,7 @@ def test_create_contract_with_too_much_args(self): ) def test_create_contract_with_string_args(self): + import pdb; pdb.set_trace() source_code = ( "contract DontWork1{ string s; constructor(string memory s_) public{ s = s_;} }" ) @@ -1295,7 +1315,6 @@ def will_decode_instruction_callback(self, state, pc): func_name, args = ABI.deserialize( "shutdown(string)", state.platform.current_transaction.data ) - print("Shutdown", to_constant(args[0])) self.manticore.shutdown() elif func_id == ABI.function_selector("can_be_true(bool)"): func_name, args = ABI.deserialize( @@ -1741,6 +1760,7 @@ def test_delegatecall_env(self): # check balances self.assertEqual(world.get_balance(0x111111111111111111111111111111111111111), 0) self.assertEqual(world.get_balance(0x222222222222222222222222222222222222222), 10) + import pdb;pdb.set_trace() self.assertEqual( world.get_balance(0x333333333333333333333333333333333333333), 100000000000000000000000 - 10, diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 2bc527049..09efd16ad 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -478,29 +478,29 @@ def testSolver(self): def testBool1(self): cs = ConstraintSet() - bf = BoolConstant(False) - bt = BoolConstant(True) + bf = BoolConstant(value=False) + bt = BoolConstant(value=True) cs.add(Operators.AND(bf, bt)) self.assertFalse(self.solver.check(cs)) def testBool2(self): cs = ConstraintSet() - bf = BoolConstant(False) - bt = BoolConstant(True) + bf = BoolConstant(value=False) + bt = BoolConstant(value=True) cs.add(Operators.AND(bf, bt, bt, bt)) self.assertFalse(self.solver.check(cs)) def testBool3(self): cs = ConstraintSet() - bf = BoolConstant(False) - bt = BoolConstant(True) + bf = BoolConstant(value=False) + bt = BoolConstant(value=True) cs.add(Operators.AND(bt, bt, bf, bt)) self.assertFalse(self.solver.check(cs)) def testBool4(self): cs = ConstraintSet() - bf = BoolConstant(False) - bt = BoolConstant(True) + bf = BoolConstant(value=False) + bt = BoolConstant(value=True) cs.add(Operators.OR(True, bf)) cs.add(Operators.OR(bt, bt, False)) self.assertTrue(self.solver.check(cs)) @@ -926,8 +926,8 @@ def testBitVector_max1_noop(self): consts.optimize = True def testBool_nonzero(self): - self.assertTrue(BoolConstant(True).__bool__()) - self.assertFalse(BoolConstant(False).__bool__()) + self.assertTrue(BoolConstant(value=True).__bool__()) + self.assertFalse(BoolConstant(value=False).__bool__()) def test_visitors(self): solver = Z3Solver.instance() @@ -1070,8 +1070,8 @@ def test_constant_folding_udiv(self): def test_simplify_OR(self): cs = ConstraintSet() - bf = BoolConstant(False) - bt = BoolConstant(True) + bf = BoolConstant(value=False) + bt = BoolConstant(value=True) var = cs.new_bool() cs.add(simplify(Operators.OR(var, var)) == var) cs.add(simplify(Operators.OR(var, bt)) == bt) From 4323beff05f73c0c25347f64a584d3a9c0a8b09a Mon Sep 17 00:00:00 2001 From: feliam Date: Thu, 14 Jan 2021 19:45:17 -0300 Subject: [PATCH 115/126] merge --- manticore/core/smtlib/expression.py | 2 +- manticore/core/smtlib/solver.py | 4 ++-- manticore/core/smtlib/visitors.py | 18 ++++++++---------- manticore/platforms/platform.py | 1 - tests/other/test_smtlibv2.py | 3 +-- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index e0c1075c7..63d41608b 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1259,7 +1259,7 @@ def select(self, index): return ArraySelect(self, index) # if a default is defined we need to check if the index was previously written - return BitVecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default)) + return local_simplify(BitVecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default))) def store(self, index, value): casted = self.cast_index(index) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 361007762..ff5ba4384 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -214,7 +214,7 @@ def __readline_and_count(self): assert self._proc.stdout buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error - # print (">",buf) + #print (">",buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -227,7 +227,7 @@ def send(self, cmd: str) -> None: :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ - # print ("<",cmd) + #print ("<",cmd) if self._debug: logger.debug(">%s", cmd) self._proc.stdout.flush() # type: ignore diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 19d559d2f..19b7bdb00 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -961,10 +961,11 @@ def visit_Operation(self, expression, *operands): @property def result(self): - return self.apply_bindings(self._stack[-1]) + return f"{self.apply_bindings(self._stack[-1])}" def apply_bindings(self, main_smtlib_str): # Python program to print topological sorting of a DAG + # Well-sortedness requirements from toposort import toposort_flatten as toposort G = {} @@ -977,25 +978,22 @@ def apply_bindings(self, main_smtlib_str): if not variables: G[name] = set() - output = "" # Build let statement + output = main_smtlib_str for name in reversed(toposort(G)): if name not in self._bindings: continue expr, smtlib = self._bindings[name] + # FIXME: too much string manipulation. Search occurrences in the Expression realm - if main_smtlib_str.count(name) + output.count(name) == 1: - main_smtlib_str = main_smtlib_str.replace(name, smtlib) + if output.count(name) <= 1: + #output = f"let (({name} {smtlib})) ({output})" output = output.replace(name, smtlib) else: - output = f"({name} {smtlib}) {output}" + output = f"(let (({name} {smtlib})) {output})" - if output: - output = f"(let ({output}) {main_smtlib_str})" - else: - output = main_smtlib_str - return output + return f"{output}" def declarations(self): result = "" diff --git a/manticore/platforms/platform.py b/manticore/platforms/platform.py index ba736a144..da3fb8754 100644 --- a/manticore/platforms/platform.py +++ b/manticore/platforms/platform.py @@ -51,7 +51,6 @@ class Platform(Eventful): _published_events = {"solve"} - def __init__(self, *, state: Optional["StateBase"] = None, **kwargs): self._state = state super().__init__(**kwargs) diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index 09efd16ad..ff7a888cc 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -951,7 +951,6 @@ def test_visitors(self): self.assertEqual(get_depth(aux), 9) self.maxDiff = 5500 - print(translate_to_smtlib(aux)) self.assertEqual( translate_to_smtlib(aux, use_bindings=False), "(select (store (store (store MEM #x00000000 #x61) #x00000001 #x62) #x00000003 (select (store (store MEM #x00000000 #x61) #x00000001 #x62) (bvadd VAR #x00000001))) (bvadd VAR ((_ zero_extend 24) (select (store (store (store MEM #x00000000 #x61) #x00000001 #x62) #x00000003 (select (store (store MEM #x00000000 #x61) #x00000001 #x62) (bvadd VAR #x00000001))) VAR))))", @@ -959,7 +958,7 @@ def test_visitors(self): self.assertEqual( translate_to_smtlib(aux, use_bindings=True), - "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62)) (!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001)))) ) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR)))))", + "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62))) (let ((!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001))))) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR))))))" ) values = arr[0:2] From 6ff1e8774eff4264f6b6c266c833a7d1f0cf2d96 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 19 Jan 2021 10:27:29 -0500 Subject: [PATCH 116/126] Ek testing dev expressions (#2388) * Black formatting * Native tests pass * Passes checked in (non-generated) wasm tests * Some ethereum test fixes * Reenable wasm tests * Fix mypy error * Use pytest timeout * mypy checked untyped defs in manticore.core.smtlib --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + examples/evm/asm_to_smtlib.py | 2 +- manticore/core/manticore.py | 1 + manticore/core/smtlib/constraints.py | 1 + manticore/core/smtlib/expression.py | 144 ++++---- manticore/core/smtlib/operators.py | 4 +- manticore/core/smtlib/solver.py | 17 +- manticore/core/smtlib/visitors.py | 70 ++-- manticore/core/state.py | 50 ++- manticore/core/workspace.py | 15 +- manticore/ethereum/manticore.py | 2 +- manticore/ethereum/parsetab.py | 475 +++++++++++++++++++++++++-- manticore/native/manticore.py | 2 +- manticore/native/state.py | 15 +- manticore/platforms/evm.py | 47 ++- manticore/platforms/platform.py | 6 +- manticore/platforms/wasm.py | 10 +- manticore/wasm/manticore.py | 2 +- mypy.ini | 6 + scripts/run_tests.sh | 2 +- setup.py | 10 +- tests/ethereum/test_detectors.py | 2 +- tests/ethereum/test_general.py | 33 +- tests/ethereum/test_regressions.py | 4 +- tests/native/test_armv7unicorn.py | 2 +- tests/native/test_models.py | 2 +- tests/native/test_state.py | 15 +- tests/native/test_workspace.py | 2 +- tests/other/test_smtlibv2.py | 2 +- 30 files changed, 736 insertions(+), 210 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e56cc640..6cabe4eac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: strategy: fail-fast: false matrix: - type: ["ethereum_truffle", "ethereum_bench", "examples", "ethereum", "ethereum_vm", "native", "other"] # "wasm", "wasm_sym", + type: ["ethereum_truffle", "ethereum_bench", "examples", "ethereum", "ethereum_vm", "native", "other", "wasm", "wasm_sym"] steps: - uses: actions/checkout@v1 - name: Set up Python 3.6 diff --git a/.gitignore b/.gitignore index 98ddc3600..01c2fbe8c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ nosetests.xml coverage.xml *,cover .hypothesis/ +mcore_* # Translations *.mo diff --git a/examples/evm/asm_to_smtlib.py b/examples/evm/asm_to_smtlib.py index c49df8cd9..9e6b94704 100644 --- a/examples/evm/asm_to_smtlib.py +++ b/examples/evm/asm_to_smtlib.py @@ -117,7 +117,7 @@ def send_funds(self, address, recipient, value): callbacks = Callbacks() # evm = world.current_vm -evm = EVM(constraints, 0x41424344454647484950, data, caller, value, code, world=world, gas=1000000) +evm = EVM(0x41424344454647484950, data, caller, value, code, world=world, gas=1000000) evm.subscribe("will_execute_instruction", callbacks.will_execute_instruction) print("CODE:") diff --git a/manticore/core/manticore.py b/manticore/core/manticore.py index 4064545aa..1650b66e3 100644 --- a/manticore/core/manticore.py +++ b/manticore/core/manticore.py @@ -59,6 +59,7 @@ description="The seed to use when randomly selecting states", ) + class ManticoreBase(Eventful): _published_events = {"solve"} diff --git a/manticore/core/smtlib/constraints.py b/manticore/core/smtlib/constraints.py index c80ad4fbe..3525be5b5 100644 --- a/manticore/core/smtlib/constraints.py +++ b/manticore/core/smtlib/constraints.py @@ -82,6 +82,7 @@ def __enter__(self) -> "ConstraintSet": return self._child def __exit__(self, ty, value, traceback) -> None: + assert self._child is not None self._child._parent = None self._child = None diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index 63d41608b..c1aa78cf3 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -21,7 +21,7 @@ import uuid import re import copy -from typing import overload, Union, Optional, Tuple, List, FrozenSet +from typing import overload, Union, Optional, Tuple, List, FrozenSet, Dict, Any, Set from functools import lru_cache @@ -83,13 +83,11 @@ def __new__(cls, clsname, bases, attrs, abstract=False): # merge the xslots of all the bases with the one defined here for base in bases: xslots = xslots.union(getattr(base, "__xslots__", ())) - attrs["__xslots__"]: Tuple[str] = tuple(xslots) + attrs["__xslots__"] = tuple(xslots) if abstract: attrs["__slots__"] = tuple() else: - attrs["__slots__"]: Tuple[str] = tuple( - map(lambda attr: attr.split("#", 1)[0], attrs["__xslots__"]) - ) + attrs["__slots__"] = tuple(map(lambda attr: attr.split("#", 1)[0], attrs["__xslots__"])) """ def h(self): print(self.__class__, self.__slots__) @@ -140,9 +138,9 @@ def taint(self) -> FrozenSet[str]: return self._taint @property - def operands(self) -> Tuple["Expression"]: + def operands(self) -> Optional[Tuple["Expression"]]: """ Hack so we can use any Expression as a node """ - return () + ... def __getstate__(self): return {attr: getattr(self, attr) for attr in self.__slots__} @@ -183,7 +181,7 @@ class Constant(Expression, abstract=True): __xslots__: Tuple[str, ...] = ("_value",) - def __init__(self, *, value: Union[bool, int, bytes, List[int]], **kwargs): + def __init__(self, *, value: Union[bool, int], **kwargs): """A constant expression has a value :param value: The constant value @@ -210,9 +208,7 @@ def __init__(self, *, operands: Tuple[Expression, ...], **kwargs): taint = kwargs.get("taint") # If taint was not forced by a keyword argument, calculate default if taint is None: - operands_taints = map(lambda x: x.taint, operands) - taint = reduce(lambda x, y: x.union(y), operands_taints, frozenset()) - kwargs["taint"] = taint + kwargs["taint"] = frozenset({y for x in operands for y in x.taint}) super().__init__(**kwargs) @property @@ -278,16 +274,17 @@ def __init__(self, *, value: bool, **kwargs): super().__init__(value=bool(value), **kwargs) def __bool__(self) -> bool: - return self._value + return bool(self._value) class BoolOperation(Bool, Operation, abstract=True): """ It's an operation that results in a Bool """ + pass - #def __init__(self, *args, **kwargs): + # def __init__(self, *args, **kwargs): # super().__init__(*args, **kwargs) - #def __xbool__(self): + # def __xbool__(self): # # FIXME: TODO: re-think is we want to be this forgiving every use of # # local_simplify looks hacky # simplified = self # local_simplify(self) @@ -572,12 +569,13 @@ def __repr__(self): class BitVecOperation(BitVec, Operation, abstract=True): """ Operations that result in a BitVec """ - def __init__(self, *, operands:Tuple[BitVec, ...], **kwargs): + + def __init__(self, *, operands: Tuple[Expression, ...], **kwargs): super().__init__(operands=operands, **kwargs) class BitVecAdd(BitVecOperation): - def __init__(self, operanda:BitVec, operandb:BitVec, **kwargs): + def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): assert operanda.size == operandb.size super().__init__(size=operanda.size, operands=(operanda, operandb), **kwargs) @@ -677,9 +675,11 @@ class BoolEqual(BoolOperation): @overload def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): ... + @overload def __init__(self, operanda: Bool, operandb: Bool, **kwargs): ... + @overload def __init__(self, operanda: "Array", operandb: "Array", **kwargs): ... @@ -719,7 +719,7 @@ def __init__(self, operanda, operandb, **kwargs): operands=(operanda, operandb), **kwargs ) -Array="Array" + class Array(Expression, abstract=True): """An Array expression is an unmutable mapping from bitvector to bitvector @@ -829,7 +829,7 @@ def cast_value(self, value: Union[BitVec, bytes, int]) -> BitVec: value = int(value) return BitVecConstant(self.value_size, value) - def write(self, offset: Union[BitVec, int], buf:Union["Array", bytes]) -> "Array": + def write(self, offset: Union[BitVec, int], buf: Union["Array", bytes]) -> "Array": """Builds a new unmutable Array instance on top of current array by writing buf at offset""" array = self @@ -837,16 +837,18 @@ def write(self, offset: Union[BitVec, int], buf:Union["Array", bytes]) -> "Array array = array.store(offset + i, value) return array - def read(self, offset:Union[BitVec, int], size: int) -> "Array": + def read(self, offset: int, size: int) -> "Array": """ A projection of the current array. """ return ArraySlice(self, offset=offset, size=size) @overload def __getitem__(self, index: Union[BitVec, int]) -> Union[BitVec, int]: ... + @overload def __getitem__(self, index: slice) -> "Array": ... + def __getitem__(self, index): """__getitem__ allows for pythonic access A = ArrayVariable(index_size=32, value_size=8) @@ -864,11 +866,11 @@ def __iter__(self): yield self[i] @staticmethod - def _compare_buffers(a:"Array", b:"Array") -> Union[Bool, bool]: + def _compare_buffers(a: "Array", b: "Array") -> Bool: """ Builds an expression that represents equality between the two arrays.""" if a.length != b.length: return BoolConstant(value=False) - cond = BoolConstant(value=True) + cond: Bool = BoolConstant(value=True) for i in range(a.length): cond = BoolAnd(cond.cast(a[i] == b[i]), cond) if cond is BoolConstant(value=False): @@ -899,7 +901,7 @@ def _fix_slice(self, index: slice): raise ExpressionError("Size could not be simplified to a constant in a slice operation") return start, stop, size.value - def _concatenate(self, array_a:"Array", array_b:"Array") -> "Array": + def _concatenate(self, array_a: "Array", array_b: "Array") -> "Array": """Build a new array from the concatenation of the operands""" new_arr = ArrayVariable( index_size=self.index_size, @@ -920,7 +922,7 @@ def __radd__(self, other): return self._concatenate(other, self) @lru_cache(maxsize=128, typed=True) - def read_BE(self, address: Union[int, BitVec], size:int) -> Union[BitVec, int]: + def read_BE(self, address: Union[int, BitVec], size: int) -> Union[BitVec, int]: address = self.cast_index(address) bytes = [] for offset in range(size): @@ -928,14 +930,16 @@ def read_BE(self, address: Union[int, BitVec], size:int) -> Union[BitVec, int]: return BitVecConcat(operands=tuple(bytes)) @lru_cache(maxsize=128, typed=True) - def read_LE(self, address: Union[int, BitVec], size:int) -> Union[BitVec, int]: + def read_LE(self, address: Union[int, BitVec], size: int) -> Union[BitVec, int]: address = self.cast_index(address) bytes = [] for offset in range(size): - bytes.append(self.get(address + offset, self._default)) - return BitVecConcat(operands=reversed(bytes)) + bytes.append(self.cast_value(self[address + offset])) + return BitVecConcat(operands=tuple(reversed(bytes))) - def write_BE(self, address:Union[int, BitVec], value:Union[int, BitVec], size:int) -> Array: + def write_BE( + self, address: Union[int, BitVec], value: Union[int, BitVec], size: int + ) -> "Array": address = self.cast_index(address) value = BitVecConstant(size=size * self.value_size, value=0).cast(value) array = self @@ -946,7 +950,9 @@ def write_BE(self, address:Union[int, BitVec], value:Union[int, BitVec], size:in ) return array - def write_LE(self, address:Union[int, BitVec], value:Union[int, BitVec], size:int) -> Array: + def write_LE( + self, address: Union[int, BitVec], value: Union[int, BitVec], size: int + ) -> "Array": address = self.cast_index(address) value = BitVec(size * self.value_size).cast(value) array = self @@ -962,7 +968,11 @@ class ArrayConstant(Array, Constant): __xslots__: Tuple[str, ...] = ("_index_size", "_value_size") def __init__( - self, *, index_size: int, value_size: int, **kwargs, + self, + *, + index_size: int, + value_size: int, + **kwargs, ): self._index_size = index_size self._value_size = value_size @@ -995,7 +1005,7 @@ def _select(self, index): ) # Index being symbolic generates a symbolic result ! - result = BitVecConstant(size=self.value_size, value=0, taint=("out_of_bounds")) + result: BitVec = BitVecConstant(size=self.value_size, value=0, taint=("out_of_bounds")) for i, c in enumerate(self.value): result = BitVecITE( index == i, BitVecConstant(size=self.value_size, value=c), result, taint=self.taint @@ -1036,7 +1046,8 @@ def length(self): return self._length def __init__( - self, *, + self, + *, index_size: int, value_size: int, length: Optional[int] = None, @@ -1153,11 +1164,26 @@ class ArrayStore(ArrayOperation): "_default#v", ) + def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): + assert index.size == array.index_size + assert value.size == array.value_size + self._written: Optional[Set[Any]] = None # Cache of the known indexes + self._concrete_cache: Dict[Any, Any] = dict() + self._length = array.length + self._default = array.default + + # recreate and reuse cache + # if isinstance(index, Constant) and isinstance(array, ArrayStore) and array._concrete_cache is not None: + # self._concrete_cache = dict(array._concrete_cache) + # self._concrete_cache[index.value] = value + + super().__init__( + operands=(array, index, value), + **kwargs, + ) + @property def concrete_cache(self): - if self._concrete_cache is not None: - return self._concrete_cache - self._concrete_cache = {} for index, value in get_items(self): if not isinstance(index, Constant): break @@ -1169,7 +1195,7 @@ def concrete_cache(self): def written(self): # Calculate only first time # This can have repeated and reused written indexes. - if self._written is None: + if not self._written: self._written = {offset for offset, _ in get_items(self)} return self._written @@ -1177,7 +1203,7 @@ def is_known(self, index): if isinstance(index, Constant) and index.value in self.concrete_cache: return BoolConstant(value=True) - is_known_index = BoolConstant(value=False) + is_known_index: Bool = BoolConstant(value=False) written = self.written for known_index in written: if isinstance(index, Constant) and isinstance(known_index, Constant): @@ -1186,23 +1212,6 @@ def is_known(self, index): is_known_index = BoolOr(is_known_index.cast(index == known_index), is_known_index) return is_known_index - def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): - assert index.size == array.index_size - assert value.size == array.value_size - self._written = None # Cache of the known indexes - self._concrete_cache = None - self._length = array.length - self._default = array.default - - # recreate and reuse cache - # if isinstance(index, Constant) and isinstance(array, ArrayStore) and array._concrete_cache is not None: - # self._concrete_cache = dict(array._concrete_cache) - # self._concrete_cache[index.value] = value - - super().__init__( - operands=(array, index, value), **kwargs, - ) - @property def length(self): return self._length @@ -1259,7 +1268,9 @@ def select(self, index): return ArraySelect(self, index) # if a default is defined we need to check if the index was previously written - return local_simplify(BitVecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default))) + return local_simplify( + BitVecITE(self.is_known(index), ArraySelect(self, index), self.cast_value(default)) + ) def store(self, index, value): casted = self.cast_index(index) @@ -1283,7 +1294,8 @@ def __init__(self, array: "Array", offset: int, size: int, **kwargs): raise ValueError("Array expected") super().__init__( - operands=(array, array.cast_index(offset), array.cast_index(size)), **kwargs, + operands=(array, array.cast_index(offset), array.cast_index(size)), + **kwargs, ) @property @@ -1320,7 +1332,9 @@ def select(self, index): def store(self, index, value): return ArraySlice( - self.array.store(index + self.offset, value), offset=self.offset, size=len(self), + self.array.store(index + self.offset, value), + offset=self.offset, + size=len(self), ) @property @@ -1359,11 +1373,19 @@ def __hash__(self): @property def underlying_variable(self): - return self._array.underlying_variable + if isinstance(self._array, ArrayVariable): + return self._array.underlying_variable + # NOTE: What to do here? + assert False + return self._array @property def name(self): - return self._array.name + if isinstance(self._array, ArrayVariable): + return self._array.name + # NOTE: What to do here? + assert False + return None @property def array(self): @@ -1525,7 +1547,11 @@ class BitVecITE(BitVecOperation): __xslots__ = BitVecOperation.__xslots__ def __init__( - self, condition: Bool, true_value: BitVec, false_value: BitVec, **kwargs, + self, + condition: Bool, + true_value: BitVec, + false_value: BitVec, + **kwargs, ): super().__init__( diff --git a/manticore/core/smtlib/operators.py b/manticore/core/smtlib/operators.py index 5f381e23d..3402dcb37 100644 --- a/manticore/core/smtlib/operators.py +++ b/manticore/core/smtlib/operators.py @@ -184,10 +184,10 @@ def ITE(cond, true_value, false_value): return false_value if isinstance(true_value, bool): - true_value = BoolConstant(true_value) + true_value = BoolConstant(value=true_value) if isinstance(false_value, bool): - false_value = BoolConstant(false_value) + false_value = BoolConstant(value=false_value) return BoolITE(cond, true_value, false_value) diff --git a/manticore/core/smtlib/solver.py b/manticore/core/smtlib/solver.py index 11be3d572..a49df3cee 100644 --- a/manticore/core/smtlib/solver.py +++ b/manticore/core/smtlib/solver.py @@ -22,7 +22,7 @@ import shlex import time from functools import lru_cache -from typing import Dict, Tuple, Sequence, Optional +from typing import Dict, Tuple, Sequence, Optional, NamedTuple from subprocess import PIPE, Popen, check_output import re from . import operators as Operators @@ -163,7 +163,10 @@ def minmax(self, constraints, x, iters=10000): return x, x -Version = collections.namedtuple("Version", "major minor patch") +class Version(NamedTuple): + major: int + minor: int + path: int class SmtlibProc: @@ -214,7 +217,7 @@ def __readline_and_count(self): assert self._proc.stdout buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error - #print (">",buf) + # print (">",buf) if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") @@ -227,7 +230,7 @@ def send(self, cmd: str) -> None: :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ - #print ("<",cmd) + # print ("<",cmd) if self._debug: logger.debug(">%s", cmd) self._proc.stdout.flush() # type: ignore @@ -645,7 +648,7 @@ def get_value(self, constraints: ConstraintSet, *expressions): #################### values = [] start = time.time() - #with constraints.related_to(*expressions) as temp_cs: + # with constraints.related_to(*expressions) as temp_cs: with constraints as temp_cs: for expression in expressions: bucket = self._cache.setdefault(hash(constraints), {}) @@ -773,8 +776,8 @@ def _solver_version(self) -> Version: Anticipated version_cmd_output format: 'Z3 version 4.4.2' 'Z3 version 4.4.5 - 64 bit - build hashcode $Z3GITHASH' """ + received_version = check_output([f"{consts.z3_bin}", "--version"]) try: - received_version = check_output([f"{consts.z3_bin}", "--version"]) Z3VERSION = re.compile( r".*(?P([0-9]+))\.(?P([0-9]+))\.(?P([0-9]+)).*" ) @@ -787,7 +790,7 @@ def _solver_version(self) -> Version: logger.warning( f"Could not parse Z3 version: '{str(received_version)}'. Assuming compatibility." ) - parsed_version = Version(float("inf"), float("inf"), float("inf")) + parsed_version = Version(sys.maxsize, sys.maxsize, sys.maxsize) return parsed_version diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 19b7bdb00..be8865896 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict +from typing import Optional, Dict, Set, Type, TYPE_CHECKING from ...utils.helpers import CacheDict from ...exceptions import SmtlibError @@ -10,6 +10,9 @@ import math from decimal import Decimal +if TYPE_CHECKING: + from . import ConstraintException + logger = logging.getLogger(__name__) @@ -55,7 +58,7 @@ def result(self): assert len(self._stack) == 1 return self._stack[-1] - def visit(self, node, use_fixed_point=False): + def visit(self, node: Expression, use_fixed_point: bool = False): assert isinstance(node, Expression) """ The entry point of the visitor. @@ -71,7 +74,7 @@ def visit(self, node, use_fixed_point=False): :param use_fixed_point: if True, it runs _methods until a fixed point is found """ cache = self._cache - visited = set() + visited: Set[Expression] = set() local_stack = [node] # initially the stack contains only the visiting node while local_stack: node = local_stack.pop() @@ -81,15 +84,18 @@ def visit(self, node, use_fixed_point=False): if node in visited: visited.remove(node) # Visited! Then there is a visited version of the operands in the stack - operands = (self.pop() for _ in range(len(node.operands))) - # Actually process the node + operands: Tuple[Expression, ...] = tuple([]) + if node.operands: + operands = tuple([self.pop() for _ in range(len(node.operands))]) + # Actually process the node value = self._method(node, *operands) self.push(value) cache[node] = value else: visited.add(node) local_stack.append(node) - local_stack.extend(node.operands) + if node.operands: + local_stack.extend(node.operands) # Repeat until the result is not changed if use_fixed_point: @@ -101,7 +107,7 @@ def visit(self, node, use_fixed_point=False): new_value = self.pop() self.push(new_value) - def _method(self, expression, *operands): + def _method(self, expression: Expression, *operands): """ Magic method to walk the mro looking for the first overloaded visiting method that returns something. @@ -124,8 +130,8 @@ def _method(self, expression, *operands): return value return self._rebuild(expression, operands) - def _changed(self, expression: Expression, operands): - changed = any(x is not y for x, y in zip(expression.operands, operands)) + def _changed(self, expression: Expression, operands: Optional[Tuple[Expression, ...]]): + changed = any(x is not y for x, y in zip(expression.operands or (), operands or ())) return changed @lru_cache(maxsize=32, typed=True) @@ -332,7 +338,7 @@ def visit_BoolGreaterOrEqual(self, expression, *operands) -> Optional[BoolConsta if all(isinstance(o, Constant) for o in operands): a = operands[0].signed_value b = operands[1].signed_value - return BoolConstant(a >= b, taint=expression.taint) + return BoolConstant(value=a >= b, taint=expression.taint) return None def visit_BitVecDiv(self, expression, *operands) -> Optional[BitVecConstant]: @@ -385,11 +391,11 @@ def visit_BoolAnd(self, expression, a, b): if isinstance(b, Constant) and b.value == True: return a - def visit_Operation(self, expression, *operands): + def visit_Operation(self, expression: Expression, *operands): """ constant folding, if all operands of an expression are a Constant do the math """ operation = self.operations.get(type(expression), None) if operation is not None and all(isinstance(o, Constant) for o in operands): - value = operation(*(x.value for x in operands)) + value = operation(*(x.value for x in operands)) # type: ignore if isinstance(expression, BitVec): return BitVecConstant(expression.size, value, taint=expression.taint) else: @@ -567,8 +573,8 @@ def visit_BitVecConcat(self, expression, *operands): op = operands[0] value = None - end = None - begining = None + end: Optional[int] = None + begining: Optional[int] = None for o in operands: # If found a non BitVecExtract, do not apply if not isinstance(o, BitVecExtract): @@ -591,7 +597,7 @@ def visit_BitVecConcat(self, expression, *operands): # update begining variable begining = o.begining - if value is not None: + if value is not None and end is not None and begining is not None: if end + 1 != value.size or begining != 0: return BitVecExtract(value, begining, end - begining + 1, taint=expression.taint) @@ -612,8 +618,9 @@ def visit_BitVecExtract(self, expression, *operands): elif isinstance(op, BitVecExtract): return BitVecExtract(op.value, op.begining + begining, size, taint=expression.taint) elif isinstance(op, BitVecConcat): - new_operands = [] + new_operands: List[BitVec] = [] for item in reversed(op.operands): + assert isinstance(item, BitVec) if size == 0: assert expression.size == sum([x.size for x in new_operands]) return BitVecConcat( @@ -660,7 +667,6 @@ def visit_BitVecMul(self, expression, *operands): if left.value == 0: return left - def visit_BitVecAdd(self, expression, *operands): """a + 0 ==> a 0 + a ==> a @@ -799,7 +805,10 @@ def visit_Expression(self, expression, *operands): assert not isinstance(expression, Operation) return expression + import time + + @lru_cache(maxsize=128, typed=True) def arithmetic_simplify(expression): start = time.time() @@ -823,9 +832,9 @@ def to_constant(expression): if isinstance(value, Constant): return value.value elif isinstance(value, Array): - if expression.index_max: + if expression.length: ba = bytearray() - for i in range(expression.index_max): + for i in range(expression.length): value_i = simplify(value[i]) if not isinstance(value_i, Constant): break @@ -867,7 +876,7 @@ def __init__(self, use_bindings=True, *args, **kw): self._unique = 0 self.use_bindings = use_bindings self._bindings_cache_exp = {} - self._bindings = {} + self._bindings: Dict[str, Any] = {} self._variables = set() def _add_binding(self, expression, smtlib): @@ -889,7 +898,7 @@ def _add_binding(self, expression, smtlib): def bindings(self): return self._bindings - translation_table = { + translation_table: Dict[Type[Operation], str] = { BoolNot: "not", BoolEqual: "=", BoolAnd: "and", @@ -956,8 +965,10 @@ def visit_Operation(self, expression, *operands): elif isinstance(expression, BitVecExtract): operation = operation % (expression.end, expression.begining) - operands = [self._add_binding(*x) for x in zip(expression.operands, operands)] - return f"({operation} {' '.join(operands)})" + bound_operands: List[str] = [ + self._add_binding(*x) for x in zip(expression.operands, operands) + ] + return f"({operation} {' '.join(bound_operands)})" @property def result(self): @@ -968,7 +979,7 @@ def apply_bindings(self, main_smtlib_str): # Well-sortedness requirements from toposort import toposort_flatten as toposort - G = {} + G: Dict[Any, Any] = {} for name, (expr, smtlib) in self._bindings.items(): variables = {v.name for v in get_variables(expr)} variables.update(re.findall(r"!a_\d+!", smtlib)) @@ -985,10 +996,9 @@ def apply_bindings(self, main_smtlib_str): continue expr, smtlib = self._bindings[name] - # FIXME: too much string manipulation. Search occurrences in the Expression realm if output.count(name) <= 1: - #output = f"let (({name} {smtlib})) ({output})" + # output = f"let (({name} {smtlib})) ({output})" output = output.replace(name, smtlib) else: output = f"(let (({name} {smtlib})) {output})" @@ -998,11 +1008,11 @@ def apply_bindings(self, main_smtlib_str): def declarations(self): result = "" for exp in self._variables: - if isinstance(exp, BitVec): + if isinstance(exp, BitVecVariable): result += f"(declare-fun {exp.name} () (_ BitVec {exp.size}))\n" - elif isinstance(exp, Bool): + elif isinstance(exp, BoolVariable): result += f"(declare-fun {exp.name} () Bool)\n" - elif isinstance(exp, Array): + elif isinstance(exp, ArrayVariable): result += f"(declare-fun {exp.name} () (Array (_ BitVec {exp.index_size}) (_ BitVec {exp.value_size})))\n" else: raise ConstraintException(f"Type not supported {exp!r}") @@ -1014,12 +1024,14 @@ def smtlib(self): result += f"(assert {self.apply_bindings(constraint_str)})\n" return result + @lru_cache(maxsize=128, typed=True) def _translate_to_smtlib(expression, use_bindings=True, **kwargs): translator = TranslatorSmtlib(use_bindings=use_bindings, **kwargs) translator.visit(expression) return translator.result + def translate_to_smtlib(expression, use_bindings=True, **kwargs): if isinstance(expression, MutableArray): expression = expression.array diff --git a/manticore/core/state.py b/manticore/core/state.py index f182096a3..efc1c0d4d 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,13 +1,17 @@ import copy import logging -from typing import Optional +from typing import Any, Dict, List, Optional, TYPE_CHECKING -from .smtlib import Bool, issymbolic, BitVecConstant, MutableArray +from .smtlib import Bool, ConstraintSet, Expression, issymbolic, BitVecConstant, MutableArray from ..utils.event import Eventful from ..utils.helpers import PickleSerializer from ..utils import config from .plugin import StateDescriptor +if TYPE_CHECKING: + from .manticore import ManticoreBase + from ..platforms.platform import Platform + consts = config.get_group("core") consts.add( "execs_per_intermittent_cb", @@ -175,17 +179,22 @@ class StateBase(Eventful): _published_events = {"execution_intermittent"} - def __init__(self, *, constraints: "Constraints", platform: "Platform", manticore: Optional["ManticoreBase"] = None, **kwargs): + def __init__( + self, + *, + constraints: ConstraintSet, + platform: "Platform", + manticore: Optional["ManticoreBase"] = None, + **kwargs, + ): super().__init__(**kwargs) self._manticore = manticore self._platform = platform self._constraints = constraints - self._platform._constraints = constraints - self._platform.state=self - self._input_symbols = list() + self._input_symbols: List[Expression] = list() self._child = None - self._context = dict() + self._context: Dict[str, Any] = dict() self._terminated_by = None self._solver = EventSolver() @@ -194,7 +203,7 @@ def __init__(self, *, constraints: "Constraints", platform: "Platform", manticor # 33 # Events are lost in serialization and fork !! self.forward_events_from(self._solver) - platform.set_state(self) + self._platform.set_state(self) def __getstate__(self): state = super().__getstate__() @@ -202,6 +211,7 @@ def __getstate__(self): state["constraints"] = self._constraints state["child"] = self._child state["context"] = self._context + state["input_symbols"] = self._input_symbols state["terminated_by"] = self._terminated_by state["exec_counter"] = self._total_exec @@ -210,11 +220,11 @@ def __getstate__(self): def __setstate__(self, state): super().__setstate__(state) self._platform = state["platform"] - self._platform.state=self self._constraints = state["constraints"] self._child = state["child"] self._context = state["context"] + self._input_symbols = state["input_symbols"] self._manticore = None self._terminated_by = state["terminated_by"] @@ -224,7 +234,7 @@ def __setstate__(self, state): # 33 # Events are lost in serialization and fork !! self.forward_events_from(self._solver) - self.platform.set_state(self) + self._platform.set_state(self) @property def id(self): @@ -238,17 +248,23 @@ def __repr__(self): def __enter__(self): assert self._child is None self._platform._constraints = None - new_state = self.__class__(constraints=self._constraints.__enter__(), platform=self._platform, manticore=self._manticore) - #Keep the same constraint + new_state = self.__class__( + constraints=self._constraints.__enter__(), + platform=self._platform, + manticore=self._manticore, + ) + # Keep the same constraint self.platform._constraints = new_state.constraints - #backup copy of the context + # backup copy of the context new_state._context = copy.copy(self._context) + new_state._input_symbols = self._input_symbols new_state._id = None new_state._total_exec = self._total_exec self.copy_eventful_state(new_state) self._child = new_state assert new_state.platform.constraints is new_state.constraints + # assert self.platform.constraints is self.constraints return new_state @@ -330,7 +346,11 @@ def new_symbolic_buffer(self, nbytes, **options): avoid_collisions = True taint = options.get("taint", frozenset()) expr = self._constraints.new_array( - name=label, length=nbytes, value_size=8, taint=taint, avoid_collisions=avoid_collisions, + name=label, + length=nbytes, + value_size=8, + taint=taint, + avoid_collisions=avoid_collisions, ) self._input_symbols.append(expr) @@ -371,7 +391,7 @@ def concretize(self, symbolic, policy, maxcount=7): than `maxcount` feasible solutions, some states will be silently ignored.** """ - assert self.constraints == self.platform.constraints + # assert self.constraints is self.platform.constraints symbolic = self.migrate_expression(symbolic) vals = [] diff --git a/manticore/core/workspace.py b/manticore/core/workspace.py index b2215ee2c..9b269374a 100644 --- a/manticore/core/workspace.py +++ b/manticore/core/workspace.py @@ -602,17 +602,11 @@ def save_testcase(self, state: StateBase, testcase: Testcase, message: str = "") # The workspaces should override `save_testcase` method # # Below is native-only - def save_input_symbols(testcase, state: StateBase): - with testcase.open_stream("input") as f: - for symbol in state.input_symbols: - # TODO - constrain=False here, so the extra migration shouldn't cause problems, right? - buf = state.solve_one(symbol) - f.write(f"{symbol.name}: {buf!r}\n") self.save_summary(testcase, state, message) self.save_trace(testcase, state) self.save_constraints(testcase, state) - save_input_symbols(testcase, state) + self.save_input_symbols(testcase, state) for stream_name, data in state.platform.generate_workspace_files().items(): with testcase.open_stream(stream_name, binary=True) as stream: @@ -662,3 +656,10 @@ def save_constraints(testcase, state: StateBase): with testcase.open_stream("smt") as f: f.write(str(state.constraints)) + @staticmethod + def save_input_symbols(testcase, state: StateBase): + with testcase.open_stream("input") as f: + for symbol in state.input_symbols: + # TODO - constrain=False here, so the extra migration shouldn't cause problems, right? + buf = state.solve_one(symbol) + f.write(f"{symbol.name}: {buf!r}\n") diff --git a/manticore/ethereum/manticore.py b/manticore/ethereum/manticore.py index 59178211a..494184a5a 100644 --- a/manticore/ethereum/manticore.py +++ b/manticore/ethereum/manticore.py @@ -1014,7 +1014,7 @@ def preconstraint_for_call_transaction( selectors = contract_metadata.function_selectors if not selectors or len(data) <= 4: - return BoolConstant(True) + return BoolConstant(value=True) symbolic_selector = data[:4] diff --git a/manticore/ethereum/parsetab.py b/manticore/ethereum/parsetab.py index 3ef2495e9..3faba9d00 100644 --- a/manticore/ethereum/parsetab.py +++ b/manticore/ethereum/parsetab.py @@ -1,50 +1,457 @@ - # parsetab.py # This file is automatically generated. Do not edit. # pylint: disable=W,C,R -_tabversion = '3.10' +_tabversion = "3.10" + +_lr_method = "LALR" -_lr_method = 'LALR' +_lr_signature = "ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n " -_lr_signature = 'ADDRESS BOOL BYTES BYTESM COMMA FIXED FIXEDMN FUNCTION INT INTN LBRAKET LPAREN NUMBER RBRAKET RPAREN STRING UFIXED UFIXEDMN UINT UINTN\n T : UINTN\n T : UINT\n T : INTN\n T : INT\n T : ADDRESS\n T : BOOL\n T : FIXEDMN\n T : UFIXEDMN\n T : FIXED\n T : UFIXED\n T : BYTESM\n T : FUNCTION\n T : BYTES\n T : STRING\n\n \n TL : T\n \n TL : T COMMA TL\n \n T : LPAREN TL RPAREN\n \n T : LPAREN RPAREN\n \n T : T LBRAKET RBRAKET\n \n T : T LBRAKET NUMBER RBRAKET\n ' - -_lr_action_items = {'UINTN':([0,16,24,],[2,2,2,]),'UINT':([0,16,24,],[3,3,3,]),'INTN':([0,16,24,],[4,4,4,]),'INT':([0,16,24,],[5,5,5,]),'ADDRESS':([0,16,24,],[6,6,6,]),'BOOL':([0,16,24,],[7,7,7,]),'FIXEDMN':([0,16,24,],[8,8,8,]),'UFIXEDMN':([0,16,24,],[9,9,9,]),'FIXED':([0,16,24,],[10,10,10,]),'UFIXED':([0,16,24,],[11,11,11,]),'BYTESM':([0,16,24,],[12,12,12,]),'FUNCTION':([0,16,24,],[13,13,13,]),'BYTES':([0,16,24,],[14,14,14,]),'STRING':([0,16,24,],[15,15,15,]),'LPAREN':([0,16,24,],[16,16,16,]),'$end':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,21,23,25,],[0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,-19,-17,-20,]),'LBRAKET':([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[17,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,17,-19,-17,-20,]),'COMMA':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,19,20,21,23,25,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-18,24,-19,-17,-20,]),'RPAREN':([2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,18,19,20,21,23,25,26,],[-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,19,23,-18,-15,-19,-17,-20,-16,]),'RBRAKET':([17,22,],[21,25,]),'NUMBER':([17,],[22,]),} +_lr_action_items = { + "UINTN": ( + [ + 0, + 16, + 24, + ], + [ + 2, + 2, + 2, + ], + ), + "UINT": ( + [ + 0, + 16, + 24, + ], + [ + 3, + 3, + 3, + ], + ), + "INTN": ( + [ + 0, + 16, + 24, + ], + [ + 4, + 4, + 4, + ], + ), + "INT": ( + [ + 0, + 16, + 24, + ], + [ + 5, + 5, + 5, + ], + ), + "ADDRESS": ( + [ + 0, + 16, + 24, + ], + [ + 6, + 6, + 6, + ], + ), + "BOOL": ( + [ + 0, + 16, + 24, + ], + [ + 7, + 7, + 7, + ], + ), + "FIXEDMN": ( + [ + 0, + 16, + 24, + ], + [ + 8, + 8, + 8, + ], + ), + "UFIXEDMN": ( + [ + 0, + 16, + 24, + ], + [ + 9, + 9, + 9, + ], + ), + "FIXED": ( + [ + 0, + 16, + 24, + ], + [ + 10, + 10, + 10, + ], + ), + "UFIXED": ( + [ + 0, + 16, + 24, + ], + [ + 11, + 11, + 11, + ], + ), + "BYTESM": ( + [ + 0, + 16, + 24, + ], + [ + 12, + 12, + 12, + ], + ), + "FUNCTION": ( + [ + 0, + 16, + 24, + ], + [ + 13, + 13, + 13, + ], + ), + "BYTES": ( + [ + 0, + 16, + 24, + ], + [ + 14, + 14, + 14, + ], + ), + "STRING": ( + [ + 0, + 16, + 24, + ], + [ + 15, + 15, + 15, + ], + ), + "LPAREN": ( + [ + 0, + 16, + 24, + ], + [ + 16, + 16, + 16, + ], + ), + "$end": ( + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 19, + 21, + 23, + 25, + ], + [ + 0, + -1, + -2, + -3, + -4, + -5, + -6, + -7, + -8, + -9, + -10, + -11, + -12, + -13, + -14, + -18, + -19, + -17, + -20, + ], + ), + "LBRAKET": ( + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 19, + 20, + 21, + 23, + 25, + ], + [ + 17, + -1, + -2, + -3, + -4, + -5, + -6, + -7, + -8, + -9, + -10, + -11, + -12, + -13, + -14, + -18, + 17, + -19, + -17, + -20, + ], + ), + "COMMA": ( + [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 19, + 20, + 21, + 23, + 25, + ], + [ + -1, + -2, + -3, + -4, + -5, + -6, + -7, + -8, + -9, + -10, + -11, + -12, + -13, + -14, + -18, + 24, + -19, + -17, + -20, + ], + ), + "RPAREN": ( + [ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 18, + 19, + 20, + 21, + 23, + 25, + 26, + ], + [ + -1, + -2, + -3, + -4, + -5, + -6, + -7, + -8, + -9, + -10, + -11, + -12, + -13, + -14, + 19, + 23, + -18, + -15, + -19, + -17, + -20, + -16, + ], + ), + "RBRAKET": ( + [ + 17, + 22, + ], + [ + 21, + 25, + ], + ), + "NUMBER": ( + [ + 17, + ], + [ + 22, + ], + ), +} _lr_action = {} for _k, _v in _lr_action_items.items(): - for _x,_y in zip(_v[0],_v[1]): - if not _x in _lr_action: _lr_action[_x] = {} - _lr_action[_x][_k] = _y + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_action: + _lr_action[_x] = {} + _lr_action[_x][_k] = _y del _lr_action_items -_lr_goto_items = {'T':([0,16,24,],[1,20,20,]),'TL':([16,24,],[18,26,]),} +_lr_goto_items = { + "T": ( + [ + 0, + 16, + 24, + ], + [ + 1, + 20, + 20, + ], + ), + "TL": ( + [ + 16, + 24, + ], + [ + 18, + 26, + ], + ), +} _lr_goto = {} for _k, _v in _lr_goto_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_goto: _lr_goto[_x] = {} - _lr_goto[_x][_k] = _y + for _x, _y in zip(_v[0], _v[1]): + if not _x in _lr_goto: + _lr_goto[_x] = {} + _lr_goto[_x][_k] = _y del _lr_goto_items _lr_productions = [ - ("S' -> T","S'",1,None,None,None), - ('T -> UINTN','T',1,'p_basic_type','abitypes.py',154), - ('T -> UINT','T',1,'p_basic_type','abitypes.py',155), - ('T -> INTN','T',1,'p_basic_type','abitypes.py',156), - ('T -> INT','T',1,'p_basic_type','abitypes.py',157), - ('T -> ADDRESS','T',1,'p_basic_type','abitypes.py',158), - ('T -> BOOL','T',1,'p_basic_type','abitypes.py',159), - ('T -> FIXEDMN','T',1,'p_basic_type','abitypes.py',160), - ('T -> UFIXEDMN','T',1,'p_basic_type','abitypes.py',161), - ('T -> FIXED','T',1,'p_basic_type','abitypes.py',162), - ('T -> UFIXED','T',1,'p_basic_type','abitypes.py',163), - ('T -> BYTESM','T',1,'p_basic_type','abitypes.py',164), - ('T -> FUNCTION','T',1,'p_basic_type','abitypes.py',165), - ('T -> BYTES','T',1,'p_basic_type','abitypes.py',166), - ('T -> STRING','T',1,'p_basic_type','abitypes.py',167), - ('TL -> T','TL',1,'p_type_list_one','abitypes.py',175), - ('TL -> T COMMA TL','TL',3,'p_type_list','abitypes.py',182), - ('T -> LPAREN TL RPAREN','T',3,'p_tuple','abitypes.py',189), - ('T -> LPAREN RPAREN','T',2,'p_tuple_empty','abitypes.py',196), - ('T -> T LBRAKET RBRAKET','T',3,'p_dynamic_type','abitypes.py',203), - ('T -> T LBRAKET NUMBER RBRAKET','T',4,'p_dynamic_fixed_type','abitypes.py',212), + ("S' -> T", "S'", 1, None, None, None), + ("T -> UINTN", "T", 1, "p_basic_type", "abitypes.py", 154), + ("T -> UINT", "T", 1, "p_basic_type", "abitypes.py", 155), + ("T -> INTN", "T", 1, "p_basic_type", "abitypes.py", 156), + ("T -> INT", "T", 1, "p_basic_type", "abitypes.py", 157), + ("T -> ADDRESS", "T", 1, "p_basic_type", "abitypes.py", 158), + ("T -> BOOL", "T", 1, "p_basic_type", "abitypes.py", 159), + ("T -> FIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 160), + ("T -> UFIXEDMN", "T", 1, "p_basic_type", "abitypes.py", 161), + ("T -> FIXED", "T", 1, "p_basic_type", "abitypes.py", 162), + ("T -> UFIXED", "T", 1, "p_basic_type", "abitypes.py", 163), + ("T -> BYTESM", "T", 1, "p_basic_type", "abitypes.py", 164), + ("T -> FUNCTION", "T", 1, "p_basic_type", "abitypes.py", 165), + ("T -> BYTES", "T", 1, "p_basic_type", "abitypes.py", 166), + ("T -> STRING", "T", 1, "p_basic_type", "abitypes.py", 167), + ("TL -> T", "TL", 1, "p_type_list_one", "abitypes.py", 175), + ("TL -> T COMMA TL", "TL", 3, "p_type_list", "abitypes.py", 182), + ("T -> LPAREN TL RPAREN", "T", 3, "p_tuple", "abitypes.py", 189), + ("T -> LPAREN RPAREN", "T", 2, "p_tuple_empty", "abitypes.py", 196), + ("T -> T LBRAKET RBRAKET", "T", 3, "p_dynamic_type", "abitypes.py", 203), + ("T -> T LBRAKET NUMBER RBRAKET", "T", 4, "p_dynamic_fixed_type", "abitypes.py", 212), ] diff --git a/manticore/native/manticore.py b/manticore/native/manticore.py index aa457407c..8891bc593 100644 --- a/manticore/native/manticore.py +++ b/manticore/native/manticore.py @@ -375,7 +375,7 @@ def _make_decree(program, concrete_start="", **kwargs): constraints = ConstraintSet() platform = decree.SDecree(constraints, program) - initial_state = State(constraints, platform) + initial_state = State(constraints=constraints, platform=platform) logger.info("Loading program %s", program) if concrete_start != "": diff --git a/manticore/native/state.py b/manticore/native/state.py index 905460816..00921ef46 100644 --- a/manticore/native/state.py +++ b/manticore/native/state.py @@ -1,13 +1,15 @@ import copy from collections import namedtuple -from typing import Any, Callable, Dict, NamedTuple, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, NamedTuple, Optional, Set, Tuple, Union, TYPE_CHECKING from .cpu.disasm import Instruction from .memory import ConcretizeMemory, MemoryException from .. import issymbolic from ..core.state import StateBase, Concretize, TerminateState -from ..core.smtlib import Expression +from ..core.smtlib import Expression, ConstraintSet +if TYPE_CHECKING: + from ..platforms.platform import Platform HookCallback = Callable[[StateBase], None] @@ -18,22 +20,19 @@ class CheckpointData(NamedTuple): class State(StateBase): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._input_symbols = list() + def __init__(self, *, constraints: ConstraintSet, platform: "Platform", **kwargs): + super().__init__(constraints=constraints, platform=platform, **kwargs) self._hooks: Dict[Optional[int], Set[HookCallback]] = {} self._after_hooks: Dict[Optional[int], Set[HookCallback]] = {} def __getstate__(self) -> Dict[str, Any]: state = super().__getstate__() - state["input_symbols"] = self._input_symbols state["hooks"] = self._hooks state["after_hooks"] = self._after_hooks return state def __setstate__(self, state: Dict[str, Any]) -> None: super().__setstate__(state) - self._input_symbols = state["input_symbols"] self._hooks = state["hooks"] self._after_hooks = state["after_hooks"] self._resub_hooks() @@ -224,7 +223,7 @@ def setstate(state: State, value): raise TerminateState(str(e), testcase=True) # Remove when code gets stable? - assert self.platform.constraints is self.constraints + # assert self.platform.constraints is self.constraints return result diff --git a/manticore/platforms/evm.py b/manticore/platforms/evm.py index 3b978b67b..f74889d13 100644 --- a/manticore/platforms/evm.py +++ b/manticore/platforms/evm.py @@ -135,7 +135,7 @@ def ceil32(x): def to_signed(i): - i &= ((1<<256)-1) + i &= (1 << 256) - 1 return Operators.ITEBV(256, Operators.EXTRACT(i, 255, 1) == 0, i, -((1 << 256) - i)) @@ -716,7 +716,7 @@ def __init__( caller, value, bytecode, - world=None, + world, gas=None, fork=DEFAULT_FORK, **kwargs, @@ -831,7 +831,7 @@ def extend_with_zeroes(b): self._valid_jmpdests = set() self._sha3 = {} self._refund = 0 - self._temp_call_gas = None + self._temp_call_gas = 0 self._failed = False def fail_if(self, failed): @@ -870,7 +870,7 @@ def constraints(self): @constraints.setter def constraints(self, constraints): self._constraints = constraints - # self.memory.constraints = constraints + self.memory.constraints = constraints @property def gas(self): @@ -1087,8 +1087,8 @@ def _pop(self): return self.stack.pop() def _consume(self, fee): - if consts.oog == "ignore": - return + # if consts.oog == "ignore": + # return # Check type and bitvec size if isinstance(fee, int): if fee > (1 << 512) - 1: @@ -1188,8 +1188,8 @@ def _push_results(self, instruction, result): assert result is None def _calculate_gas(self, *arguments): - if consts.oog == "ignore": - return 0 + # if consts.oog == "ignore": + # return 0 start = time.time() current = self.instruction @@ -1267,13 +1267,15 @@ def _check_jmpdest(self): if isinstance(should_check_jumpdest, Constant): should_check_jumpdest = should_check_jumpdest.value elif issymbolic(should_check_jumpdest): - self._publish("will_solve", self.constraints, should_check_jumpdest, "get_all_values") + self._publish( + "will_solve", self.world.constraints, should_check_jumpdest, "get_all_values" + ) should_check_jumpdest_solutions = SelectedSolver.instance().get_all_values( - self.constraints, should_check_jumpdest + self.world.constraints, should_check_jumpdest ) self._publish( "did_solve", - self.constraints, + self.world.constraints, should_check_jumpdest, "get_all_values", should_check_jumpdest_solutions, @@ -1422,6 +1424,7 @@ def setstate(state, value): raise def read_buffer(self, offset, size): + size = simplify(size) if issymbolic(size) and not isinstance(size, Constant): raise EVMException("Symbolic size not supported") if isinstance(size, Constant): @@ -2195,7 +2198,12 @@ def CREATE2(self, endowment, memory_start, memory_length, salt): ) self.world.start_transaction( - "CREATE", address, data=data, caller=self.address, value=value, gas=self.gas, + "CREATE", + address, + data=data, + caller=self.address, + value=value, + gas=self.gas, ) raise StartTx() @@ -2477,9 +2485,10 @@ class EVMWorld(Platform): "solve", } - def __init__(self, fork=DEFAULT_FORK, **kwargs): + def __init__(self, constraints, fork=DEFAULT_FORK, **kwargs): super().__init__(**kwargs) self._world_state = {} + self._constraints = constraints self._callstack: List[ Tuple[Transaction, List[EVMLog], Set[int], Union[bytearray, MutableArray], EVM] ] = [] @@ -2496,6 +2505,7 @@ def __getstate__(self): state["_pending_transaction"] = self._pending_transaction state["_logs"] = self._logs state["_world_state"] = self._world_state + state["_constraints"] = self._constraints state["_callstack"] = self._callstack state["_deleted_accounts"] = self._deleted_accounts state["_transactions"] = self._transactions @@ -2506,6 +2516,7 @@ def __getstate__(self): def __setstate__(self, state): super().__setstate__(state) + self._constraints = state["_constraints"] self._pending_transaction = state["_pending_transaction"] self._world_state = state["_world_state"] self._deleted_accounts = state["_deleted_accounts"] @@ -2518,6 +2529,14 @@ def __setstate__(self, state): for _, _, _, _, vm in self._callstack: self.forward_events_from(vm) + @property + def constraints(self): + return self._constraints + + @constraints.setter + def constraints(self, constraints): + self._constraints = constraints + def try_simplify_to_constant(self, data): concrete_data = bytearray() # for c in data: @@ -2734,7 +2753,7 @@ def _close_transaction(self, result, data=None, rollback=False): tx, logs, deleted_accounts, account_storage, vm = self._callstack.pop() # Keep constraints gathered in the last vm - self._state.constraints = vm.constraints + self.constraints = vm.constraints # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-211.md if data is not None and self.current_vm is not None: diff --git a/manticore/platforms/platform.py b/manticore/platforms/platform.py index da3fb8754..43a4e5aee 100644 --- a/manticore/platforms/platform.py +++ b/manticore/platforms/platform.py @@ -5,6 +5,7 @@ from typing import Any, Callable, TypeVar from ..utils.event import Eventful +from ..core.state import StateBase logger = logging.getLogger(__name__) @@ -51,11 +52,11 @@ class Platform(Eventful): _published_events = {"solve"} - def __init__(self, *, state: Optional["StateBase"] = None, **kwargs): + def __init__(self, *, state: Optional[StateBase] = None, **kwargs): self._state = state super().__init__(**kwargs) - def set_state(self, state: "StateBase"): + def set_state(self, state: StateBase): self._state = state state.forward_events_from(self) @@ -70,6 +71,7 @@ def __getstate__(self): state = super().__getstate__() return state + class NativePlatform(Platform): def __init__(self, path, **kwargs): super().__init__(**kwargs) diff --git a/manticore/platforms/wasm.py b/manticore/platforms/wasm.py index 9665fc9ff..53039db16 100644 --- a/manticore/platforms/wasm.py +++ b/manticore/platforms/wasm.py @@ -65,6 +65,14 @@ def __init__(self, filename, name="self", **kwargs): self.forward_events_from(self.instance) self.forward_events_from(self.instance.executor) + @property + def constraints(self): + return self._constraints + + @constraints.setter + def constraints(self, constraints): + self._constraints = constraints + def __getstate__(self): state = super().__getstate__() state["modules"] = self.modules @@ -83,7 +91,7 @@ def __setstate__(self, state): self.store = state["store"] self.stack = state["stack"] self.advice = state["advice"] - self.constraints = state["constraints"] + self._constraints = state["constraints"] self.instantiated = state["instantiated"] self.module_names = state["module_names"] self.default_module = state["default_module"] diff --git a/manticore/wasm/manticore.py b/manticore/wasm/manticore.py index a90f89463..5e313dc21 100644 --- a/manticore/wasm/manticore.py +++ b/manticore/wasm/manticore.py @@ -244,6 +244,6 @@ def _make_wasm_bin(program, env={}, sup_env={}, **kwargs) -> State: exec_start=kwargs.get("exec_start", False), stub_missing=kwargs.get("stub_missing", True), ) - initial_state = State(constraints, platform) + initial_state = State(constraints=constraints, platform=platform) return initial_state diff --git a/mypy.ini b/mypy.ini index 322afe080..2de7aeefd 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,6 +4,9 @@ files = manticore, tests, examples/**/*.py # TODO: LOTS OF ERRORS # check_untyped_defs = True +[mypy-manticore.core.smtlib.*] +check_untyped_defs = True + # Generated file [mypy-manticore.ethereum.parsetab] ignore_errors = True @@ -36,6 +39,9 @@ ignore_missing_imports = True [mypy-rlp.*] ignore_missing_imports = True +[mypy-setuptools.*] +ignore_missing_imports = True + [mypy-toposort.*] ignore_missing_imports = True diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 9957a2182..508779e1a 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -114,7 +114,7 @@ run_tests_from_dir() { DIR=$1 COVERAGE_RCFILE=$GITHUB_WORKSPACE/.coveragerc echo "Running only the tests from 'tests/$DIR' directory" - pytest --durations=100 --cov=manticore --cov-config=$GITHUB_WORKSPACE/.coveragerc -n auto "tests/$DIR" + pytest --timeout=1200 --durations=100 --cov=manticore --cov-config=$GITHUB_WORKSPACE/.coveragerc -n auto "tests/$DIR" RESULT=$? return $RESULT } diff --git a/setup.py b/setup.py index acfb0437f..9feafa598 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,15 @@ def rtd_dependent_deps(): # Development dependencies without keystone dev_noks = ( native_deps - + ["coverage", "Sphinx", "pytest==5.3.0", "pytest-xdist==1.30.0", "pytest-cov==2.8.1", "jinja2"] + + [ + "coverage", + "Sphinx", + "pytest==5.3.0", + "pytest-timeout==1.4.2", + "pytest-xdist==1.30.0", + "pytest-cov==2.8.1", + "jinja2", + ] + lint_deps + auto_test_deps ) diff --git a/tests/ethereum/test_detectors.py b/tests/ethereum/test_detectors.py index 9ca957c8e..8f17a0c45 100644 --- a/tests/ethereum/test_detectors.py +++ b/tests/ethereum/test_detectors.py @@ -35,7 +35,7 @@ def make_mock_evm_state(): cs = ConstraintSet() - fakestate = State(cs, EVMWorld(cs)) + fakestate = State(constraints=cs, platform=EVMWorld(cs)) return fakestate diff --git a/tests/ethereum/test_general.py b/tests/ethereum/test_general.py index d364c6b76..2242ea1f3 100644 --- a/tests/ethereum/test_general.py +++ b/tests/ethereum/test_general.py @@ -415,7 +415,7 @@ def _make(self): data = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" gas = 1000000 - new_vm = evm.EVM(constraints, address, data, caller, value, bytecode, gas=gas, world=world) + new_vm = evm.EVM(address, data, caller, value, bytecode, gas=gas, world=world) return constraints, world, new_vm def test_str(self): @@ -487,12 +487,25 @@ def test_to_sig(self): self.assertEqual(evm.to_signed(-2), -2) self.assertEqual(evm.to_signed(2), 2) self.assertEqual(evm.to_signed(0), 0) - self.assertEqual(evm.to_signed(0x8000000000000000000000000000000000000000000000000000000000000001), -0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff) - self.assertEqual(evm.to_signed(0x8000000000000000000000000000000000000000000000000000000000000002), -0x7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe) - self.assertEqual(evm.to_signed(0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff), 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff) - self.assertEqual(evm.to_signed(0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff), -1) - self.assertEqual(evm.to_signed( 0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe), -2) - + self.assertEqual( + evm.to_signed(0x8000000000000000000000000000000000000000000000000000000000000001), + -0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF, + ) + self.assertEqual( + evm.to_signed(0x8000000000000000000000000000000000000000000000000000000000000002), + -0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE, + ) + self.assertEqual( + evm.to_signed(0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF), + 0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF, + ) + self.assertEqual( + evm.to_signed(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF), -1 + ) + self.assertEqual( + evm.to_signed(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE), -2 + ) + class EthTests(unittest.TestCase): def setUp(self): @@ -574,7 +587,6 @@ def test_create_contract_with_too_much_args(self): ) def test_create_contract_with_string_args(self): - import pdb; pdb.set_trace() source_code = ( "contract DontWork1{ string s; constructor(string memory s_) public{ s = s_;} }" ) @@ -796,7 +808,7 @@ def test_gen_testcase_only_if(self): for ext in ("summary", "constraints", "pkl", "tx.json", "tx", "trace", "logs") } - expected_files.add("state_00000001.pkl") + expected_files.add("state_00000002.pkl") actual_files = set((fn for fn in os.listdir(self.mevm.workspace) if not fn.startswith("."))) self.assertEqual(actual_files, expected_files) @@ -1668,7 +1680,7 @@ def test_jmpdest_check(self): bytecode = world.get_code(address) gas = 100000 - new_vm = evm.EVM(constraints, address, data, caller, value, bytecode, world=world, gas=gas) + new_vm = evm.EVM(address, data, caller, value, bytecode, world=world, gas=gas) result = None returndata = "" @@ -1760,7 +1772,6 @@ def test_delegatecall_env(self): # check balances self.assertEqual(world.get_balance(0x111111111111111111111111111111111111111), 0) self.assertEqual(world.get_balance(0x222222222222222222222222222222222222222), 10) - import pdb;pdb.set_trace() self.assertEqual( world.get_balance(0x333333333333333333333333333333333333333), 100000000000000000000000 - 10, diff --git a/tests/ethereum/test_regressions.py b/tests/ethereum/test_regressions.py index f55ff457a..a4c1bb99c 100644 --- a/tests/ethereum/test_regressions.py +++ b/tests/ethereum/test_regressions.py @@ -279,7 +279,7 @@ def test_addmod(self): caller = 0x42424242424242424242 value = 0 bytecode = "" - vm = evm.EVM(constraints, address, data, caller, value, bytecode, gas=23000) + vm = evm.EVM(address, data, caller, value, bytecode, evm.EVMWorld(constraints), gas=23000) self.assertEqual(vm.ADDMOD(12323, 2343, 20), 6) self.assertEqual(vm.ADDMOD(12323, 2343, 0), 0) @@ -336,7 +336,7 @@ def test_mulmod(self): caller = 0x42424242424242424242 value = 0 bytecode = "" - vm = evm.EVM(constraints, address, data, caller, value, bytecode, gas=23000) + vm = evm.EVM(address, data, caller, value, bytecode, evm.EVMWorld(constraints), gas=23000) self.assertEqual(vm.MULMOD(12323, 2343, 20), 9) self.assertEqual(vm.MULMOD(12323, 2343, 0), 0) diff --git a/tests/native/test_armv7unicorn.py b/tests/native/test_armv7unicorn.py index 41574328c..86fdd9efb 100644 --- a/tests/native/test_armv7unicorn.py +++ b/tests/native/test_armv7unicorn.py @@ -1493,7 +1493,7 @@ def get_state(cls): constraints = ConstraintSet() dirname = os.path.dirname(__file__) platform = linux.SLinux(os.path.join(dirname, "binaries", "basic_linux_amd64")) - cls.state = State(constraints, platform) + cls.state = State(constraints=constraints, platform=platform) cls.cpu = platform._mk_proc("armv7") return (cls.cpu, cls.state) diff --git a/tests/native/test_models.py b/tests/native/test_models.py index 2b41cafbf..ffc531117 100644 --- a/tests/native/test_models.py +++ b/tests/native/test_models.py @@ -47,7 +47,7 @@ def f(): class ModelTest(unittest.TestCase): dirname = os.path.dirname(__file__) l = linux.SLinux(os.path.join(dirname, "binaries", "basic_linux_amd64")) - state = State(ConstraintSet(), l) + state = State(constraints=ConstraintSet(), platform=l) stack_top = state.cpu.RSP def _clear_constraints(self): diff --git a/tests/native/test_state.py b/tests/native/test_state.py index c4f2dc3a6..e6c8f642b 100644 --- a/tests/native/test_state.py +++ b/tests/native/test_state.py @@ -4,6 +4,7 @@ from contextlib import redirect_stdout from manticore.core.state import StateBase +from manticore.platforms.platform import Platform from manticore.utils.event import Eventful from manticore.platforms import linux from manticore.native.state import State @@ -37,7 +38,7 @@ def memory(self): return self._memory -class FakePlatform(Eventful): +class FakePlatform(Platform): def __init__(self): super().__init__() self._constraints = None @@ -75,7 +76,7 @@ class StateTest(unittest.TestCase): def setUp(self): dirname = os.path.dirname(__file__) l = linux.Linux(os.path.join(dirname, "binaries", "basic_linux_amd64")) - self.state = State(ConstraintSet(), l) + self.state = State(constraints=ConstraintSet(), platform=l) def test_solve_one(self): val = 42 @@ -116,7 +117,7 @@ def test_policy_one(self): def test_state(self): constraints = ConstraintSet() - initial_state = State(constraints, FakePlatform()) + initial_state = State(constraints=constraints, platform=FakePlatform()) arr = initial_state.symbolicate_buffer("+" * 100, label="SYMBA") initial_state.constrain(arr[0] > 0x41) @@ -153,15 +154,15 @@ def test_new_bad_symbolic_value(self): def test_tainted_symbolic_buffer(self): taint = ("TEST_TAINT",) expr = self.state.new_symbolic_buffer(64, taint=taint) - self.assertEqual(expr.taint, frozenset(taint)) + self.assertEqual(frozenset(expr.taint), frozenset(taint)) def test_tainted_symbolic_value(self): taint = ("TEST_TAINT",) expr = self.state.new_symbolic_value(64, taint=taint) - self.assertEqual(expr.taint, frozenset(taint)) + self.assertEqual(frozenset(expr.taint), frozenset(taint)) def test_state_hook(self): - initial_state = State(ConstraintSet(), FakePlatform()) + initial_state = State(constraints=ConstraintSet(), platform=FakePlatform()) def fake_hook(_: StateBase) -> None: return None @@ -224,7 +225,7 @@ def testContextSerialization(self): new_file = "" new_new_file = "" constraints = ConstraintSet() - initial_state = State(constraints, FakePlatform()) + initial_state = State(constraints=constraints, platform=FakePlatform()) initial_state.context["step"] = 10 initial_file = pickle_dumps(initial_state) with initial_state as new_state: diff --git a/tests/native/test_workspace.py b/tests/native/test_workspace.py index c8af276b1..43f5ac0cd 100644 --- a/tests/native/test_workspace.py +++ b/tests/native/test_workspace.py @@ -60,7 +60,7 @@ def setUp(self): # self.manager.start(lambda: signal.signal(signal.SIGINT, signal.SIG_IGN)) dirname = os.path.dirname(__file__) l = linux.Linux(os.path.join(dirname, "binaries", "basic_linux_amd64")) - self.state = State(ConstraintSet(), l) + self.state = State(constraints=ConstraintSet(), platform=l) # self.lock = self.manager.Condition() def test_workspace_save_load(self): diff --git a/tests/other/test_smtlibv2.py b/tests/other/test_smtlibv2.py index ff7a888cc..c43571799 100644 --- a/tests/other/test_smtlibv2.py +++ b/tests/other/test_smtlibv2.py @@ -958,7 +958,7 @@ def test_visitors(self): self.assertEqual( translate_to_smtlib(aux, use_bindings=True), - "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62))) (let ((!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001))))) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR))))))" + "(let ((!a_2! (store (store MEM #x00000000 #x61) #x00000001 #x62))) (let ((!a_4! (store !a_2! #x00000003 (select !a_2! (bvadd VAR #x00000001))))) (select !a_4! (bvadd VAR ((_ zero_extend 24) (select !a_4! VAR))))))", ) values = arr[0:2] From 8e0797df6b5728fc876ae0cf8430a75708ad4a0b Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 19 Jan 2021 12:27:53 -0300 Subject: [PATCH 117/126] Add comment --- manticore/core/smtlib/visitors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 19b7bdb00..aac801d5c 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -125,6 +125,7 @@ def _method(self, expression, *operands): return self._rebuild(expression, operands) def _changed(self, expression: Expression, operands): + # False if no operands changed = any(x is not y for x, y in zip(expression.operands, operands)) return changed From 4843f90a40864e2ec84726df26bc06bd0ea0ac02 Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 19 Jan 2021 18:36:24 -0300 Subject: [PATCH 118/126] Remove unused/wrong operator --- manticore/core/smtlib/operators.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/manticore/core/smtlib/operators.py b/manticore/core/smtlib/operators.py index 3402dcb37..0fc1715fc 100644 --- a/manticore/core/smtlib/operators.py +++ b/manticore/core/smtlib/operators.py @@ -237,14 +237,6 @@ def SDIV(a, b): return int(math.trunc(float(a) / float(b))) -def SMOD(a, b): - if isinstance(a, BitVec): - return a.smod(b) - elif isinstance(b, BitVec): - return b.rsmod(a) - return int(math.fmod(a, b)) - - def SREM(a, b): if isinstance(a, BitVec): return a.srem(b) From d27068cd9c2aab9c418a8e998a33165cf4f0ddeb Mon Sep 17 00:00:00 2001 From: feliam Date: Tue, 2 Feb 2021 11:17:04 -0300 Subject: [PATCH 119/126] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a55f86745..ddcc0caa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -440,7 +440,7 @@ Thanks to our external contributors! - Ethereum: Support for Solidity `bytesM` and `bytes` types - Ethereum: Beta API for preconstraining inputs (`ManticoreEVM.constrain`) - Improved performance for smtlib module -- Ability to transparently operate on bytearray and symbolic buffer (MutableArray) types (e.g: concatenate, slice) +- Ability to transparently operate on bytearray and symbolic buffer (ArrayProxy) types (e.g: concatenate, slice) ### Changed From 49d82da3c85b6f8e8cfb800a7d679143cfd6b338 Mon Sep 17 00:00:00 2001 From: feliam Date: Mon, 8 Feb 2021 13:04:52 -0300 Subject: [PATCH 120/126] fix cache --- manticore/core/smtlib/expression.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index c1aa78cf3..c1cddba27 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1168,7 +1168,7 @@ def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size self._written: Optional[Set[Any]] = None # Cache of the known indexes - self._concrete_cache: Dict[Any, Any] = dict() + self._concrete_cache: Dict[Any, Any] = None self._length = array.length self._default = array.default @@ -1184,11 +1184,13 @@ def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): @property def concrete_cache(self): - for index, value in get_items(self): - if not isinstance(index, Constant): - break - if index.value not in self._concrete_cache: - self._concrete_cache[index.value] = value + if self._concrete_cache is None: + self._concrete_cache = {} + for index, value in get_items(self): + if not isinstance(index, Constant): + break + if index.value not in self._concrete_cache: + self._concrete_cache[index.value] = value return self._concrete_cache @property From 2c6cffa8e28530b982d4198229c0c61d119c9c8c Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Thu, 18 Feb 2021 16:40:28 -0500 Subject: [PATCH 121/126] CC --- manticore/core/smtlib/visitors.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/manticore/core/smtlib/visitors.py b/manticore/core/smtlib/visitors.py index 7d337c701..da256cbd5 100644 --- a/manticore/core/smtlib/visitors.py +++ b/manticore/core/smtlib/visitors.py @@ -5,8 +5,10 @@ from .expression import * from functools import lru_cache import copy +import io import logging import operator +import time import math from decimal import Decimal @@ -211,9 +213,6 @@ def get_depth(exp): return visitor.result -import io - - class PrettyPrinter(Visitor): def __init__(self, depth=None, **kwargs): super().__init__(**kwargs) @@ -807,9 +806,6 @@ def visit_Expression(self, expression, *operands): return expression -import time - - @lru_cache(maxsize=128, typed=True) def arithmetic_simplify(expression): start = time.time() From a81f8e87f3f744fa65af1b9f3c08dbb3fcf44a62 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Fri, 19 Feb 2021 18:01:24 -0500 Subject: [PATCH 122/126] Upgrade mypy and linting fixes --- manticore/core/smtlib/expression.py | 2 +- manticore/core/state.py | 6 +-- manticore/native/cpu/aarch64.py | 4 +- manticore/native/cpu/x86.py | 2 +- manticore/native/state.py | 7 +-- manticore/platforms/platform.py | 7 +-- mypy.ini | 41 +--------------- setup.py | 2 +- tests/auto_generators/flags.py | 4 +- tests/auto_generators/make_dump.py | 76 +++++++++++++++-------------- tests/auto_generators/make_tests.py | 3 +- tests/wasm/json2mc.py | 35 ++++++++----- 12 files changed, 83 insertions(+), 106 deletions(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index c1cddba27..f65001e4d 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -1168,7 +1168,7 @@ def __init__(self, array: Array, index: BitVec, value: BitVec, **kwargs): assert index.size == array.index_size assert value.size == array.value_size self._written: Optional[Set[Any]] = None # Cache of the known indexes - self._concrete_cache: Dict[Any, Any] = None + self._concrete_cache: Optional[Dict[Any, Any]] = None self._length = array.length self._default = array.default diff --git a/manticore/core/state.py b/manticore/core/state.py index efc1c0d4d..3ddd22dda 100644 --- a/manticore/core/state.py +++ b/manticore/core/state.py @@ -1,6 +1,6 @@ import copy import logging -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import Any, Dict, List, Optional, TypeVar, TYPE_CHECKING from .smtlib import Bool, ConstraintSet, Expression, issymbolic, BitVecConstant, MutableArray from ..utils.event import Eventful @@ -172,8 +172,8 @@ class StateBase(Eventful): """ Representation of a unique program state/path. - :param ConstraintSet constraints: Initial constraints - :param Platform platform: Initial operating system state + :param constraints: Initial constraints + :param platform: Initial operating system state :ivar dict context: Local context for arbitrary data storage """ diff --git a/manticore/native/cpu/aarch64.py b/manticore/native/cpu/aarch64.py index 2957c5fb3..ce861b753 100644 --- a/manticore/native/cpu/aarch64.py +++ b/manticore/native/cpu/aarch64.py @@ -40,7 +40,7 @@ class Aarch64InvalidInstruction(CpuException): # See "C1.2.4 Condition code". -Condspec = collections.namedtuple("CondSpec", "inverse func") +Condspec = collections.namedtuple("Condspec", "inverse func") COND_MAP = { cs.arm64.ARM64_CC_EQ: Condspec(cs.arm64.ARM64_CC_NE, lambda n, z, c, v: z == 1), cs.arm64.ARM64_CC_NE: Condspec(cs.arm64.ARM64_CC_EQ, lambda n, z, c, v: z == 0), @@ -75,7 +75,7 @@ class Aarch64InvalidInstruction(CpuException): class Aarch64RegisterFile(RegisterFile): - Regspec = collections.namedtuple("RegSpec", "parent size") + Regspec = collections.namedtuple("Regspec", "parent size") # Register table. _table = {} diff --git a/manticore/native/cpu/x86.py b/manticore/native/cpu/x86.py index 5076bc15a..730514df4 100644 --- a/manticore/native/cpu/x86.py +++ b/manticore/native/cpu/x86.py @@ -144,7 +144,7 @@ def new_method(cpu, *args, **kw_args): class AMD64RegFile(RegisterFile): - Regspec = collections.namedtuple("RegSpec", "register_id ty offset size reset") + Regspec = collections.namedtuple("Regspec", "register_id ty offset size reset") _flags = {"CF": 0, "PF": 2, "AF": 4, "ZF": 6, "SF": 7, "IF": 9, "DF": 10, "OF": 11} _table = { "CS": Regspec("CS", int, 0, 16, False), diff --git a/manticore/native/state.py b/manticore/native/state.py index bb747920c..d8db868fa 100644 --- a/manticore/native/state.py +++ b/manticore/native/state.py @@ -11,9 +11,10 @@ from ..platforms import linux_syscalls if TYPE_CHECKING: - from ..platforms.platform import Platform + from ..platforms.linux import Linux + from ..platforms.decree import Decree -HookCallback = Callable[[StateBase], None] +HookCallback = Callable[["State"], None] logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ class CheckpointData(NamedTuple): class State(StateBase): - def __init__(self, *, constraints: ConstraintSet, platform: "Platform", **kwargs): + def __init__(self, *, constraints: ConstraintSet, platform: Union["Linux", "Decree"], **kwargs): super().__init__(constraints=constraints, platform=platform, **kwargs) self._hooks: Dict[Optional[int], Set[HookCallback]] = {} self._after_hooks: Dict[Optional[int], Set[HookCallback]] = {} diff --git a/manticore/platforms/platform.py b/manticore/platforms/platform.py index 43a4e5aee..e92194db9 100644 --- a/manticore/platforms/platform.py +++ b/manticore/platforms/platform.py @@ -1,11 +1,10 @@ import logging -from typing import Optional - from functools import wraps -from typing import Any, Callable, TypeVar +from typing import Any, Callable, TypeVar, Optional from ..utils.event import Eventful from ..core.state import StateBase +from ..native.cpu.abstractcpu import Cpu logger = logging.getLogger(__name__) @@ -50,6 +49,8 @@ class Platform(Eventful): Base class for all platforms e.g. operating systems or virtual machines. """ + current: Any + _published_events = {"solve"} def __init__(self, *, state: Optional[StateBase] = None, **kwargs): diff --git a/mypy.ini b/mypy.ini index c15877c4e..8597e3660 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,7 @@ [mypy] python_version = 3.6 files = manticore, tests, examples/**/*.py +ignore_missing_imports = True # TODO: LOTS OF ERRORS # check_untyped_defs = True @@ -11,45 +12,5 @@ check_untyped_defs = True [mypy-manticore.ethereum.parsetab] ignore_errors = True -# 3rd-party libraries with no typing information -[mypy-capstone.*] -ignore_missing_imports = True - -[mypy-crytic_compile.*] -ignore_missing_imports = True - -[mypy-elftools.*] -ignore_missing_imports = True - -[mypy-sha3.*] -ignore_missing_imports = True - -[mypy-pyevmasm.*] -ignore_missing_imports = True - -[mypy-unicorn.*] -ignore_missing_imports = True - -[mypy-keystone.*] -ignore_missing_imports = True - -[mypy-ply.*] -ignore_missing_imports = True - -[mypy-rlp.*] -ignore_missing_imports = True - -[mypy-setuptools.*] -ignore_missing_imports = True - -[mypy-toposort.*] -ignore_missing_imports = True - -[mypy-prettytable.*] -ignore_missing_imports = True - -[mypy-wasm.*] -ignore_missing_imports = True - [mypy-manticore.core.state_pb2] ignore_errors = True diff --git a/setup.py b/setup.py index 031678517..ef8968c84 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def rtd_dependent_deps(): # (we need to know how to import a given native dependency so we can check if native dependencies are installed) native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc2"] -lint_deps = ["black==20.8b1", "mypy==0.790"] +lint_deps = ["black==20.8b1", "mypy==0.812"] auto_test_deps = ["py-evm"] diff --git a/tests/auto_generators/flags.py b/tests/auto_generators/flags.py index 62e8f7eeb..99f026ef2 100644 --- a/tests/auto_generators/flags.py +++ b/tests/auto_generators/flags.py @@ -1,4 +1,6 @@ -flags = { +from typing import Dict, List + +flags: Dict[str, Dict[str, List[str]]] = { "AAA": { "undefined": ["OF", "SF", "ZF", "PF"], "defined": ["AF", "CF"], diff --git a/tests/auto_generators/make_dump.py b/tests/auto_generators/make_dump.py index 259342ad3..76e3dd4dd 100644 --- a/tests/auto_generators/make_dump.py +++ b/tests/auto_generators/make_dump.py @@ -5,8 +5,10 @@ import sys import time import subprocess -from capstone import * -from capstone.x86 import * +from typing import Any, Dict +from capstone import Cs +from capstone.x86 import CS_ARCH_X86, CS_MODE_32, CS_MODE_64, X86_OP_MEM, X86_OP_REG, X86_OP_IMM +import capstone.x86 as csr from flags import flags flags_maks = { @@ -228,7 +230,7 @@ def read_operand(o): groups = map(instruction.group_name, instruction.groups) PC = {"i386": "EIP", "amd64": "RIP"}[arch] - registers = {PC: gdb.getR(PC)} + registers: Dict[Any, Any] = {PC: gdb.getR(PC)} memory = {} # save the encoded instruction @@ -246,11 +248,11 @@ def read_operand(o): if instruction.insn_name().upper() in ["PUSHF", "PUSHFD"]: registers["EFLAGS"] = gdb.getR("EFLAGS") - if instruction.insn_name().upper() in ["XLAT", "XLATB"]: - registers["AL"] = gdb.getR("AL") - registers[B] = gdb.getR(B) - address = registers[B] + registers["AL"] - memory[address] = chr(gdb.getByte(address)) + # if instruction.insn_name().upper() in ["XLAT", "XLATB"]: + # registers["AL"] = gdb.getR("AL") + # registers[B] = gdb.getR(B) + # address = registers[B] + registers["AL"] + # memory[address] = chr(gdb.getByte(address)) if instruction.insn_name().upper() in ["BTC", "BTR", "BTS", "BT"]: if instruction.operands[0].type == X86_OP_MEM: @@ -310,34 +312,34 @@ def read_operand(o): # registers[reg_name] = gdb.getR(reg_name) reg_sizes = { - X86_REG_AH: X86_REG_AX, - X86_REG_AL: X86_REG_AX, - X86_REG_AX: X86_REG_EAX, - X86_REG_EAX: X86_REG_RAX, - X86_REG_RAX: X86_REG_INVALID, - X86_REG_BH: X86_REG_BX, - X86_REG_BL: X86_REG_BX, - X86_REG_BX: X86_REG_EBX, - X86_REG_EBX: X86_REG_RBX, - X86_REG_RBX: X86_REG_INVALID, - X86_REG_CH: X86_REG_CX, - X86_REG_CL: X86_REG_CX, - X86_REG_CX: X86_REG_ECX, - X86_REG_ECX: X86_REG_RCX, - X86_REG_RCX: X86_REG_INVALID, - X86_REG_DH: X86_REG_DX, - X86_REG_DL: X86_REG_DX, - X86_REG_DX: X86_REG_EDX, - X86_REG_EDX: X86_REG_RDX, - X86_REG_RDX: X86_REG_INVALID, - X86_REG_DIL: X86_REG_EDI, - X86_REG_DI: X86_REG_EDI, - X86_REG_EDI: X86_REG_RDI, - X86_REG_RDI: X86_REG_INVALID, - X86_REG_SIL: X86_REG_ESI, - X86_REG_SI: X86_REG_ESI, - X86_REG_ESI: X86_REG_RSI, - X86_REG_RSI: X86_REG_INVALID, + csr.X86_REG_AH: csr.X86_REG_AX, + csr.X86_REG_AL: csr.X86_REG_AX, + csr.X86_REG_AX: csr.X86_REG_EAX, + csr.X86_REG_EAX: csr.X86_REG_RAX, + csr.X86_REG_RAX: csr.X86_REG_INVALID, + csr.X86_REG_BH: csr.X86_REG_BX, + csr.X86_REG_BL: csr.X86_REG_BX, + csr.X86_REG_BX: csr.X86_REG_EBX, + csr.X86_REG_EBX: csr.X86_REG_RBX, + csr.X86_REG_RBX: csr.X86_REG_INVALID, + csr.X86_REG_CH: csr.X86_REG_CX, + csr.X86_REG_CL: csr.X86_REG_CX, + csr.X86_REG_CX: csr.X86_REG_ECX, + csr.X86_REG_ECX: csr.X86_REG_RCX, + csr.X86_REG_RCX: csr.X86_REG_INVALID, + csr.X86_REG_DH: csr.X86_REG_DX, + csr.X86_REG_DL: csr.X86_REG_DX, + csr.X86_REG_DX: csr.X86_REG_EDX, + csr.X86_REG_EDX: csr.X86_REG_RDX, + csr.X86_REG_RDX: csr.X86_REG_INVALID, + csr.X86_REG_DIL: csr.X86_REG_EDI, + csr.X86_REG_DI: csr.X86_REG_EDI, + csr.X86_REG_EDI: csr.X86_REG_RDI, + csr.X86_REG_RDI: csr.X86_REG_INVALID, + csr.X86_REG_SIL: csr.X86_REG_ESI, + csr.X86_REG_SI: csr.X86_REG_ESI, + csr.X86_REG_ESI: csr.X86_REG_RSI, + csr.X86_REG_RSI: csr.X86_REG_INVALID, } # There is a capstone branch that should fix all these annoyances... soon # https://github.com/aquynh/capstone/tree/next @@ -387,7 +389,7 @@ def read_operand(o): registers[reg_name] = gdb.getR(reg_name) address += o.mem.scale * registers[reg_name] address = address & ({"i386": 0xFFFFFFFF, "amd64": 0xFFFFFFFFFFFFFFFF}[arch]) - for i in xrange(address, address + o.size): + for i in range(address, address + o.size): memory[i] = chr(gdb.getByte(i)) # gather PRE info diff --git a/tests/auto_generators/make_tests.py b/tests/auto_generators/make_tests.py index 318b35a27..b56af1ae7 100644 --- a/tests/auto_generators/make_tests.py +++ b/tests/auto_generators/make_tests.py @@ -1,5 +1,6 @@ from __future__ import print_function import sys +from typing import Any, Dict import random tests = [] @@ -15,7 +16,7 @@ random.shuffle(tests) -op_count = {} +op_count: Dict[str, int] = {} test_dic = {} for test in tests: try: diff --git a/tests/wasm/json2mc.py b/tests/wasm/json2mc.py index 3611cc1c4..0294cd746 100644 --- a/tests/wasm/json2mc.py +++ b/tests/wasm/json2mc.py @@ -3,6 +3,7 @@ from jinja2 import Environment, FileSystemLoader from base64 import b64encode from copy import copy +from typing import Any, List parser = argparse.ArgumentParser("Generate Manticore tests from the WASM Spec") parser.add_argument("filename", type=argparse.FileType("r"), help="JSON file output from wast2json") @@ -60,7 +61,7 @@ def escape_null(in_str: str): template = env.get_template("test_template.jinja2") -modules = [] +modules: List[Any] = [] registered_modules = {} imports = [] current_module = None @@ -100,14 +101,16 @@ def escape_null(in_str: str): mod_name=d["action"].get("module", None), ) elif d["action"]["type"] == "get": - modules[current_module].add_test( - d["action"]["field"], - d["line"], - [], - convert_types(d["expected"]), - "assert_global", - d["action"].get("module", None), - ) + if current_module: + modules[current_module].add_test( + d["action"]["field"], + d["line"], + [], + convert_types(d["expected"]), + "assert_global", + d["action"].get("module", None), + ) + current_module = None else: raise NotImplementedError("assert_return with action type: " + d["action"]["type"]) elif d["type"] == "assert_return_arithmetic_nan": @@ -155,10 +158,16 @@ def escape_null(in_str: str): if maybe_name: # This is an alias for another registered module imports.append({"type": "alias", "alias": d["as"], "orig": maybe_name}) else: # This is an alias for the current module - imports.append( - {"type": "import", "name": d["as"], "filename": modules[current_module].filename} - ) - modules[current_module].registered_name = d["as"] + if current_module: + imports.append( + { + "type": "import", + "name": d["as"], + "filename": modules[current_module].filename, + } + ) + modules[current_module].registered_name = d["as"] + current_module = None if current_module: modules[current_module].imports = copy(imports) From 6610f0c506dd68273c2935f99c54067bbbdff6ec Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 23 Feb 2021 10:32:24 -0500 Subject: [PATCH 123/126] Make sure concatenation variable is unique --- manticore/core/smtlib/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index f65001e4d..ee95fe5d3 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -907,7 +907,7 @@ def _concatenate(self, array_a: "Array", array_b: "Array") -> "Array": index_size=self.index_size, length=len(array_a) + len(array_b), value_size=self.value_size, - name="concatenation", + name=f"concatenation{uuid.uuid1()}", ) for index in range(len(array_a)): new_arr = new_arr.store(index, local_simplify(array_a[index])) From dda6d6aafa33d2ffc77af0cc3f497776be014247 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 23 Feb 2021 10:51:38 -0500 Subject: [PATCH 124/126] Don't pin pytest dependencies --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index ef8968c84..4377f0e35 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,10 @@ def rtd_dependent_deps(): + [ "coverage", "Sphinx", - "pytest==5.3.0", - "pytest-timeout==1.4.2", - "pytest-xdist==1.30.0", - "pytest-cov==2.8.1", + "pytest", + "pytest-timeout", + "pytest-xdist", + "pytest-cov", "jinja2", ] + lint_deps From 58136f04313eb070138c2560deffffd62b2e96e7 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 23 Feb 2021 11:24:51 -0500 Subject: [PATCH 125/126] Implement __bool__ for BoolEqual. Return True by default --- manticore/core/smtlib/expression.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/manticore/core/smtlib/expression.py b/manticore/core/smtlib/expression.py index ee95fe5d3..5cf70775f 100644 --- a/manticore/core/smtlib/expression.py +++ b/manticore/core/smtlib/expression.py @@ -687,6 +687,16 @@ def __init__(self, operanda: "Array", operandb: "Array", **kwargs): def __init__(self, operanda, operandb, **kwargs): super().__init__(operands=(operanda, operandb), **kwargs) + def __bool__(self): + from .visitors import simplify + + x = simplify(self) + if isinstance(x, Constant): + return bool(x.value) + # NOTE: True is the default return value for objects that don't implement __bool__ + # https://docs.python.org/3.6/reference/datamodel.html#object.__bool__ + return True + class BoolGreaterThan(BoolOperation): def __init__(self, operanda: BitVec, operandb: BitVec, **kwargs): From f2055be67379b357ec1cbfd0f4e30d53945c76e1 Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Tue, 23 Feb 2021 12:40:20 -0500 Subject: [PATCH 126/126] Fix some wasm tests due to bad changes made to generator script --- tests/wasm/generate_tests.sh | 4 ++-- tests/wasm/json2mc.py | 35 +++++++++++++---------------------- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/wasm/generate_tests.sh b/tests/wasm/generate_tests.sh index 0b5082bd4..c11d273aa 100755 --- a/tests/wasm/generate_tests.sh +++ b/tests/wasm/generate_tests.sh @@ -4,14 +4,14 @@ touch __init__.py if ! [ -x "$(command -v wast2json)" ]; then - wget -nc -nv -O wabt.tgz -c https://github.com/WebAssembly/wabt/releases/download/1.0.12/wabt-1.0.12-linux.tar.gz + wget -nv -O wabt.tgz -c https://github.com/WebAssembly/wabt/releases/download/1.0.12/wabt-1.0.12-linux.tar.gz tar --wildcards --strip=1 -xf wabt.tgz 'wabt-*/wast2json' rm wabt.tgz else cp "$(command -v wast2json)" . fi -wget -nc -nv -O spec.zip -c https://github.com/WebAssembly/spec/archive/opam-1.1.zip +wget -nv -O spec.zip -c https://github.com/WebAssembly/spec/archive/opam-1.1.zip unzip -q -j spec.zip 'spec-*/test/core/*' -d . rm run.py README.md diff --git a/tests/wasm/json2mc.py b/tests/wasm/json2mc.py index 0294cd746..a5165c438 100644 --- a/tests/wasm/json2mc.py +++ b/tests/wasm/json2mc.py @@ -3,7 +3,6 @@ from jinja2 import Environment, FileSystemLoader from base64 import b64encode from copy import copy -from typing import Any, List parser = argparse.ArgumentParser("Generate Manticore tests from the WASM Spec") parser.add_argument("filename", type=argparse.FileType("r"), help="JSON file output from wast2json") @@ -61,7 +60,7 @@ def escape_null(in_str: str): template = env.get_template("test_template.jinja2") -modules: List[Any] = [] +modules = [] # type: ignore registered_modules = {} imports = [] current_module = None @@ -101,16 +100,14 @@ def escape_null(in_str: str): mod_name=d["action"].get("module", None), ) elif d["action"]["type"] == "get": - if current_module: - modules[current_module].add_test( - d["action"]["field"], - d["line"], - [], - convert_types(d["expected"]), - "assert_global", - d["action"].get("module", None), - ) - current_module = None + modules[current_module].add_test( # type: ignore + d["action"]["field"], + d["line"], + [], + convert_types(d["expected"]), + "assert_global", + d["action"].get("module", None), + ) else: raise NotImplementedError("assert_return with action type: " + d["action"]["type"]) elif d["type"] == "assert_return_arithmetic_nan": @@ -158,16 +155,10 @@ def escape_null(in_str: str): if maybe_name: # This is an alias for another registered module imports.append({"type": "alias", "alias": d["as"], "orig": maybe_name}) else: # This is an alias for the current module - if current_module: - imports.append( - { - "type": "import", - "name": d["as"], - "filename": modules[current_module].filename, - } - ) - modules[current_module].registered_name = d["as"] - current_module = None + imports.append( + {"type": "import", "name": d["as"], "filename": modules[current_module].filename} # type: ignore + ) + modules[current_module].registered_name = d["as"] # type: ignore if current_module: modules[current_module].imports = copy(imports)