From dcc15e5a16b483bf05154c7d38ae1260fb64a745 Mon Sep 17 00:00:00 2001 From: ggrieco-tob Date: Fri, 3 Sep 2021 09:19:46 +0200 Subject: [PATCH 01/21] first step to remove manticore and avoid any logger issues --- etheno/__main__.py | 49 +--------- etheno/manticoreclient.py | 193 -------------------------------------- etheno/manticorelogger.py | 26 ----- etheno/manticoreutils.py | 148 ----------------------------- 4 files changed, 1 insertion(+), 415 deletions(-) delete mode 100644 etheno/manticoreclient.py delete mode 100644 etheno/manticorelogger.py delete mode 100644 etheno/manticoreutils.py diff --git a/etheno/__main__.py b/etheno/__main__.py index 747c822..ae403e9 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -19,14 +19,6 @@ from . import parity from . import truffle -try: - from .manticoreclient import ManticoreClient - from . import manticoreutils - MANTICORE_INSTALLED = True -except ModuleNotFoundError: - MANTICORE_INSTALLED = False - - def main(argv = None): parser = argparse.ArgumentParser(description='An Ethereum JSON RPC multiplexer and Manticore wrapper') parser.add_argument('--debug', action='store_true', default=False, help='Enable debugging from within the web server') @@ -36,9 +28,6 @@ def main(argv = None): parser.add_argument('-b', '--balance', type=float, default=100.0, help='Default balance (in Ether) to seed to each account (default=100.0)') parser.add_argument('-c', '--gas-price', type=int, default=None, help='Default gas price (default=20000000000)') parser.add_argument('-i', '--network-id', type=int, default=None, help='Specify a network ID (default is the network ID of the master client)') - parser.add_argument('-m', '--manticore', action='store_true', default=False, help='Run all transactions through manticore') - parser.add_argument('-r', '--manticore-script', type=argparse.FileType('rb'), default=None, help='Instead of running automated detectors and analyses, run this Manticore script') - parser.add_argument('--manticore-max-depth', type=int, default=None, help='Maximum state depth for Manticore to explore') parser.add_argument('-e', '--echidna', action='store_true', default=False, help='Fuzz the clients using transactions generated by Echidna') parser.add_argument('--fuzz-limit', type=int, default=None, help='The maximum number of transactions for Echidna to generate (default=unlimited)') parser.add_argument('--fuzz-contract', type=str, default=None, help='Path to a Solidity contract to have Echidna use for fuzzing (default is to use a builtin generic Echidna fuzzing contract)') @@ -259,27 +248,6 @@ def main(argv = None): for client in args.raw: ETHENO.add_client(RawTransactionClient(RpcProxyClient(client), accounts)) - manticore_client = None - if args.manticore: - if not MANTICORE_INSTALLED: - ETHENO.logger.error('Manticore is not installed! Running Etheno with Manticore requires Manticore version 0.2.2 or newer. Reinstall Etheno with Manticore support by running `pip3 install --user \'etheno[manticore]\'`, or install Manticore separately with `pip3 install --user \'manticore\'`') - sys.exit(1) - new_enough = manticoreutils.manticore_is_new_enough() - if new_enough is None: - ETHENO.logger.warning(f"Unknown Manticore version {manticoreutils.manticore_version()}; it may not be new enough to have Etheno support!") - elif not new_enough: - ETHENO.logger.error(f"The version of Manticore installed is {manticoreutils.manticore_version()}, but the minimum required version with Etheno support is 0.2.2. We will try to proceed, but things might not work correctly! Please upgrade Manticore.") - manticore_client = ManticoreClient() - ETHENO.add_client(manticore_client) - if args.manticore_max_depth is not None: - manticore_client.manticore.register_detector(manticoreutils.StopAtDepth(args.manticore_max_depth)) - if manticoreutils.manticore_is_new_enough(0, 2, 4): - # the verbosity static method was deprecated - from manticore.utils.log import set_verbosity - set_verbosity(getattr(logger, args.log_level)) - else: - manticore_client.manticore.verbosity(getattr(logger, args.log_level)) - if args.truffle: truffle_controller = truffle.Truffle(truffle_cmd=args.truffle_cmd, parent_logger=ETHENO.logger) @@ -296,22 +264,7 @@ def truffle_thread(): for plugin in ETHENO.plugins: plugin.finalize() - if manticore_client is not None: - if args.manticore_script is not None: - f = args.manticore_script - code = compile(f.read(), f.name, 'exec') - exec(code, { - 'manticore': manticore_client.manticore, - 'manticoreutils': manticoreutils, - 'logger': logger.EthenoLogger(os.path.basename(args.manticore_script.name), parent=manticore_client.logger) - }) - else: - manticoreutils.register_all_detectors(manticore_client.manticore) - manticore_client.multi_tx_analysis() - manticore_client.manticore.finalize() - manticore_client.logger.info("Results are in %s" % manticore_client.manticore.workspace) - ETHENO.shutdown() - elif not ETHENO.clients and not ETHENO.plugins: + if not ETHENO.clients and not ETHENO.plugins: ETHENO.logger.info("No clients or plugins running; exiting...") ETHENO.shutdown() diff --git a/etheno/manticoreclient.py b/etheno/manticoreclient.py deleted file mode 100644 index 0c3355d..0000000 --- a/etheno/manticoreclient.py +++ /dev/null @@ -1,193 +0,0 @@ -import logging -import time - -import builtins -import sys - -# ####BEGIN#### -# Horrible hack to workaround Manticore's global logging system. -# This can be removed after https://github.com/trailofbits/manticore/issues/1369 -# is resolved. -from . import manticorelogger - -oldimport = builtins.__import__ -def manticoreimport(name, *args, **kwargs): - if name == 'manticore.utils.log': - manticorelogger.__name__ = 'manticore.utils.log' - sys.modules[name] = manticorelogger - return manticorelogger - else: - return oldimport(name, *args, **kwargs) - -builtins.__import__ = manticoreimport -try: - import manticore.utils.log - import manticore.utils -finally: - builtins.__import__ = oldimport - -manticore.utils.log = manticorelogger -# ####END#### - -from manticore.ethereum import ManticoreEVM -from manticore.exceptions import NoAliveStates -import manticore - -from . import logger -from . import threadwrapper -from .client import EthenoClient, jsonrpc, DATA, QUANTITY -from .etheno import _CONTROLLER -from .manticoreutils import manticore_is_new_enough - -def encode_hex(data): - if data is None: - return None - elif isinstance(data, int) or isinstance(data, long): - encoded = hex(data) - if encoded[-1] == 'L': - encoded = encoded[:-1] - return encoded - else: - return "0x%s" % data.encode('hex') - -class ManticoreClient(EthenoClient): - def __init__(self, manticore=None): - self._assigned_manticore = manticore - self._manticore = None - self.contracts = [] - self.short_name = 'Manticore' - self._accounts_to_create = [] - - @property - def manticore(self): - if self._manticore is None: - if self._assigned_manticore is None: - # we do lazy evaluation of ManticoreClient.manticore so self.log_directory will be assigned already - if self.log_directory is None: - workspace = None - else: - workspace = self.log_directory - self._assigned_manticore = ManticoreEVM(workspace_url=workspace) - self._manticore = threadwrapper.MainThreadWrapper(self._assigned_manticore, _CONTROLLER) - self._finalize_manticore() - return self._manticore - - def _finalize_manticore(self): - if not self._manticore: - return - for balance, address in self._accounts_to_create: - self._manticore.create_account(balance=balance, address=address) - self._accounts_to_create = [] - self.reassign_manticore_loggers() - self.logger.cleanup_empty = True - - def create_account(self, balance, address): - self._accounts_to_create.append((balance, address)) - self._finalize_manticore() - - def reassign_manticore_loggers(self): - # Manticore uses a global to track its loggers: - manticore.utils.log.ETHENO_LOGGER = self.logger - manticore_loggers = (name for name in logging.root.manager.loggerDict if name.startswith('manticore')) - logger_parents = {} - for name in sorted(manticore_loggers): - sep = name.rfind('.') - if sep > 0: - path = name[:sep] - parent = logger_parents[path] - displayname = name[len(path)+1:] - else: - parent = self.logger - displayname = name - m_logger = logger.EthenoLogger(name, parent=parent, cleanup_empty=True, displayname=displayname) - m_logger.propagate = False - logger_parents[name] = m_logger - - @jsonrpc(from_addr = QUANTITY, to = QUANTITY, gas = QUANTITY, gasPrice = QUANTITY, value = QUANTITY, data = DATA, nonce = QUANTITY, RETURN = DATA) - def eth_sendTransaction(self, from_addr, to = None, gas = 90000, gasPrice = None, value = 0, data = None, nonce = None, rpc_client_result = None): - if to is None or to == 0: - # we are creating a new contract - if rpc_client_result is not None: - tx_hash = rpc_client_result['result'] - while True: - receipt = self.etheno.master_client.post({ - 'id' : "%s_receipt" % rpc_client_result['id'], - 'method' : 'eth_getTransactionReceipt', - 'params' : [tx_hash] - }) - if 'result' in receipt and receipt['result']: - address = int(receipt['result']['contractAddress'], 16) - break - # The transaction is still pending - time.sleep(1.0) - else: - address = None - contract_address = self.manticore.create_contract(owner = from_addr, balance = value, init=data) - self.contracts.append(contract_address) - self.logger.info(f"Manticore contract created: {encode_hex(contract_address.address)}") - #self.logger.info("Block number: %s" % self.manticore.world.block_number()) - else: - self.manticore.transaction(address = to, data = data, caller=from_addr, value = value) - # Just mimic the result from the master client - # We need to return something valid to appease the differential tester - return rpc_client_result - - @jsonrpc(TX_HASH = QUANTITY) - def eth_getTransactionReceipt(self, tx_hash, rpc_client_result = None): - # Mimic the result from the master client - # to appease the differential tester - return rpc_client_result - - def multi_tx_analysis(self, contract_address = None, tx_limit=None, tx_use_coverage=True, args=None): - if contract_address is None: - for contract_address in self.contracts: - self.multi_tx_analysis( - contract_address=contract_address, - tx_limit=tx_limit, - tx_use_coverage=tx_use_coverage, - args=args - ) - return - - tx_account = self.etheno.accounts - - current_coverage = 0 - tx_no = 0 - if manticore_is_new_enough(0, 3, 0): - shutdown_test = 'is_killed' - else: - shutdown_test = 'is_shutdown' - - while (current_coverage < 100 or not tx_use_coverage) and not getattr(self.manticore, shutdown_test)(): - try: - self.logger.info("Starting symbolic transaction: %d" % tx_no) - - # run_symbolic_tx - symbolic_data = self.manticore.make_symbolic_buffer(320) - symbolic_value = self.manticore.make_symbolic_value() - self.manticore.transaction(caller=tx_account[min(tx_no, len(tx_account) - 1)], - address=contract_address, - data=symbolic_data, - value=symbolic_value) - if manticore_is_new_enough(0, 3, 0): - # TODO: find the equivalent functions to get state counts in v0.3.0 - pass - else: - self.logger.info("%d alive states, %d terminated states" % (self.manticore.count_running_states(), self.manticore.count_terminated_states())) - except NoAliveStates: - break - - # Check if the maximun number of tx was reached - if tx_limit is not None and tx_no + 1 >= tx_limit: - break - - # Check if coverage has improved or not - if tx_use_coverage: - prev_coverage = current_coverage - current_coverage = self.manticore.global_coverage(contract_address) - found_new_coverage = prev_coverage < current_coverage - - if not found_new_coverage: - break - - tx_no += 1 diff --git a/etheno/manticorelogger.py b/etheno/manticorelogger.py deleted file mode 100644 index 2245223..0000000 --- a/etheno/manticorelogger.py +++ /dev/null @@ -1,26 +0,0 @@ -# This is a horrible hack that is used to replace manticore.utils.log -# Remove this once https://github.com/trailofbits/manticore/issues/1369 -# is resolved. - -ETHENO_LOGGER = None - -@property -def manticore_verbosity(): - return ETHENO_LOGGER.log_level - -@property -def DEFAULT_LOG_LEVEL(): - return ETHENO_LOGGER.log_level - -def set_verbosity(setting): - pass - #global manticore_verbosity - #manticore_verbosity = min(max(setting, 0), len(get_levels()) - 1) - #for logger_name in all_loggers: - # logger = logging.getLogger(logger_name) - # # min because more verbosity == lower numbers - # # This means if you explicitly call setLevel somewhere else in the source, and it's *more* - # # verbose, it'll stay that way even if manticore_verbosity is 0. - # logger.setLevel(min(get_verbosity(logger_name), logger.getEffectiveLevel())) - -all_loggers = set() diff --git a/etheno/manticoreutils.py b/etheno/manticoreutils.py deleted file mode 100644 index 22455fb..0000000 --- a/etheno/manticoreutils.py +++ /dev/null @@ -1,148 +0,0 @@ -import inspect -import itertools -import pkg_resources - -# Import manticoreclient before we load any actual Manticore classes. -# We don't need it here, but we do rely on it to hook in the Manticore loggers: -from . import manticoreclient -del manticoreclient - -from manticore.core.smtlib.operators import AND -from manticore.ethereum import ManticoreEVM, Detector -import manticore.ethereum.detectors - - -def manticore_version(): - return pkg_resources.get_distribution('manticore').version - - -def manticore_is_new_enough(*required_version): - """Checks if Manticore is newer than the given version. Returns True or False if known, or None if uncertain.""" - if required_version is None or len(required_version) == 0: - required_version = (0, 2, 2) - try: - version = manticore_version() - version = list(map(int, version.split('.'))) - for v, required in itertools.zip_longest(version, required_version, fillvalue=0): - if v < required: - return False - elif v > required: - return True - except Exception: - return None - return True - - -"""Detectors that should not be included in the results from `get_detectors()` (e.g., because they are buggy)""" -if manticore_is_new_enough(0, 2, 3): - # At some point after Manticore 0.2.2, these all stopped working: - DETECTOR_BLACKLIST = { - manticore.ethereum.detectors.DetectDelegatecall, - manticore.ethereum.detectors.DetectEnvInstruction, - manticore.ethereum.detectors.DetectExternalCallAndLeak, - manticore.ethereum.detectors.DetectIntegerOverflow, - manticore.ethereum.detectors.DetectInvalid, - manticore.ethereum.detectors.DetectRaceCondition, - manticore.ethereum.detectors.DetectReentrancyAdvanced, - manticore.ethereum.detectors.DetectReentrancySimple, - manticore.ethereum.detectors.DetectSuicidal, - manticore.ethereum.detectors.DetectUninitializedMemory, - manticore.ethereum.detectors.DetectUninitializedStorage, - manticore.ethereum.detectors.DetectUnusedRetVal - } -else: - DETECTOR_BLACKLIST = set() - - -def get_detectors(): - for name, obj in inspect.getmembers(manticore.ethereum.detectors): - if inspect.isclass(obj)\ - and issubclass(obj, manticore.ethereum.detectors.Detector)\ - and obj != manticore.ethereum.detectors.Detector\ - and obj not in DETECTOR_BLACKLIST: - yield obj - - -def register_all_detectors(manticore): - for detector in get_detectors(): - try: - manticore.register_detector(detector()) - except Exception as e: - manticore.logger.warning(f"Unable to register detector {detector!r}: {e!s}") - - -class StopAtDepth(Detector): - """This just aborts explorations that are too deep""" - - def __init__(self, max_depth): - self.max_depth = max_depth - - stop_at_death = self - - def will_start_run_callback(*args): - with stop_at_death.manticore.locked_context('seen_rep', dict) as reps: - reps.clear() - - # this callback got renamed to `will_run_callback` in Manticore 0.3.0 - if manticore_is_new_enough(0, 3, 0): - self.will_run_callback = will_start_run_callback - else: - self.will_start_run_callback = will_start_run_callback - - super().__init__() - - def will_decode_instruction_callback(self, state, pc): - world = state.platform - with self.manticore.locked_context('seen_rep', dict) as reps: - item = (world.current_transaction.sort == 'CREATE', world.current_transaction.address, pc) - if item not in reps: - reps[item] = 0 - reps[item] += 1 - if reps[item] > self.max_depth: - state.abandon() - - -class ManticoreTest: - def __init__(self, state, expression): - self.state = state - self.expression = expression - - def __bool__(self): - return self.can_be_true() - - def can_be_true(self): - return self.state.can_be_true(self.expression) - - def _solve_one(self, *variables, initial_state): - with initial_state as state: - state.constrain(self.expression) - for v in variables: - value = state.solve_one(v) - yield value - state.constrain(v == value) - - def solve_one(self, *variables): - """Finds a solution to the state and returns all of the variables in that solution""" - return self._solve_one(*variables, initial_state=self.state) - - def solve_all(self, *variables): - """Enumerates all solutions to the state for the given variables""" - with self.state as state: - while state.can_be_true(self.expression): - solution = tuple(self._solve_one(*variables, initial_state=state)) - if len(solution) < len(variables): - break - yield solution - state.constrain(AND(*(v != s for v, s in zip(variables, solution)))) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - -if __name__ == '__main__': - print('Available Manticore Detectors:') - for detector in get_detectors(): - print(" %s" % detector) From 7828352b49d70ee58e73e69838cb6425ea81ad51 Mon Sep 17 00:00:00 2001 From: ggrieco-tob Date: Fri, 3 Sep 2021 09:35:37 +0200 Subject: [PATCH 02/21] more stuff removed --- README.md | 41 +----------- .../ExploitMetaCoinManticoreScript.py | 31 --------- examples/BrokenMetaCoin/LICENSE | 22 ------- examples/BrokenMetaCoin/README.md | 9 --- .../BrokenMetaCoin/contracts/ConvertLib.sol | 8 --- .../BrokenMetaCoin/contracts/MetaCoin.sol | 48 -------------- .../BrokenMetaCoin/contracts/Migrations.sol | 23 ------- .../migrations/1_initial_migration.js | 5 -- .../migrations/2_deploy_contracts.js | 8 --- examples/BrokenMetaCoin/run_etheno.sh | 16 ----- examples/BrokenMetaCoin/test/TestMetacoin.sol | 25 -------- examples/BrokenMetaCoin/test/metacoin.js | 63 ------------------- examples/BrokenMetaCoin/truffle.js | 9 --- setup.py | 3 - 14 files changed, 3 insertions(+), 308 deletions(-) delete mode 100644 examples/BrokenMetaCoin/ExploitMetaCoinManticoreScript.py delete mode 100644 examples/BrokenMetaCoin/LICENSE delete mode 100644 examples/BrokenMetaCoin/README.md delete mode 100644 examples/BrokenMetaCoin/contracts/ConvertLib.sol delete mode 100644 examples/BrokenMetaCoin/contracts/MetaCoin.sol delete mode 100644 examples/BrokenMetaCoin/contracts/Migrations.sol delete mode 100644 examples/BrokenMetaCoin/migrations/1_initial_migration.js delete mode 100644 examples/BrokenMetaCoin/migrations/2_deploy_contracts.js delete mode 100755 examples/BrokenMetaCoin/run_etheno.sh delete mode 100644 examples/BrokenMetaCoin/test/TestMetacoin.sol delete mode 100644 examples/BrokenMetaCoin/test/metacoin.js delete mode 100644 examples/BrokenMetaCoin/truffle.js diff --git a/README.md b/README.md index 2f3a70b..6a231e4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@
-Etheno is the Ethereum testing Swiss Army knife. It’s a JSON RPC multiplexer, analysis tool wrapper, and test integration tool. It eliminates the complexity of setting up analysis tools like [Manticore](https://github.com/trailofbits/manticore/) and [Echidna](https://github.com/trailofbits/echidna) on large, multi-contract projects. In particular, custom Manticore analysis scripts require less code, are simpler to write, and integrate with Truffle. +Etheno is the Ethereum testing Swiss Army knife. It’s a JSON RPC multiplexer, analysis tool wrapper, and test integration tool. It eliminates the complexity of setting up analysis tools like [Echidna](https://github.com/trailofbits/echidna) on large, multi-contract projects. If you are a smart contract developer, you should use Etheno to test your contracts. If you are an Ethereum client developer, you should use Etheno to perform differential testing on your implementation. For example, Etheno is [capable of automatically reproducing](examples/ConstantinopleGasUsage) the Constantinople gas usage consensus bug that caused a fork on Ropsten. @@ -19,13 +19,8 @@ Etheno is named after the Greek goddess [Stheno](https://en.wikipedia.org/wiki/S * API for filtering and modifying JSON RPC calls * Enables differential testing by sending JSON RPC sequences to multiple Ethereum clients * Deploy to and interact with multiple networks at the same time -* **Analysis Tool Wrapper**: Etheno provides a JSON RPC client for advanced analysis tools like [Manticore](https://github.com/trailofbits/manticore/) - * Lowers barrier to entry for using advanced analysis tools - * No need for custom scripts to set up account and contract state - * Analyze arbitrary transactions without Solidity source code * **Integration with Test Frameworks** like Ganache and Truffle * Run a local test network with a single command - * Use Truffle migrations to bootstrap Manticore analyses * Symbolic semantic annotations within unit tests ## Quickstart @@ -35,10 +30,6 @@ Use our prebuilt Docker container to quickly install and try Etheno: ``` docker pull trailofbits/etheno docker run -it trailofbits/etheno - -# Run one of the examples -etheno@982abdc96791:~$ cd examples/BrokenMetaCoin/ -etheno@982abdc96791:~/examples/BrokenMetaCoin$ etheno --truffle --ganache --manticore --manticore-max-depth 2 --manticore-script ExploitMetaCoinManticoreScript.py ``` Alternatively, natively install Etheno in a few shell commands: @@ -52,7 +43,7 @@ pip3 install --user etheno # Use the Etheno CLI cd /path/to/a/truffle/project -etheno --manticore --ganache --truffle +etheno --ganache --truffle ``` ## Usage @@ -70,7 +61,7 @@ etheno https://client1.url.com:1234/ https://client2.url.com:8545/ http://client * `--port` or `-p` allows you to specify a port on which to run Etheno’s JSON RPC server (default is 8545) * `--run-publicly` allows incoming JSON RPC connections from external computers on the network * `--debug` will run a web-based interactive debugger in the event that an internal Etheno client throws an exception while processing a JSON RPC call; this should _never_ be used in conjunction with `--run-publicly` -* `--master` or `-s` will set the “master” client, which will be used for synchronizing with Etheno clients like Manticore. If a master is not explicitly provided, it defaults to the first client listed. +* `--master` or `-s` will set the “master” client, which will be used for synchronizing with Etheno clients. If a master is not explicitly provided, it defaults to the first client listed. * `--raw`, when prefixed before a client URL, will cause Etheno to auto-sign all transactions and submit then to the client as raw transactions ### Geth and Parity Integration @@ -123,17 +114,6 @@ By default, Echidna deploys a generic fuzz testing contract to all clients, enum * `--fuzz-limit` limits the number of transactions that Echidna will emit * `--fuzz-contract` lets the user specify a custom contract for Echidna to deploy and fuzz -### Manticore Client - -Manticore—which, by itself, does not implement a JSON RPC interface—can be run as an Etheno client, synchronizing its accounts with Etheno’s master client and symbolically executing all transactions sent to Etheno. -``` -etheno --manticore -``` -This alone will not run any Manticore analyses; they must either be run manually, or automated through [the `--truffle` command](#truffle-integration); - -* `--manticore-verbosity` sets Manticore’s logging verbosity (default is 3) -* `--manticore-max-depth` sets the maximum state depth for Manticore to explore; if omitted, Manticore will have no depth limit - ### Truffle Integration Truffle migrations can automatically be run within a Truffle project: @@ -141,20 +121,6 @@ Truffle migrations can automatically be run within a Truffle project: etheno --truffle ``` -When combined with the `--manticore` option, this will automatically run Manticore’s default analyses on all contracts created once the Truffle migration completes: -``` -etheno --truffle --manticore -``` - -This requires a master JSON RPC client, so will most often be used in conjunction with Ganache. If a local Ganache server is not running, you can simply add that to the command: -``` -etheno --truffle --manticore --ganache -``` - -If you would like to run a custom Manticore script instead of the standard Manticore analysis and detectors, it can be specified using the `--manticore-script` or `-r` command. - -This script does not need to import Manticore or create a `ManticoreEVM` object; Etheno will run the script with a global variable called `manticore` that already contains all of the accounts and contracts automatically provisioned. See the [`BrokenMetaCoin` Manticore script](examples/BrokenMetaCoin/ExploitMetaCoinManticoreScript.py) for an example. - Additional arguments can be passed to Truffle using `--truffle-args`. ### Logging @@ -174,7 +140,6 @@ saved: ## Requirements * Python 3.6 or newer -* [Manticore](https://github.com/trailofbits/manticore/) release 0.2.2 or newer * [Flask](http://flask.pocoo.org/), which is used to run the JSON RPC server ### Optional Requirements diff --git a/examples/BrokenMetaCoin/ExploitMetaCoinManticoreScript.py b/examples/BrokenMetaCoin/ExploitMetaCoinManticoreScript.py deleted file mode 100644 index ad512a3..0000000 --- a/examples/BrokenMetaCoin/ExploitMetaCoinManticoreScript.py +++ /dev/null @@ -1,31 +0,0 @@ -# global variables `logger`, `manticore`, and `manticoreutils` are provided by Etheno - -# No need to set up accounts or contracts the way we usually do with Manticore alone! -# They are already pre-provisioned in the `manticore` object -# and we can simply access them from there: - -# The Truffle migrations deploy three contracts: [Migrations contract, ConvertLib, MetaCoin] -contract_account = list(manticore.contract_accounts.values())[2] - -# The contract was loaded from bytecode, so we need to manually set the ABI: -contract_account.add_function('setMetadata(uint256,uint256)') - -# Create symbolic variables for which Manticore will discover values: -key1 = manticore.make_symbolic_value(name='key1') -value1 = manticore.make_symbolic_value(name='val1') -key2 = manticore.make_symbolic_value(name='key2') - -# Make two calls to the `setMetadata` function of the `MetaCoin` contract -# using the symbolic variables: -contract_account.setMetadata(key1, value1) -contract_account.setMetadata(key2, 1) - -for st in manticore.all_states: - # The value we want to overwrite is the `balances` mapping in storage slot 0 - balances_value = st.platform.get_storage_data(contract_account.address, 0) - with manticoreutils.ManticoreTest(st, balances_value == 1) as test: - for k1, v1, k2 in test.solve_all(key1, value1, key2): - result = f"\nFound a way to overwrite balances! Check {manticore.workspace}\n" - result += f" setMetadata({hex(k1)}, {hex(v1)})\n" - result += f" setMetadata({hex(k2)}, 0x1)\n" - logger.info(result) diff --git a/examples/BrokenMetaCoin/LICENSE b/examples/BrokenMetaCoin/LICENSE deleted file mode 100644 index bb98c92..0000000 --- a/examples/BrokenMetaCoin/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Truffle - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/examples/BrokenMetaCoin/README.md b/examples/BrokenMetaCoin/README.md deleted file mode 100644 index 02c0fc2..0000000 --- a/examples/BrokenMetaCoin/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Broken MetaCoin Truffle and Manticore Example - -This example is the same `MetaCoin` project used in the Truffle -documentation and tutorials, however, we have added common Solidity -errors to the code, including some that are quite subtle. - -A [`run_etheno.sh`](run_etheno.sh) script is provided to give some -examples of how one might use Etheno and Manticore to automatically -discover the bugs in this project. diff --git a/examples/BrokenMetaCoin/contracts/ConvertLib.sol b/examples/BrokenMetaCoin/contracts/ConvertLib.sol deleted file mode 100644 index 5d83fa9..0000000 --- a/examples/BrokenMetaCoin/contracts/ConvertLib.sol +++ /dev/null @@ -1,8 +0,0 @@ -pragma solidity ^0.5.0; - -library ConvertLib{ - function convert(uint amount,uint conversionRate) public pure returns (uint convertedAmount) - { - return amount * conversionRate; - } -} diff --git a/examples/BrokenMetaCoin/contracts/MetaCoin.sol b/examples/BrokenMetaCoin/contracts/MetaCoin.sol deleted file mode 100644 index 25494f2..0000000 --- a/examples/BrokenMetaCoin/contracts/MetaCoin.sol +++ /dev/null @@ -1,48 +0,0 @@ -pragma solidity ^0.5.0; - -import "./ConvertLib.sol"; - -/** - * This is a simple example token with several vulnerabilities added. - */ -contract MetaCoin { - mapping (address => uint) balances; - uint256[] metadata; - - event Transfer(address indexed _from, address indexed _to, uint256 _value); - - constructor() public { - balances[tx.origin] = 10000; - } - - function setMetadata(uint256 key, uint256 value) public { - if (metadata.length <= key) { - metadata.length = key + 1; - } - metadata[key] = value; - } - - function getMetadata(uint256 key) public view returns (uint256) { - return metadata[key]; - } - - function backdoor() public { - selfdestruct(msg.sender); - } - - function sendCoin(address receiver, uint amount) public returns(bool sufficient) { - if (balances[msg.sender] < amount) return false; - balances[msg.sender] -= amount; - balances[receiver] += amount; - emit Transfer(msg.sender, receiver, amount); - return true; - } - - function getBalanceInEth(address addr) public view returns(uint){ - return ConvertLib.convert(getBalance(addr),2); - } - - function getBalance(address addr) public view returns(uint) { - return balances[addr]; - } -} diff --git a/examples/BrokenMetaCoin/contracts/Migrations.sol b/examples/BrokenMetaCoin/contracts/Migrations.sol deleted file mode 100644 index 89b4d5c..0000000 --- a/examples/BrokenMetaCoin/contracts/Migrations.sol +++ /dev/null @@ -1,23 +0,0 @@ -pragma solidity ^0.5.0; - -contract Migrations { - address public owner; - uint public last_completed_migration; - - modifier restricted() { - if (msg.sender == owner) _; - } - - constructor() public { - owner = msg.sender; - } - - function setCompleted(uint completed) public restricted { - last_completed_migration = completed; - } - - function upgrade(address new_address) public restricted { - Migrations upgraded = Migrations(new_address); - upgraded.setCompleted(last_completed_migration); - } -} diff --git a/examples/BrokenMetaCoin/migrations/1_initial_migration.js b/examples/BrokenMetaCoin/migrations/1_initial_migration.js deleted file mode 100644 index 4d5f3f9..0000000 --- a/examples/BrokenMetaCoin/migrations/1_initial_migration.js +++ /dev/null @@ -1,5 +0,0 @@ -var Migrations = artifacts.require("./Migrations.sol"); - -module.exports = function(deployer) { - deployer.deploy(Migrations); -}; diff --git a/examples/BrokenMetaCoin/migrations/2_deploy_contracts.js b/examples/BrokenMetaCoin/migrations/2_deploy_contracts.js deleted file mode 100644 index b3dc3e9..0000000 --- a/examples/BrokenMetaCoin/migrations/2_deploy_contracts.js +++ /dev/null @@ -1,8 +0,0 @@ -var ConvertLib = artifacts.require("./ConvertLib.sol"); -var MetaCoin = artifacts.require("./MetaCoin.sol"); - -module.exports = function(deployer) { - deployer.deploy(ConvertLib); - deployer.link(ConvertLib, MetaCoin); - deployer.deploy(MetaCoin); -}; diff --git a/examples/BrokenMetaCoin/run_etheno.sh b/examples/BrokenMetaCoin/run_etheno.sh deleted file mode 100755 index b77bcb4..0000000 --- a/examples/BrokenMetaCoin/run_etheno.sh +++ /dev/null @@ -1,16 +0,0 @@ -# First, remove the Truffle build directory. -# This shouldn't be necessary, but Truffle will often fail with -# confusing error messages if it is upgraded between builds. -# So, we just rebuild everything from scratch each time to ensure -# that it always works. -rm -rf build - -echo "Running the custom Manticore script ExploitMetaCoinManticoreScript.py" -# Set the max depth for Manticore to 2 because this script only needs to -# find a sequence of two transactions to exploit the bug -etheno --manticore --truffle --ganache --manticore-max-depth 2 -r ExploitMetaCoinManticoreScript.py - -echo "Running a full Manticore analysis with standard vulnerability detectors (this can take roughly 30 minutes)" -# Set the max depth for Manticore to 2 because we can get ~98% coverage -# with that setting, and it drastically reduces compute time -etheno -m -t -g --manticore-max-depth 2 diff --git a/examples/BrokenMetaCoin/test/TestMetacoin.sol b/examples/BrokenMetaCoin/test/TestMetacoin.sol deleted file mode 100644 index 7af110c..0000000 --- a/examples/BrokenMetaCoin/test/TestMetacoin.sol +++ /dev/null @@ -1,25 +0,0 @@ -pragma solidity ^0.4.2; - -import "truffle/Assert.sol"; -import "truffle/DeployedAddresses.sol"; -import "../contracts/MetaCoin.sol"; - -contract TestMetacoin { - - function testInitialBalanceUsingDeployedContract() public { - MetaCoin meta = MetaCoin(DeployedAddresses.MetaCoin()); - - uint expected = 10000; - - Assert.equal(meta.getBalance(tx.origin), expected, "Owner should have 10000 MetaCoin initially"); - } - - function testInitialBalanceWithNewMetaCoin() public { - MetaCoin meta = new MetaCoin(); - - uint expected = 10000; - - Assert.equal(meta.getBalance(tx.origin), expected, "Owner should have 10000 MetaCoin initially"); - } - -} diff --git a/examples/BrokenMetaCoin/test/metacoin.js b/examples/BrokenMetaCoin/test/metacoin.js deleted file mode 100644 index c61c093..0000000 --- a/examples/BrokenMetaCoin/test/metacoin.js +++ /dev/null @@ -1,63 +0,0 @@ -var MetaCoin = artifacts.require("./MetaCoin.sol"); - -contract('MetaCoin', function(accounts) { - it("should put 10000 MetaCoin in the first account", function() { - return MetaCoin.deployed().then(function(instance) { - return instance.getBalance.call(accounts[0]); - }).then(function(balance) { - assert.equal(balance.valueOf(), 10000, "10000 wasn't in the first account"); - }); - }); - it("should call a function that depends on a linked library", function() { - var meta; - var metaCoinBalance; - var metaCoinEthBalance; - - return MetaCoin.deployed().then(function(instance) { - meta = instance; - return meta.getBalance.call(accounts[0]); - }).then(function(outCoinBalance) { - metaCoinBalance = outCoinBalance.toNumber(); - return meta.getBalanceInEth.call(accounts[0]); - }).then(function(outCoinBalanceEth) { - metaCoinEthBalance = outCoinBalanceEth.toNumber(); - }).then(function() { - assert.equal(metaCoinEthBalance, 2 * metaCoinBalance, "Library function returned unexpected function, linkage may be broken"); - }); - }); - it("should send coin correctly", function() { - var meta; - - // Get initial balances of first and second account. - var account_one = accounts[0]; - var account_two = accounts[1]; - - var account_one_starting_balance; - var account_two_starting_balance; - var account_one_ending_balance; - var account_two_ending_balance; - - var amount = 10; - - return MetaCoin.deployed().then(function(instance) { - meta = instance; - return meta.getBalance.call(account_one); - }).then(function(balance) { - account_one_starting_balance = balance.toNumber(); - return meta.getBalance.call(account_two); - }).then(function(balance) { - account_two_starting_balance = balance.toNumber(); - return meta.sendCoin(account_two, amount, {from: account_one}); - }).then(function() { - return meta.getBalance.call(account_one); - }).then(function(balance) { - account_one_ending_balance = balance.toNumber(); - return meta.getBalance.call(account_two); - }).then(function(balance) { - account_two_ending_balance = balance.toNumber(); - - assert.equal(account_one_ending_balance, account_one_starting_balance - amount, "Amount wasn't correctly taken from the sender"); - assert.equal(account_two_ending_balance, account_two_starting_balance + amount, "Amount wasn't correctly sent to the receiver"); - }); - }); -}); diff --git a/examples/BrokenMetaCoin/truffle.js b/examples/BrokenMetaCoin/truffle.js deleted file mode 100644 index aed5f84..0000000 --- a/examples/BrokenMetaCoin/truffle.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = { - networks: { - development: { - host: "127.0.0.1", - port: 8545, - network_id: "*" - } - } -}; diff --git a/setup.py b/setup.py index bde9f4c..f1154bd 100644 --- a/setup.py +++ b/setup.py @@ -19,9 +19,6 @@ 'pycryptodome>=3.4.7,<4.0.0', 'setuptools' ], - extras_require={ - 'manticore': ['manticore>=0.2.2'] - }, entry_points={ 'console_scripts': [ 'etheno = etheno.__main__:main' From 83bb5246b8d00d7496cd8a3c256254dd07c35f62 Mon Sep 17 00:00:00 2001 From: Rappie Date: Tue, 12 Apr 2022 13:24:24 +0200 Subject: [PATCH 03/21] Use "0x00" as data if there is no data in the transaction. This happens with simple eth transfer transactions. --- etheno/jsonrpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etheno/jsonrpc.py b/etheno/jsonrpc.py index 4f91040..84244b4 100644 --- a/etheno/jsonrpc.py +++ b/etheno/jsonrpc.py @@ -147,7 +147,7 @@ def after_post(self, post_data, result): contract_address = result['result']['contractAddress'] self.handle_contract_created(original_transaction['from'], contract_address, result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'], value) else: - self.handle_function_call(original_transaction['from'], original_transaction['to'], result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'], value) + self.handle_function_call(original_transaction['from'], original_transaction['to'], result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'] if 'data' in original_transaction else '0x00', value) class EventSummaryExportPlugin(EventSummaryPlugin): From 4ab67d9d38d454f97ff0971d29f6070cc2a3712d Mon Sep 17 00:00:00 2001 From: Rappie Date: Sun, 24 Apr 2022 18:06:46 +0200 Subject: [PATCH 04/21] Changed '0x00' to '0x' --- etheno/jsonrpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etheno/jsonrpc.py b/etheno/jsonrpc.py index 84244b4..a7f25be 100644 --- a/etheno/jsonrpc.py +++ b/etheno/jsonrpc.py @@ -147,7 +147,7 @@ def after_post(self, post_data, result): contract_address = result['result']['contractAddress'] self.handle_contract_created(original_transaction['from'], contract_address, result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'], value) else: - self.handle_function_call(original_transaction['from'], original_transaction['to'], result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'] if 'data' in original_transaction else '0x00', value) + self.handle_function_call(original_transaction['from'], original_transaction['to'], result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'] if 'data' in original_transaction else '0x', value) class EventSummaryExportPlugin(EventSummaryPlugin): From c41574fe8f374bb1814e550d57a505a439794c8d Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Fri, 27 May 2022 06:18:06 -0400 Subject: [PATCH 05/21] removed manticore-based if statement in main.py --- README.md | 1 - etheno/__main__.py | 7 ++++--- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6a231e4..e0ddb73 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ Etheno is named after the Greek goddess [Stheno](https://en.wikipedia.org/wiki/S * Deploy to and interact with multiple networks at the same time * **Integration with Test Frameworks** like Ganache and Truffle * Run a local test network with a single command - * Symbolic semantic annotations within unit tests ## Quickstart diff --git a/etheno/__main__.py b/etheno/__main__.py index 5a1bc96..e424e0d 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -20,7 +20,7 @@ from . import truffle def main(argv = None): - parser = argparse.ArgumentParser(description='An Ethereum JSON RPC multiplexer and Manticore wrapper') + parser = argparse.ArgumentParser(description='An Ethereum JSON RPC multiplexer, differential fuzzer, and test framework integration tool.') parser.add_argument('--debug', action='store_true', default=False, help='Enable debugging from within the web server') parser.add_argument('--run-publicly', action='store_true', default=False, help='Allow the web server to accept external connections') parser.add_argument('-p', '--port', type=int, default=GETH_DEFAULT_RPC_PORT, help='Port on which to run the JSON RPC webserver (default=%d)' % GETH_DEFAULT_RPC_PORT) @@ -314,8 +314,9 @@ def truffle_thread(): thread = Thread(target=truffle_thread) thread.start() - if args.run_differential and (ETHENO.master_client is not None) and \ - next(filter(lambda c: not isinstance(c, ManticoreClient), ETHENO.clients), False): + # Without Manticore integration the only client types are geth, parity, and command-line raw/regular clients. + # So checking len() >= 2 should be sufficient. + if args.run_differential and (ETHENO.master_client is not None) and len(ETHENO.clients) >= 2: # There are at least two non-Manticore clients running ETHENO.logger.info("Initializing differential tests to compare clients %s" % ', '.join( map(str, [ETHENO.master_client] + ETHENO.clients) diff --git a/setup.py b/setup.py index f1154bd..65a3bda 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='etheno', - description='Etheno is a JSON RPC multiplexer, Manticore wrapper, differential fuzzer, and test framework integration tool.', + description='Etheno is a JSON RPC multiplexer, differential fuzzer, and test framework integration tool.', url='https://github.com/trailofbits/etheno', author='Trail of Bits', version='0.2.4', From 7ebf1175f48e888dc4e3f042b0ce6d0303c9551c Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Fri, 27 May 2022 06:21:16 -0400 Subject: [PATCH 06/21] fix if statement bc master_client is also technically a client --- etheno/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etheno/__main__.py b/etheno/__main__.py index e424e0d..10b8dc5 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -316,7 +316,7 @@ def truffle_thread(): # Without Manticore integration the only client types are geth, parity, and command-line raw/regular clients. # So checking len() >= 2 should be sufficient. - if args.run_differential and (ETHENO.master_client is not None) and len(ETHENO.clients) >= 2: + if args.run_differential and (ETHENO.master_client is not None) and len(ETHENO.clients) >= 1: # There are at least two non-Manticore clients running ETHENO.logger.info("Initializing differential tests to compare clients %s" % ', '.join( map(str, [ETHENO.master_client] + ETHENO.clients) From 8cf9e8aa97a5fd60c3461232da4327017bb4d3c0 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Fri, 27 May 2022 06:37:30 -0400 Subject: [PATCH 07/21] added small todo --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 09123b8..2f0bd86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,7 @@ WORKDIR /home/etheno COPY --chown=etheno:etheno LICENSE setup.py etheno/ COPY --chown=etheno:etheno etheno/*.py etheno/etheno/ +# TODO: get rid of manticore --user flag here. will need to validate that it doesn't break anything. RUN cd etheno && \ pip3 install --no-cache-dir --user '.[manticore]' && \ cd .. && \ From d8b13fef8591ea51a01c361adc4196bf5ddac01f Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Fri, 27 May 2022 07:02:49 -0400 Subject: [PATCH 08/21] initial removal of echidna features --- Dockerfile | 9 - etheno/__main__.py | 27 --- etheno/contracts.py | 3 +- etheno/echidna.py | 199 ------------------ examples/ConstantinopleGasUsage/README.md | 23 -- .../ConstantinopleGasUsage/constantinople.sol | 15 -- examples/ConstantinopleGasUsage/run_etheno.sh | 1 - 7 files changed, 2 insertions(+), 275 deletions(-) delete mode 100644 etheno/echidna.py delete mode 100644 examples/ConstantinopleGasUsage/README.md delete mode 100644 examples/ConstantinopleGasUsage/constantinople.sol delete mode 100755 examples/ConstantinopleGasUsage/run_etheno.sh diff --git a/Dockerfile b/Dockerfile index 09123b8..6fa84e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,15 +31,6 @@ RUN curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - && sudo apt- RUN npm install --production -g ganache-cli truffle && npm --force cache clean -# BEGIN Install Echidna - -COPY --from=trailofbits/echidna:latest /root/.local/bin/echidna-test /usr/local/bin/echidna-test - -RUN update-locale LANG=en_US.UTF-8 && locale-gen en_US.UTF-8 -ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 - -# END Install Echidna - RUN useradd -m etheno RUN usermod -aG sudo etheno USER etheno diff --git a/etheno/__main__.py b/etheno/__main__.py index 06d2030..5951898 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -7,7 +7,6 @@ from .client import RpcProxyClient from .differentials import DifferentialTester -from .echidna import echidna_exists, EchidnaPlugin, install_echidna from .etheno import app, EthenoView, GETH_DEFAULT_RPC_PORT, ETHENO, VERSION_NAME from .genesis import Account, make_accounts, make_genesis from .jsonrpc import EventSummaryExportPlugin, JSONRPCExportPlugin @@ -49,13 +48,6 @@ def main(argv=None): help='Instead of running automated detectors and analyses, run this Manticore script') parser.add_argument('--manticore-max-depth', type=int, default=None, help='Maximum state depth for Manticore to explore') - parser.add_argument('-e', '--echidna', action='store_true', default=False, - help='Fuzz the clients using transactions generated by Echidna') - parser.add_argument('--fuzz-limit', type=int, default=None, - help='The maximum number of transactions for Echidna to generate (default=unlimited)') - parser.add_argument('--fuzz-contract', type=str, default=None, - help='Path to a Solidity contract to have Echidna use for fuzzing (default is to use a builtin ' - 'generic Echidna fuzzing contract)') parser.add_argument('-t', '--truffle', action='store_true', default=False, help='Run the truffle migrations in the current directory and exit') parser.add_argument('--truffle-cmd', type=str, default='truffle', help='Command to run truffle (default=truffle)') @@ -162,18 +154,6 @@ def main(argv=None): if args.export_summary is not None: ETHENO.add_plugin(EventSummaryExportPlugin(args.export_summary)) - - # First, see if we need to install Echidna: - if args.echidna: - if not echidna_exists(): - if not ynprompt('Echidna does not appear to be installed.\nWould you like to have Etheno attempt to ' - 'install it now? [yN] '): - sys.exit(1) - install_echidna() - if not echidna_exists(): - ETHENO.logger.error('Etheno failed to install Echidna. Please install it manually ' - 'https://github.com/trailofbits/echidna') - sys.exit(1) if args.genesis is None: # Set defaults since no genesis was supplied @@ -390,13 +370,6 @@ def truffle_thread(): )) ETHENO.add_plugin(DifferentialTester()) - if args.echidna: - contract_source = None - if args.fuzz_contract is not None: - with open(args.fuzz_contract, 'rb') as c: - contract_source = c.read() - ETHENO.add_plugin(EchidnaPlugin(transaction_limit=args.fuzz_limit, contract_source=contract_source)) - had_plugins = len(ETHENO.plugins) > 0 if ETHENO.master_client is None and not ETHENO.clients and not ETHENO.plugins: diff --git a/etheno/contracts.py b/etheno/contracts.py index 5c21e17..48ef16a 100644 --- a/etheno/contracts.py +++ b/etheno/contracts.py @@ -2,9 +2,10 @@ from .etheno import EthenoPlugin from .utils import format_hex_address +# TODO: what is this file for? class ContractSynchronizer(EthenoPlugin): def __init__(self, source_client, contract_address): - if isintsance(source_client, str): + if isinstance(source_client, str): source_client = RpcProxyClient(source_client) self.source = source_client self.contract = format_hex_address(contract_address, True) diff --git a/etheno/echidna.py b/etheno/echidna.py deleted file mode 100644 index 679291b..0000000 --- a/etheno/echidna.py +++ /dev/null @@ -1,199 +0,0 @@ -import os -import subprocess -import tempfile -from typing import Optional, Union - -from .ascii_escapes import decode -from .etheno import EthenoPlugin -from .utils import ConstantTemporaryFile, format_hex_address - -ECHIDNA_CONTRACT = b'''pragma solidity ^0.4.24; -contract C { - mapping(int => int) public s; - int public stored = 1337; - function save(int key, int value) public { - s[key] = value; - } - function remove(int key) public { - delete s[key]; - } - function setStored(int value) public { - stored = value; - } - function f(uint, int, int[]) public { } - function g(bool, int, address[]) public { } - function echidna_() public returns (bool) { - return true; - } -} -''' - -ECHIDNA_CONFIG = b'''outputRawTxs: true\nquiet: true\ndashboard: false\ngasLimit: 0xfffff\n''' - - -def echidna_exists(): - return subprocess.call(['/usr/bin/env', 'echidna-test', '--help'], stdout=subprocess.DEVNULL) == 0 - - -def stack_exists(): - return subprocess.call(['/usr/bin/env', 'stack', '--help'], stdout=subprocess.DEVNULL) == 0 - - -def git_exists(): - return subprocess.call(['/usr/bin/env', 'git', '--version'], stdout=subprocess.DEVNULL) == 0 - - -def install_echidna(allow_reinstall: bool = False): - if not allow_reinstall and echidna_exists(): - return - elif not git_exists(): - raise Exception('Git must be installed in order to install Echidna') - elif not stack_exists(): - raise Exception('Haskell Stack must be installed in order to install Echidna. On macOS you can easily install ' - 'it using Homebrew: `brew install haskell-stack`') - - with tempfile.TemporaryDirectory() as path: - subprocess.check_call(['/usr/bin/env', 'git', 'clone', 'https://github.com/trailofbits/echidna.git', path]) - # TODO: Once the `dev-etheno` branch is merged into `master`, we can remove this: - subprocess.call(['/usr/bin/env', 'git', 'checkout', 'dev-etheno'], cwd=path) - subprocess.check_call(['/usr/bin/env', 'stack', 'install'], cwd=path) - - -def decode_binary_json(text: Union[str, bytes]) -> Optional[bytes]: - orig = text - text = decode(text).strip() - if not text.startswith(b'['): - return None - offset = len(orig) - len(text) - orig = text - text = text[1:].strip() - offset += len(orig) - len(text) - if text[:1] != b'"': - raise ValueError( - f"Malformed JSON list! Expected '\"' but instead got '{text[0:1].decode()}' at offset {offset}" - ) - text = text[1:] - offset += 1 - if text[-1:] != b']': - raise ValueError( - f"Malformed JSON list! Expected ']' but instead got '{chr(text[-1])}' at offset {offset + len(text) - 1}" - ) - text = text[:-1].strip() - if text[-1:] != b'"': - raise ValueError( - f"Malformed JSON list! Expected '\"' but instead got '{chr(text[-1])}' at offset {offset + len(text) - 1}" - ) - return text[:-1] - - -class EchidnaPlugin(EthenoPlugin): - def __init__(self, transaction_limit: Optional[int] = None, contract_source: Optional[bytes] = None): - self._transaction: int = 0 - self.limit: Optional[int] = transaction_limit - self.contract_address = None - if contract_source is None: - self.contract_source: bytes = ECHIDNA_CONTRACT - else: - self.contract_source = contract_source - self.contract_bytecode = None - - def added(self): - # Wait until the plugin was added to Etheno so its logger is initialized - self.contract_bytecode = self.compile(self.contract_source) - - def run(self): - if not self.etheno.accounts: - self.logger.info("Etheno does not know about any accounts, so Echidna has nothing to do!") - self._shutdown() - return - elif self.contract_source is None: - self.logger.error("Error compiling source contract") - self._shutdown() - # First, deploy the testing contract: - self.logger.info('Deploying Echidna test contract...') - self.contract_address = format_hex_address(self.etheno.deploy_contract(self.etheno.accounts[0], - self.contract_bytecode), True) - if self.contract_address is None: - self.logger.error('Unable to deploy Echidna test contract!') - self._shutdown() - return - self.logger.info("Deployed Echidna test contract to %s" % self.contract_address) - config = self.logger.make_constant_logged_file(ECHIDNA_CONFIG, prefix='echidna', suffix='.yaml') - sol = self.logger.make_constant_logged_file( - self.contract_source, prefix='echidna', suffix='.sol') # type: ignore - echidna_args = ['/usr/bin/env', 'echidna-test', self.logger.to_log_path(sol), '--config', - self.logger.to_log_path(config)] - run_script = self.logger.make_constant_logged_file(' '.join(echidna_args), prefix='run_echidna', suffix='.sh') - # make the script executable: - os.chmod(run_script, 0o755) - - echidna = subprocess.Popen(echidna_args, stderr=subprocess.DEVNULL, stdout=subprocess.PIPE, bufsize=1, - universal_newlines=True, cwd=self.log_directory) - while self.limit is None or self._transaction < self.limit: - line = echidna.stdout.readline() - if line != b'': - txn = decode_binary_json(line) - if txn is None: - continue - self.emit_transaction(txn) - else: - break - self._shutdown() - - def _shutdown(self): - etheno = self.etheno - self.etheno.remove_plugin(self) - etheno.shutdown() - - def compile(self, solidity): - with ConstantTemporaryFile(solidity, prefix='echidna', suffix='.sol') as contract: - solc = subprocess.Popen(['/usr/bin/env', 'solc', '--bin', contract], stderr=subprocess.PIPE, - stdout=subprocess.PIPE, bufsize=1, universal_newlines=True) - errors = solc.stderr.read().strip() - output = solc.stdout.read() - if solc.wait() != 0: - self.logger.error(f"{errors}\n{output}") - return None - elif errors: - if solidity == ECHIDNA_CONTRACT: - # no need to raise a warning with our own contract: - self.logger.debug(errors) - else: - self.logger.warning(errors) - binary_key = 'Binary:' - offset = output.find(binary_key) - if offset < 0: - self.logger.error(f"Could not parse `solc` output:\n{output}") - return None - code = hex(int(output[offset+len(binary_key):].strip(), 16)) - self.logger.debug(f"Compiled contract code: {code}") - return code - - def emit_transaction(self, txn): - self._transaction += 1 - transaction = { - 'id': 1, - 'jsonrpc': '2.0', - 'method': 'eth_sendTransaction', - 'params' : [{ - 'from': format_hex_address(self.etheno.accounts[0], True), - 'to': self.contract_address, - 'gasPrice': "0x%x" % self.etheno.master_client.get_gas_price(), - 'value': '0x0', - 'data': "0x%s" % txn.hex() - }] - } - gas = self.etheno.estimate_gas(transaction) - if gas is None: - self.logger.warning(f"All clients were unable to estimate the gas cost for transaction {self._transaction}." - f" This typically means that Echidna emitted a transaction that is too large.") - return - gas = "0x%x" % gas - self.logger.info(f"Estimated gas cost for Transaction {self._transaction}: {gas}") - transaction['params'][0]['gas'] = gas - self.logger.info("Emitting Transaction %d" % self._transaction) - self.etheno.post(transaction) - - -if __name__ == '__main__': - install_echidna(allow_reinstall=True) diff --git a/examples/ConstantinopleGasUsage/README.md b/examples/ConstantinopleGasUsage/README.md deleted file mode 100644 index 460331f..0000000 --- a/examples/ConstantinopleGasUsage/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Constantinople Gas Usage Consensus Bug - -This example is able to automatically reproduce [the Constantinople -gas usage -discrepancy](https://github.com/paritytech/parity-ethereum/pull/9746) -that caused a hard-fork on Ropsten in October of 2018. This bug was -related to how clients interpreted [a new -EIP](https://eips.ethereum.org/EIPS/eip-1283) changing how gas refunds -are accounted across calls. - -Run this example by using the included -[`run_etheno.sh`](run_etheno.sh) script. - -This example uses [Echidna](https://github.com/trailofbits/echidna), a -property-based fuzzer, so results are nondeterminstic. But generally -running this example should result in at least one failed differential -test. You can get additional details of the transaction that triggered -the bug by examining `log/DifferentialTester/GAS_USAGE/FAILED.log`. - -Note that this example was tested with Geth 1.8.17-stable and Parity -v2.0.8-stable. Newer versions of these clients will likely have -patched the Constantinople consensus bug and Etheno's differential -tester will therefore pass all tests. \ No newline at end of file diff --git a/examples/ConstantinopleGasUsage/constantinople.sol b/examples/ConstantinopleGasUsage/constantinople.sol deleted file mode 100644 index 17a7e57..0000000 --- a/examples/ConstantinopleGasUsage/constantinople.sol +++ /dev/null @@ -1,15 +0,0 @@ -pragma solidity ^0.5.4; -contract C { - int public stored = 1337; - function setStored(int value) public { - stored = value; - } - function increment() public { - int newValue = stored + 1; - stored = 0; - address(this).call(abi.encodeWithSignature("setStored(int256)", newValue)); - } - function echidna_() public returns (bool) { - return true; - } -} diff --git a/examples/ConstantinopleGasUsage/run_etheno.sh b/examples/ConstantinopleGasUsage/run_etheno.sh deleted file mode 100755 index c3fa179..0000000 --- a/examples/ConstantinopleGasUsage/run_etheno.sh +++ /dev/null @@ -1 +0,0 @@ -etheno --echidna --fuzz-limit 20 --fuzz-contract constantinople.sol --accounts 2 --parity --geth --constantinople --log-dir log From 6fb7cca57b96d816219eb5032996f0fabd9aa519 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 17 Jun 2022 15:23:42 -0400 Subject: [PATCH 09/21] Add a `pip-audit` workflow --- .github/workflows/pip-audit.yml | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/pip-audit.yml diff --git a/.github/workflows/pip-audit.yml b/.github/workflows/pip-audit.yml new file mode 100644 index 0000000..3d3c1ac --- /dev/null +++ b/.github/workflows/pip-audit.yml @@ -0,0 +1,37 @@ +name: Scan dependencies for vulnerabilities with pip-audit + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "0 12 * * *" + +jobs: + pip-audit: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install project + run: | + python -m venv /tmp/pip-audit-env + source /tmp/pip-audit-env/bin/activate + + python -m pip install --upgrade pip + python -m pip install . + + + - name: Run pip-audit + uses: trailofbits/gh-action-pip-audit@v0.0.4 + with: + virtual-environment: /tmp/pip-audit-env + From 9d69e66b0381732f6b15392b685f8a7152dd7dde Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Mon, 27 Jun 2022 17:04:35 -0400 Subject: [PATCH 10/21] final removal of manticore references --- Dockerfile | 4 ++-- etheno/__main__.py | 2 +- etheno/logger.py | 7 ------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f0bd86..c902f44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,9 +59,9 @@ WORKDIR /home/etheno COPY --chown=etheno:etheno LICENSE setup.py etheno/ COPY --chown=etheno:etheno etheno/*.py etheno/etheno/ -# TODO: get rid of manticore --user flag here. will need to validate that it doesn't break anything. + RUN cd etheno && \ - pip3 install --no-cache-dir --user '.[manticore]' && \ + pip3 install --no-cache-dir && \ cd .. && \ rm -rf etheno diff --git a/etheno/__main__.py b/etheno/__main__.py index 10b8dc5..e526c24 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -315,7 +315,7 @@ def truffle_thread(): thread.start() # Without Manticore integration the only client types are geth, parity, and command-line raw/regular clients. - # So checking len() >= 2 should be sufficient. + # So checking len() >= 1 should be sufficient. if args.run_differential and (ETHENO.master_client is not None) and len(ETHENO.clients) >= 1: # There are at least two non-Manticore clients running ETHENO.logger.info("Initializing differential tests to compare clients %s" % ', '.join( diff --git a/etheno/logger.py b/etheno/logger.py index 10298f0..8594bec 100644 --- a/etheno/logger.py +++ b/etheno/logger.py @@ -108,13 +108,6 @@ def getLogger(name: str): ret = ETHENO_LOGGERS[name] else: ret = _LOGGING_GETLOGGER(name) - # ####BEGIN#### - # Horrible hack to workaround Manticore's global logging system. - # This can be removed after https://github.com/trailofbits/manticore/issues/1369 - # is resolved. - if name.startswith('manticore'): - ret.propagate = False - # ####END#### return ret From 63dd9c32f1eab4741e3b718c5cc6bfbdf2a6f720 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Mon, 27 Jun 2022 17:14:42 -0400 Subject: [PATCH 11/21] Release candidate 1 (rc-1) will be the next release of Etheno --- etheno/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etheno/__main__.py b/etheno/__main__.py index 06d2030..7672bf6 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -163,6 +163,7 @@ def main(argv=None): if args.export_summary is not None: ETHENO.add_plugin(EventSummaryExportPlugin(args.export_summary)) + # First, see if we need to install Echidna: if args.echidna: if not echidna_exists(): From 4ded2e912537a45e6413b5fa2e241f924623ddf0 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 28 Jun 2022 11:39:14 -0400 Subject: [PATCH 12/21] commented out command line args and added back echidna.py file --- etheno/__main__.py | 3 + etheno/echidna.py | 199 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 etheno/echidna.py diff --git a/etheno/__main__.py b/etheno/__main__.py index 260a203..00301a9 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -28,9 +28,12 @@ def main(argv = None): parser.add_argument('-b', '--balance', type=float, default=100.0, help='Default balance (in Ether) to seed to each account (default=100.0)') parser.add_argument('-c', '--gas-price', type=int, default=None, help='Default gas price (default=20000000000)') parser.add_argument('-i', '--network-id', type=int, default=None, help='Specify a network ID (default is the network ID of the master client)') + ''' + We might need the Echidna feature later for differential fuzz testing but for reducing confusion we will remove the command-line arguments parser.add_argument('-e', '--echidna', action='store_true', default=False, help='Fuzz the clients using transactions generated by Echidna') parser.add_argument('--fuzz-limit', type=int, default=None, help='The maximum number of transactions for Echidna to generate (default=unlimited)') parser.add_argument('--fuzz-contract', type=str, default=None, help='Path to a Solidity contract to have Echidna use for fuzzing (default is to use a builtin generic Echidna fuzzing contract)') + ''' parser.add_argument('-t', '--truffle', action='store_true', default=False, help='Run the truffle migrations in the current directory and exit') parser.add_argument('--truffle-cmd', type=str, default='truffle', help='Command to run truffle (default=truffle)') parser.add_argument('--truffle-args', type=str, default='migrate', diff --git a/etheno/echidna.py b/etheno/echidna.py new file mode 100644 index 0000000..169961d --- /dev/null +++ b/etheno/echidna.py @@ -0,0 +1,199 @@ +import os +import subprocess +import tempfile +from typing import Optional, Union + +from .ascii_escapes import decode +from .etheno import EthenoPlugin +from .utils import ConstantTemporaryFile, format_hex_address + +ECHIDNA_CONTRACT = b'''pragma solidity ^0.4.24; +contract C { + mapping(int => int) public s; + int public stored = 1337; + function save(int key, int value) public { + s[key] = value; + } + function remove(int key) public { + delete s[key]; + } + function setStored(int value) public { + stored = value; + } + function f(uint, int, int[]) public { } + function g(bool, int, address[]) public { } + function echidna_() public returns (bool) { + return true; + } +} +''' + +ECHIDNA_CONFIG = b'''outputRawTxs: true\nquiet: true\ndashboard: false\ngasLimit: 0xfffff\n''' + + +def echidna_exists(): + return subprocess.call(['/usr/bin/env', 'echidna-test', '--help'], stdout=subprocess.DEVNULL) == 0 + + +def stack_exists(): + return subprocess.call(['/usr/bin/env', 'stack', '--help'], stdout=subprocess.DEVNULL) == 0 + + +def git_exists(): + return subprocess.call(['/usr/bin/env', 'git', '--version'], stdout=subprocess.DEVNULL) == 0 + + +def install_echidna(allow_reinstall: bool = False): + if not allow_reinstall and echidna_exists(): + return + elif not git_exists(): + raise Exception('Git must be installed in order to install Echidna') + elif not stack_exists(): + raise Exception('Haskell Stack must be installed in order to install Echidna. On macOS you can easily install ' + 'it using Homebrew: `brew install haskell-stack`') + + with tempfile.TemporaryDirectory() as path: + subprocess.check_call(['/usr/bin/env', 'git', 'clone', 'https://github.com/trailofbits/echidna.git', path]) + # TODO: Once the `dev-etheno` branch is merged into `master`, we can remove this: + subprocess.call(['/usr/bin/env', 'git', 'checkout', 'dev-etheno'], cwd=path) + subprocess.check_call(['/usr/bin/env', 'stack', 'install'], cwd=path) + + +def decode_binary_json(text: Union[str, bytes]) -> Optional[bytes]: + orig = text + text = decode(text).strip() + if not text.startswith(b'['): + return None + offset = len(orig) - len(text) + orig = text + text = text[1:].strip() + offset += len(orig) - len(text) + if text[:1] != b'"': + raise ValueError( + f"Malformed JSON list! Expected '\"' but instead got '{text[0:1].decode()}' at offset {offset}" + ) + text = text[1:] + offset += 1 + if text[-1:] != b']': + raise ValueError( + f"Malformed JSON list! Expected ']' but instead got '{chr(text[-1])}' at offset {offset + len(text) - 1}" + ) + text = text[:-1].strip() + if text[-1:] != b'"': + raise ValueError( + f"Malformed JSON list! Expected '\"' but instead got '{chr(text[-1])}' at offset {offset + len(text) - 1}" + ) + return text[:-1] + + +class EchidnaPlugin(EthenoPlugin): + def __init__(self, transaction_limit: Optional[int] = None, contract_source: Optional[bytes] = None): + self._transaction: int = 0 + self.limit: Optional[int] = transaction_limit + self.contract_address = None + if contract_source is None: + self.contract_source: bytes = ECHIDNA_CONTRACT + else: + self.contract_source = contract_source + self.contract_bytecode = None + + def added(self): + # Wait until the plugin was added to Etheno so its logger is initialized + self.contract_bytecode = self.compile(self.contract_source) + + def run(self): + if not self.etheno.accounts: + self.logger.info("Etheno does not know about any accounts, so Echidna has nothing to do!") + self._shutdown() + return + elif self.contract_source is None: + self.logger.error("Error compiling source contract") + self._shutdown() + # First, deploy the testing contract: + self.logger.info('Deploying Echidna test contract...') + self.contract_address = format_hex_address(self.etheno.deploy_contract(self.etheno.accounts[0], + self.contract_bytecode), True) + if self.contract_address is None: + self.logger.error('Unable to deploy Echidna test contract!') + self._shutdown() + return + self.logger.info("Deployed Echidna test contract to %s" % self.contract_address) + config = self.logger.make_constant_logged_file(ECHIDNA_CONFIG, prefix='echidna', suffix='.yaml') + sol = self.logger.make_constant_logged_file( + self.contract_source, prefix='echidna', suffix='.sol') # type: ignore + echidna_args = ['/usr/bin/env', 'echidna-test', self.logger.to_log_path(sol), '--config', + self.logger.to_log_path(config)] + run_script = self.logger.make_constant_logged_file(' '.join(echidna_args), prefix='run_echidna', suffix='.sh') + # make the script executable: + os.chmod(run_script, 0o755) + + echidna = subprocess.Popen(echidna_args, stderr=subprocess.DEVNULL, stdout=subprocess.PIPE, bufsize=1, + universal_newlines=True, cwd=self.log_directory) + while self.limit is None or self._transaction < self.limit: + line = echidna.stdout.readline() + if line != b'': + txn = decode_binary_json(line) + if txn is None: + continue + self.emit_transaction(txn) + else: + break + self._shutdown() + + def _shutdown(self): + etheno = self.etheno + self.etheno.remove_plugin(self) + etheno.shutdown() + + def compile(self, solidity): + with ConstantTemporaryFile(solidity, prefix='echidna', suffix='.sol') as contract: + solc = subprocess.Popen(['/usr/bin/env', 'solc', '--bin', contract], stderr=subprocess.PIPE, + stdout=subprocess.PIPE, bufsize=1, universal_newlines=True) + errors = solc.stderr.read().strip() + output = solc.stdout.read() + if solc.wait() != 0: + self.logger.error(f"{errors}\n{output}") + return None + elif errors: + if solidity == ECHIDNA_CONTRACT: + # no need to raise a warning with our own contract: + self.logger.debug(errors) + else: + self.logger.warning(errors) + binary_key = 'Binary:' + offset = output.find(binary_key) + if offset < 0: + self.logger.error(f"Could not parse `solc` output:\n{output}") + return None + code = hex(int(output[offset+len(binary_key):].strip(), 16)) + self.logger.debug(f"Compiled contract code: {code}") + return code + + def emit_transaction(self, txn): + self._transaction += 1 + transaction = { + 'id': 1, + 'jsonrpc': '2.0', + 'method': 'eth_sendTransaction', + 'params' : [{ + 'from': format_hex_address(self.etheno.accounts[0], True), + 'to': self.contract_address, + 'gasPrice': "0x%x" % self.etheno.master_client.get_gas_price(), + 'value': '0x0', + 'data': "0x%s" % txn.hex() + }] + } + gas = self.etheno.estimate_gas(transaction) + if gas is None: + self.logger.warning(f"All clients were unable to estimate the gas cost for transaction {self._transaction}." + f" This typically means that Echidna emitted a transaction that is too large.") + return + gas = "0x%x" % gas + self.logger.info(f"Estimated gas cost for Transaction {self._transaction}: {gas}") + transaction['params'][0]['gas'] = gas + self.logger.info("Emitting Transaction %d" % self._transaction) + self.etheno.post(transaction) + + +if __name__ == '__main__': + install_echidna(allow_reinstall=True) \ No newline at end of file From 084097cdfa2f3877c1a38300769dda12baaf1432 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 28 Jun 2022 11:43:38 -0400 Subject: [PATCH 13/21] created venv and added back some removed code --- .gitignore | 1 + etheno/__main__.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d7423af..1afe0aa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc build dist +venv/ \ No newline at end of file diff --git a/etheno/__main__.py b/etheno/__main__.py index 00301a9..1f16431 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -7,6 +7,7 @@ from .client import RpcProxyClient from .differentials import DifferentialTester +from .echidna import echidna_exists, EchidnaPlugin, install_echidna from .etheno import app, EthenoView, GETH_DEFAULT_RPC_PORT, ETHENO, VERSION_NAME from .genesis import Account, make_accounts, make_genesis from .jsonrpc import EventSummaryExportPlugin, JSONRPCExportPlugin @@ -139,7 +140,21 @@ def main(argv = None): if args.export_summary is not None: ETHENO.add_plugin(EventSummaryExportPlugin(args.export_summary)) - + + # First, see if we need to install Echidna: + ''' + Commenting this out and can re-use if needed + if args.echidna: + if not echidna_exists(): + if not ynprompt('Echidna does not appear to be installed.\nWould you like to have Etheno attempt to ' + 'install it now? [yN] '): + sys.exit(1) + install_echidna() + if not echidna_exists(): + ETHENO.logger.error('Etheno failed to install Echidna. Please install it manually ' + 'https://github.com/trailofbits/echidna') + sys.exit(1) + ''' if args.genesis is None: # Set defaults since no genesis was supplied if args.accounts is None: @@ -313,7 +328,16 @@ def truffle_thread(): map(str, [ETHENO.master_client] + ETHENO.clients) )) ETHENO.add_plugin(DifferentialTester()) - + + ''' + Keeping in case we want it later + if args.echidna: + contract_source = None + if args.fuzz_contract is not None: + with open(args.fuzz_contract, 'rb') as c: + contract_source = c.read() + ETHENO.add_plugin(EchidnaPlugin(transaction_limit=args.fuzz_limit, contract_source=contract_source)) + ''' had_plugins = len(ETHENO.plugins) > 0 if ETHENO.master_client is None and not ETHENO.clients and not ETHENO.plugins: From f9f3714095868405b1855eae5174030e62908b9a Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 28 Jun 2022 12:30:26 -0400 Subject: [PATCH 14/21] something with eggs --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1afe0aa..63195df 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.pyc build dist -venv/ \ No newline at end of file +venv/ +*egg* \ No newline at end of file From 47ddf600b88a5312daf17f0509409c6eb8c1dcc6 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Wed, 29 Jun 2022 10:29:37 -0400 Subject: [PATCH 15/21] initial commit of updating ganache to v7.0+ --- Dockerfile | 3 ++- etheno/__main__.py | 21 ++++++++++++++------- etheno/ganache.py | 6 +++--- setup.py | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95c6738..5c645db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,8 @@ RUN DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:ethereum/ethereum & RUN curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - && sudo apt-get install -y --no-install-recommends nodejs && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN npm install --production -g ganache-cli truffle && npm --force cache clean +# TODO: need to check whether this installation is correct +RUN npm install --production -g ganache truffle && npm --force cache clean RUN useradd -m etheno RUN usermod -aG sudo etheno diff --git a/etheno/__main__.py b/etheno/__main__.py index 1f16431..62dd880 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -19,15 +19,20 @@ from . import parity from . import truffle +# Constant for converting whole units to wei +ETHER = 1e18 + def main(argv = None): parser = argparse.ArgumentParser(description='An Ethereum JSON RPC multiplexer, differential fuzzer, and test framework integration tool.') parser.add_argument('--debug', action='store_true', default=False, help='Enable debugging from within the web server') parser.add_argument('--run-publicly', action='store_true', default=False, help='Allow the web server to accept external connections') parser.add_argument('-p', '--port', type=int, default=GETH_DEFAULT_RPC_PORT, help='Port on which to run the JSON RPC webserver (default=%d)' % GETH_DEFAULT_RPC_PORT) - parser.add_argument('-a', '--accounts', type=int, default=None, help='Number of accounts to create in the client (default=10)') - parser.add_argument('-b', '--balance', type=float, default=100.0, help='Default balance (in Ether) to seed to each account (default=100.0)') - parser.add_argument('-c', '--gas-price', type=int, default=None, help='Default gas price (default=20000000000)') + parser.add_argument('-a', '--accounts', type=int, default=10, help='Number of accounts to create in the client (default=10)') + parser.add_argument('-b', '--balance', type=float, default=1000.0, help='Default balance (in Ether) to seed to each account (default=100.0)') + # TODO: do we really need a gas price specified for ganache? is there a use case here? + parser.add_argument('-c', '--gas-price', type=int, default=20000000000, help='Default gas price (default=20000000000)') + # TODO: networkID can have a default value it seems like parser.add_argument('-i', '--network-id', type=int, default=None, help='Specify a network ID (default is the network ID of the master client)') ''' We might need the Echidna feature later for differential fuzz testing but for reducing confusion we will remove the command-line arguments @@ -42,7 +47,7 @@ def main(argv = None): parser.add_argument('-g', '--ganache', action='store_true', default=False, help='Run Ganache as a master JSON RPC client (cannot be used in conjunction with --master)') parser.add_argument('--ganache-cmd', type=str, default=None, help='Specify a command that runs Ganache ' - '(default="/usr/bin/env ganache-cli")') + '(default="/usr/bin/env ganache")') parser.add_argument('--ganache-args', type=str, default=None, help='Additional arguments to pass to Ganache') parser.add_argument('--ganache-port', type=int, default=None, @@ -164,6 +169,7 @@ def main(argv = None): accounts = [] + # TODO: args.gas_price is not set if a genesis file is provided if args.genesis: with open(args.genesis, 'rb') as f: genesis = json.load(f) @@ -186,7 +192,7 @@ def main(argv = None): # We will generate it further below once we've resolved all of the parameters genesis = None - accounts += make_accounts(args.accounts, default_balance=int(args.balance * 1000000000000000000)) + accounts += make_accounts(args.accounts, default_balance=int(args.balance * ETHER)) if genesis is not None: # add the new accounts to the genesis @@ -214,13 +220,14 @@ def main(argv = None): if args.network_id is None: args.network_id = 0x657468656E6F # 'etheno' in hex - ganache_accounts = ["--account=%s,0x%x" % (acct.private_key, acct.balance) for acct in accounts] + # Have to use hex() so that string is hex-encoded (prefixed with 0x) that is necessary for Ganache v7.0+ + # https://github.com/trufflesuite/ganache/discussions/1075 + ganache_accounts = ["--account=%s,0x%x" % (hex(acct.private_key), acct.balance) for acct in accounts] ganache_args = ganache_accounts + ['-g', str(args.gas_price), '-i', str(args.network_id)] if args.ganache_args is not None: ganache_args += shlex.split(args.ganache_args) - ganache_instance = ganache.Ganache(cmd=args.ganache_cmd, args=ganache_args, port=args.ganache_port) ETHENO.master_client = ganache.GanacheClient(ganache_instance) diff --git a/etheno/ganache.py b/etheno/ganache.py index 9446c36..79f72b1 100644 --- a/etheno/ganache.py +++ b/etheno/ganache.py @@ -19,7 +19,7 @@ def __init__(self, cmd=None, args=None, port=8546): if cmd is not None: cmd = shlex.split(cmd) else: - cmd = ['/usr/bin/env', 'ganache-cli'] + cmd = ['/usr/bin/env', 'ganache'] if args is None: args = [] self.args = cmd + ['-d', '-p', str(port)] + args @@ -29,8 +29,8 @@ def __init__(self, cmd=None, args=None, port=8546): def start(self): if self.ganache: return - if shutil.which("ganache-cli") is None: - raise ValueError("`ganache-cli` is not installed! Install it by running `npm -g i ganache-cli`") + if shutil.which("ganache") is None: + raise ValueError("`ganache` is not installed! Install it by running `npm -g i ganache`") if self._client: self.ganache = PtyLogger(self._client.logger, self.args) self.ganache.start() diff --git a/setup.py b/setup.py index 65a3bda..62d1666 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ description='Etheno is a JSON RPC multiplexer, differential fuzzer, and test framework integration tool.', url='https://github.com/trailofbits/etheno', author='Trail of Bits', - version='0.2.4', + version='0.2.5', packages=find_packages(), python_requires='>=3.6', install_requires=[ From acadd428e3537a1e7f839f340a1b5ba8c724d506 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Thu, 30 Jun 2022 12:51:37 -0400 Subject: [PATCH 16/21] Added some todo and one bug fix --- etheno/client.py | 3 ++- etheno/jsonrpc.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/etheno/client.py b/etheno/client.py index a493c87..69d4a22 100644 --- a/etheno/client.py +++ b/etheno/client.py @@ -179,7 +179,8 @@ def create_account(self, balance: int = 0, address: Optional[int] = None): def wait_until_running(self): pass - + + # TODO: need to ensure that JSON RPC calls match latest API spec def post(self, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: ret = self.client.post(data) if ret is not None and 'error' in ret: diff --git a/etheno/jsonrpc.py b/etheno/jsonrpc.py index a7f25be..9d9f045 100644 --- a/etheno/jsonrpc.py +++ b/etheno/jsonrpc.py @@ -143,11 +143,14 @@ def after_post(self, post_data, result): else: value = original_transaction['value'] if 'to' not in result['result'] or result['result']['to'] is None: + print(original_transaction) + print(result) # this transaction is creating a contract: + # TODO: key errors are likely here...need to figure out a better way to do error handling contract_address = result['result']['contractAddress'] - self.handle_contract_created(original_transaction['from'], contract_address, result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'], value) + self.handle_contract_created(original_transaction['from'], contract_address, result['result']['gasUsed'], result['result']['effectiveGasPrice'], original_transaction['data'], value) else: - self.handle_function_call(original_transaction['from'], original_transaction['to'], result['result']['gasUsed'], original_transaction['gasPrice'], original_transaction['data'] if 'data' in original_transaction else '0x', value) + self.handle_function_call(original_transaction['from'], original_transaction['to'], result['result']['gasUsed'], result['result']['effectiveGasPrice'], original_transaction['data'] if 'data' in original_transaction else '0x', value) class EventSummaryExportPlugin(EventSummaryPlugin): From 40bfb8fc0ac2f413ae9fcd1dc4cfd8ce7b7993dc Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 5 Jul 2022 11:27:42 -0400 Subject: [PATCH 17/21] Minor bugfix during event logging --- etheno/jsonrpc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/etheno/jsonrpc.py b/etheno/jsonrpc.py index 9d9f045..112bd49 100644 --- a/etheno/jsonrpc.py +++ b/etheno/jsonrpc.py @@ -3,7 +3,7 @@ from .etheno import EthenoPlugin from .utils import format_hex_address - +from .client import JSONRPCError # source: https://ethereum.stackexchange.com/a/83855 import rlp @@ -119,6 +119,11 @@ def after_post(self, post_data, result): result = result[0] if 'method' not in post_data: return + # Fixes bug that occurs when a JSONRPCError is attempted to be logged + if isinstance(result, JSONRPCError): + self.logger.error(f'Received a JSON RPC Error when logging transaction...skipping event logging') + return + elif (post_data['method'] == 'eth_sendTransaction' or post_data['method'] == 'eth_sendRawTransaction') and 'result' in result: try: transaction_hash = int(result['result'], 16) From 6d7296ff63b92c39db43f85c94c49e34296774f0 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 5 Jul 2022 13:09:02 -0400 Subject: [PATCH 18/21] Updated flask shutdown methodology, allow unlimited contract size by default --- etheno/etheno.py | 22 ++++++++++++++++------ etheno/ganache.py | 2 +- etheno/jsonrpc.py | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/etheno/etheno.py b/etheno/etheno.py index 74bf2eb..eadcb98 100644 --- a/etheno/etheno.py +++ b/etheno/etheno.py @@ -1,6 +1,7 @@ import pkg_resources from threading import Thread from typing import Any, Dict, List, Optional +from werkzeug.serving import make_server from flask import Flask, jsonify, request, abort from flask.views import MethodView @@ -31,6 +32,8 @@ def to_account_address(raw_address: int) -> str: _CONTROLLER = threadwrapper.MainThreadController() +# Using 'werkzeug.server.shutdown' is deprecated after Flask 2.1.x +# TODO: Will probably remove this function @app.route('/shutdown') def _etheno_shutdown(): # shut down the Flask server @@ -320,6 +323,9 @@ def shutdown(self, port: int = GETH_DEFAULT_RPC_PORT): for client in self.clients: client.shutdown() self.logger.close() + _CONTROLLER.quit() + """ + Won't need this if shutdown() function is removed from urllib.request import urlopen import socket import urllib @@ -329,19 +335,22 @@ def shutdown(self, port: int = GETH_DEFAULT_RPC_PORT): pass except urllib.error.URLError: pass + """ def run(self, debug=True, run_publicly=False, port=GETH_DEFAULT_RPC_PORT): # Manticore only works in the main thread, so use a threadsafe wrapper: - def flask_thread(): + def server_thread(): if run_publicly: host='0.0.0.0' else: - host = None + host = "localhost" # Do not use the reloader, because Flask needs to run in the main thread to use the reloader - app.run(debug=debug, host=host, port=port, use_reloader=False) - thread = Thread(target=flask_thread) + server = make_server(host=host, port=port, app=app, threaded=True) + return server + # app.run(debug=debug, host=host, port=port, use_reloader=False) + server = server_thread() + thread = Thread(target=server.serve_forever) thread.start() - self.logger.info("Etheno v%s" % VERSION) for plugin in self.plugins: @@ -349,9 +358,10 @@ def flask_thread(): _CONTROLLER.run() self.shutdown() + self.logger.info("Shutting Etheno down") + server.shutdown() thread.join() - ETHENO = Etheno() diff --git a/etheno/ganache.py b/etheno/ganache.py index 79f72b1..9a19df6 100644 --- a/etheno/ganache.py +++ b/etheno/ganache.py @@ -22,7 +22,7 @@ def __init__(self, cmd=None, args=None, port=8546): cmd = ['/usr/bin/env', 'ganache'] if args is None: args = [] - self.args = cmd + ['-d', '-p', str(port)] + args + self.args = cmd + ['-d', '-p', str(port), '--chain.allowUnlimitedContractSize'] + args self.ganache = None self._client = None diff --git a/etheno/jsonrpc.py b/etheno/jsonrpc.py index 112bd49..4176f45 100644 --- a/etheno/jsonrpc.py +++ b/etheno/jsonrpc.py @@ -121,7 +121,7 @@ def after_post(self, post_data, result): return # Fixes bug that occurs when a JSONRPCError is attempted to be logged if isinstance(result, JSONRPCError): - self.logger.error(f'Received a JSON RPC Error when logging transaction...skipping event logging') + self.logger.info(f'Received a JSON RPC Error when logging transaction...skipping event logging') return elif (post_data['method'] == 'eth_sendTransaction' or post_data['method'] == 'eth_sendRawTransaction') and 'result' in result: From 1bb8ce6f996bf05a985a0d63792a1b57227f1d30 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 5 Jul 2022 13:37:14 -0400 Subject: [PATCH 19/21] updated flask version and added some todos --- etheno/logger.py | 7 ++++--- etheno/synchronization.py | 1 + setup.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/etheno/logger.py b/etheno/logger.py index 8594bec..ff82e5e 100644 --- a/etheno/logger.py +++ b/etheno/logger.py @@ -4,7 +4,7 @@ import tempfile import threading import time -from typing import Callable, List, Optional, Union +from typing import Callable, List, Optional, Union, Any import ptyprocess @@ -33,7 +33,7 @@ class CGAColors(enum.Enum): NOTSET: CGAColors.BLUE } - +# TODO: seems like this function can be removed, no references? def formatter_message(message: str, use_color: bool = True) -> str: if use_color: message = message.replace("$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ) @@ -307,7 +307,8 @@ def __init__(self, logger: logging.Logger, *streams, newline_char=b'\n'): self._newline_char = newline_char self._buffers = [b'' for i in range(len(streams))] self._done: bool = False - self.log: Callable[[logging.Logger, Union[str, bytes]], ...] = lambda lgr, message: lgr.info(message) + # TODO: Made a small change here due to the ellipses not being allowed, make sure it does not create any other issues + self.log: Callable[[logging.Logger, Union[str, bytes]], Any] = lambda lgr, message: lgr.info(message) def is_done(self) -> bool: return self._done diff --git a/etheno/synchronization.py b/etheno/synchronization.py index 81eb9fe..0486e59 100644 --- a/etheno/synchronization.py +++ b/etheno/synchronization.py @@ -58,6 +58,7 @@ def __init__(self, client): self._client = client def create_account(self, balance = 0, address = None): + # TODO: not sure what the data field is supposed to do here if self._client == self._client.etheno.master_client: return self._old_create_account(data) try: diff --git a/setup.py b/setup.py index 62d1666..7d8d515 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ install_requires=[ 'ptyprocess', 'pysha3>=1.0.2', - 'flask>=1.0.2', + 'flask>=2.1.0', 'web3', # The following two requirements are for our fork of `keyfile.py`, # but they should already be satisfied by the `web3` requirement From 411cd0a6d232f5e56b83627bc9cafe63e00c492d Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 5 Jul 2022 13:43:17 -0400 Subject: [PATCH 20/21] remove print statements --- etheno/jsonrpc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/etheno/jsonrpc.py b/etheno/jsonrpc.py index 4176f45..47b3047 100644 --- a/etheno/jsonrpc.py +++ b/etheno/jsonrpc.py @@ -148,8 +148,6 @@ def after_post(self, post_data, result): else: value = original_transaction['value'] if 'to' not in result['result'] or result['result']['to'] is None: - print(original_transaction) - print(result) # this transaction is creating a contract: # TODO: key errors are likely here...need to figure out a better way to do error handling contract_address = result['result']['contractAddress'] From caa6758b622695d91099b401206ee304b09bb768 Mon Sep 17 00:00:00 2001 From: Anish Naik Date: Tue, 5 Jul 2022 13:57:07 -0400 Subject: [PATCH 21/21] pin web3 version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d8d515..13cbefb 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ 'ptyprocess', 'pysha3>=1.0.2', 'flask>=2.1.0', - 'web3', + 'web3>=3.16.4', # The following two requirements are for our fork of `keyfile.py`, # but they should already be satisfied by the `web3` requirement 'cytoolz>=0.9.0,<1.0.0',