diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca8f3d09..98fb36c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "windows-2022"] - type: ["brownie", "buidler", "dapp", "embark", "hardhat", "solc", "truffle", "waffle", "foundry", "standard", "solc_multi_file", "hardhat_multi_file"] + type: ["brownie", "buidler", "dapp", "embark", "hardhat", "solc", "truffle", "waffle", "foundry", "standard", "vyper", "solc_multi_file", "hardhat_multi_file"] exclude: # Currently broken, tries to pull git:// which is blocked by GH - type: embark diff --git a/crytic_compile/crytic_compile.py b/crytic_compile/crytic_compile.py index 5c238598..4df96647 100644 --- a/crytic_compile/crytic_compile.py +++ b/crytic_compile/crytic_compile.py @@ -21,7 +21,9 @@ artifact_path, ) from crytic_compile.compilation_unit import CompilationUnit -from crytic_compile.platform import all_platforms, solc_standard_json +from crytic_compile.platform import all_platforms +from crytic_compile.platform.solc_standard_json import SolcStandardJson +from crytic_compile.platform.vyper import VyperStandardJson from crytic_compile.platform.abstract_platform import AbstractPlatform from crytic_compile.platform.all_export import PLATFORMS_EXPORT from crytic_compile.platform.solc import Solc @@ -676,18 +678,14 @@ def compile_all(target: str, **kwargs: str) -> List[CryticCompile]: **kwargs: optional arguments. Used: "solc_standard_json" Raises: - ValueError: If the target could not be compiled + NotImplementedError: If the target could not be compiled Returns: List[CryticCompile]: Returns a list of CryticCompile instances for all compilations which occurred. """ use_solc_standard_json = kwargs.get("solc_standard_json", False) - # Attempt to perform glob expansion of target/filename - globbed_targets = glob.glob(target, recursive=True) - # Check if the target refers to a valid target already. - # If it does not, we assume it's a glob pattern. compilations: List[CryticCompile] = [] if os.path.isfile(target) or is_supported(target): if target.endswith(".zip"): @@ -699,28 +697,33 @@ def compile_all(target: str, **kwargs: str) -> List[CryticCompile]: compilations = load_from_zip(tmp.name) else: compilations.append(CryticCompile(target, **kwargs)) - elif os.path.isdir(target) or len(globbed_targets) > 0: - # We create a new glob to find solidity files at this path (in case this is a directory) - filenames = glob.glob(os.path.join(target, "*.sol")) - if not filenames: - filenames = glob.glob(os.path.join(target, "*.vy")) - if not filenames: - filenames = globbed_targets - + elif os.path.isdir(target): + solidity_filenames = glob.glob(os.path.join(target, "*.sol")) + vyper_filenames = glob.glob(os.path.join(target, "*.vy")) # Determine if we're using --standard-solc option to # aggregate many files into a single compilation. if use_solc_standard_json: # If we're using standard solc, then we generated our # input to create a single compilation with all files - standard_json = solc_standard_json.SolcStandardJson() - for filename in filenames: - standard_json.add_source_file(filename) - compilations.append(CryticCompile(standard_json, **kwargs)) + solc_standard_json = SolcStandardJson() + solc_standard_json.add_source_files(solidity_filenames) + compilations.append(CryticCompile(solc_standard_json, **kwargs)) else: # We compile each file and add it to our compilations. - for filename in filenames: + for filename in solidity_filenames: compilations.append(CryticCompile(filename, **kwargs)) + + if vyper_filenames: + vyper_standard_json = VyperStandardJson() + vyper_standard_json.add_source_files(vyper_filenames) + compilations.append(CryticCompile(vyper_standard_json, **kwargs)) else: - raise ValueError(f"{str(target)} is not a file or directory.") + raise NotImplementedError() + # TODO split glob into language + # # Attempt to perform glob expansion of target/filename + # globbed_targets = glob.glob(target, recursive=True) + # print(globbed_targets) + + # raise ValueError(f"{str(target)} is not a file or directory.") return compilations diff --git a/crytic_compile/platform/all_platforms.py b/crytic_compile/platform/all_platforms.py index 624629e2..5e2fe1ba 100644 --- a/crytic_compile/platform/all_platforms.py +++ b/crytic_compile/platform/all_platforms.py @@ -14,6 +14,6 @@ from .solc_standard_json import SolcStandardJson from .standard import Standard from .truffle import Truffle -from .vyper import Vyper +from .vyper import VyperStandardJson from .waffle import Waffle from .foundry import Foundry diff --git a/crytic_compile/platform/solc_standard_json.py b/crytic_compile/platform/solc_standard_json.py index d7006f9a..beadf442 100644 --- a/crytic_compile/platform/solc_standard_json.py +++ b/crytic_compile/platform/solc_standard_json.py @@ -405,6 +405,15 @@ def add_source_file(self, file_path: str) -> None: """ add_source_file(self._json, file_path) + def add_source_files(self, files_path: List[str]) -> None: + """Append files + + Args: + files_path (List[str]): files to append + """ + for file_path in files_path: + add_source_file(self._json, file_path) + def add_remapping(self, remapping: str) -> None: """Append our remappings diff --git a/crytic_compile/platform/vyper.py b/crytic_compile/platform/vyper.py index 49133fd9..4d6fc13f 100644 --- a/crytic_compile/platform/vyper.py +++ b/crytic_compile/platform/vyper.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger("CryticCompile") -class Vyper(AbstractPlatform): +class VyperStandardJson(AbstractPlatform): """ Vyper platform """ @@ -34,6 +34,28 @@ class Vyper(AbstractPlatform): PROJECT_URL = "https://github.com/vyperlang/vyper" TYPE = Type.VYPER + def __init__(self, target: Optional[Path] = None, **_kwargs: str): + super().__init__(str(target), **_kwargs) + self.standard_json_input = { + "language": "Vyper", + "sources": {}, + "settings": { + "outputSelection": { + "*": { + "*": [ + "abi", + "devdoc", + "userdoc", + "evm.bytecode", + "evm.deployedBytecode", + "evm.deployedBytecode.sourceMap", + ], + "": ["ast"], + } + } + }, + } + def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None: """Compile the target @@ -44,46 +66,67 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None: """ target = self._target + # If the target was a directory `add_source_file` should have been called + # by `compile_all`. Otherwise, we should have a single file target. + if self._target is not None and os.path.isfile(self._target): + self.add_source_files([target]) - vyper = kwargs.get("vyper", "vyper") - - targets_json = _run_vyper(target, vyper) + vyper_bin = kwargs.get("vyper", "vyper") - assert "version" in targets_json + compilation_artifacts = _run_vyper_standard_json(self.standard_json_input, vyper_bin) compilation_unit = CompilationUnit(crytic_compile, str(target)) + compiler_version = compilation_artifacts["compiler"].split("-")[1] + if compiler_version != "0.3.7": + LOGGER.info("Vyper != 0.3.7 support is a best effort and might fail") compilation_unit.compiler_version = CompilerVersion( - compiler="vyper", version=targets_json["version"], optimized=False + compiler="vyper", version=compiler_version, optimized=False ) - assert target in targets_json - - info = targets_json[target] - filename = convert_filename(target, _relative_to_short, crytic_compile) - - contract_name = Path(target).parts[-1] + for source_file, contract_info in compilation_artifacts["contracts"].items(): + filename = convert_filename(source_file, _relative_to_short, crytic_compile) + source_unit = compilation_unit.create_source_unit(filename) + for contract_name, contract_metadata in contract_info.items(): + source_unit.add_contract_name(contract_name) + compilation_unit.filename_to_contracts[filename].add(contract_name) + + source_unit.abis[contract_name] = contract_metadata["abi"] + source_unit.bytecodes_init[contract_name] = contract_metadata["evm"]["bytecode"][ + "object" + ].replace("0x", "") + # Vyper does not provide the source mapping for the init bytecode + source_unit.srcmaps_init[contract_name] = [] + source_unit.srcmaps_runtime[contract_name] = contract_metadata["evm"][ + "deployedBytecode" + ]["sourceMap"].split(";") + source_unit.bytecodes_runtime[contract_name] = contract_metadata["evm"][ + "deployedBytecode" + ]["object"].replace("0x", "") + source_unit.natspec[contract_name] = Natspec( + contract_metadata["userdoc"], contract_metadata["devdoc"] + ) + + for source_file, ast in compilation_artifacts["sources"].items(): + filename = convert_filename(source_file, _relative_to_short, crytic_compile) + source_unit = compilation_unit.create_source_unit(filename) + source_unit.ast = ast + + def add_source_files(self, file_paths: List[str]) -> None: + """ + Append files - source_unit = compilation_unit.create_source_unit(filename) + Args: + file_paths (List[str]): files to append - source_unit.add_contract_name(contract_name) - compilation_unit.filename_to_contracts[filename].add(contract_name) - source_unit.abis[contract_name] = info["abi"] - source_unit.bytecodes_init[contract_name] = info["bytecode"].replace("0x", "") - source_unit.bytecodes_runtime[contract_name] = info["bytecode_runtime"].replace("0x", "") - # Vyper does not provide the source mapping for the init bytecode - source_unit.srcmaps_init[contract_name] = [] - # info["source_map"]["pc_pos_map"] contains the source mapping in a simpler format - # However pc_pos_map_compressed" seems to follow solc's format, so for convenience - # We store the same - # TODO: create SourceMapping class, so that srcmaps_runtime would store an class - # That will give more flexebility to different compilers - source_unit.srcmaps_runtime[contract_name] = info["source_map"]["pc_pos_map_compressed"] + Returns: - # Natspec not yet handled for vyper - source_unit.natspec[contract_name] = Natspec({}, {}) + """ - ast = _get_vyper_ast(target, vyper) - source_unit.ast = ast + for file_path in file_paths: + with open(file_path, "r", encoding="utf8") as f: + self.standard_json_input["sources"][file_path] = { # type: ignore + "content": f.read(), + } def clean(self, **_kwargs: str) -> None: """Clean compilation artifacts @@ -129,16 +172,15 @@ def _guessed_tests(self) -> List[str]: return [] -def _run_vyper( - filename: str, vyper: str, env: Optional[Dict] = None, working_dir: Optional[str] = None +def _run_vyper_standard_json( + standard_json_input: Dict, vyper: str, env: Optional[Dict] = None ) -> Dict: - """Run vyper + """Run vyper and write compilation output to a file Args: - filename (str): vyper file + standard_json_input (Dict): Dict containing the vyper standard json input vyper (str): vyper binary env (Optional[Dict], optional): Environment variables. Defaults to None. - working_dir (Optional[str], optional): Working directory. Defaults to None. Raises: InvalidCompilation: If vyper failed to run @@ -146,81 +188,29 @@ def _run_vyper( Returns: Dict: Vyper json compilation artifact """ - if not os.path.isfile(filename): - raise InvalidCompilation(f"{filename} does not exist (are you in the correct directory?)") - - cmd = [vyper, filename, "-f", "combined_json"] - - additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {} - stderr = "" - LOGGER.info( - "'%s' running", - " ".join(cmd), - ) - try: - with subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - executable=shutil.which(cmd[0]), - **additional_kwargs, - ) as process: - stdout, stderr = process.communicate() - res = stdout.split(b"\n") - res = res[-2] - return json.loads(res) - except OSError as error: - # pylint: disable=raise-missing-from - raise InvalidCompilation(error) - except json.decoder.JSONDecodeError: - # pylint: disable=raise-missing-from - raise InvalidCompilation(f"Invalid vyper compilation\n{stderr}") - - -def _get_vyper_ast( - filename: str, vyper: str, env: Optional[Dict] = None, working_dir: Optional[str] = None -) -> Dict: - """Get ast from vyper - - Args: - filename (str): vyper file - vyper (str): vyper binary - env (Dict, optional): Environment variables. Defaults to None. - working_dir (str, optional): Working directory. Defaults to None. - - Raises: - InvalidCompilation: If vyper failed to run - - Returns: - Dict: [description] - """ - if not os.path.isfile(filename): - raise InvalidCompilation(f"{filename} does not exist (are you in the correct directory?)") - - cmd = [vyper, filename, "-f", "ast"] - - additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {} - stderr = "" - try: - with subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - executable=shutil.which(cmd[0]), - **additional_kwargs, - ) as process: - stdout, stderr = process.communicate() - res = stdout.split(b"\n") - res = res[-2] - return json.loads(res) - except json.decoder.JSONDecodeError: - # pylint: disable=raise-missing-from - raise InvalidCompilation(f"Invalid vyper compilation\n{stderr}") - except Exception as exception: - # pylint: disable=raise-missing-from - raise InvalidCompilation(exception) + cmd = [vyper, "--standard-json"] + + with subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + executable=shutil.which(cmd[0]), + ) as process: + + stdout_b, stderr_b = process.communicate(json.dumps(standard_json_input).encode("utf-8")) + stdout, _stderr = ( + stdout_b.decode(), + stderr_b.decode(errors="backslashreplace"), + ) # convert bytestrings to unicode strings + + vyper_standard_output = json.loads(stdout) + if "errors" in vyper_standard_output: + # TODO format errors + raise InvalidCompilation(vyper_standard_output["errors"]) + + return vyper_standard_output def _relative_to_short(relative: Path) -> Path: diff --git a/scripts/ci_test_vyper.sh b/scripts/ci_test_vyper.sh new file mode 100755 index 00000000..9be8fa31 --- /dev/null +++ b/scripts/ci_test_vyper.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +### Test vyper integration + +pip install vyper + +echo "Testing vyper integration of $(realpath "$(which crytic-compile)")" + +cd tests/vyper || exit 255 + +if ! crytic-compile auction.vy --export-formats standard +then echo "vyper test failed" && exit 255 +else echo "vyper test passed" && exit 0 +fi diff --git a/tests/vyper/auction.vy b/tests/vyper/auction.vy new file mode 100644 index 00000000..1486d2d8 --- /dev/null +++ b/tests/vyper/auction.vy @@ -0,0 +1,178 @@ +# Taken from https://github.com/vyperlang/vyper/blob/9136169468f317a53b4e7448389aa315f90b95ba/examples/auctions/blind_auction.vy +# Blind Auction. Adapted to Vyper from [Solidity by Example](https://github.com/ethereum/solidity/blob/develop/docs/solidity-by-example.rst#blind-auction-1) + +struct Bid: + blindedBid: bytes32 + deposit: uint256 + +# Note: because Vyper does not allow for dynamic arrays, we have limited the +# number of bids that can be placed by one address to 128 in this example +MAX_BIDS: constant(int128) = 128 + +# Event for logging that auction has ended +event AuctionEnded: + highestBidder: address + highestBid: uint256 + +# Auction parameters +beneficiary: public(address) +biddingEnd: public(uint256) +revealEnd: public(uint256) + +# Set to true at the end of auction, disallowing any new bids +ended: public(bool) + +# Final auction state +highestBid: public(uint256) +highestBidder: public(address) + +# State of the bids +bids: HashMap[address, Bid[128]] +bidCounts: HashMap[address, int128] + +# Allowed withdrawals of previous bids +pendingReturns: HashMap[address, uint256] + + +# Create a blinded auction with `_biddingTime` seconds bidding time and +# `_revealTime` seconds reveal time on behalf of the beneficiary address +# `_beneficiary`. +@external +def __init__(_beneficiary: address, _biddingTime: uint256, _revealTime: uint256): + self.beneficiary = _beneficiary + self.biddingEnd = block.timestamp + _biddingTime + self.revealEnd = self.biddingEnd + _revealTime + + +# Place a blinded bid with: +# +# _blindedBid = keccak256(concat( +# convert(value, bytes32), +# convert(fake, bytes32), +# secret) +# ) +# +# The sent ether is only refunded if the bid is correctly revealed in the +# revealing phase. The bid is valid if the ether sent together with the bid is +# at least "value" and "fake" is not true. Setting "fake" to true and sending +# not the exact amount are ways to hide the real bid but still make the +# required deposit. The same address can place multiple bids. +@external +@payable +def bid(_blindedBid: bytes32): + # Check if bidding period is still open + assert block.timestamp < self.biddingEnd + + # Check that payer hasn't already placed maximum number of bids + numBids: int128 = self.bidCounts[msg.sender] + assert numBids < MAX_BIDS + + # Add bid to mapping of all bids + self.bids[msg.sender][numBids] = Bid({ + blindedBid: _blindedBid, + deposit: msg.value + }) + self.bidCounts[msg.sender] += 1 + + +# Returns a boolean value, `True` if bid placed successfully, `False` otherwise. +@internal +def placeBid(bidder: address, _value: uint256) -> bool: + # If bid is less than highest bid, bid fails + if (_value <= self.highestBid): + return False + + # Refund the previously highest bidder + if (self.highestBidder != empty(address)): + self.pendingReturns[self.highestBidder] += self.highestBid + + # Place bid successfully and update auction state + self.highestBid = _value + self.highestBidder = bidder + + return True + + +# Reveal your blinded bids. You will get a refund for all correctly blinded +# invalid bids and for all bids except for the totally highest. +@external +def reveal(_numBids: int128, _values: uint256[128], _fakes: bool[128], _secrets: bytes32[128]): + # Check that bidding period is over + assert block.timestamp > self.biddingEnd + + # Check that reveal end has not passed + assert block.timestamp < self.revealEnd + + # Check that number of bids being revealed matches log for sender + assert _numBids == self.bidCounts[msg.sender] + + # Calculate refund for sender + refund: uint256 = 0 + for i in range(MAX_BIDS): + # Note that loop may break sooner than 128 iterations if i >= _numBids + if (i >= _numBids): + break + + # Get bid to check + bidToCheck: Bid = (self.bids[msg.sender])[i] + + # Check against encoded packet + value: uint256 = _values[i] + fake: bool = _fakes[i] + secret: bytes32 = _secrets[i] + blindedBid: bytes32 = keccak256(concat( + convert(value, bytes32), + convert(fake, bytes32), + secret + )) + + # Bid was not actually revealed + # Do not refund deposit + assert blindedBid == bidToCheck.blindedBid + + # Add deposit to refund if bid was indeed revealed + refund += bidToCheck.deposit + if (not fake and bidToCheck.deposit >= value): + if (self.placeBid(msg.sender, value)): + refund -= value + + # Make it impossible for the sender to re-claim the same deposit + zeroBytes32: bytes32 = empty(bytes32) + bidToCheck.blindedBid = zeroBytes32 + + # Send refund if non-zero + if (refund != 0): + send(msg.sender, refund) + + +# Withdraw a bid that was overbid. +@external +def withdraw(): + # Check that there is an allowed pending return. + pendingAmount: uint256 = self.pendingReturns[msg.sender] + if (pendingAmount > 0): + # If so, set pending returns to zero to prevent recipient from calling + # this function again as part of the receiving call before `transfer` + # returns (see the remark above about conditions -> effects -> + # interaction). + self.pendingReturns[msg.sender] = 0 + + # Then send return + send(msg.sender, pendingAmount) + + +# End the auction and send the highest bid to the beneficiary. +@external +def auctionEnd(): + # Check that reveal end has passed + assert block.timestamp > self.revealEnd + + # Check that auction has not already been marked as ended + assert not self.ended + + # Log auction ending and set flag + log AuctionEnded(self.highestBidder, self.highestBid) + self.ended = True + + # Transfer funds to beneficiary + send(self.beneficiary, self.highestBid) \ No newline at end of file