Merge pull request #492 from crytic/feat/vyper-standard-json

feat: compile vyper 0.3.7 via standard json input

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "windows-2022"]
-        type: ["brownie", "buidler", "dapp", "embark", "hardhat", "solc", "truffle", "waffle", "foundry", "standard"]
+        type: ["brownie", "buidler", "dapp", "embark", "hardhat", "solc", "truffle", "waffle", "foundry", "standard", "vyper"]
         exclude:
           # Currently broken, tries to pull git:// which is blocked by GH
           - type: embark

crytic_compile/crytic_compile.py

@@ -21,7 +21,9 @@
     artifact_path,
 )
 from crytic_compile.compilation_unit import CompilationUnit
-from crytic_compile.platform import all_platforms, solc_standard_json
+from crytic_compile.platform import all_platforms
+from crytic_compile.platform.solc_standard_json import SolcStandardJson
+from crytic_compile.platform.vyper import VyperStandardJson
 from crytic_compile.platform.abstract_platform import AbstractPlatform
 from crytic_compile.platform.all_export import PLATFORMS_EXPORT
 from crytic_compile.platform.solc import Solc
@@ -676,18 +678,14 @@ def compile_all(target: str, **kwargs: str) -> List[CryticCompile]:
         **kwargs: optional arguments. Used: "solc_standard_json"
 
     Raises:
-        ValueError: If the target could not be compiled
+        NotImplementedError: If the target could not be compiled
 
     Returns:
         List[CryticCompile]: Returns a list of CryticCompile instances for all compilations which occurred.
     """
     use_solc_standard_json = kwargs.get("solc_standard_json", False)
 
-    # Attempt to perform glob expansion of target/filename
-    globbed_targets = glob.glob(target, recursive=True)
-
     # Check if the target refers to a valid target already.
-    # If it does not, we assume it's a glob pattern.
     compilations: List[CryticCompile] = []
     if os.path.isfile(target) or is_supported(target):
         if target.endswith(".zip"):
@@ -699,28 +697,33 @@ def compile_all(target: str, **kwargs: str) -> List[CryticCompile]:
                     compilations = load_from_zip(tmp.name)
         else:
             compilations.append(CryticCompile(target, **kwargs))
-    elif os.path.isdir(target) or len(globbed_targets) > 0:
-        # We create a new glob to find solidity files at this path (in case this is a directory)
-        filenames = glob.glob(os.path.join(target, "*.sol"))
-        if not filenames:
-            filenames = glob.glob(os.path.join(target, "*.vy"))
-            if not filenames:
-                filenames = globbed_targets
-
+    elif os.path.isdir(target):
+        solidity_filenames = glob.glob(os.path.join(target, "*.sol"))
+        vyper_filenames = glob.glob(os.path.join(target, "*.vy"))
         # Determine if we're using --standard-solc option to
         # aggregate many files into a single compilation.
         if use_solc_standard_json:
             # If we're using standard solc, then we generated our
             # input to create a single compilation with all files
-            standard_json = solc_standard_json.SolcStandardJson()
-            for filename in filenames:
-                standard_json.add_source_file(filename)
-            compilations.append(CryticCompile(standard_json, **kwargs))
+            solc_standard_json = SolcStandardJson()
+            solc_standard_json.add_source_files(solidity_filenames)
+            compilations.append(CryticCompile(solc_standard_json, **kwargs))
         else:
             # We compile each file and add it to our compilations.
-            for filename in filenames:
+            for filename in solidity_filenames:
                 compilations.append(CryticCompile(filename, **kwargs))
+
+        if vyper_filenames:
+            vyper_standard_json = VyperStandardJson()
+            vyper_standard_json.add_source_files(vyper_filenames)
+            compilations.append(CryticCompile(vyper_standard_json, **kwargs))
     else:
-        raise ValueError(f"{str(target)} is not a file or directory.")
+        raise NotImplementedError()
+        # TODO split glob into language
+        # # Attempt to perform glob expansion of target/filename
+        # globbed_targets = glob.glob(target, recursive=True)
+        # print(globbed_targets)
+
+        # raise ValueError(f"{str(target)} is not a file or directory.")
 
     return compilations

crytic_compile/platform/all_platforms.py

@@ -14,6 +14,6 @@
 from .solc_standard_json import SolcStandardJson
 from .standard import Standard
 from .truffle import Truffle
-from .vyper import Vyper
+from .vyper import VyperStandardJson
 from .waffle import Waffle
 from .foundry import Foundry

crytic_compile/platform/solc_standard_json.py

@@ -405,6 +405,15 @@ def add_source_file(self, file_path: str) -> None:
         """
         add_source_file(self._json, file_path)
 
+    def add_source_files(self, files_path: List[str]) -> None:
+        """Append files
+
+        Args:
+            files_path (List[str]): files to append
+        """
+        for file_path in files_path:
+            add_source_file(self._json, file_path)
+
     def add_remapping(self, remapping: str) -> None:
         """Append our remappings
 

crytic_compile/platform/vyper.py

@@ -25,7 +25,7 @@
 LOGGER = logging.getLogger("CryticCompile")
 
 
-class Vyper(AbstractPlatform):
+class VyperStandardJson(AbstractPlatform):
     """
     Vyper platform
     """
@@ -34,6 +34,28 @@ class Vyper(AbstractPlatform):
     PROJECT_URL = "https://github.com/vyperlang/vyper"
     TYPE = Type.VYPER
 
+    def __init__(self, target: Optional[Path] = None, **_kwargs: str):
+        super().__init__(str(target), **_kwargs)
+        self.standard_json_input = {
+            "language": "Vyper",
+            "sources": {},
+            "settings": {
+                "outputSelection": {
+                    "*": {
+                        "*": [
+                            "abi",
+                            "devdoc",
+                            "userdoc",
+                            "evm.bytecode",
+                            "evm.deployedBytecode",
+                            "evm.deployedBytecode.sourceMap",
+                        ],
+                        "": ["ast"],
+                    }
+                }
+            },
+        }
+
     def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
         """Compile the target
 
@@ -44,46 +66,67 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
 
         """
         target = self._target
+        # If the target was a directory `add_source_file` should have been called
+        # by `compile_all`. Otherwise, we should have a single file target.
+        if self._target is not None and os.path.isfile(self._target):
+            self.add_source_files([target])
 
-        vyper = kwargs.get("vyper", "vyper")
-
-        targets_json = _run_vyper(target, vyper)
+        vyper_bin = kwargs.get("vyper", "vyper")
 
-        assert "version" in targets_json
+        compilation_artifacts = _run_vyper_standard_json(self.standard_json_input, vyper_bin)
         compilation_unit = CompilationUnit(crytic_compile, str(target))
 
+        compiler_version = compilation_artifacts["compiler"].split("-")[1]
+        if compiler_version != "0.3.7":
+            LOGGER.info("Vyper != 0.3.7 support is a best effort and might fail")
         compilation_unit.compiler_version = CompilerVersion(
-            compiler="vyper", version=targets_json["version"], optimized=False
+            compiler="vyper", version=compiler_version, optimized=False
         )
 
-        assert target in targets_json
-
-        info = targets_json[target]
-        filename = convert_filename(target, _relative_to_short, crytic_compile)
-
-        contract_name = Path(target).parts[-1]
+        for source_file, contract_info in compilation_artifacts["contracts"].items():
+            filename = convert_filename(source_file, _relative_to_short, crytic_compile)
+            source_unit = compilation_unit.create_source_unit(filename)
+            for contract_name, contract_metadata in contract_info.items():
+                source_unit.add_contract_name(contract_name)
+                compilation_unit.filename_to_contracts[filename].add(contract_name)
+
+                source_unit.abis[contract_name] = contract_metadata["abi"]
+                source_unit.bytecodes_init[contract_name] = contract_metadata["evm"]["bytecode"][
+                    "object"
+                ].replace("0x", "")
+                # Vyper does not provide the source mapping for the init bytecode
+                source_unit.srcmaps_init[contract_name] = []
+                source_unit.srcmaps_runtime[contract_name] = contract_metadata["evm"][
+                    "deployedBytecode"
+                ]["sourceMap"].split(";")
+                source_unit.bytecodes_runtime[contract_name] = contract_metadata["evm"][
+                    "deployedBytecode"
+                ]["object"].replace("0x", "")
+                source_unit.natspec[contract_name] = Natspec(
+                    contract_metadata["userdoc"], contract_metadata["devdoc"]
+                )
+
+        for source_file, ast in compilation_artifacts["sources"].items():
+            filename = convert_filename(source_file, _relative_to_short, crytic_compile)
+            source_unit = compilation_unit.create_source_unit(filename)
+            source_unit.ast = ast
+
+    def add_source_files(self, file_paths: List[str]) -> None:
+        """
+        Append files
 
-        source_unit = compilation_unit.create_source_unit(filename)
+        Args:
+            file_paths (List[str]): files to append
 
-        source_unit.add_contract_name(contract_name)
-        compilation_unit.filename_to_contracts[filename].add(contract_name)
-        source_unit.abis[contract_name] = info["abi"]
-        source_unit.bytecodes_init[contract_name] = info["bytecode"].replace("0x", "")
-        source_unit.bytecodes_runtime[contract_name] = info["bytecode_runtime"].replace("0x", "")
-        # Vyper does not provide the source mapping for the init bytecode
-        source_unit.srcmaps_init[contract_name] = []
-        # info["source_map"]["pc_pos_map"] contains the source mapping in a simpler format
-        # However pc_pos_map_compressed" seems to follow solc's format, so for convenience
-        # We store the same
-        # TODO: create SourceMapping class, so that srcmaps_runtime would store an class
-        # That will give more flexebility to different compilers
-        source_unit.srcmaps_runtime[contract_name] = info["source_map"]["pc_pos_map_compressed"]
+        Returns:
 
-        # Natspec not yet handled for vyper
-        source_unit.natspec[contract_name] = Natspec({}, {})
+        """
 
-        ast = _get_vyper_ast(target, vyper)
-        source_unit.ast = ast
+        for file_path in file_paths:
+            with open(file_path, "r", encoding="utf8") as f:
+                self.standard_json_input["sources"][file_path] = {  # type: ignore
+                    "content": f.read(),
+                }
 
     def clean(self, **_kwargs: str) -> None:
         """Clean compilation artifacts
@@ -129,98 +172,45 @@ def _guessed_tests(self) -> List[str]:
         return []
 
 
-def _run_vyper(
-    filename: str, vyper: str, env: Optional[Dict] = None, working_dir: Optional[str] = None
+def _run_vyper_standard_json(
+    standard_json_input: Dict, vyper: str, env: Optional[Dict] = None
 ) -> Dict:
-    """Run vyper
+    """Run vyper and write compilation output to a file
 
     Args:
-        filename (str): vyper file
+        standard_json_input (Dict): Dict containing the vyper standard json input
         vyper (str): vyper binary
         env (Optional[Dict], optional): Environment variables. Defaults to None.
-        working_dir (Optional[str], optional): Working directory. Defaults to None.
 
     Raises:
         InvalidCompilation: If vyper failed to run
 
     Returns:
         Dict: Vyper json compilation artifact
     """
-    if not os.path.isfile(filename):
-        raise InvalidCompilation(f"{filename} does not exist (are you in the correct directory?)")
-
-    cmd = [vyper, filename, "-f", "combined_json"]
-
-    additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {}
-    stderr = ""
-    LOGGER.info(
-        "'%s' running",
-        " ".join(cmd),
-    )
-    try:
-        with subprocess.Popen(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            env=env,
-            executable=shutil.which(cmd[0]),
-            **additional_kwargs,
-        ) as process:
-            stdout, stderr = process.communicate()
-            res = stdout.split(b"\n")
-            res = res[-2]
-            return json.loads(res)
-    except OSError as error:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(error)
-    except json.decoder.JSONDecodeError:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(f"Invalid vyper compilation\n{stderr}")
-
-
-def _get_vyper_ast(
-    filename: str, vyper: str, env: Optional[Dict] = None, working_dir: Optional[str] = None
-) -> Dict:
-    """Get ast from vyper
-
-    Args:
-        filename (str): vyper file
-        vyper (str): vyper binary
-        env (Dict, optional): Environment variables. Defaults to None.
-        working_dir (str, optional): Working directory. Defaults to None.
-
-    Raises:
-        InvalidCompilation: If vyper failed to run
-
-    Returns:
-        Dict: [description]
-    """
-    if not os.path.isfile(filename):
-        raise InvalidCompilation(f"{filename} does not exist (are you in the correct directory?)")
-
-    cmd = [vyper, filename, "-f", "ast"]
-
-    additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {}
-    stderr = ""
-    try:
-        with subprocess.Popen(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            env=env,
-            executable=shutil.which(cmd[0]),
-            **additional_kwargs,
-        ) as process:
-            stdout, stderr = process.communicate()
-            res = stdout.split(b"\n")
-            res = res[-2]
-            return json.loads(res)
-    except json.decoder.JSONDecodeError:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(f"Invalid vyper compilation\n{stderr}")
-    except Exception as exception:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(exception)
+    cmd = [vyper, "--standard-json"]
+
+    with subprocess.Popen(
+        cmd,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+        executable=shutil.which(cmd[0]),
+    ) as process:
+
+        stdout_b, stderr_b = process.communicate(json.dumps(standard_json_input).encode("utf-8"))
+        stdout, _stderr = (
+            stdout_b.decode(),
+            stderr_b.decode(errors="backslashreplace"),
+        )  # convert bytestrings to unicode strings
+
+        vyper_standard_output = json.loads(stdout)
+        if "errors" in vyper_standard_output:
+            # TODO format errors
+            raise InvalidCompilation(vyper_standard_output["errors"])
+
+        return vyper_standard_output
 
 
 def _relative_to_short(relative: Path) -> Path:

scripts/ci_test_vyper.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+### Test vyper integration
+
+pip install vyper
+
+echo "Testing vyper integration of $(realpath "$(which crytic-compile)")"
+
+cd tests/vyper || exit 255
+
+if ! crytic-compile auction.vy --export-formats standard
+then echo "vyper test failed" && exit 255
+else echo "vyper test passed" && exit 0
+fi