From 5f395215a24da52cdbb05405b784307d589179e4 Mon Sep 17 00:00:00 2001 From: Nick Gheorghita Date: Tue, 25 Jun 2019 14:38:11 -0600 Subject: [PATCH 1/2] Implement write_to_disk for ipfs backends --- ethpm/backends/base.py | 9 ++++++++ ethpm/backends/http.py | 21 ++++++++++++++--- ethpm/backends/ipfs.py | 22 +++++++++++++++++- ethpm/backends/registry.py | 8 +++++++ tests/ethpm/backends/test_http_backends.py | 27 +++++++++++++++------- tests/ethpm/backends/test_ipfs_backends.py | 18 +++++++++++++++ 6 files changed, 93 insertions(+), 12 deletions(-) diff --git a/ethpm/backends/base.py b/ethpm/backends/base.py index 8735b39..ab84789 100644 --- a/ethpm/backends/base.py +++ b/ethpm/backends/base.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from pathlib import Path from typing import Union from eth_typing import URI @@ -34,3 +35,11 @@ def fetch_uri_contents(self, uri: URI) -> Union[bytes, URI]: Fetch the contents stored at a URI. """ pass + + @abstractmethod + def write_to_disk(self, uri: URI, target_path: Path) -> None: + """ + Writes the contents of target URI to target path. + Raises exception if target path exists. + """ + pass diff --git a/ethpm/backends/http.py b/ethpm/backends/http.py index bc63f1d..5e896ae 100644 --- a/ethpm/backends/http.py +++ b/ethpm/backends/http.py @@ -1,5 +1,8 @@ import base64 import json +from pathlib import Path +import shutil +import tempfile from eth_typing import URI import requests @@ -18,6 +21,10 @@ class GithubOverHTTPSBackend(BaseURIBackend): Base class for all URIs pointing to a content-addressed Github URI. """ + @property + def base_uri(self) -> str: + return GITHUB_API_AUTHORITY + def can_resolve_uri(self, uri: URI) -> bool: return is_valid_content_addressed_github_uri(uri) @@ -44,6 +51,14 @@ def fetch_uri_contents(self, uri: URI) -> bytes: validate_blob_uri_contents(decoded_contents, uri) return decoded_contents - @property - def base_uri(self) -> str: - return GITHUB_API_AUTHORITY + def write_to_disk(self, uri: URI, target_path: Path) -> None: + contents = self.fetch_uri_contents(uri) + if target_path.exists(): + raise CannotHandleURI( + f"Github blob: {uri} cannot be written to disk since target path ({target_path}) " + "already exists. Please provide a target_path that does not exist." + ) + with tempfile.NamedTemporaryFile() as temp: + temp.write(contents) + temp.seek(0) + shutil.copyfile(temp.name, target_path) diff --git a/ethpm/backends/ipfs.py b/ethpm/backends/ipfs.py index 596672b..fdcef7a 100644 --- a/ethpm/backends/ipfs.py +++ b/ethpm/backends/ipfs.py @@ -1,8 +1,11 @@ from abc import abstractmethod import os from pathlib import Path +import shutil +import tempfile from typing import Dict, List, Type +from eth_typing import URI from eth_utils import import_string, to_bytes import ipfshttpclient @@ -49,6 +52,18 @@ def pin_assets(self, file_or_dir_path: Path) -> List[Dict[str, str]]: """ pass + def write_to_disk(self, uri: URI, target_path: Path) -> None: + contents = self.fetch_uri_contents(uri) + if target_path.exists(): + raise CannotHandleURI( + f"IPFS uri: {uri} cannot be written to disk since target path ({target_path}) " + "already exists. Please provide a target_path that does not exist." + ) + with tempfile.NamedTemporaryFile() as temp: + temp.write(contents) + temp.seek(0) + shutil.copyfile(temp.name, target_path) + class IPFSOverHTTPBackend(BaseIPFSBackend): """ @@ -59,7 +74,7 @@ class IPFSOverHTTPBackend(BaseIPFSBackend): def __init__(self) -> None: self.client = ipfshttpclient.connect(self.base_uri) - def fetch_uri_contents(self, uri: str) -> bytes: + def fetch_uri_contents(self, uri: URI) -> bytes: ipfs_hash = extract_ipfs_path_from_uri(uri) contents = self.client.cat(ipfs_hash) validation_hash = generate_file_hash(contents) @@ -108,6 +123,11 @@ def fetch_uri_contents(self, uri: str) -> bytes: "IPFS gateway is currently disabled, please use a different IPFS backend." ) + def write_to_disk(self, uri: URI, target_path: Path) -> None: + raise CannotHandleURI( + "IPFS gateway is currently disabled, please use a different IPFS backend." + ) + class InfuraIPFSBackend(IPFSOverHTTPBackend): """ diff --git a/ethpm/backends/registry.py b/ethpm/backends/registry.py index 38bb8c0..2b61a2b 100644 --- a/ethpm/backends/registry.py +++ b/ethpm/backends/registry.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from eth_typing import URI from web3 import Web3 @@ -6,6 +7,7 @@ from ethpm.backends.base import BaseURIBackend from ethpm.constants import INFURA_API_KEY +from ethpm.exceptions import CannotHandleURI from ethpm.utils.registry import fetch_standard_registry_abi from ethpm.utils.uri import parse_registry_uri from ethpm.validation import is_valid_registry_uri @@ -41,3 +43,9 @@ def fetch_uri_contents(self, uri: str) -> URI: self.w3.pm.set_registry(address) _, _, manifest_uri = self.w3.pm.get_release_data(pkg_name, pkg_version) return manifest_uri + + def write_to_disk(self, uri: str, target_path: Path) -> None: + raise CannotHandleURI( + "Registry backends are not allowed to write resolved uris to disk. " + "Please use a different backend." + ) diff --git a/tests/ethpm/backends/test_http_backends.py b/tests/ethpm/backends/test_http_backends.py index 802ae2f..a5274fe 100644 --- a/tests/ethpm/backends/test_http_backends.py +++ b/tests/ethpm/backends/test_http_backends.py @@ -6,19 +6,15 @@ from ethpm.constants import GITHUB_API_AUTHORITY from ethpm.exceptions import CannotHandleURI, ValidationError +BLOB_URI = "https://api.github.com/repos/ethpm/py-ethpm/git/blobs/a7232a93f1e9e75d606f6c1da18aa16037e03480" -@pytest.mark.parametrize( - "uri", - ( - "https://api.github.com/repos/ethpm/py-ethpm/git/blobs/a7232a93f1e9e75d606f6c1da18aa16037e03480", - ), -) -def test_github_over_https_backend_fetch_uri_contents(uri, owned_contract, w3): + +def test_github_over_https_backend_fetch_uri_contents(owned_contract, w3): # these tests may occassionally fail CI as a result of their network requests backend = GithubOverHTTPSBackend() assert backend.base_uri == GITHUB_API_AUTHORITY # integration with Package.from_uri - owned_package = Package.from_uri(uri, w3) + owned_package = Package.from_uri(BLOB_URI, w3) assert owned_package.name == "owned" @@ -26,3 +22,18 @@ def test_github_over_https_backend_raises_error_with_invalid_content_hash(w3): invalid_uri = "https://api.github.com/repos/ethpm/py-ethpm/git/blobs/a7232a93f1e9e75d606f6c1da18aa16037e03123" with pytest.raises(HTTPError): Package.from_uri(invalid_uri, w3) + + +def test_github_backend_writes_to_disk(tmp_path): + backend = GithubOverHTTPSBackend() + contents = backend.fetch_uri_contents(BLOB_URI) + target_path = tmp_path / "github_uri.txt" + backend.write_to_disk(BLOB_URI, target_path) + assert target_path.read_bytes() == contents + + +def test_github_backend_write_to_disk_raises_exception_if_target_exists(tmp_path): + target_path = tmp_path / "test.txt" + target_path.touch() + with pytest.raises(CannotHandleURI, match="cannot be written to disk."): + GithubOverHTTPSBackend().write_to_disk(BLOB_URI, target_path) diff --git a/tests/ethpm/backends/test_ipfs_backends.py b/tests/ethpm/backends/test_ipfs_backends.py index ed48416..051d9b8 100644 --- a/tests/ethpm/backends/test_ipfs_backends.py +++ b/tests/ethpm/backends/test_ipfs_backends.py @@ -13,6 +13,7 @@ get_ipfs_backend_class, ) from ethpm.constants import INFURA_GATEWAY_MULTIADDR +from ethpm.exceptions import CannotHandleURI OWNED_MANIFEST_PATH = V2_PACKAGES_DIR / "owned" / "1.0.0.json" @@ -116,3 +117,20 @@ def test_pin_assets_to_dummy_backend(dummy_ipfs_backend): assert "StandardToken.sol" in dir_names assert "QmRJHLmPVct2rbBpdGjP3xkXbF7romQigtmcs8TRfV1yC7" in dir_hashes assert "2865" in dir_sizes + + +def test_ipfs_backend_write_to_disk(tmp_path): + ipfs_uri = "ipfs://Qme4otpS88NV8yQi8TfTP89EsQC5bko3F5N1yhRoi6cwGV" + backend = LocalIPFSBackend() + contents = backend.fetch_uri_contents(ipfs_uri) + target_path = tmp_path / "ipfs_uri.txt" + backend.write_to_disk(ipfs_uri, target_path) + assert target_path.read_bytes() == contents + + +def test_ipfs_backend_write_to_disk_raises_exception_if_target_exists(tmp_path): + ipfs_uri = "ipfs://Qme4otpS88NV8yQi8TfTP89EsQC5bko3F5N1yhRoi6cwGV" + target_path = tmp_path / "test.txt" + target_path.touch() + with pytest.raises(CannotHandleURI, match="cannot be written to disk."): + LocalIPFSBackend().write_to_disk(ipfs_uri, target_path) From b3babbe3f695510882e4fe960c652036012c91ec Mon Sep 17 00:00:00 2001 From: Nick Gheorghita Date: Thu, 27 Jun 2019 14:21:48 -0600 Subject: [PATCH 2/2] Move write_to_disk definition to BaseURIBackend class --- ethpm/backends/base.py | 16 ++++++++++++++-- ethpm/backends/http.py | 15 --------------- ethpm/backends/ipfs.py | 14 -------------- ethpm/backends/registry.py | 8 -------- 4 files changed, 14 insertions(+), 39 deletions(-) diff --git a/ethpm/backends/base.py b/ethpm/backends/base.py index ab84789..905eb7d 100644 --- a/ethpm/backends/base.py +++ b/ethpm/backends/base.py @@ -1,9 +1,13 @@ from abc import ABC, abstractmethod from pathlib import Path +import shutil +import tempfile from typing import Union from eth_typing import URI +from ethpm.exceptions import CannotHandleURI + class BaseURIBackend(ABC): """ @@ -36,10 +40,18 @@ def fetch_uri_contents(self, uri: URI) -> Union[bytes, URI]: """ pass - @abstractmethod def write_to_disk(self, uri: URI, target_path: Path) -> None: """ Writes the contents of target URI to target path. Raises exception if target path exists. """ - pass + contents = self.fetch_uri_contents(uri) + if target_path.exists(): + raise CannotHandleURI( + f"Uri: {uri} cannot be written to disk since target path ({target_path}) " + "already exists. Please provide a target_path that does not exist." + ) + with tempfile.NamedTemporaryFile() as temp: + temp.write(contents) + temp.seek(0) + shutil.copyfile(temp.name, target_path) diff --git a/ethpm/backends/http.py b/ethpm/backends/http.py index 5e896ae..d89063f 100644 --- a/ethpm/backends/http.py +++ b/ethpm/backends/http.py @@ -1,8 +1,5 @@ import base64 import json -from pathlib import Path -import shutil -import tempfile from eth_typing import URI import requests @@ -50,15 +47,3 @@ def fetch_uri_contents(self, uri: URI) -> bytes: decoded_contents = base64.b64decode(contents["content"]) validate_blob_uri_contents(decoded_contents, uri) return decoded_contents - - def write_to_disk(self, uri: URI, target_path: Path) -> None: - contents = self.fetch_uri_contents(uri) - if target_path.exists(): - raise CannotHandleURI( - f"Github blob: {uri} cannot be written to disk since target path ({target_path}) " - "already exists. Please provide a target_path that does not exist." - ) - with tempfile.NamedTemporaryFile() as temp: - temp.write(contents) - temp.seek(0) - shutil.copyfile(temp.name, target_path) diff --git a/ethpm/backends/ipfs.py b/ethpm/backends/ipfs.py index fdcef7a..678093b 100644 --- a/ethpm/backends/ipfs.py +++ b/ethpm/backends/ipfs.py @@ -1,8 +1,6 @@ from abc import abstractmethod import os from pathlib import Path -import shutil -import tempfile from typing import Dict, List, Type from eth_typing import URI @@ -52,18 +50,6 @@ def pin_assets(self, file_or_dir_path: Path) -> List[Dict[str, str]]: """ pass - def write_to_disk(self, uri: URI, target_path: Path) -> None: - contents = self.fetch_uri_contents(uri) - if target_path.exists(): - raise CannotHandleURI( - f"IPFS uri: {uri} cannot be written to disk since target path ({target_path}) " - "already exists. Please provide a target_path that does not exist." - ) - with tempfile.NamedTemporaryFile() as temp: - temp.write(contents) - temp.seek(0) - shutil.copyfile(temp.name, target_path) - class IPFSOverHTTPBackend(BaseIPFSBackend): """ diff --git a/ethpm/backends/registry.py b/ethpm/backends/registry.py index 2b61a2b..38bb8c0 100644 --- a/ethpm/backends/registry.py +++ b/ethpm/backends/registry.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from eth_typing import URI from web3 import Web3 @@ -7,7 +6,6 @@ from ethpm.backends.base import BaseURIBackend from ethpm.constants import INFURA_API_KEY -from ethpm.exceptions import CannotHandleURI from ethpm.utils.registry import fetch_standard_registry_abi from ethpm.utils.uri import parse_registry_uri from ethpm.validation import is_valid_registry_uri @@ -43,9 +41,3 @@ def fetch_uri_contents(self, uri: str) -> URI: self.w3.pm.set_registry(address) _, _, manifest_uri = self.w3.pm.get_release_data(pkg_name, pkg_version) return manifest_uri - - def write_to_disk(self, uri: str, target_path: Path) -> None: - raise CannotHandleURI( - "Registry backends are not allowed to write resolved uris to disk. " - "Please use a different backend." - )