From 78cbeb847593185906b0d2c7f854af4476b2e72e Mon Sep 17 00:00:00 2001 From: Nick Gheorghita Date: Tue, 25 Jun 2019 14:38:11 -0600 Subject: [PATCH] Implement write_to_disk for ipfs backends --- ethpm/backends/base.py | 9 +++++++++ ethpm/backends/http.py | 21 ++++++++++++++++++--- ethpm/backends/ipfs.py | 22 +++++++++++++++++++++- ethpm/backends/registry.py | 8 ++++++++ tests/ethpm/backends/test_http_backends.py | 9 +++++++++ tests/ethpm/backends/test_ipfs_backends.py | 9 +++++++++ 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/ethpm/backends/base.py b/ethpm/backends/base.py index 8735b39..ab84789 100644 --- a/ethpm/backends/base.py +++ b/ethpm/backends/base.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from pathlib import Path from typing import Union from eth_typing import URI @@ -34,3 +35,11 @@ def fetch_uri_contents(self, uri: URI) -> Union[bytes, URI]: Fetch the contents stored at a URI. """ pass + + @abstractmethod + def write_to_disk(self, uri: URI, target_path: Path) -> None: + """ + Writes the contents of target URI to target path. + Raises exception if target path exists. + """ + pass diff --git a/ethpm/backends/http.py b/ethpm/backends/http.py index bc63f1d..5e896ae 100644 --- a/ethpm/backends/http.py +++ b/ethpm/backends/http.py @@ -1,5 +1,8 @@ import base64 import json +from pathlib import Path +import shutil +import tempfile from eth_typing import URI import requests @@ -18,6 +21,10 @@ class GithubOverHTTPSBackend(BaseURIBackend): Base class for all URIs pointing to a content-addressed Github URI. """ + @property + def base_uri(self) -> str: + return GITHUB_API_AUTHORITY + def can_resolve_uri(self, uri: URI) -> bool: return is_valid_content_addressed_github_uri(uri) @@ -44,6 +51,14 @@ def fetch_uri_contents(self, uri: URI) -> bytes: validate_blob_uri_contents(decoded_contents, uri) return decoded_contents - @property - def base_uri(self) -> str: - return GITHUB_API_AUTHORITY + def write_to_disk(self, uri: URI, target_path: Path) -> None: + contents = self.fetch_uri_contents(uri) + if target_path.exists(): + raise CannotHandleURI( + f"Github blob: {uri} cannot be written to disk since target path ({target_path}) " + "already exists. Please provide a target_path that does not exist." + ) + with tempfile.NamedTemporaryFile() as temp: + temp.write(contents) + temp.seek(0) + shutil.copyfile(temp.name, target_path) diff --git a/ethpm/backends/ipfs.py b/ethpm/backends/ipfs.py index 596672b..fdcef7a 100644 --- a/ethpm/backends/ipfs.py +++ b/ethpm/backends/ipfs.py @@ -1,8 +1,11 @@ from abc import abstractmethod import os from pathlib import Path +import shutil +import tempfile from typing import Dict, List, Type +from eth_typing import URI from eth_utils import import_string, to_bytes import ipfshttpclient @@ -49,6 +52,18 @@ def pin_assets(self, file_or_dir_path: Path) -> List[Dict[str, str]]: """ pass + def write_to_disk(self, uri: URI, target_path: Path) -> None: + contents = self.fetch_uri_contents(uri) + if target_path.exists(): + raise CannotHandleURI( + f"IPFS uri: {uri} cannot be written to disk since target path ({target_path}) " + "already exists. Please provide a target_path that does not exist." + ) + with tempfile.NamedTemporaryFile() as temp: + temp.write(contents) + temp.seek(0) + shutil.copyfile(temp.name, target_path) + class IPFSOverHTTPBackend(BaseIPFSBackend): """ @@ -59,7 +74,7 @@ class IPFSOverHTTPBackend(BaseIPFSBackend): def __init__(self) -> None: self.client = ipfshttpclient.connect(self.base_uri) - def fetch_uri_contents(self, uri: str) -> bytes: + def fetch_uri_contents(self, uri: URI) -> bytes: ipfs_hash = extract_ipfs_path_from_uri(uri) contents = self.client.cat(ipfs_hash) validation_hash = generate_file_hash(contents) @@ -108,6 +123,11 @@ def fetch_uri_contents(self, uri: str) -> bytes: "IPFS gateway is currently disabled, please use a different IPFS backend." ) + def write_to_disk(self, uri: URI, target_path: Path) -> None: + raise CannotHandleURI( + "IPFS gateway is currently disabled, please use a different IPFS backend." + ) + class InfuraIPFSBackend(IPFSOverHTTPBackend): """ diff --git a/ethpm/backends/registry.py b/ethpm/backends/registry.py index 38bb8c0..2b61a2b 100644 --- a/ethpm/backends/registry.py +++ b/ethpm/backends/registry.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from eth_typing import URI from web3 import Web3 @@ -6,6 +7,7 @@ from ethpm.backends.base import BaseURIBackend from ethpm.constants import INFURA_API_KEY +from ethpm.exceptions import CannotHandleURI from ethpm.utils.registry import fetch_standard_registry_abi from ethpm.utils.uri import parse_registry_uri from ethpm.validation import is_valid_registry_uri @@ -41,3 +43,9 @@ def fetch_uri_contents(self, uri: str) -> URI: self.w3.pm.set_registry(address) _, _, manifest_uri = self.w3.pm.get_release_data(pkg_name, pkg_version) return manifest_uri + + def write_to_disk(self, uri: str, target_path: Path) -> None: + raise CannotHandleURI( + "Registry backends are not allowed to write resolved uris to disk. " + "Please use a different backend." + ) diff --git a/tests/ethpm/backends/test_http_backends.py b/tests/ethpm/backends/test_http_backends.py index 802ae2f..14b663e 100644 --- a/tests/ethpm/backends/test_http_backends.py +++ b/tests/ethpm/backends/test_http_backends.py @@ -26,3 +26,12 @@ def test_github_over_https_backend_raises_error_with_invalid_content_hash(w3): invalid_uri = "https://api.github.com/repos/ethpm/py-ethpm/git/blobs/a7232a93f1e9e75d606f6c1da18aa16037e03123" with pytest.raises(HTTPError): Package.from_uri(invalid_uri, w3) + + +def test_github_backend_writes_to_disk(tmp_path): + uri = "https://api.github.com/repos/ethpm/py-ethpm/git/blobs/a7232a93f1e9e75d606f6c1da18aa16037e03480" + backend = GithubOverHTTPSBackend() + contents = backend.fetch_uri_contents(uri) + target_path = tmp_path / "github_uri.txt" + backend.write_to_disk(uri, target_path) + assert target_path.read_bytes() == contents diff --git a/tests/ethpm/backends/test_ipfs_backends.py b/tests/ethpm/backends/test_ipfs_backends.py index ed48416..143edcd 100644 --- a/tests/ethpm/backends/test_ipfs_backends.py +++ b/tests/ethpm/backends/test_ipfs_backends.py @@ -116,3 +116,12 @@ def test_pin_assets_to_dummy_backend(dummy_ipfs_backend): assert "StandardToken.sol" in dir_names assert "QmRJHLmPVct2rbBpdGjP3xkXbF7romQigtmcs8TRfV1yC7" in dir_hashes assert "2865" in dir_sizes + + +def test_ipfs_backend_write_to_disk(tmp_path): + ipfs_uri = "ipfs://QmeD2s7KaBUoGYTP1eutHBmBkMMMoycdfiyGMx2DKrWXyV" + backend = LocalIPFSBackend() + contents = backend.fetch_uri_contents(ipfs_uri) + target_path = tmp_path / "ipfs_uri.txt" + backend.write_to_disk(ipfs_uri, target_path) + assert target_path.read_bytes() == contents