Merge pull request #310 from valory-xyz/feat/package-sync

Add support for syncing local packages with the IPFS registry

.github/pull_request_template.md

@@ -50,7 +50,7 @@ _Put an `x` in the boxes that apply._
 - [ ] I have read the [CONTRIBUTING](../CONTRIBUTING.md) doc
 - [ ] I am making a pull request against the `main` branch (left side), from `develop`
 - [ ] Lint and unit tests pass locally and in CI
-- [ ] I have checked the fingerprint hashes are correct by running (`aea hash all --check`)
+- [ ] I have checked the fingerprint hashes are correct by running (`aea hash all` and `aea packages lock --check`)
 - [ ] I have regenerated the latest API docs
 - [ ] I built the documentation and updated it with the latest changes
 - [ ] I have added an item in `HISTORY.md` for this release

CONTRIBUTING.md

@@ -95,7 +95,7 @@ We have various commands which are helpful during development.
 
       make dir=cli tdir=cli test-sub
 
-- When making changes to one of the `packages`, then use `aea hash all` and `aea hash all --packages-dir=./tests/data/packages ` to generate the latest hashes.
+- When making changes to one of the `packages`, then use `make hashes` to generate the latest hashes.
 
 ### Go Development
 

Makefile

@@ -73,6 +73,8 @@ test-sub-p:
 hashes:
 	python -m aea.cli hash all
 	python -m aea.cli hash all --packages-dir=./tests/data/packages
+	python -m aea.cli packages lock
+	python -m aea.cli --registry-path=./tests/data/packages packages lock
 
 .PHONY: test-all
 test-all:

aea/cli/core.py

@@ -52,6 +52,7 @@
 from aea.cli.local_registry_sync import local_registry_sync
 from aea.cli.login import login
 from aea.cli.logout import logout
+from aea.cli.packages import package_manager
 from aea.cli.plugin import with_plugins
 from aea.cli.publish import publish
 from aea.cli.push import push
@@ -147,3 +148,4 @@ def cli(
 cli.add_command(generate_all_protocols)
 cli.add_command(check_packages)
 cli.add_command(push_all)
+cli.add_command(package_manager)

aea/cli/ipfs_hash.py

@@ -28,11 +28,10 @@
 import sys
 import traceback
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Tuple, cast
+from typing import Callable, Dict, Optional, Tuple, cast
 
 import click
 
-from aea.cli.utils.constants import HASHES_FILE
 from aea.configurations.base import PackageConfiguration, PackageType
 from aea.configurations.constants import (
     AGENT,
@@ -44,19 +43,11 @@
 from aea.configurations.loader import load_configuration_object
 from aea.helpers.cid import DEFAULT_ENCODING, to_v0, to_v1
 from aea.helpers.dependency_tree import DependencyTree, dump_yaml, load_yaml, to_plural
-from aea.helpers.fingerprint import check_fingerprint, update_fingerprint
-from aea.helpers.io import from_csv, to_csv
+from aea.helpers.fingerprint import update_fingerprint
 from aea.helpers.ipfs.base import IPFSHashOnly
 from aea.helpers.yaml_utils import yaml_dump, yaml_dump_all
 
 
-def package_type_and_path(package_path: Path) -> Tuple[PackageType, Path]:
-    """Extract the package type from the path."""
-    item_type_plural = package_path.parent.name
-    item_type_singular = item_type_plural[:-1]
-    return PackageType(item_type_singular), package_path
-
-
 def to_package_id(package_path: Path) -> Tuple[PackageId, Path]:
     """Extract the package type from the path."""
     item_type_plural = package_path.parent.name
@@ -83,17 +74,6 @@ def package_id_and_path(
     return package_id, package_path
 
 
-def get_all_packages(packages_dir: Path) -> List[Tuple[PackageType, Path]]:
-    """Get all the hashable package of the repository."""
-
-    all_packages = []
-    for package_type, config_file in PACKAGE_TYPE_TO_CONFIG_FILE.items():
-        for package_dir in packages_dir.glob(f"**/{config_file}"):
-            all_packages.append((PackageType(package_type), package_dir.parent))
-
-    return all_packages
-
-
 def sort_configuration_file(config: PackageConfiguration) -> None:
     """Sort the order of the fields in the configuration files."""
     # load config file to get ignore patterns, dump again immediately to impose ordering
@@ -207,7 +187,7 @@ def update_hashes(
         [PackageType, Path], PackageConfiguration
     ] = load_configuration,
 ) -> int:
-    """Process all AEA packages, update fingerprint, and update hashes.csv files."""
+    """Process all AEA packages and update fingerprint."""
     return_code = 0
     package_hashes: Dict[str, str] = {}
 
@@ -245,86 +225,10 @@ def update_hashes(
                 if vendor is not None and package_id.author != vendor:
                     continue
                 package_hashes[key] = package_hash
-
-        to_csv(package_hashes, packages_dir / HASHES_FILE)
         click.echo("Done!")
-
-    except Exception:  # pylint: disable=broad-except
-        traceback.print_exc()
-        return_code = 1
-
-    return return_code
-
-
-def check_same_ipfs_hash(
-    configuration: PackageConfiguration,
-    package_type: PackageType,
-    all_expected_hashes: Dict[str, str],
-    no_wrap: bool = False,
-) -> bool:
-    """
-    Compute actual package hash and compare with expected hash.
-
-    :param configuration: the configuration object of the package.
-    :param package_type: the type of package.
-    :param all_expected_hashes: the dictionary of all the expected hashes.
-    :param no_wrap: Whether to use the wrapper node or not.
-    :return: True if the IPFS hash match, False otherwise.
-    """
-
-    key, actual_hash = hash_package(configuration, package_type, no_wrap=no_wrap)
-    expected_hash = all_expected_hashes.get(key)
-    if expected_hash is None:
-        click.echo(f"Hash not found for {configuration.package_id}, skipping check.")
-        return True
-
-    result = actual_hash == expected_hash
-    if not result:
-        click.echo(
-            f"IPFS Hashes do not match for {configuration.name} in {configuration.directory}"
-        )
-        click.echo(f"Expected: {expected_hash}")
-        click.echo(f"Actual:   {actual_hash}")
-
-    return result
-
-
-def check_hashes(
-    packages_dir: Path,
-    no_wrap: bool = False,
-    vendor: Optional[str] = None,
-    config_loader: Callable[
-        [PackageType, Path], PackageConfiguration
-    ] = load_configuration,
-) -> int:
-    """Check fingerprints and outer hash of all AEA packages."""
-
-    return_code = 0
-    failed = False
-
-    try:
-        packages = get_all_packages(packages_dir)
-        expected_package_hashes = from_csv(packages_dir / HASHES_FILE)
-        packages += list(map(package_type_and_path, SCAFFOLD_PACKAGES))
-
-        for package_type, package_path in packages:
-            configuration_obj = config_loader(package_type, package_path)
-            if vendor is not None and configuration_obj.author != vendor:
-                continue
-
-            failed = failed or not check_fingerprint(configuration_obj)
-            failed = failed or not check_same_ipfs_hash(
-                configuration_obj, package_type, expected_package_hashes, no_wrap
-            )
-
     except Exception:  # pylint: disable=broad-except
         traceback.print_exc()
-        failed = True
-
-    if failed:
         return_code = 1
-    else:
-        click.echo("OK!")
 
     return return_code
 
@@ -342,19 +246,14 @@ def hash_group() -> None:
 )
 @click.option("--vendor", type=str)
 @click.option("--no-wrap", is_flag=True)
-@click.option("--check", is_flag=True)
 def generate_all(
     packages_dir: Path,
     vendor: Optional[str],
     no_wrap: bool,
-    check: bool,
 ) -> None:
     """Generate IPFS hashes."""
     packages_dir = Path(packages_dir).absolute()
-    if check:
-        return_code = check_hashes(packages_dir, no_wrap, vendor=vendor)
-    else:
-        return_code = update_hashes(packages_dir, no_wrap, vendor=vendor)
+    return_code = update_hashes(packages_dir, no_wrap, vendor=vendor)
     sys.exit(return_code)