From 0334b4a0ad37d088a50f5ed82b72fd89c9540cd8 Mon Sep 17 00:00:00 2001 From: antazoey Date: Thu, 31 Oct 2024 17:22:12 -0500 Subject: [PATCH 01/28] perf: ape compile --help wasnt fast (#2362) --- src/ape/cli/options.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ape/cli/options.py b/src/ape/cli/options.py index b2d7829440..88c715c040 100644 --- a/src/ape/cli/options.py +++ b/src/ape/cli/options.py @@ -1,4 +1,5 @@ import inspect +import sys from collections.abc import Callable from functools import partial from pathlib import Path @@ -528,6 +529,11 @@ def handle_parse_result(self, ctx, opts, args): def _project_callback(ctx, param, val): + if "--help" in sys.argv or "-h" in sys.argv: + # Perf: project option is eager; have to check sys.argv to + # know to exit early when only doing --help. + return + from ape.utils.basemodel import ManagerAccessMixin pm = None From aa1c02c96702639fc6405dc0cf64e649c9ade5b9 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 1 Nov 2024 11:40:51 -0500 Subject: [PATCH 02/28] fix: `ape plugins list` bad auth and unnecessary requests (#2365) --- src/ape/utils/_github.py | 32 +++++++++++++-- src/ape_plugins/_cli.py | 3 +- tests/functional/utils/test_github.py | 59 ++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/src/ape/utils/_github.py b/src/ape/utils/_github.py index b5da772cc4..0d23091551 100644 --- a/src/ape/utils/_github.py +++ b/src/ape/utils/_github.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Optional, Union -from requests import Session +from requests import HTTPError, Session from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry @@ -224,8 +224,34 @@ def _get(self, url: str, params: Optional[dict] = None) -> Any: def _request(self, method: str, url: str, **kwargs) -> Any: url = f"{self.API_URL_PREFIX}/{url}" response = self.__session.request(method, url, **kwargs) - response.raise_for_status() - return response.json() + + try: + response.raise_for_status() + except HTTPError as err: + if err.response.status_code == 401 and self.__session.headers.get("Authorization"): + token = self.__session.headers["Authorization"] + del self.__session.headers["Authorization"] + response = self.__session.request(method, url, **kwargs) + try: + response.raise_for_status() # Raise exception if the retry also fails + except HTTPError: + # Even without the Authorization token, the request still failed. + # Raise the original error in this case. Also, put back token just in case. + self.__session.headers["Authorization"] = token + raise err + else: + # The request failed with Authorization but succeeded without. + # Let the user know their token is likely expired. + logger.warning( + "Requests are not authorized! GITHUB_ACCESS_TOKEN is likely expired; " + "received 401 when attempted to use it. If you need GitHub authorization, " + "try resetting your token." + ) + return response.json() + + else: + # Successful response status code! + return response.json() github_client = _GithubClient() diff --git a/src/ape_plugins/_cli.py b/src/ape_plugins/_cli.py index b4c27b1a81..ecf5a80937 100644 --- a/src/ape_plugins/_cli.py +++ b/src/ape_plugins/_cli.py @@ -112,7 +112,8 @@ def _display_all_callback(ctx, param, value): help="Display all plugins installed and available (including Core)", ) def _list(cli_ctx, to_display): - metadata = PluginMetadataList.load(cli_ctx.plugin_manager) + include_available = PluginType.AVAILABLE in to_display + metadata = PluginMetadataList.load(cli_ctx.plugin_manager, include_available=include_available) if output := metadata.to_str(include=to_display): click.echo(output) if not metadata.installed and not metadata.third_party: diff --git a/tests/functional/utils/test_github.py b/tests/functional/utils/test_github.py index 4418a5f72a..206a2cf5f2 100644 --- a/tests/functional/utils/test_github.py +++ b/tests/functional/utils/test_github.py @@ -1,7 +1,7 @@ from pathlib import Path import pytest -from requests.exceptions import ConnectTimeout +from requests.exceptions import ConnectTimeout, HTTPError from ape.utils._github import _GithubClient from ape.utils.os import create_tempdir @@ -98,3 +98,60 @@ def test_get_org_repos(self, github_client, mock_session): params = call.kwargs["params"] # Show we are fetching more than the default 30 per page. assert params == {"per_page": 100, "page": 1} + + def test_available_plugins(self, mocker, github_client, mock_session): + response1 = mocker.MagicMock() + response1.json.return_value = [{"name": "ape-myplugin"}] + response2 = mocker.MagicMock() + response2.json.return_value = [] + + def get_org_repos(method, url, **kwargs): + if kwargs["params"]["page"] == 1: + return response1 + else: + # End. + return response2 + + mock_session.request.side_effect = get_org_repos + actual = github_client.available_plugins + assert actual == {"ape_myplugin"} + + def test_available_plugins_handles_401(self, mocker, github_client, mock_session, ape_caplog): + """ + When you get a 401 from using a token, Ape's GitHub client should not + only warn the user but retry the request w/o authorization, as it likely + will still work. + """ + mock_session.headers = {"Authorization": "token mytoken"} + + response1 = mocker.MagicMock() + response1.json.return_value = [{"name": "ape-myplugin"}] + response2 = mocker.MagicMock() + response2.json.return_value = [] + + bad_auth_response = mocker.MagicMock() + bad_auth_response.status_code = 401 + bad_auth_response.raise_for_status.side_effect = HTTPError(response=bad_auth_response) + + def get_org_repos(method, url, **kwargs): + if mock_session.headers.get("Authorization") == "token mytoken": + return bad_auth_response + elif kwargs["params"]["page"] == 1: + return response1 + else: + # End. + return response2 + + mock_session.request.side_effect = get_org_repos + actual = github_client.available_plugins + + # Still works, even with bad auth. + assert actual == {"ape_myplugin"} + + # Show we got our log message. + expected = ( + "Requests are not authorized! GITHUB_ACCESS_TOKEN is likely " + "expired; received 401 when attempted to use it. If you need " + "GitHub authorization, try resetting your token." + ) + assert ape_caplog.head == expected From 2515e642c018c0268d634a2052d731699c5e3053 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 1 Nov 2024 16:25:25 -0500 Subject: [PATCH 03/28] perf: make `ape console --help` faster (#2366) --- src/ape/cli/choices.py | 6 +++--- src/ape/cli/commands.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/ape/cli/choices.py b/src/ape/cli/choices.py index d880e262bc..dd3685ab67 100644 --- a/src/ape/cli/choices.py +++ b/src/ape/cli/choices.py @@ -373,12 +373,12 @@ def __init__( @property def base_type(self) -> type["ProviderAPI"]: - # perf: property exists to delay import ProviderAPI at init time. - from ape.api.providers import ProviderAPI - if self._base_type is not None: return self._base_type + # perf: property exists to delay import ProviderAPI at init time. + from ape.api.providers import ProviderAPI + self._base_type = ProviderAPI return ProviderAPI diff --git a/src/ape/cli/commands.py b/src/ape/cli/commands.py index ea6110d42c..3cf62b7d3f 100644 --- a/src/ape/cli/commands.py +++ b/src/ape/cli/commands.py @@ -71,10 +71,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def parse_args(self, ctx: "Context", args: list[str]) -> list[str]: - from ape.api.providers import ProviderAPI - arguments = args # Renamed for better pdb support. - base_type = ProviderAPI if self._use_cls_types else str + base_type: Optional[type] = None if self._use_cls_types else str if existing_option := next( iter( x @@ -85,13 +83,20 @@ def parse_args(self, ctx: "Context", args: list[str]) -> list[str]: ), None, ): + if base_type is None: + from ape.api.providers import ProviderAPI + + base_type = ProviderAPI + # Checking instance above, not sure why mypy still mad. existing_option.type.base_type = base_type # type: ignore else: # Add the option automatically. + # NOTE: Local import here only avoids circular import issues. from ape.cli.options import NetworkOption + # NOTE: None base-type will default to `ProviderAPI`. option = NetworkOption(base_type=base_type, callback=self._network_callback) self.params.append(option) From f8edd7c41bd4a96a33ef86d15134ecb439670938 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 1 Nov 2024 17:04:10 -0500 Subject: [PATCH 04/28] perf: make `ape test --help` faster (#2368) --- src/ape_test/__init__.py | 15 +++-- src/ape_test/_cli.py | 97 ++------------------------ src/ape_test/_watch.py | 105 +++++++++++++++++++++++++++++ tests/functional/test_test.py | 30 +++++++++ tests/integration/cli/test_test.py | 21 +----- 5 files changed, 151 insertions(+), 117 deletions(-) create mode 100644 src/ape_test/_watch.py diff --git a/src/ape_test/__init__.py b/src/ape_test/__init__.py index eabb36d1c9..3c9d5b0d46 100644 --- a/src/ape_test/__init__.py +++ b/src/ape_test/__init__.py @@ -5,20 +5,23 @@ @plugins.register(plugins.Config) def config_class(): - module = import_module("ape_test.config") - return module.ApeTestConfig + from ape_test.config import ApeTestConfig + + return ApeTestConfig @plugins.register(plugins.AccountPlugin) def account_types(): - module = import_module("ape_test.accounts") - return module.TestAccountContainer, module.TestAccount + from ape_test.accounts import TestAccount, TestAccountContainer + + return TestAccountContainer, TestAccount @plugins.register(plugins.ProviderPlugin) def providers(): - module = import_module("ape_test.provider") - yield "ethereum", "local", module.LocalProvider + from ape_test.provider import LocalProvider + + yield "ethereum", "local", LocalProvider def __getattr__(name: str): diff --git a/src/ape_test/_cli.py b/src/ape_test/_cli.py index 22541d51d6..a98baaffd5 100644 --- a/src/ape_test/_cli.py +++ b/src/ape_test/_cli.py @@ -1,80 +1,14 @@ import sys -import threading -import time -from datetime import datetime, timedelta -from functools import cached_property +from collections.abc import Iterable from pathlib import Path -from subprocess import run as run_subprocess from typing import Any import click import pytest from click import Command -from watchdog import events -from watchdog.observers import Observer from ape.cli.options import ape_cli_context from ape.logging import LogLevel, _get_level -from ape.utils.basemodel import ManagerAccessMixin as access - -# Copied from https://github.com/olzhasar/pytest-watcher/blob/master/pytest_watcher/watcher.py -trigger_lock = threading.Lock() -trigger = None - - -def emit_trigger(): - """ - Emits trigger to run pytest - """ - - global trigger - - with trigger_lock: - trigger = datetime.now() - - -class EventHandler(events.FileSystemEventHandler): - EVENTS_WATCHED = ( - events.EVENT_TYPE_CREATED, - events.EVENT_TYPE_DELETED, - events.EVENT_TYPE_MODIFIED, - events.EVENT_TYPE_MOVED, - ) - - def dispatch(self, event: events.FileSystemEvent) -> None: - if event.event_type in self.EVENTS_WATCHED: - self.process_event(event) - - @cached_property - def _extensions_to_watch(self) -> list[str]: - return [".py", *access.compiler_manager.registered_compilers.keys()] - - def _is_path_watched(self, filepath: str) -> bool: - """ - Check if file should trigger pytest run - """ - return any(map(filepath.endswith, self._extensions_to_watch)) - - def process_event(self, event: events.FileSystemEvent) -> None: - if self._is_path_watched(event.src_path): - emit_trigger() - - -def _run_ape_test(*pytest_args): - return run_subprocess(["ape", "test", *[f"{a}" for a in pytest_args]]) - - -def _run_main_loop(delay: float, *pytest_args: str) -> None: - global trigger - - now = datetime.now() - if trigger and now - trigger > timedelta(seconds=delay): - _run_ape_test(*pytest_args) - - with trigger_lock: - trigger = None - - time.sleep(delay) def _validate_pytest_args(*pytest_args) -> list[str]: @@ -176,25 +110,7 @@ def cli(cli_ctx, watch, watch_folders, watch_delay, pytest_args): pytest_arg_ls = _validate_pytest_args(*pytest_arg_ls) if watch: - event_handler = _create_event_handler() - observer = _create_observer() - - for folder in watch_folders: - if folder.is_dir(): - observer.schedule(event_handler, folder, recursive=True) - else: - cli_ctx.logger.warning(f"Folder '{folder}' doesn't exist or isn't a folder.") - - observer.start() - - try: - _run_ape_test(*pytest_arg_ls) - while True: - _run_main_loop(watch_delay, *pytest_arg_ls) - - finally: - observer.stop() - observer.join() + _run_with_observer(watch_folders, watch_delay, *pytest_arg_ls) else: return_code = pytest.main([*pytest_arg_ls], ["ape_test"]) @@ -203,11 +119,8 @@ def cli(cli_ctx, watch, watch_folders, watch_delay, pytest_args): sys.exit(return_code) -def _create_event_handler(): +def _run_with_observer(watch_folders: Iterable[Path], watch_delay: float, *pytest_arg_ls: str): # Abstracted for testing purposes. - return EventHandler() - + from ape_test._watch import run_with_observer as run -def _create_observer(): - # Abstracted for testing purposes. - return Observer() + run(watch_folders, watch_delay, *pytest_arg_ls) diff --git a/src/ape_test/_watch.py b/src/ape_test/_watch.py new file mode 100644 index 0000000000..95627cf4f4 --- /dev/null +++ b/src/ape_test/_watch.py @@ -0,0 +1,105 @@ +import threading +import time +from collections.abc import Iterable +from datetime import datetime, timedelta +from functools import cached_property +from pathlib import Path +from subprocess import run as run_subprocess + +from watchdog import events +from watchdog.observers import Observer + +from ape.logging import logger + +# Copied from https://github.com/olzhasar/pytest-watcher/blob/master/pytest_watcher/watcher.py +trigger_lock = threading.Lock() +trigger = None + + +def run_with_observer(watch_folders: Iterable[Path], watch_delay: float, *pytest_arg_ls: str): + event_handler = _create_event_handler() + observer = _create_observer() + + for folder in watch_folders: + if folder.is_dir(): + observer.schedule(event_handler, folder, recursive=True) + else: + logger.warning(f"Folder '{folder}' doesn't exist or isn't a folder.") + + observer.start() + + try: + _run_ape_test(*pytest_arg_ls) + while True: + _run_main_loop(watch_delay, *pytest_arg_ls) + + finally: + observer.stop() + observer.join() + + +def emit_trigger(): + """ + Emits trigger to run pytest + """ + + global trigger + + with trigger_lock: + trigger = datetime.now() + + +class EventHandler(events.FileSystemEventHandler): + EVENTS_WATCHED = ( + events.EVENT_TYPE_CREATED, + events.EVENT_TYPE_DELETED, + events.EVENT_TYPE_MODIFIED, + events.EVENT_TYPE_MOVED, + ) + + def dispatch(self, event: events.FileSystemEvent) -> None: + if event.event_type in self.EVENTS_WATCHED: + self.process_event(event) + + @cached_property + def _extensions_to_watch(self) -> list[str]: + from ape.utils.basemodel import ManagerAccessMixin as access + + return [".py", *access.compiler_manager.registered_compilers.keys()] + + def _is_path_watched(self, filepath: str) -> bool: + """ + Check if file should trigger pytest run + """ + return any(map(filepath.endswith, self._extensions_to_watch)) + + def process_event(self, event: events.FileSystemEvent) -> None: + if self._is_path_watched(event.src_path): + emit_trigger() + + +def _run_ape_test(*pytest_args): + return run_subprocess(["ape", "test", *[f"{a}" for a in pytest_args]]) + + +def _run_main_loop(delay: float, *pytest_args: str) -> None: + global trigger + + now = datetime.now() + if trigger and now - trigger > timedelta(seconds=delay): + _run_ape_test(*pytest_args) + + with trigger_lock: + trigger = None + + time.sleep(delay) + + +def _create_event_handler(): + # Abstracted for testing purposes. + return EventHandler() + + +def _create_observer(): + # Abstracted for testing purposes. + return Observer() diff --git a/tests/functional/test_test.py b/tests/functional/test_test.py index 56ee09c88a..e063973e0e 100644 --- a/tests/functional/test_test.py +++ b/tests/functional/test_test.py @@ -1,8 +1,11 @@ +from pathlib import Path + import pytest from ape.exceptions import ConfigError from ape.pytest.runners import PytestApeRunner from ape_test import ApeTestConfig +from ape_test._watch import run_with_observer class TestApeTestConfig: @@ -33,3 +36,30 @@ def test_connect_to_mainnet_by_default(mocker): ) with pytest.raises(ConfigError, match=expected): runner._connect() + + +def test_watch(mocker): + mock_event_handler = mocker.MagicMock() + event_handler_patch = mocker.patch("ape_test._watch._create_event_handler") + event_handler_patch.return_value = mock_event_handler + + mock_observer = mocker.MagicMock() + observer_patch = mocker.patch("ape_test._watch._create_observer") + observer_patch.return_value = mock_observer + + run_subprocess_patch = mocker.patch("ape_test._watch.run_subprocess") + run_main_loop_patch = mocker.patch("ape_test._watch._run_main_loop") + run_main_loop_patch.side_effect = SystemExit # Avoid infinite loop. + + # Only passing `-s` so we have an extra arg to test. + with pytest.raises(SystemExit): + run_with_observer((Path("contracts"),), 0.1, "-s") + + # The observer started, then the main runner exits, and the observer stops + joins. + assert mock_observer.start.call_count == 1 + assert mock_observer.stop.call_count == 1 + assert mock_observer.join.call_count == 1 + + # NOTE: We had a bug once where the args it received were not strings. + # (wasn't deconstructing), so this check is important. + run_subprocess_patch.assert_called_once_with(["ape", "test", "-s"]) diff --git a/tests/integration/cli/test_test.py b/tests/integration/cli/test_test.py index f7c09a615f..90cf0b184c 100644 --- a/tests/integration/cli/test_test.py +++ b/tests/integration/cli/test_test.py @@ -435,27 +435,10 @@ def test_fails(): @skip_projects_except("with-contracts") def test_watch(mocker, integ_project, runner, ape_cli): - mock_event_handler = mocker.MagicMock() - event_handler_patch = mocker.patch("ape_test._cli._create_event_handler") - event_handler_patch.return_value = mock_event_handler - - mock_observer = mocker.MagicMock() - observer_patch = mocker.patch("ape_test._cli._create_observer") - observer_patch.return_value = mock_observer - - run_subprocess_patch = mocker.patch("ape_test._cli.run_subprocess") - run_main_loop_patch = mocker.patch("ape_test._cli._run_main_loop") - run_main_loop_patch.side_effect = SystemExit # Avoid infinite loop. + runner_patch = mocker.patch("ape_test._cli._run_with_observer") # Only passing `-s` so we have an extra arg to test. result = runner.invoke(ape_cli, ("test", "--watch", "-s")) assert result.exit_code == 0 - # The observer started, then the main runner exits, and the observer stops + joins. - assert mock_observer.start.call_count == 1 - assert mock_observer.stop.call_count == 1 - assert mock_observer.join.call_count == 1 - - # NOTE: We had a bug once where the args it received were not strings. - # (wasn't deconstructing), so this check is important. - run_subprocess_patch.assert_called_once_with(["ape", "test", "-s"]) + runner_patch.assert_called_once_with((Path("contracts"), Path("tests")), 0.5, "-s") From 26dcf1311badd32b2713d5cc8550f39f1591c341 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 1 Nov 2024 17:19:50 -0500 Subject: [PATCH 05/28] perf: make `ape plugins --help` faster (#2367) --- src/ape_plugins/_cli.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/ape_plugins/_cli.py b/src/ape_plugins/_cli.py index ecf5a80937..59f351ee66 100644 --- a/src/ape_plugins/_cli.py +++ b/src/ape_plugins/_cli.py @@ -1,23 +1,16 @@ import subprocess import sys from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import click from packaging.version import Version from ape.cli.options import ape_cli_context, skip_confirmation_option from ape.logging import logger -from ape.plugins._utils import ( - PIP_COMMAND, - ModifyPluginResultHandler, - PluginMetadata, - PluginMetadataList, - PluginType, - ape_version, - get_plugin_dists, -) -from ape.utils.misc import load_config + +if TYPE_CHECKING: + from ape.plugins._utils import PluginMetadata @click.group(short_help="Manage ape plugins") @@ -33,7 +26,10 @@ def plugins_argument(): or plugins loaded from the local config file. """ - def load_from_file(ctx, file_path: Path) -> list[PluginMetadata]: + def load_from_file(ctx, file_path: Path) -> list["PluginMetadata"]: + from ape.plugins._utils import PluginMetadata + from ape.utils.misc import load_config + if file_path.is_dir(): name_options = ( "ape-config.yaml", @@ -55,6 +51,8 @@ def load_from_file(ctx, file_path: Path) -> list[PluginMetadata]: return [] def callback(ctx, param, value: tuple[str]): + from ape.plugins._utils import PluginMetadata + res = [] if not value: ctx.obj.abort("You must give at least one requirement to install.") @@ -93,6 +91,8 @@ def upgrade_option(help: str = "", **kwargs): def _display_all_callback(ctx, param, value): + from ape.plugins._utils import PluginType + return ( (PluginType.CORE, PluginType.INSTALLED, PluginType.THIRD_PARTY, PluginType.AVAILABLE) if value @@ -112,6 +112,8 @@ def _display_all_callback(ctx, param, value): help="Display all plugins installed and available (including Core)", ) def _list(cli_ctx, to_display): + from ape.plugins._utils import PluginMetadataList, PluginType + include_available = PluginType.AVAILABLE in to_display metadata = PluginMetadataList.load(cli_ctx.plugin_manager, include_available=include_available) if output := metadata.to_str(include=to_display): @@ -128,7 +130,7 @@ def _list(cli_ctx, to_display): @plugins_argument() @skip_confirmation_option("Don't ask for confirmation to install the plugins") @upgrade_option(help="Upgrade the plugin to the newest available version") -def install(cli_ctx, plugins: list[PluginMetadata], skip_confirmation: bool, upgrade: bool): +def install(cli_ctx, plugins: list["PluginMetadata"], skip_confirmation: bool, upgrade: bool): """Install plugins""" failures_occurred = False @@ -170,6 +172,7 @@ def install(cli_ctx, plugins: list[PluginMetadata], skip_confirmation: bool, upg @skip_confirmation_option("Don't ask for confirmation to install the plugins") def uninstall(cli_ctx, plugins, skip_confirmation): """Uninstall plugins""" + from ape.plugins._utils import ModifyPluginResultHandler failures_occurred = False did_warn_about_version = False @@ -217,6 +220,7 @@ def update(): """ Update Ape and all plugins to the next version """ + from ape.plugins._utils import ape_version _change_version(ape_version.next_version_range) @@ -249,6 +253,8 @@ def _install(name, spec, exit_on_fail: bool = True) -> int: Returns: The process return-code. """ + from ape.plugins._utils import PIP_COMMAND + arguments = [*PIP_COMMAND, "install", f"{name}{spec}", "--quiet"] # Run the installation process and capture output for error checking @@ -281,6 +287,8 @@ def _change_version(spec: str): # This will also update core Ape. # NOTE: It is possible plugins may depend on each other and may update in # an order causing some error codes to pop-up, so we ignore those for now. + from ape.plugins._utils import get_plugin_dists + plugin_retcode = 0 for plugin in get_plugin_dists(): logger.info(f"Updating {plugin} ...") From 50299ab7bc1f40d97ce2b7a72b2792b210f8348c Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 1 Nov 2024 17:59:34 -0500 Subject: [PATCH 06/28] perf: use `p1.relative(p2)` when it makes sense (#2369) --- src/ape/managers/project.py | 6 ++---- src/ape/pytest/coverage.py | 4 ++-- src/ape/utils/os.py | 12 ++++++------ src/ape_pm/compiler.py | 3 +-- tests/functional/test_compilers.py | 2 +- tests/functional/utils/test_os.py | 15 --------------- 6 files changed, 12 insertions(+), 30 deletions(-) diff --git a/src/ape/managers/project.py b/src/ape/managers/project.py index 2f6f658945..1b154a573c 100644 --- a/src/ape/managers/project.py +++ b/src/ape/managers/project.py @@ -43,7 +43,7 @@ def _path_to_source_id(path: Path, root_path: Path) -> str: - return f"{get_relative_path(path.absolute(), root_path.absolute())}" + return f"{path.relative_to(root_path)}" class SourceManager(BaseManager): @@ -495,9 +495,7 @@ def _compile( ): self._compile_contracts(needs_compile) - src_ids = [ - f"{get_relative_path(Path(p).absolute(), self.project.path)}" for p in path_ls_final - ] + src_ids = [f"{Path(p).relative_to(self.project.path)}" for p in path_ls_final] for contract_type in (self.project.manifest.contract_types or {}).values(): if contract_type.source_id and contract_type.source_id in src_ids: yield ContractContainer(contract_type) diff --git a/src/ape/pytest/coverage.py b/src/ape/pytest/coverage.py index 768c385492..eebe47c670 100644 --- a/src/ape/pytest/coverage.py +++ b/src/ape/pytest/coverage.py @@ -7,7 +7,7 @@ from ape.logging import logger from ape.utils.basemodel import ManagerAccessMixin from ape.utils.misc import get_current_timestamp_ms -from ape.utils.os import get_full_extension, get_relative_path +from ape.utils.os import get_full_extension from ape.utils.trace import parse_coverage_tables if TYPE_CHECKING: @@ -92,7 +92,7 @@ def cover( self, src_path: Path, pcs: Iterable[int], inc_fn_hits: bool = True ) -> tuple[set[int], list[str]]: if hasattr(self.project, "path"): - source_id = str(get_relative_path(src_path.absolute(), self.project.path)) + source_id = f"{src_path.relative_to(self.project.path)}" else: source_id = str(src_path) diff --git a/src/ape/utils/os.py b/src/ape/utils/os.py index f66d9736c1..bc6e3f37dd 100644 --- a/src/ape/utils/os.py +++ b/src/ape/utils/os.py @@ -34,7 +34,12 @@ def get_relative_path(target: Path, anchor: Path) -> Path: Compute the relative path of ``target`` relative to ``anchor``, which may or may not share a common ancestor. - **NOTE**: Both paths must be absolute. + **NOTE ON PERFORMANCE**: Both paths must be absolute to + use this method. If you know both methods are absolute, + this method is a performance boost. If you have to first + call `.absolute()` on the paths, use + `target.relative_to(anchor)` instead; as it will be + faster in that case. Args: target (pathlib.Path): The path we are interested in. @@ -43,11 +48,6 @@ def get_relative_path(target: Path, anchor: Path) -> Path: Returns: pathlib.Path: The new path to the target path from the anchor path. """ - if not target.is_absolute(): - raise ValueError("'target' must be an absolute path.") - if not anchor.is_absolute(): - raise ValueError("'anchor' must be an absolute path.") - # Calculate common prefix length common_parts = 0 for target_part, anchor_part in zip(target.parts, anchor.parts): diff --git a/src/ape_pm/compiler.py b/src/ape_pm/compiler.py index ebd5ea97ee..98a4d5c377 100644 --- a/src/ape_pm/compiler.py +++ b/src/ape_pm/compiler.py @@ -11,7 +11,6 @@ from ape.api.compiler import CompilerAPI from ape.exceptions import CompilerError, ContractLogicError from ape.logging import logger -from ape.utils.os import get_relative_path if TYPE_CHECKING: from ape.managers.project import ProjectManager @@ -41,7 +40,7 @@ def compile( ) -> Iterator[ContractType]: project = project or self.local_project source_ids = { - p: f"{get_relative_path(p, project.path.absolute())}" if p.is_absolute() else str(p) + p: f"{p.relative_to(project.path)}" if p.is_absolute() else str(p) for p in contract_filepaths } logger.info(f"Compiling {', '.join(source_ids.values())}.") diff --git a/tests/functional/test_compilers.py b/tests/functional/test_compilers.py index 58db5f44b8..5e9ed7ac2c 100644 --- a/tests/functional/test_compilers.py +++ b/tests/functional/test_compilers.py @@ -77,7 +77,7 @@ def test_compile(compilers, project_with_contract, factory): Testing both stringified paths and path-object paths. """ path = next(iter(project_with_contract.sources.paths)) - actual = compilers.compile((factory(path),)) + actual = compilers.compile((factory(path),), project=project_with_contract) contract_name = path.stem assert contract_name in [x.name for x in actual] diff --git a/tests/functional/utils/test_os.py b/tests/functional/utils/test_os.py index 0acca31efc..5aa5c34785 100644 --- a/tests/functional/utils/test_os.py +++ b/tests/functional/utils/test_os.py @@ -23,21 +23,6 @@ def test_get_relative_path_from_project(): assert actual == expected -def test_get_relative_path_given_relative_path(): - relative_script_path = Path("../deploy.py") - with pytest.raises(ValueError) as err: - get_relative_path(relative_script_path, _TEST_DIRECTORY_PATH) - - assert str(err.value) == "'target' must be an absolute path." - - relative_project_path = Path("../This/is/a/test") - - with pytest.raises(ValueError) as err: - get_relative_path(_TEST_FILE_PATH, relative_project_path) - - assert str(err.value) == "'anchor' must be an absolute path." - - def test_get_relative_path_same_path(): actual = get_relative_path(_TEST_FILE_PATH, _TEST_FILE_PATH) assert actual == Path() From 75e0d8ec654004042173c0cb3e178bc6153bd358 Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 4 Nov 2024 07:38:52 -0600 Subject: [PATCH 07/28] feat: Add `--code` option to `ape console` (#2370) --- docs/userguides/console.md | 15 ++++++++++ src/ape_console/_cli.py | 40 +++++++++++++++++++++++---- tests/integration/cli/test_console.py | 12 ++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/docs/userguides/console.md b/docs/userguides/console.md index b9d8690d94..27b6cf62a2 100644 --- a/docs/userguides/console.md +++ b/docs/userguides/console.md @@ -164,3 +164,18 @@ Out[3]: '0.00040634 ETH' In [4]: %bal 0xE3747e6341E0d3430e6Ea9e2346cdDCc2F8a4b5b Out[4]: '0.00040634 ETH' ``` + +## Executing Code + +You can also use the `ape console` to execute programs directly from strings. +This is similar to the `python -c|--code` option except it will display the output cell. +Anything available in `ape console` is also available in `ape console --code`. + +```shell +ape console -c 'project.name' +Out[1]: 'my-project' +ape console -c 'x = 3\nx + 1' +Out[1]: 4 +ape console -c 'networks.active_provider.name' +Out[1]: 'test' +``` diff --git a/src/ape_console/_cli.py b/src/ape_console/_cli.py index 07855986d1..c3eef924fa 100644 --- a/src/ape_console/_cli.py +++ b/src/ape_console/_cli.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Optional, cast import click +from IPython import InteractiveShell from ape.cli.commands import ConnectedProviderCommand from ape.cli.options import ape_cli_context, project_option @@ -21,6 +22,18 @@ CONSOLE_EXTRAS_FILENAME = "ape_console_extras.py" +def _code_callback(ctx, param, value) -> list[str]: + if not value: + return value + + # NOTE: newlines are escaped in code automatically, so we + # need to de-escape them. Any actually escaped newlines + # will still be escaped. + value = value.replace("\\n", "\n").replace("\\t", "\t").replace("\\b", "\b") + + return value.splitlines() + + @click.command( cls=ConnectedProviderCommand, short_help="Load the console", @@ -28,10 +41,11 @@ ) @ape_cli_context() @project_option(hidden=True) # Hidden as mostly used for test purposes. -def cli(cli_ctx, project): +@click.option("-c", "--code", help="Program passed in as a string", callback=_code_callback) +def cli(cli_ctx, project, code): """Opens a console for the local project.""" verbose = cli_ctx.logger.level == logging.DEBUG - return console(project=project, verbose=verbose) + return console(project=project, verbose=verbose, code=code) def import_extras_file(file_path) -> ModuleType: @@ -95,6 +109,7 @@ def console( verbose: bool = False, extra_locals: Optional[dict] = None, embed: bool = False, + code: Optional[list[str]] = None, ): import IPython from IPython.terminal.ipapp import Config as IPythonConfig @@ -149,16 +164,24 @@ def console( # Required for click.testing.CliRunner support. embed = True - _launch_console(namespace, ipy_config, embed, banner) + _launch_console(namespace, ipy_config, embed, banner, code=code) -def _launch_console(namespace: dict, ipy_config: "IPythonConfig", embed: bool, banner: str): +def _launch_console( + namespace: dict, + ipy_config: "IPythonConfig", + embed: bool, + banner: str, + code: Optional[list[str]], +): import IPython from ape_console.config import ConsoleConfig ipython_kwargs = {"user_ns": namespace, "config": ipy_config} - if embed: + if code: + _execute_code(code, **ipython_kwargs) + elif embed: IPython.embed(**ipython_kwargs, colors="Neutral", banner1=banner) else: ipy_config.TerminalInteractiveShell.colors = "Neutral" @@ -169,3 +192,10 @@ def _launch_console(namespace: dict, ipy_config: "IPythonConfig", embed: bool, b ipy_config.InteractiveShellApp.extensions.extend(console_config.plugins) IPython.start_ipython(**ipython_kwargs, argv=()) + + +def _execute_code(code: list[str], **ipython_kwargs): + shell = InteractiveShell.instance(**ipython_kwargs) + # NOTE: Using `store_history=True` just so the cell IDs are accurate. + for line in code: + shell.run_cell(line, store_history=True) diff --git a/tests/integration/cli/test_console.py b/tests/integration/cli/test_console.py index fac1c5ec0f..171ac3311b 100644 --- a/tests/integration/cli/test_console.py +++ b/tests/integration/cli/test_console.py @@ -310,3 +310,15 @@ def test_console_natspecs(integ_project, solidity_contract_type, console_runner) assert all(ln in actual for ln in expected_method.splitlines()) assert all(ln in actual for ln in expected_event.splitlines()) + + +@skip_projects_except("with-contracts") +def test_console_code(integ_project, mocker, console_runner): + """ + Testing the -c | --code option. + """ + result = console_runner.invoke( + "--project", f"{integ_project.path}", "--code", "chain\nx = 3\nx + 1" + ) + expected = "Out[1]: \nOut[3]: 4\n" + assert result.output == expected From f833e468e0818a33d30cddd4f441b55932ce2785 Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 4 Nov 2024 08:53:53 -0600 Subject: [PATCH 08/28] feat: lookup network in evmchains; plugin-less networks, adhoc networks w/ correct name (#2328) --- docs/userguides/networks.md | 21 ++++++++++-- src/ape/api/networks.py | 34 +++++++++++++++---- src/ape/managers/networks.py | 33 ++++++++++++++---- src/ape/managers/project.py | 1 - src/ape_ethereum/provider.py | 21 +++++++++++- tests/functional/geth/test_network_manager.py | 14 ++++++++ 6 files changed, 105 insertions(+), 19 deletions(-) diff --git a/docs/userguides/networks.md b/docs/userguides/networks.md index ae6237b0fb..9e91b89db6 100644 --- a/docs/userguides/networks.md +++ b/docs/userguides/networks.md @@ -1,8 +1,9 @@ # Networks When interacting with a blockchain, you will have to select an ecosystem (e.g. Ethereum, Arbitrum, or Fantom), a network (e.g. Mainnet or Sepolia) and a provider (e.g. Eth-Tester, Node (Geth), or Alchemy). -Networks are part of ecosystems and typically defined in plugins. -For example, the `ape-ethereum` plugin comes with Ape and can be used for handling EVM-like behavior. +The `ape-ethereum` ecosystem and network(s) plugin comes with Ape and can be used for handling EVM-like behavior. +Networks are part of ecosystems and typically defined in plugins or custom-network configurations. +However, Ape works out-of-the-box (in a limited way) with any network defined in the [evmchains](https://github.com/ApeWorX/evmchains) library. ## Selecting a Network @@ -25,7 +26,7 @@ ape test --network ethereum:local:foundry ape console --network arbitrum:testnet:alchemy # NOTICE: All networks, even from other ecosystems, use this. ``` -To see all possible values for `--network`, run the command: +To see all networks that work with the `--network` flag (besides those _only_ defined in `evmchains`), run the command: ```shell ape networks list @@ -100,6 +101,20 @@ ape networks list In the remainder of this guide, any example below using Ethereum, you can replace with an L2 ecosystem's name and network combination. +## evmchains Networks + +If a network is in the [evmchains](https://github.com/ApeWorX/evmchains) library, it will work in Ape automatically, even without a plugin or any custom configuration for that network. + +```shell +ape console --network moonbeam +``` + +This works because the `moonbeam` network data is available in the `evmchains` library, and Ape is able to look it up. + +```{warning} +Support for networks from evm-chains alone may be limited and require additional configuration to work in production use-cases. +``` + ## Custom Network Connection You can add custom networks to Ape without creating a plugin. diff --git a/src/ape/api/networks.py b/src/ape/api/networks.py index 5f965428f1..07c249135c 100644 --- a/src/ape/api/networks.py +++ b/src/ape/api/networks.py @@ -12,6 +12,7 @@ ) from eth_pydantic_types import HexBytes from eth_utils import keccak, to_int +from evmchains import PUBLIC_CHAIN_META from pydantic import model_validator from ape.exceptions import ( @@ -109,7 +110,7 @@ def data_folder(self) -> Path: """ return self.config_manager.DATA_FOLDER / self.name - @cached_property + @property def custom_network(self) -> "NetworkAPI": """ A :class:`~ape.api.networks.NetworkAPI` for custom networks where the @@ -125,13 +126,11 @@ def custom_network(self) -> "NetworkAPI": if ethereum_class is None: raise NetworkError("Core Ethereum plugin missing.") - request_header = self.config_manager.REQUEST_HEADER - init_kwargs = {"name": "ethereum", "request_header": request_header} - ethereum = ethereum_class(**init_kwargs) # type: ignore + init_kwargs = {"name": "ethereum"} + evm_ecosystem = ethereum_class(**init_kwargs) # type: ignore return NetworkAPI( name="custom", - ecosystem=ethereum, - request_header=request_header, + ecosystem=evm_ecosystem, _default_provider="node", _is_custom=True, ) @@ -301,6 +300,11 @@ def networks(self) -> dict[str, "NetworkAPI"]: network_api._is_custom = True networks[net_name] = network_api + # Add any remaining networks from EVM chains here (but don't override). + # NOTE: Only applicable to EVM-based ecosystems, of course. + # Otherwise, this is a no-op. + networks = {**self._networks_from_evmchains, **networks} + return networks @cached_property @@ -311,6 +315,17 @@ def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]: if ecosystem_name == self.name } + @cached_property + def _networks_from_evmchains(self) -> dict[str, "NetworkAPI"]: + # NOTE: Purposely exclude plugins here so we also prefer plugins. + return { + network_name: create_network_type(data["chainId"], data["chainId"])( + name=network_name, ecosystem=self + ) + for network_name, data in PUBLIC_CHAIN_META.get(self.name, {}).items() + if network_name not in self._networks_from_plugins + } + def __post_init__(self): if len(self.networks) == 0: raise NetworkError("Must define at least one network in ecosystem") @@ -1057,7 +1072,6 @@ def providers(self): # -> dict[str, Partial[ProviderAPI]] Returns: dict[str, partial[:class:`~ape.api.providers.ProviderAPI`]] """ - from ape.plugins._utils import clean_plugin_name providers = {} @@ -1089,6 +1103,12 @@ def providers(self): # -> dict[str, Partial[ProviderAPI]] network=self, ) + # Any EVM-chain works with node provider. + if "node" not in providers and self.name in self.ecosystem._networks_from_evmchains: + # NOTE: Arbitrarily using sepolia to access the Node class. + node_provider_cls = self.network_manager.ethereum.sepolia.get_provider("node").__class__ + providers["node"] = partial(node_provider_cls, name="node", network=self) + return providers def _get_plugin_providers(self): diff --git a/src/ape/managers/networks.py b/src/ape/managers/networks.py index 8297ce43a3..a30c63b06e 100644 --- a/src/ape/managers/networks.py +++ b/src/ape/managers/networks.py @@ -2,6 +2,8 @@ from functools import cached_property from typing import TYPE_CHECKING, Optional, Union +from evmchains import PUBLIC_CHAIN_META + from ape.api.networks import EcosystemAPI, NetworkAPI, ProviderContextManager from ape.exceptions import EcosystemNotFoundError, NetworkError, NetworkNotFoundError from ape.managers.base import BaseManager @@ -53,7 +55,6 @@ def active_provider(self) -> Optional["ProviderAPI"]: """ The currently connected provider if one exists. Otherwise, returns ``None``. """ - return self._active_provider @active_provider.setter @@ -164,7 +165,6 @@ def ecosystem_names(self) -> set[str]: """ The set of all ecosystem names in ``ape``. """ - return set(self.ecosystems) @property @@ -236,7 +236,8 @@ def ecosystems(self) -> dict[str, EcosystemAPI]: existing_cls = plugin_ecosystems[base_ecosystem_name] ecosystem_cls = existing_cls.model_copy( - update={"name": ecosystem_name}, cache_clear=("_networks_from_plugins",) + update={"name": ecosystem_name}, + cache_clear=("_networks_from_plugins", "_networks_from_evmchains"), ) plugin_ecosystems[ecosystem_name] = ecosystem_cls @@ -437,10 +438,29 @@ def get_ecosystem(self, ecosystem_name: str) -> EcosystemAPI: :class:`~ape.api.networks.EcosystemAPI` """ - if ecosystem_name not in self.ecosystem_names: - raise EcosystemNotFoundError(ecosystem_name, options=self.ecosystem_names) + if ecosystem_name in self.ecosystem_names: + return self.ecosystems[ecosystem_name] - return self.ecosystems[ecosystem_name] + elif ecosystem_name.lower().replace(" ", "-") in PUBLIC_CHAIN_META: + ecosystem_name = ecosystem_name.lower().replace(" ", "-") + symbol = None + for net in PUBLIC_CHAIN_META[ecosystem_name].values(): + if not (native_currency := net.get("nativeCurrency")): + continue + + if "symbol" not in native_currency: + continue + + symbol = native_currency["symbol"] + break + + symbol = symbol or "ETH" + + # Is an EVM chain, can automatically make a class using evm-chains. + evm_class = self._plugin_ecosystems["ethereum"].__class__ + return evm_class(name=ecosystem_name, fee_token_symbol=symbol) + + raise EcosystemNotFoundError(ecosystem_name, options=self.ecosystem_names) def get_provider_from_choice( self, @@ -548,7 +568,6 @@ def parse_network_choice( Returns: :class:`~api.api.networks.ProviderContextManager` """ - provider = self.get_provider_from_choice( network_choice=network_choice, provider_settings=provider_settings ) diff --git a/src/ape/managers/project.py b/src/ape/managers/project.py index 1b154a573c..9fd69439db 100644 --- a/src/ape/managers/project.py +++ b/src/ape/managers/project.py @@ -1940,7 +1940,6 @@ def reconfigure(self, **overrides): self._config_override = overrides _ = self.config - self.account_manager.test_accounts.reset() def extract_manifest(self) -> PackageManifest: diff --git a/src/ape_ethereum/provider.py b/src/ape_ethereum/provider.py index c1ff49e704..8359729a09 100644 --- a/src/ape_ethereum/provider.py +++ b/src/ape_ethereum/provider.py @@ -16,7 +16,7 @@ from eth_pydantic_types import HexBytes from eth_typing import BlockNumber, HexStr from eth_utils import add_0x_prefix, is_hex, to_hex -from evmchains import get_random_rpc +from evmchains import PUBLIC_CHAIN_META, get_random_rpc from pydantic.dataclasses import dataclass from requests import HTTPError from web3 import HTTPProvider, IPCProvider, Web3 @@ -1524,9 +1524,16 @@ def _complete_connect(self): for option in ("earliest", "latest"): try: block = self.web3.eth.get_block(option) # type: ignore[arg-type] + except ExtraDataLengthError: is_likely_poa = True break + + except Exception: + # Some chains are "light" and we may not be able to detect + # if it need PoA middleware. + continue + else: is_likely_poa = ( "proofOfAuthorityData" in block @@ -1540,6 +1547,18 @@ def _complete_connect(self): self.network.verify_chain_id(chain_id) + # Correct network name, if using custom-URL approach. + if self.network.name == "custom": + for ecosystem_name, network in PUBLIC_CHAIN_META.items(): + for network_name, meta in network.items(): + if "chainId" not in meta or meta["chainId"] != chain_id: + continue + + # Network found. + self.network.name = network_name + self.network.ecosystem.name = ecosystem_name + break + def disconnect(self): self._call_trace_approach = None self._web3 = None diff --git a/tests/functional/geth/test_network_manager.py b/tests/functional/geth/test_network_manager.py index 18d5274f6e..8c2d5986aa 100644 --- a/tests/functional/geth/test_network_manager.py +++ b/tests/functional/geth/test_network_manager.py @@ -37,3 +37,17 @@ def test_fork_upstream_provider(networks, mock_geth_sepolia, geth_provider, mock geth_provider.provider_settings["uri"] = orig else: del geth_provider.provider_settings["uri"] + + +@geth_process_test +@pytest.mark.parametrize( + "connection_str", ("moonbeam:moonriver", "https://moonriver.api.onfinality.io/public") +) +def test_parse_network_choice_evmchains(networks, connection_str): + """ + Show we can (without having a plugin installed) connect to a network + that evm-chains knows about. + """ + with networks.parse_network_choice(connection_str) as moon_provider: + assert moon_provider.network.name == "moonriver" + assert moon_provider.network.ecosystem.name == "moonbeam" From b71c810568ab321a3c7d9bb267b95e503d5bfff7 Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 4 Nov 2024 10:23:12 -0600 Subject: [PATCH 09/28] perf: make contracts load faster (#2371) --- src/ape/contracts/__init__.py | 6 ++- src/ape/contracts/base.py | 73 +++++++++++++++++--------------- src/ape_node/provider.py | 2 +- tests/functional/test_project.py | 6 ++- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/ape/contracts/__init__.py b/src/ape/contracts/__init__.py index 8ebd5e04f2..535514e7af 100644 --- a/src/ape/contracts/__init__.py +++ b/src/ape/contracts/__init__.py @@ -1,4 +1,8 @@ -from .base import ContractContainer, ContractEvent, ContractInstance, ContractLog, ContractNamespace +def __getattr__(name: str): + import ape.contracts.base as module + + return getattr(module, name) + __all__ = [ "ContractContainer", diff --git a/src/ape/contracts/base.py b/src/ape/contracts/base.py index 9c73fb8158..3c6be64c0b 100644 --- a/src/ape/contracts/base.py +++ b/src/ape/contracts/base.py @@ -7,14 +7,10 @@ from typing import TYPE_CHECKING, Any, Optional, Union import click -import pandas as pd from eth_pydantic_types import HexBytes from eth_utils import to_hex -from ethpm_types.abi import EventABI, MethodABI -from ethpm_types.contract_type import ABI_W_SELECTOR_T, ContractType -from IPython.lib.pretty import for_type +from ethpm_types.abi import EventABI -from ape.api.accounts import AccountAPI from ape.api.address import Address, BaseAddress from ape.api.query import ( ContractCreation, @@ -34,7 +30,6 @@ MissingDeploymentBytecodeError, ) from ape.logging import get_rich_console, logger -from ape.types.address import AddressType from ape.types.events import ContractLog, LogFilter, MockContractLog from ape.utils.abi import StructParser, _enrich_natspec from ape.utils.basemodel import ( @@ -49,9 +44,12 @@ from ape.utils.misc import log_instead_of_fail if TYPE_CHECKING: - from ethpm_types.abi import ConstructorABI, ErrorABI + from ethpm_types.abi import ConstructorABI, ErrorABI, MethodABI + from ethpm_types.contract_type import ABI_W_SELECTOR_T, ContractType + from pandas import DataFrame from ape.api.transactions import ReceiptAPI, TransactionAPI + from ape.types.address import AddressType class ContractConstructor(ManagerAccessMixin): @@ -90,7 +88,7 @@ def serialize_transaction(self, *args, **kwargs) -> "TransactionAPI": def __call__(self, private: bool = False, *args, **kwargs) -> "ReceiptAPI": txn = self.serialize_transaction(*args, **kwargs) - if "sender" in kwargs and isinstance(kwargs["sender"], AccountAPI): + if "sender" in kwargs and hasattr(kwargs["sender"], "call"): sender = kwargs["sender"] return sender.call(txn, **kwargs) elif "sender" not in kwargs and self.account_manager.default_sender is not None: @@ -104,7 +102,7 @@ def __call__(self, private: bool = False, *args, **kwargs) -> "ReceiptAPI": class ContractCall(ManagerAccessMixin): - def __init__(self, abi: MethodABI, address: AddressType) -> None: + def __init__(self, abi: "MethodABI", address: "AddressType") -> None: super().__init__() self.abi = abi self.address = address @@ -140,9 +138,9 @@ def __call__(self, *args, **kwargs) -> Any: class ContractMethodHandler(ManagerAccessMixin): contract: "ContractInstance" - abis: list[MethodABI] + abis: list["MethodABI"] - def __init__(self, contract: "ContractInstance", abis: list[MethodABI]) -> None: + def __init__(self, contract: "ContractInstance", abis: list["MethodABI"]) -> None: super().__init__() self.contract = contract self.abis = abis @@ -320,7 +318,7 @@ def estimate_gas_cost(self, *args, **kwargs) -> int: return self.transact.estimate_gas_cost(*arguments, **kwargs) -def _select_method_abi(abis: list[MethodABI], args: Union[tuple, list]) -> MethodABI: +def _select_method_abi(abis: list["MethodABI"], args: Union[tuple, list]) -> "MethodABI": args = args or [] selected_abi = None for abi in abis: @@ -335,13 +333,10 @@ def _select_method_abi(abis: list[MethodABI], args: Union[tuple, list]) -> Metho class ContractTransaction(ManagerAccessMixin): - abi: MethodABI - address: AddressType - - def __init__(self, abi: MethodABI, address: AddressType) -> None: + def __init__(self, abi: "MethodABI", address: "AddressType") -> None: super().__init__() - self.abi = abi - self.address = address + self.abi: "MethodABI" = abi + self.address: "AddressType" = address @log_instead_of_fail(default="") def __repr__(self) -> str: @@ -362,7 +357,7 @@ def __call__(self, *args, **kwargs) -> "ReceiptAPI": txn = self.serialize_transaction(*args, **kwargs) private = kwargs.get("private", False) - if "sender" in kwargs and isinstance(kwargs["sender"], AccountAPI): + if "sender" in kwargs and hasattr(kwargs["sender"], "call"): return kwargs["sender"].call(txn, **kwargs) txn = self.provider.prepare_transaction(txn) @@ -441,6 +436,7 @@ def _as_transaction(self, *args) -> ContractTransaction: ) +# TODO: In Ape 0.9 - make not a BaseModel - no reason to. class ContractEvent(BaseInterfaceModel): """ The types of events on a :class:`~ape.contracts.base.ContractInstance`. @@ -616,7 +612,7 @@ def query( stop_block: Optional[int] = None, step: int = 1, engine_to_use: Optional[str] = None, - ) -> pd.DataFrame: + ) -> "DataFrame": """ Iterate through blocks for log events @@ -635,6 +631,8 @@ def query( Returns: pd.DataFrame """ + # perf: pandas import is really slow. Avoid importing at module level. + import pandas as pd if start_block < 0: start_block = self.chain_manager.blocks.height + start_block @@ -800,7 +798,7 @@ def poll_logs( class ContractTypeWrapper(ManagerAccessMixin): - contract_type: ContractType + contract_type: "ContractType" base_path: Optional[Path] = None @property @@ -812,7 +810,7 @@ def selector_identifiers(self) -> dict[str, str]: return self.contract_type.selector_identifiers @property - def identifier_lookup(self) -> dict[str, ABI_W_SELECTOR_T]: + def identifier_lookup(self) -> dict[str, "ABI_W_SELECTOR_T"]: """ Provides a mapping of method, error, and event selector identifiers to ABI Types. @@ -898,6 +896,9 @@ def repr_pretty_for_assignment(cls, *args, **kwargs): info = _get_info() error_type.info = error_type.__doc__ = info # type: ignore if info: + # perf: Avoid forcing everyone to import from IPython. + from IPython.lib.pretty import for_type + error_type._repr_pretty_ = repr_pretty_for_assignment # type: ignore # Register the dynamically-created type with IPython so it integrates. @@ -922,8 +923,8 @@ class ContractInstance(BaseAddress, ContractTypeWrapper): def __init__( self, - address: AddressType, - contract_type: ContractType, + address: "AddressType", + contract_type: "ContractType", txn_hash: Optional[Union[str, HexBytes]] = None, ) -> None: super().__init__() @@ -957,7 +958,9 @@ def __call__(self, *args, **kwargs) -> "ReceiptAPI": return super().__call__(*args, **kwargs) @classmethod - def from_receipt(cls, receipt: "ReceiptAPI", contract_type: ContractType) -> "ContractInstance": + def from_receipt( + cls, receipt: "ReceiptAPI", contract_type: "ContractType" + ) -> "ContractInstance": """ Create a contract instance from the contract deployment receipt. """ @@ -997,7 +1000,7 @@ def __repr__(self) -> str: return f"<{contract_name} {self.address}>" @property - def address(self) -> AddressType: + def address(self) -> "AddressType": """ The address of the contract. @@ -1009,7 +1012,7 @@ def address(self) -> AddressType: @cached_property def _view_methods_(self) -> dict[str, ContractCallHandler]: - view_methods: dict[str, list[MethodABI]] = dict() + view_methods: dict[str, list["MethodABI"]] = dict() for abi in self.contract_type.view_methods: if abi.name in view_methods: @@ -1028,7 +1031,7 @@ def _view_methods_(self) -> dict[str, ContractCallHandler]: @cached_property def _mutable_methods_(self) -> dict[str, ContractTransactionHandler]: - mutable_methods: dict[str, list[MethodABI]] = dict() + mutable_methods: dict[str, list["MethodABI"]] = dict() for abi in self.contract_type.mutable_methods: if abi.name in mutable_methods: @@ -1075,7 +1078,7 @@ def call_view_method(self, method_name: str, *args, **kwargs) -> Any: else: # Didn't find anything that matches - name = self.contract_type.name or ContractType.__name__ + name = self.contract_type.name or "ContractType" raise ApeAttributeError(f"'{name}' has no attribute '{method_name}'.") def invoke_transaction(self, method_name: str, *args, **kwargs) -> "ReceiptAPI": @@ -1110,7 +1113,7 @@ def invoke_transaction(self, method_name: str, *args, **kwargs) -> "ReceiptAPI": else: # Didn't find anything that matches - name = self.contract_type.name or ContractType.__name__ + name = self.contract_type.name or "ContractType" raise ApeAttributeError(f"'{name}' has no attribute '{method_name}'.") def get_event_by_signature(self, signature: str) -> ContractEvent: @@ -1168,7 +1171,7 @@ def get_error_by_signature(self, signature: str) -> type[CustomError]: @cached_property def _events_(self) -> dict[str, list[ContractEvent]]: - events: dict[str, list[EventABI]] = {} + events: dict[str, list["EventABI"]] = {} for abi in self.contract_type.events: if abi.name in events: @@ -1339,7 +1342,7 @@ class ContractContainer(ContractTypeWrapper, ExtraAttributesMixin): contract_container = project.MyContract # Assuming there is a contract named "MyContract" """ - def __init__(self, contract_type: ContractType) -> None: + def __init__(self, contract_type: "ContractType") -> None: self.contract_type = contract_type @log_instead_of_fail(default="") @@ -1404,7 +1407,7 @@ def deployments(self): return self.chain_manager.contracts.get_deployments(self) def at( - self, address: AddressType, txn_hash: Optional[Union[str, HexBytes]] = None + self, address: "AddressType", txn_hash: Optional[Union[str, HexBytes]] = None ) -> ContractInstance: """ Get a contract at the given address. @@ -1473,7 +1476,7 @@ def deploy(self, *args, publish: bool = False, **kwargs) -> ContractInstance: if kwargs.get("value") and not self.contract_type.constructor.is_payable: raise MethodNonPayableError("Sending funds to a non-payable constructor.") - if "sender" in kwargs and isinstance(kwargs["sender"], AccountAPI): + if "sender" in kwargs and hasattr(kwargs["sender"], "call"): # Handle account-related preparation if needed, such as signing receipt = self._cache_wrap(lambda: kwargs["sender"].call(txn, **kwargs)) @@ -1533,7 +1536,7 @@ def declare(self, *args, **kwargs) -> "ReceiptAPI": transaction = self.provider.network.ecosystem.encode_contract_blueprint( self.contract_type, *args, **kwargs ) - if "sender" in kwargs and isinstance(kwargs["sender"], AccountAPI): + if "sender" in kwargs and hasattr(kwargs["sender"], "call"): return kwargs["sender"].call(transaction) receipt = self.provider.send_transaction(transaction) diff --git a/src/ape_node/provider.py b/src/ape_node/provider.py index 95bd54d2b7..aa7fb0f2ee 100644 --- a/src/ape_node/provider.py +++ b/src/ape_node/provider.py @@ -208,7 +208,7 @@ def disconnect(self): def _clean(self): if self._data_dir.is_dir(): - shutil.rmtree(self._data_dir) + shutil.rmtree(self._data_dir, ignore_errors=True) # dir must exist when initializing chain. self._data_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/functional/test_project.py b/tests/functional/test_project.py index bb0ee3de41..b432fc1b94 100644 --- a/tests/functional/test_project.py +++ b/tests/functional/test_project.py @@ -673,8 +673,12 @@ class TestProject: def test_init(self, with_dependencies_project_path): # Purpose not using `project_with_contracts` fixture. project = Project(with_dependencies_project_path) - project.manifest_path.unlink(missing_ok=True) assert project.path == with_dependencies_project_path + project.manifest_path.unlink(missing_ok=True) + + # Re-init to show it doesn't create the manifest file. + project = Project(with_dependencies_project_path) + # Manifest should have been created by default. assert not project.manifest_path.is_file() From 0a227e188445835735dfccefccf246501364a674 Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 4 Nov 2024 10:48:40 -0600 Subject: [PATCH 10/28] perf: make `ape run --help` faster (#2364) --- src/ape_run/_cli.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/ape_run/_cli.py b/src/ape_run/_cli.py index 94e30d361d..6dc0153295 100644 --- a/src/ape_run/_cli.py +++ b/src/ape_run/_cli.py @@ -8,19 +8,18 @@ from typing import Any, Union import click -from click import Command, Context, Option from ape.cli.commands import ConnectedProviderCommand from ape.cli.options import _VERBOSITY_VALUES, _create_verbosity_kwargs, verbosity_option from ape.exceptions import ApeException, handle_ape_exception from ape.logging import logger -from ape.utils.basemodel import ManagerAccessMixin as access -from ape.utils.os import get_relative_path, use_temp_sys_path -from ape_console._cli import console @contextmanager def use_scripts_sys_path(path: Path): + # perf: avoid importing at top of module so `--help` is faster. + from ape.utils.os import use_temp_sys_path + # First, ensure there is not an existing scripts module. scripts = sys.modules.get("scripts") if scripts: @@ -70,7 +69,9 @@ def __init__(self, *args, **kwargs): self._command_called = None self._has_warned_missing_hook: set[Path] = set() - def invoke(self, ctx: Context) -> Any: + def invoke(self, ctx: click.Context) -> Any: + from ape.utils.basemodel import ManagerAccessMixin as access + try: return super().invoke(ctx) except Exception as err: @@ -95,7 +96,8 @@ def invoke(self, ctx: Context) -> Any: raise def _get_command(self, filepath: Path) -> Union[click.Command, click.Group, None]: - relative_filepath = get_relative_path(filepath, access.local_project.path) + scripts_folder = Path.cwd() / "scripts" + relative_filepath = filepath.relative_to(scripts_folder) # First load the code module by compiling it # NOTE: This does not execute the module @@ -122,14 +124,14 @@ def _get_command(self, filepath: Path) -> Union[click.Command, click.Group, None self._namespace[filepath.stem] = cli_ns cli_obj = cli_ns["cli"] - if not isinstance(cli_obj, Command): + if not isinstance(cli_obj, click.Command): logger.warning("Found `cli()` method but it is not a click command.") return None params = [getattr(x, "name", None) for x in cli_obj.params] if "verbosity" not in params: option_kwargs = _create_verbosity_kwargs() - option = Option(_VERBOSITY_VALUES, **option_kwargs) + option = click.Option(_VERBOSITY_VALUES, **option_kwargs) cli_obj.params.append(option) cli_obj.name = filepath.stem if cli_obj.name in ("cli", "", None) else cli_obj.name @@ -175,13 +177,16 @@ def call(): @property def commands(self) -> dict[str, Union[click.Command, click.Group]]: - if not access.local_project.scripts_folder.is_dir(): + # perf: Don't reference `.local_project.scripts_folder` here; + # it's too slow when doing just doing `--help`. + scripts_folder = Path.cwd() / "scripts" + if not scripts_folder.is_dir(): return {} - return self._get_cli_commands(access.local_project.scripts_folder) + return self._get_cli_commands(scripts_folder) def _get_cli_commands(self, base_path: Path) -> dict: - commands: dict[str, Command] = {} + commands: dict[str, click.Command] = {} for filepath in base_path.iterdir(): if filepath.stem.startswith("_"): @@ -194,6 +199,7 @@ def _get_cli_commands(self, base_path: Path) -> dict: subcommands = self._get_cli_commands(filepath) for subcommand in subcommands.values(): group.add_command(subcommand) + commands[filepath.stem] = group if filepath.suffix == ".py": @@ -223,6 +229,8 @@ def result_callback(self, result, interactive: bool): # type: ignore[override] return result def _launch_console(self): + from ape.utils.basemodel import ManagerAccessMixin as access + trace = inspect.trace() trace_frames = [ x for x in trace if x.filename.startswith(str(access.local_project.scripts_folder)) @@ -247,6 +255,8 @@ def _launch_console(self): if frame: del frame + from ape_console._cli import console + return console(project=access.local_project, extra_locals=extra_locals, embed=True) From 6a8a73728cfa5543b6ec2a49f49b5d5666eb97f7 Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 4 Nov 2024 11:06:04 -0600 Subject: [PATCH 11/28] fix: recursion error when a bad URI was configured in `node:` (#2372) --- src/ape_ethereum/provider.py | 9 +++++---- tests/functional/geth/test_provider.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/ape_ethereum/provider.py b/src/ape_ethereum/provider.py index 8359729a09..58bba04dde 100644 --- a/src/ape_ethereum/provider.py +++ b/src/ape_ethereum/provider.py @@ -1340,7 +1340,7 @@ def uri(self) -> str: else: raise TypeError(f"Not an URI: {uri}") - config = self.config.model_dump().get(self.network.ecosystem.name, None) + config: dict = self.config.get(self.network.ecosystem.name, None) if config is None: if rpc := self._get_random_rpc(): return rpc @@ -1351,7 +1351,7 @@ def uri(self) -> str: raise ProviderError(f"Please configure a URL for '{self.network_choice}'.") # Use value from config file - network_config = config.get(self.network.name) or DEFAULT_SETTINGS + network_config: dict = (config or {}).get(self.network.name) or DEFAULT_SETTINGS if "url" in network_config: raise ConfigError("Unknown provider setting 'url'. Did you mean 'uri'?") @@ -1370,10 +1370,11 @@ def uri(self) -> str: settings_uri = network_config.get(key, DEFAULT_SETTINGS["uri"]) if _is_uri(settings_uri): + # Is true if HTTP, WS, or IPC. return settings_uri - # Likely was an IPC Path (or websockets) and will connect that way. - return super().http_uri or "" + # Is not HTTP, WS, or IPC. Raise an error. + raise ConfigError(f"Invalid URI (not HTTP, WS, or IPC): {settings_uri}") @property def http_uri(self) -> Optional[str]: diff --git a/tests/functional/geth/test_provider.py b/tests/functional/geth/test_provider.py index 57cb676451..93e8c550ca 100644 --- a/tests/functional/geth/test_provider.py +++ b/tests/functional/geth/test_provider.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from typing import cast @@ -16,6 +17,7 @@ from ape.exceptions import ( APINotImplementedError, BlockNotFoundError, + ConfigError, ContractLogicError, NetworkMismatchError, ProviderError, @@ -127,6 +129,23 @@ def test_uri_non_dev_and_not_configured(mocker, ethereum): assert actual == expected +def test_uri_invalid(geth_provider, project, ethereum): + settings = geth_provider.provider_settings + geth_provider.provider_settings = {} + value = "I AM NOT A URI OF ANY KIND!" + config = {"node": {"ethereum": {"local": {"uri": value}}}} + + try: + with project.temp_config(**config): + # Assert we use the config value. + expected = rf"Invalid URI \(not HTTP, WS, or IPC\): {re.escape(value)}" + with pytest.raises(ConfigError, match=expected): + _ = geth_provider.uri + + finally: + geth_provider.provider_settings = settings + + @geth_process_test def test_repr_connected(geth_provider): actual = repr(geth_provider) From 04ac2f1490172df7e6366cc1e87612bebac8888b Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 5 Nov 2024 08:48:51 -0600 Subject: [PATCH 12/28] perf: localize import for `ape console` (#2374) --- src/ape_console/_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ape_console/_cli.py b/src/ape_console/_cli.py index c3eef924fa..a3d3431cac 100644 --- a/src/ape_console/_cli.py +++ b/src/ape_console/_cli.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, Optional, cast import click -from IPython import InteractiveShell from ape.cli.commands import ConnectedProviderCommand from ape.cli.options import ape_cli_context, project_option @@ -195,6 +194,8 @@ def _launch_console( def _execute_code(code: list[str], **ipython_kwargs): + from IPython import InteractiveShell + shell = InteractiveShell.instance(**ipython_kwargs) # NOTE: Using `store_history=True` just so the cell IDs are accurate. for line in code: From a84b6922f79e98e61eec5a7cc2949e3ce58ab290 Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 5 Nov 2024 09:09:22 -0600 Subject: [PATCH 13/28] test: rerun random URL connection test if need-be (#2373) --- setup.py | 1 + tests/functional/geth/test_network_manager.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 932989e652..8b2f7d5e57 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ "pytest-cov>=4.0.0,<5", # Coverage analyzer plugin "pytest-mock", # For creating mocks "pytest-benchmark", # For performance tests + "pytest-rerunfailures", # For flakey tests "pytest-timeout>=2.2.0,<3", # For avoiding timing out during tests "hypothesis>=6.2.0,<7.0", # Strategy-based fuzzer "hypothesis-jsonschema==0.19.0", # JSON Schema fuzzer extension diff --git a/tests/functional/geth/test_network_manager.py b/tests/functional/geth/test_network_manager.py index 8c2d5986aa..a0949ee245 100644 --- a/tests/functional/geth/test_network_manager.py +++ b/tests/functional/geth/test_network_manager.py @@ -39,6 +39,8 @@ def test_fork_upstream_provider(networks, mock_geth_sepolia, geth_provider, mock del geth_provider.provider_settings["uri"] +# NOTE: Test is flakey because random URLs may be offline when test runs; avoid CI failure. +@pytest.mark.flaky(reruns=5) @geth_process_test @pytest.mark.parametrize( "connection_str", ("moonbeam:moonriver", "https://moonriver.api.onfinality.io/public") From 0c9e5f16e6da98fc504f65e7312c1832e94b0602 Mon Sep 17 00:00:00 2001 From: omahs <73983677+omahs@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:18:31 +0100 Subject: [PATCH 14/28] docs: fix typos (#2375) --- src/ape/api/accounts.py | 2 +- src/ape/api/providers.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ape/api/accounts.py b/src/ape/api/accounts.py index c8c357ffb6..c5e3ba1dc0 100644 --- a/src/ape/api/accounts.py +++ b/src/ape/api/accounts.py @@ -394,7 +394,7 @@ def prepare_transaction(self, txn: TransactionAPI) -> TransactionAPI: :class:`~ape.api.transactions.TransactionAPI` """ - # NOTE: Allow overriding nonce, assume user understand what this does + # NOTE: Allow overriding nonce, assume user understands what this does if txn.nonce is None: txn.nonce = self.nonce elif txn.nonce < self.nonce: diff --git a/src/ape/api/providers.py b/src/ape/api/providers.py index b0e1695f0b..559646f70e 100644 --- a/src/ape/api/providers.py +++ b/src/ape/api/providers.py @@ -60,7 +60,7 @@ class BlockAPI(BaseInterfaceModel): An abstract class representing a block and its attributes. """ - # NOTE: All fields in this class (and it's subclasses) should not be `Optional` + # NOTE: All fields in this class (and its subclasses) should not be `Optional` # except the edge cases noted below num_transactions: HexInt = 0 @@ -231,7 +231,7 @@ def connection_str(self) -> str: @abstractmethod def connect(self): """ - Connect a to a provider, such as start-up a process or create an HTTP connection. + Connect to a provider, such as start-up a process or create an HTTP connection. """ @abstractmethod @@ -352,7 +352,7 @@ def network_choice(self) -> str: def make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any: """ Make a raw RPC request to the provider. - Advanced featues such as tracing may utilize this to by-pass unnecessary + Advanced features such as tracing may utilize this to by-pass unnecessary class-serializations. """ @@ -933,7 +933,7 @@ def auto_mine(self) -> bool: @abstractmethod def auto_mine(self) -> bool: """ - Enable or disbale automine. + Enable or disable automine. """ def _increment_call_func_coverage_hit_count(self, txn: TransactionAPI): From e31c42695032ac076fe6a041065d5f957f3b8982 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 8 Nov 2024 18:54:19 -0600 Subject: [PATCH 15/28] perf: `ape_node` plugin load time improvement (#2378) --- docs/userguides/developing_plugins.md | 8 ++++++++ src/ape_node/__init__.py | 20 +++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/userguides/developing_plugins.md b/docs/userguides/developing_plugins.md index 199e970cbb..f1a407cbea 100644 --- a/docs/userguides/developing_plugins.md +++ b/docs/userguides/developing_plugins.md @@ -61,6 +61,9 @@ from ape import plugins # Here, we register our provider plugin so we can use it in 'ape'. @plugins.register(plugins.ProviderPlugin) def providers(): + # NOTE: By keeping this import local, we avoid slower plugin load times. + from ape_my_plugin.provider import MyProvider + # NOTE: 'MyProvider' defined in a prior code-block. yield "ethereum", "local", MyProvider ``` @@ -69,6 +72,11 @@ This decorator hooks into ape core and ties everything together by looking for a Then, it will loop through these potential `ape` plugins and see which ones have created a plugin type registration. If the plugin type registration is found, then `ape` knows this package is a plugin and attempts to process it according to its registration interface. +```{warning} +Ensure your plugin's `__init__.py` file imports quickly by keeping all expensive imports in the hook functions locally. +This helps Ape register plugins faster, which is required when checking for API implementations. +``` + ### CLI Plugins The `ape` CLI is built using the python package [click](https://palletsprojects.com/p/click/). diff --git a/src/ape_node/__init__.py b/src/ape_node/__init__.py index f76e94a7c0..26c45369cb 100644 --- a/src/ape_node/__init__.py +++ b/src/ape_node/__init__.py @@ -1,16 +1,17 @@ from ape import plugins -from .provider import EthereumNetworkConfig, EthereumNodeConfig, GethDev, Node -from .query import OtterscanQueryEngine - @plugins.register(plugins.Config) def config_class(): + from ape_node.provider import EthereumNodeConfig + return EthereumNodeConfig @plugins.register(plugins.ProviderPlugin) def providers(): + from ape_node.provider import EthereumNetworkConfig, GethDev, Node + networks_dict = EthereumNetworkConfig().model_dump() networks_dict.pop("local") for network_name in networks_dict: @@ -21,9 +22,22 @@ def providers(): @plugins.register(plugins.QueryPlugin) def query_engines(): + from ape_node.query import OtterscanQueryEngine + yield OtterscanQueryEngine +def __getattr__(name: str): + if name == "OtterscanQueryEngine": + from ape_node.query import OtterscanQueryEngine + + return OtterscanQueryEngine + + import ape_node.provider as module + + return getattr(module, name) + + __all__ = [ "EthereumNetworkConfig", "EthereumNodeConfig", From 9573398923266b5442b79f8d224690ba94dfb0e2 Mon Sep 17 00:00:00 2001 From: antazoey Date: Thu, 21 Nov 2024 01:34:57 +0700 Subject: [PATCH 16/28] fix: auto fork evmchains networks and better access (#2380) --- .pre-commit-config.yaml | 2 +- setup.py | 2 +- src/ape/api/networks.py | 19 +++++++++- src/ape/managers/networks.py | 47 +++++++++++++----------- tests/functional/test_network_api.py | 5 +++ tests/functional/test_network_manager.py | 21 ++++++++++- 6 files changed, 69 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0a339b0ec..db391b7e31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,7 +37,7 @@ repos: ] - repo: https://github.com/executablebooks/mdformat - rev: 0.7.18 + rev: 0.7.19 hooks: - id: mdformat additional_dependencies: [mdformat-gfm, mdformat-frontmatter, mdformat-pyproject] diff --git a/setup.py b/setup.py index 8b2f7d5e57..27b8013b7f 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ "flake8-pydantic", # For detecting issues with Pydantic models "flake8-type-checking", # Detect imports to move in/out of type-checking blocks "isort>=5.13.2,<6", # Import sorting linter - "mdformat>=0.7.18", # Auto-formatter for markdown + "mdformat==0.7.18", # Auto-formatter for markdown "mdformat-gfm>=0.3.5", # Needed for formatting GitHub-flavored markdown "mdformat-frontmatter>=0.4.1", # Needed for frontmatters-style headers in issue templates "mdformat-pyproject>=0.0.1", # Allows configuring in pyproject.toml diff --git a/src/ape/api/networks.py b/src/ape/api/networks.py index 07c249135c..601b2633a7 100644 --- a/src/ape/api/networks.py +++ b/src/ape/api/networks.py @@ -318,13 +318,29 @@ def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]: @cached_property def _networks_from_evmchains(self) -> dict[str, "NetworkAPI"]: # NOTE: Purposely exclude plugins here so we also prefer plugins. - return { + networks = { network_name: create_network_type(data["chainId"], data["chainId"])( name=network_name, ecosystem=self ) for network_name, data in PUBLIC_CHAIN_META.get(self.name, {}).items() if network_name not in self._networks_from_plugins } + forked_networks: dict[str, ForkedNetworkAPI] = {} + for network_name, network in networks.items(): + if network_name.endswith("-fork"): + # Already a fork. + continue + + fork_network_name = f"{network_name}-fork" + if any(x == fork_network_name for x in networks): + # The forked version of this network is already known. + continue + + forked_networks[fork_network_name] = ForkedNetworkAPI( + name=fork_network_name, ecosystem=self + ) + + return {**networks, **forked_networks} def __post_init__(self): if len(self.networks) == 0: @@ -535,7 +551,6 @@ def get_network(self, network_name: str) -> "NetworkAPI": Returns: :class:`~ape.api.networks.NetworkAPI` """ - names = {network_name, network_name.replace("-", "_"), network_name.replace("_", "-")} networks = self.networks for name in names: diff --git a/src/ape/managers/networks.py b/src/ape/managers/networks.py index a30c63b06e..a046f90b79 100644 --- a/src/ape/managers/networks.py +++ b/src/ape/managers/networks.py @@ -241,7 +241,7 @@ def ecosystems(self) -> dict[str, EcosystemAPI]: ) plugin_ecosystems[ecosystem_name] = ecosystem_cls - return plugin_ecosystems + return {**plugin_ecosystems, **self._evmchains_ecosystems} @cached_property def _plugin_ecosystems(self) -> dict[str, EcosystemAPI]: @@ -249,6 +249,30 @@ def _plugin_ecosystems(self) -> dict[str, EcosystemAPI]: plugins = self.plugin_manager.ecosystems return {n: cls(name=n) for n, cls in plugins} # type: ignore[operator] + @cached_property + def _evmchains_ecosystems(self) -> dict[str, EcosystemAPI]: + ecosystems: dict[str, EcosystemAPI] = {} + for name in PUBLIC_CHAIN_META: + ecosystem_name = name.lower().replace(" ", "-") + symbol = None + for net in PUBLIC_CHAIN_META[ecosystem_name].values(): + if not (native_currency := net.get("nativeCurrency")): + continue + + if "symbol" not in native_currency: + continue + + symbol = native_currency["symbol"] + break + + symbol = symbol or "ETH" + + # Is an EVM chain, can automatically make a class using evm-chains. + evm_class = self._plugin_ecosystems["ethereum"].__class__ + ecosystems[name] = evm_class(name=ecosystem_name, fee_token_symbol=symbol) + + return ecosystems + def create_custom_provider( self, connection_str: str, @@ -437,29 +461,9 @@ def get_ecosystem(self, ecosystem_name: str) -> EcosystemAPI: Returns: :class:`~ape.api.networks.EcosystemAPI` """ - if ecosystem_name in self.ecosystem_names: return self.ecosystems[ecosystem_name] - elif ecosystem_name.lower().replace(" ", "-") in PUBLIC_CHAIN_META: - ecosystem_name = ecosystem_name.lower().replace(" ", "-") - symbol = None - for net in PUBLIC_CHAIN_META[ecosystem_name].values(): - if not (native_currency := net.get("nativeCurrency")): - continue - - if "symbol" not in native_currency: - continue - - symbol = native_currency["symbol"] - break - - symbol = symbol or "ETH" - - # Is an EVM chain, can automatically make a class using evm-chains. - evm_class = self._plugin_ecosystems["ethereum"].__class__ - return evm_class(name=ecosystem_name, fee_token_symbol=symbol) - raise EcosystemNotFoundError(ecosystem_name, options=self.ecosystem_names) def get_provider_from_choice( @@ -606,7 +610,6 @@ def set_default_ecosystem(self, ecosystem_name: str): ecosystem_name (str): The name of the ecosystem to set as the default. """ - if ecosystem_name in self.ecosystem_names: self._default_ecosystem_name = ecosystem_name diff --git a/tests/functional/test_network_api.py b/tests/functional/test_network_api.py index 7578a975fa..8ce476f646 100644 --- a/tests/functional/test_network_api.py +++ b/tests/functional/test_network_api.py @@ -349,3 +349,8 @@ def supports_chain(cls, chain_id): ] assert network.explorer is not None assert network.explorer.name == NAME + + +def test_evm_chains_auto_forked_networks_exist(networks): + # NOTE: Moonbeam networks exist in evmchains only; that is how Ape knows about them. + assert isinstance(networks.moonbeam.moonriver_fork, ForkedNetworkAPI) diff --git a/tests/functional/test_network_manager.py b/tests/functional/test_network_manager.py index 73b1acd86f..aabece0481 100644 --- a/tests/functional/test_network_manager.py +++ b/tests/functional/test_network_manager.py @@ -240,7 +240,7 @@ def test_parse_network_choice_multiple_contexts( def test_getattr_ecosystem_with_hyphenated_name(networks, ethereum): - networks.ecosystems["hyphen-in-name"] = networks.ecosystems["ethereum"] + networks._plugin_ecosystems["hyphen-in-name"] = networks.ecosystems["ethereum"] assert networks.hyphen_in_name # Make sure does not raise AttributeError del networks.ecosystems["hyphen-in-name"] @@ -438,7 +438,26 @@ def test_custom_networks_defined_in_non_local_project(custom_networks_config_dic with ape.Project.create_temporary_project(config_override=custom_networks) as temp_project: nm = temp_project.network_manager + + # Tests `.get_ecosystem()` for custom networks. ecosystem = nm.get_ecosystem(eco_name) assert ecosystem.name == eco_name + network = ecosystem.get_network(net_name) assert network.name == net_name + + +def test_get_ecosystem(networks): + ethereum = networks.get_ecosystem("ethereum") + assert isinstance(ethereum, EcosystemAPI) + assert ethereum.name == "ethereum" + + +def test_get_ecosystem_from_evmchains(networks): + """ + Show we can call `.get_ecosystem()` for an ecosystem only + defined in evmchains. + """ + moonbeam = networks.get_ecosystem("moonbeam") + assert isinstance(moonbeam, EcosystemAPI) + assert moonbeam.name == "moonbeam" From 74c77466707c2955db9cdc3359ce7093ae251138 Mon Sep 17 00:00:00 2001 From: antazoey Date: Thu, 21 Nov 2024 07:51:10 +0700 Subject: [PATCH 17/28] docs: handle mdformat updates (#2381) --- docs/userguides/console.md | 6 +++--- setup.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/userguides/console.md b/docs/userguides/console.md index 27b6cf62a2..e7b04e4d79 100644 --- a/docs/userguides/console.md +++ b/docs/userguides/console.md @@ -51,7 +51,7 @@ Follow [this guide](./networks.html) for more information on networks in Ape. ## Namespace Extras -You can also create scripts to be included in the console namespace by adding a file (`ape_console_extras.py`) to your root project directory. All non-internal symbols from this file will be included in the console namespace. Internal symbols are prefixed by an underscore (`_`). +You can also create scripts to be included in the console namespace by adding a file (`ape_console_extras.py`) to your root project directory. All non-internal symbols from this file will be included in the console namespace. Internal symbols are prefixed by an underscore (`_`). An example file might look something like this: @@ -75,7 +75,7 @@ Out[2]: '0x68f768988e9bd4be971d527f72483f321975fa52aff9692b6d0e0af71fb77aaf' ### Init Function -If you include a function named `ape_init_extras`, it will be executed with the symbols from the existing namespace being provided as keyword arguments. This allows you to alter the scripts namespace using locals already included in the Ape namespace. If you return a `dict`, these values will be added to the console namespace. For example, you could set up an initialized Web3.py object by using one from an existing Ape Provider. +If you include a function named `ape_init_extras`, it will be executed with the symbols from the existing namespace being provided as keyword arguments. This allows you to alter the scripts namespace using locals already included in the Ape namespace. If you return a `dict`, these values will be added to the console namespace. For example, you could set up an initialized Web3.py object by using one from an existing Ape Provider. ```python def ape_init_extras(chain): @@ -91,7 +91,7 @@ Out[1]: 1 ### Global Extras -You can also add an `ape_console_extras.py` file to the global ape data directory (`$HOME/.ape/ape_console_extras.py`) and it will execute regardless of what project context you are in. This may be useful for variables and utility functions you use across all of your projects. +You can also add an `ape_console_extras.py` file to the global ape data directory (`$HOME/.ape/ape_console_extras.py`) and it will execute regardless of what project context you are in. This may be useful for variables and utility functions you use across all of your projects. ## Configure diff --git a/setup.py b/setup.py index 27b8013b7f..b9934ee8dc 100644 --- a/setup.py +++ b/setup.py @@ -37,10 +37,10 @@ "flake8-pydantic", # For detecting issues with Pydantic models "flake8-type-checking", # Detect imports to move in/out of type-checking blocks "isort>=5.13.2,<6", # Import sorting linter - "mdformat==0.7.18", # Auto-formatter for markdown + "mdformat>=0.7.19", # Auto-formatter for markdown "mdformat-gfm>=0.3.5", # Needed for formatting GitHub-flavored markdown "mdformat-frontmatter>=0.4.1", # Needed for frontmatters-style headers in issue templates - "mdformat-pyproject>=0.0.1", # Allows configuring in pyproject.toml + "mdformat-pyproject>=0.0.2", # Allows configuring in pyproject.toml ], "doc": ["sphinx-ape"], "release": [ # `release` GitHub Action job uses this From 442ad01d629658424a5d96febb51e6637e0a77c4 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 22 Nov 2024 00:40:22 +0700 Subject: [PATCH 18/28] fix(unreleased): bug where evm-chains ecosystems overrode plugin ecosystems. (#2382) --- setup.py | 2 +- src/ape/managers/networks.py | 2 +- tests/functional/test_network_manager.py | 32 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b9934ee8dc..60ee60fae3 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ "packaging>=23.0,<24", "pandas>=2.2.2,<3", "pluggy>=1.3,<2", - "pydantic>=2.6.4,<3", + "pydantic>=2.6.4,<2.10", "pydantic-settings>=2.5.2,<3", "pytest>=8.0,<9.0", "python-dateutil>=2.8.2,<3", diff --git a/src/ape/managers/networks.py b/src/ape/managers/networks.py index a046f90b79..06fb23003f 100644 --- a/src/ape/managers/networks.py +++ b/src/ape/managers/networks.py @@ -241,7 +241,7 @@ def ecosystems(self) -> dict[str, EcosystemAPI]: ) plugin_ecosystems[ecosystem_name] = ecosystem_cls - return {**plugin_ecosystems, **self._evmchains_ecosystems} + return {**self._evmchains_ecosystems, **plugin_ecosystems} @cached_property def _plugin_ecosystems(self) -> dict[str, EcosystemAPI]: diff --git a/tests/functional/test_network_manager.py b/tests/functional/test_network_manager.py index aabece0481..e302b37142 100644 --- a/tests/functional/test_network_manager.py +++ b/tests/functional/test_network_manager.py @@ -254,6 +254,38 @@ def test_getattr_custom_ecosystem(networks, custom_networks_config_dict, project assert isinstance(actual, EcosystemAPI) +def test_getattr_plugin_ecosystem_same_as_evm_chains(mocker, networks): + """ + Simulated having a plugin ecosystem installed. + """ + optimismy = mocker.MagicMock() + networks._plugin_ecosystems["optimismy"] = optimismy + actual = networks.optimismy + del networks._plugin_ecosystems["optimismy"] + assert actual == optimismy + + +def test_getattr_evm_chains_ecosystem(networks): + """ + Show we can getattr evm-chains only ecosystems. + """ + actual = networks.moonbeam + assert actual.name == "moonbeam" + + +def test_getattr_plugin_ecosystem_same_name_as_evm_chains(mocker, networks): + """ + Show when an ecosystem is both in evm-chains and an installed plugin + that Ape prefers the installed plugin. + """ + moonbeam_plugin = mocker.MagicMock() + networks._plugin_ecosystems["moonbeam"] = moonbeam_plugin + actual = networks.moonbeam + del networks._plugin_ecosystems["moonbeam"] + assert actual == moonbeam_plugin + assert actual != networks._evmchains_ecosystems["moonbeam"] + + @pytest.mark.parametrize("scheme", ("http", "https")) def test_create_custom_provider_http(networks, scheme): provider = networks.create_custom_provider(f"{scheme}://example.com") From 2073b529cebc80e47a2ad5a527ccdd3a579f806b Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 22 Nov 2024 00:53:49 +0700 Subject: [PATCH 19/28] perf: move no-capture check to the inside of the `ConfigWrapper` constructor; avoid when only `--help` (#2376) --- src/ape/pytest/config.py | 12 ++++++++++++ src/ape/pytest/plugin.py | 5 ----- tests/functional/test_test.py | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/ape/pytest/config.py b/src/ape/pytest/config.py index a60cf15fd9..1a83963a7d 100644 --- a/src/ape/pytest/config.py +++ b/src/ape/pytest/config.py @@ -27,6 +27,18 @@ class ConfigWrapper(ManagerAccessMixin): def __init__(self, pytest_config: "PytestConfig"): self.pytest_config = pytest_config + if not self.verbosity: + # Enable verbose output if stdout capture is disabled + self.verbosity = self.pytest_config.getoption("capture") == "no" + # else: user has already changes verbosity to an equal or higher level; avoid downgrading. + + @property + def verbosity(self) -> int: + return self.pytest_config.option.verbose + + @verbosity.setter + def verbosity(self, value): + self.pytest_config.option.verbose = value @cached_property def supports_tracing(self) -> bool: diff --git a/src/ape/pytest/plugin.py b/src/ape/pytest/plugin.py index e23dd5bf32..f364ec9f13 100644 --- a/src/ape/pytest/plugin.py +++ b/src/ape/pytest/plugin.py @@ -68,11 +68,6 @@ def is_module(v): except AttributeError: pass - if not config.option.verbose: - # Enable verbose output if stdout capture is disabled - config.option.verbose = config.getoption("capture") == "no" - # else: user has already changes verbosity to an equal or higher level; avoid downgrading. - if "--help" in config.invocation_params.args: # perf: Don't bother setting up runner if only showing help. return diff --git a/tests/functional/test_test.py b/tests/functional/test_test.py index e063973e0e..a5909cfa9f 100644 --- a/tests/functional/test_test.py +++ b/tests/functional/test_test.py @@ -3,6 +3,7 @@ import pytest from ape.exceptions import ConfigError +from ape.pytest.config import ConfigWrapper from ape.pytest.runners import PytestApeRunner from ape_test import ApeTestConfig from ape_test._watch import run_with_observer @@ -18,6 +19,32 @@ def test_balance_set_from_currency_str(self): assert actual == expected +class TestConfigWrapper: + def test_verbosity(self, mocker): + """ + Show it returns the same as pytest_config's. + """ + pytest_cfg = mocker.MagicMock() + pytest_cfg.option.verbose = False + wrapper = ConfigWrapper(pytest_cfg) + assert wrapper.verbosity is False + + def test_verbosity_when_no_capture(self, mocker): + """ + Shows we enable verbose output when no-capture is set. + """ + + def get_opt(name: str): + return "no" if name == "capture" else None + + pytest_cfg = mocker.MagicMock() + pytest_cfg.option.verbose = False # Start off as False + pytest_cfg.getoption.side_effect = get_opt + + wrapper = ConfigWrapper(pytest_cfg) + assert wrapper.verbosity is True + + def test_connect_to_mainnet_by_default(mocker): """ Tests the condition where mainnet is configured as the default network From 9cc65bb386e004b94e6a1e0744cf36de7a116218 Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 22 Nov 2024 03:25:37 +0700 Subject: [PATCH 20/28] feat: support Pydantic 2.10 (#2383) --- setup.py | 4 ++-- src/ape/api/query.py | 2 +- src/ape/contracts/base.py | 4 ++-- src/ape/managers/chain.py | 4 ++-- src/ape/managers/converters.py | 2 +- src/ape/utils/basemodel.py | 4 ++-- tests/functional/test_project.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 60ee60fae3..fffa3f86fe 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ "packaging>=23.0,<24", "pandas>=2.2.2,<3", "pluggy>=1.3,<2", - "pydantic>=2.6.4,<2.10", + "pydantic>=2.10.0,<3", "pydantic-settings>=2.5.2,<3", "pytest>=8.0,<9.0", "python-dateutil>=2.8.2,<3", @@ -128,7 +128,7 @@ "web3[tester]>=6.17.2,<7", # ** Dependencies maintained by ApeWorX ** "eip712>=0.2.10,<0.3", - "ethpm-types>=0.6.17,<0.7", + "ethpm-types>=0.6.19,<0.7", "eth_pydantic_types>=0.1.3,<0.2", "evmchains>=0.1.0,<0.2", "evm-trace>=0.2.3,<0.3", diff --git a/src/ape/api/query.py b/src/ape/api/query.py index 8732a3d404..5198bd5324 100644 --- a/src/ape/api/query.py +++ b/src/ape/api/query.py @@ -23,7 +23,7 @@ @cache def _basic_columns(Model: type[BaseInterfaceModel]) -> set[str]: - columns = set(Model.model_fields) + columns = set(Model.__pydantic_fields__) # TODO: Remove once `ReceiptAPI` fields cleaned up for better processing if Model == ReceiptAPI: diff --git a/src/ape/contracts/base.py b/src/ape/contracts/base.py index 3c6be64c0b..3e0b27145b 100644 --- a/src/ape/contracts/base.py +++ b/src/ape/contracts/base.py @@ -649,7 +649,7 @@ def query( f"the chain length ({self.chain_manager.blocks.height})." ) query: dict = { - "columns": list(ContractLog.model_fields) if columns[0] == "*" else columns, + "columns": list(ContractLog.__pydantic_fields__) if columns[0] == "*" else columns, "event": self.abi, "start_block": start_block, "stop_block": stop_block, @@ -720,7 +720,7 @@ def range( addresses = list(set([contract_address] + (extra_addresses or []))) contract_event_query = ContractEventQuery( - columns=list(ContractLog.model_fields.keys()), + columns=list(ContractLog.__pydantic_fields__), contract=addresses, event=self.abi, search_topics=search_topics, diff --git a/src/ape/managers/chain.py b/src/ape/managers/chain.py index 4c14bad123..e5e99a087d 100644 --- a/src/ape/managers/chain.py +++ b/src/ape/managers/chain.py @@ -433,7 +433,7 @@ def __getitem_int(self, index: int) -> ReceiptAPI: next( self.query_manager.query( AccountTransactionQuery( - columns=list(ReceiptAPI.model_fields), + columns=list(ReceiptAPI.__pydantic_fields__), account=self.address, start_nonce=index, stop_nonce=index, @@ -471,7 +471,7 @@ def __getitem_slice(self, indices: slice) -> list[ReceiptAPI]: list( self.query_manager.query( AccountTransactionQuery( - columns=list(ReceiptAPI.model_fields), + columns=list(ReceiptAPI.__pydantic_fields__), account=self.address, start_nonce=start, stop_nonce=stop - 1, diff --git a/src/ape/managers/converters.py b/src/ape/managers/converters.py index 85c48f6e69..6fe23788b8 100644 --- a/src/ape/managers/converters.py +++ b/src/ape/managers/converters.py @@ -419,7 +419,7 @@ def convert_method_args( return converted_arguments def convert_method_kwargs(self, kwargs) -> dict: - fields = TransactionAPI.model_fields + fields = TransactionAPI.__pydantic_fields__ def get_real_type(type_): all_types = getattr(type_, "_typevar_types", []) diff --git a/src/ape/utils/basemodel.py b/src/ape/utils/basemodel.py index cabac370df..ba92547759 100644 --- a/src/ape/utils/basemodel.py +++ b/src/ape/utils/basemodel.py @@ -4,7 +4,7 @@ import inspect from abc import ABC -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from importlib import import_module from pathlib import Path from sys import getrecursionlimit @@ -413,7 +413,7 @@ class BaseModel(EthpmTypesBaseModel): def model_copy( self: "Model", *, - update: Optional[dict[str, Any]] = None, + update: Optional[Mapping[str, Any]] = None, deep: bool = False, cache_clear: Optional[Sequence[str]] = None, ) -> "Model": diff --git a/tests/functional/test_project.py b/tests/functional/test_project.py index b432fc1b94..9dc31a7db4 100644 --- a/tests/functional/test_project.py +++ b/tests/functional/test_project.py @@ -8,7 +8,6 @@ from eth_utils import to_hex from ethpm_types import Compiler, ContractType, PackageManifest, Source from ethpm_types.manifest import PackageName -from pydantic_core import Url import ape from ape import Project @@ -305,7 +304,8 @@ def test_meta(project): assert project.meta.license == "MIT" assert project.meta.description == "Zoologist meme protocol" assert project.meta.keywords == ["Indiana", "Knight's Templar"] - assert project.meta.links == {"apeworx.io": Url("https://apeworx.io")} + assert len(project.meta.links) == 1 + assert f"{project.meta.links['apeworx.io']}" == "https://apeworx.io/" def test_extract_manifest(tmp_project, mock_sepolia, vyper_contract_instance): From 25c596643a83744f73691017995ef5562a5ee2ea Mon Sep 17 00:00:00 2001 From: antazoey Date: Sat, 23 Nov 2024 05:27:22 +0700 Subject: [PATCH 21/28] feat: support local `geth --dev` 1.14.12 (#2386) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 236f005ecd..eed3dd1541 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -86,7 +86,7 @@ jobs: - name: Install Geth uses: gacts/install-geth-tools@v1 with: - version: 1.14.5 + version: 1.14.12 - name: Install Dependencies run: | From a25349ef08be97e2198e1f9c6b40f829ff9c021b Mon Sep 17 00:00:00 2001 From: leopardracer <136604165+leopardracer@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:12:52 +0200 Subject: [PATCH 22/28] fix: typos in documentation files (#2387) --- docs/userguides/projects.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/userguides/projects.md b/docs/userguides/projects.md index 7eddef55f3..a341919bb5 100644 --- a/docs/userguides/projects.md +++ b/docs/userguides/projects.md @@ -14,7 +14,7 @@ project # The root project directory └── ape-config.yaml # The ape project configuration file ``` -Notice that you can configure you ape project using the `ape-config.yaml` file. +Notice that you can configure your ape project using the `ape-config.yaml` file. See the [configuration guide](./config.html) for a more detailed explanation of settings you can adjust. ## The Local Project From 02108545153db302b6eab399d7a4e95d2578ea75 Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 26 Nov 2024 06:13:31 +0700 Subject: [PATCH 23/28] fix: Could not start a Geth process with only WS or IPC (#2377) --- setup.py | 2 +- src/ape_ethereum/provider.py | 1 - src/ape_node/provider.py | 163 +++++++++++++++++++------ tests/functional/geth/test_provider.py | 73 ++++++++++- 4 files changed, 192 insertions(+), 47 deletions(-) diff --git a/setup.py b/setup.py index fffa3f86fe..37ae1886f4 100644 --- a/setup.py +++ b/setup.py @@ -123,7 +123,7 @@ "eth-typing", "eth-utils", "hexbytes", - "py-geth>=5.0.0-beta.2,<6", + "py-geth>=5.1.0,<6", "trie>=3.0.1,<4", # Peer: stricter pin needed for uv support. "web3[tester]>=6.17.2,<7", # ** Dependencies maintained by ApeWorX ** diff --git a/src/ape_ethereum/provider.py b/src/ape_ethereum/provider.py index 58bba04dde..eba0c2d3cc 100644 --- a/src/ape_ethereum/provider.py +++ b/src/ape_ethereum/provider.py @@ -1352,7 +1352,6 @@ def uri(self) -> str: # Use value from config file network_config: dict = (config or {}).get(self.network.name) or DEFAULT_SETTINGS - if "url" in network_config: raise ConfigError("Unknown provider setting 'url'. Did you mean 'uri'?") elif "http_uri" in network_config: diff --git a/src/ape_node/provider.py b/src/ape_node/provider.py index aa7fb0f2ee..e7b49415ac 100644 --- a/src/ape_node/provider.py +++ b/src/ape_node/provider.py @@ -3,6 +3,7 @@ from pathlib import Path from subprocess import DEVNULL, PIPE, Popen from typing import TYPE_CHECKING, Any, Optional, Union +from urllib.parse import urlparse from eth_utils import add_0x_prefix, to_hex from evmchains import get_random_rpc @@ -13,7 +14,6 @@ from pydantic_settings import SettingsConfigDict from requests.exceptions import ConnectionError from web3.middleware import geth_poa_middleware as ExtraDataToPOAMiddleware -from yarl import URL from ape.api.config import PluginConfig from ape.api.providers import SubprocessProvider, TestProviderAPI @@ -90,13 +90,17 @@ def create_genesis_data(alloc: Alloc, chain_id: int) -> "GenesisDataTypedDict": class GethDevProcess(BaseGethProcess): """ A developer-configured geth that only exists until disconnected. + (Implementation detail of the local node provider). """ def __init__( self, data_dir: Path, - hostname: str = DEFAULT_HOSTNAME, - port: int = DEFAULT_PORT, + hostname: Optional[str] = None, + port: Optional[int] = None, + ipc_path: Optional[Path] = None, + ws_hostname: Optional[str] = None, + ws_port: Optional[str] = None, mnemonic: str = DEFAULT_TEST_MNEMONIC, number_of_accounts: int = DEFAULT_NUMBER_OF_TEST_ACCOUNTS, chain_id: int = DEFAULT_TEST_CHAIN_ID, @@ -111,23 +115,33 @@ def __init__( raise NodeSoftwareNotInstalledError() self._data_dir = data_dir - self._hostname = hostname - self._port = port self.is_running = False self._auto_disconnect = auto_disconnect - geth_kwargs = construct_test_chain_kwargs( - data_dir=self.data_dir, - geth_executable=executable, - rpc_addr=hostname, - rpc_port=f"{port}", - network_id=f"{chain_id}", - ws_enabled=False, - ws_addr=None, - ws_origins=None, - ws_port=None, - ws_api=None, - ) + kwargs_ctor: dict = { + "data_dir": self.data_dir, + "geth_executable": executable, + "network_id": f"{chain_id}", + } + if hostname is not None: + kwargs_ctor["rpc_addr"] = hostname + if port is not None: + kwargs_ctor["rpc_port"] = f"{port}" + if ws_hostname: + kwargs_ctor["ws_enabled"] = True + kwargs_ctor["ws_addr"] = ws_hostname + if ws_port: + kwargs_ctor["ws_enabled"] = True + kwargs_ctor["ws_port"] = f"{ws_port}" + if ipc_path is not None: + kwargs_ctor["ipc_path"] = f"{ipc_path}" + if not kwargs_ctor.get("ws_enabled"): + kwargs_ctor["ws_api"] = None + kwargs_ctor["ws_enabled"] = False + kwargs_ctor["ws_addr"] = None + kwargs_ctor["ws_port"] = None + + geth_kwargs = construct_test_chain_kwargs(**kwargs_ctor) # Ensure a clean data-dir. self._clean() @@ -147,38 +161,70 @@ def __init__( @classmethod def from_uri(cls, uri: str, data_folder: Path, **kwargs): - parsed_uri = URL(uri) - - if parsed_uri.host not in ("localhost", "127.0.0.1"): - raise ConnectionError(f"Unable to start Geth on non-local host {parsed_uri.host}.") - - port = parsed_uri.port if parsed_uri.port is not None else DEFAULT_PORT mnemonic = kwargs.get("mnemonic", DEFAULT_TEST_MNEMONIC) number_of_accounts = kwargs.get("number_of_accounts", DEFAULT_NUMBER_OF_TEST_ACCOUNTS) balance = kwargs.get("initial_balance", DEFAULT_TEST_ACCOUNT_BALANCE) extra_accounts = [a.lower() for a in kwargs.get("extra_funded_accounts", [])] - return cls( - data_folder, - auto_disconnect=kwargs.get("auto_disconnect", True), - executable=kwargs.get("executable"), - extra_funded_accounts=extra_accounts, - hd_path=kwargs.get("hd_path", DEFAULT_TEST_HD_PATH), - hostname=parsed_uri.host, - initial_balance=balance, - mnemonic=mnemonic, - number_of_accounts=number_of_accounts, - port=port, - ) + process_kwargs = { + "auto_disconnect": kwargs.get("auto_disconnect", True), + "executable": kwargs.get("executable"), + "extra_funded_accounts": extra_accounts, + "hd_path": kwargs.get("hd_path", DEFAULT_TEST_HD_PATH), + "initial_balance": balance, + "mnemonic": mnemonic, + "number_of_accounts": number_of_accounts, + } + + parsed_uri = urlparse(uri) + if not parsed_uri.netloc: + path = Path(parsed_uri.path) + if path.suffix == ".ipc": + # Was given an IPC path. + process_kwargs["ipc_path"] = path + + else: + raise ConnectionError(f"Unrecognized path type: '{path}'.") + + elif hostname := parsed_uri.hostname: + if hostname not in ("localhost", "127.0.0.1"): + raise ConnectionError( + f"Unable to start Geth on non-local host {parsed_uri.hostname}." + ) + + if parsed_uri.scheme.startswith("ws"): + process_kwargs["ws_hostname"] = hostname + process_kwargs["ws_port"] = parsed_uri.port or DEFAULT_PORT + elif parsed_uri.scheme.startswith("http"): + process_kwargs["hostname"] = hostname or DEFAULT_HOSTNAME + process_kwargs["port"] = parsed_uri.port or DEFAULT_PORT + else: + raise ConnectionError(f"Unsupported scheme: '{parsed_uri.scheme}'.") + + return cls(data_folder, **process_kwargs) @property def data_dir(self) -> str: return f"{self._data_dir}" + @property + def _hostname(self) -> Optional[str]: + return self.geth_kwargs.get("rpc_addr") + + @property + def _port(self) -> Optional[str]: + return self.geth_kwargs.get("rpc_port") + + @property + def _ws_hostname(self) -> Optional[str]: + return self.geth_kwargs.get("ws_addr") + + @property + def _ws_port(self) -> Optional[str]: + return self.geth_kwargs.get("ws_port") + def connect(self, timeout: int = 60): - home = str(Path.home()) - ipc_path = self.ipc_path.replace(home, "$HOME") - logger.info(f"Starting geth (HTTP='{self._hostname}:{self._port}', IPC={ipc_path}).") + self._log_connection() self.start() self.wait_for_rpc(timeout=timeout) @@ -186,6 +232,28 @@ def connect(self, timeout: int = 60): if self._auto_disconnect: atexit.register(self.disconnect) + def _log_connection(self): + home = str(Path.home()) + ipc_path = self.ipc_path.replace(home, "$HOME") + + http_log = "" + if self._hostname: + http_log = f"HTTP={self._hostname}" + if port := self._port: + http_log = f"{http_log}:{port}" + + ipc_log = f"IPC={ipc_path}" + + ws_log = "" + if self._ws_hostname: + ws_log = f"WS={self._ws_hostname}" + if port := self._ws_port: + ws_log = f"{ws_log}:{port}" + + connection_logs = ", ".join(x for x in (http_log, ipc_log, ws_log) if x) + + logger.info(f"Starting geth ({connection_logs}).") + def start(self): if self.is_running: return @@ -230,6 +298,21 @@ class EthereumNetworkConfig(PluginConfig): model_config = SettingsConfigDict(extra="allow") + @field_validator("local", mode="before") + @classmethod + def _validate_local(cls, value): + value = value or {} + if not value: + return {**DEFAULT_SETTINGS.copy(), "chain_id": DEFAULT_TEST_CHAIN_ID} + + if "chain_id" not in value: + value["chain_id"] = DEFAULT_TEST_CHAIN_ID + if "uri" not in value and "ipc_path" in value or "ws_uri" in value or "http_uri" in value: + # No need to add default HTTP URI if was given only IPC Path + return {**{k: v for k, v in DEFAULT_SETTINGS.items() if k != "uri"}, **value} + + return {**DEFAULT_SETTINGS, **value} + class EthereumNodeConfig(PluginConfig): """ @@ -384,8 +467,8 @@ def _create_process(self) -> GethDevProcess: extra_accounts = list({a.lower() for a in extra_accounts}) test_config["extra_funded_accounts"] = extra_accounts test_config["initial_balance"] = self.test_config.balance - - return GethDevProcess.from_uri(self.uri, self.data_dir, **test_config) + uri = self.ws_uri or self.uri + return GethDevProcess.from_uri(uri, self.data_dir, **test_config) def disconnect(self): # Must disconnect process first. diff --git a/tests/functional/geth/test_provider.py b/tests/functional/geth/test_provider.py index 93e8c550ca..3b7a8bee7c 100644 --- a/tests/functional/geth/test_provider.py +++ b/tests/functional/geth/test_provider.py @@ -45,6 +45,11 @@ def web3_factory(mocker): return mocker.patch("ape_ethereum.provider._create_web3") +@pytest.fixture +def process_factory_patch(mocker): + return mocker.patch("ape_node.provider.GethDevProcess.from_uri") + + @pytest.fixture def tx_for_call(geth_contract): return DynamicFeeTransaction.model_validate( @@ -129,6 +134,7 @@ def test_uri_non_dev_and_not_configured(mocker, ethereum): assert actual == expected +@geth_process_test def test_uri_invalid(geth_provider, project, ethereum): settings = geth_provider.provider_settings geth_provider.provider_settings = {} @@ -255,7 +261,18 @@ def test_connect_to_chain_that_started_poa(mock_web3, web3_factory, ethereum): @geth_process_test -def test_connect_using_only_ipc_for_uri(project, networks, geth_provider): +def test_connect_using_only_ipc_for_uri_already_connected(project, networks, geth_provider): + """ + Shows we can remote-connect to a node that is already running when it exposes its IPC path. + """ + ipc_path = geth_provider.ipc_path + with project.temp_config(node={"ethereum": {"local": {"uri": f"{ipc_path}"}}}): + with networks.ethereum.local.use_provider("node") as node: + assert node.uri == f"{ipc_path}" + + +@geth_process_test +def test_connect_using_ipc(process_factory_patch, project, networks, geth_provider): ipc_path = geth_provider.ipc_path with project.temp_config(node={"ethereum": {"local": {"uri": f"{ipc_path}"}}}): with networks.ethereum.local.use_provider("node") as node: @@ -794,9 +811,8 @@ def test_trace_approach_config(project): @geth_process_test -def test_start(mocker, convert, project, geth_provider): +def test_start(process_factory_patch, convert, project, geth_provider): amount = convert("100_000 ETH", int) - spy = mocker.spy(GethDevProcess, "from_uri") with project.temp_config(test={"balance": amount}): try: @@ -804,10 +820,57 @@ def test_start(mocker, convert, project, geth_provider): except Exception: pass # Exceptions are fine here. - actual = spy.call_args[1]["balance"] - assert actual == amount + actual = process_factory_patch.call_args[1]["balance"] + assert actual == amount + + +@geth_process_test +@pytest.mark.parametrize("key", ("uri", "ws_uri")) +def test_start_from_ws_uri(process_factory_patch, project, geth_provider, key): + uri = "ws://localhost:5677" + + with project.temp_config(node={"ethereum": {"local": {key: uri}}}): + try: + geth_provider.start() + except Exception: + pass # Exceptions are fine here. + + actual = process_factory_patch.call_args[0][0] # First "arg" + assert actual == uri @geth_process_test def test_auto_mine(geth_provider): assert geth_provider.auto_mine is True + + +@geth_process_test +def test_geth_dev_from_uri_http(data_folder): + geth_dev = GethDevProcess.from_uri("http://localhost:6799", data_folder) + kwargs = geth_dev.geth_kwargs + assert kwargs["rpc_addr"] == "localhost" + assert kwargs["rpc_port"] == "6799" + assert kwargs["ws_enabled"] is False + assert kwargs.get("ws_api") is None + assert kwargs.get("ws_addr") is None + assert kwargs.get("ws_port") is None + + +@geth_process_test +def test_geth_dev_from_uri_ws(data_folder): + geth_dev = GethDevProcess.from_uri("ws://localhost:6799", data_folder) + kwargs = geth_dev.geth_kwargs + assert kwargs.get("rpc_addr") is None + assert kwargs["ws_enabled"] is True + assert kwargs["ws_addr"] == "localhost" + assert kwargs["ws_port"] == "6799" + + +@geth_process_test +def test_geth_dev_from_uri_ipc(data_folder): + geth_dev = GethDevProcess.from_uri("path/to/geth.ipc", data_folder) + kwargs = geth_dev.geth_kwargs + assert kwargs["ipc_path"] == "path/to/geth.ipc" + assert kwargs.get("ws_api") is None + assert kwargs.get("ws_addr") is None + assert kwargs.get("rpc_addr") is None From 589e57472be81841325925e15b7044a4161d90f3 Mon Sep 17 00:00:00 2001 From: Maks Date: Tue, 26 Nov 2024 22:03:44 +0100 Subject: [PATCH 24/28] chore: Fix Grammar and Spelling Issues (#2389) --- docs/userguides/config.md | 2 +- docs/userguides/proxy.md | 4 ++-- tests/README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/userguides/config.md b/docs/userguides/config.md index 9e8809d928..0eb62f5897 100644 --- a/docs/userguides/config.md +++ b/docs/userguides/config.md @@ -6,7 +6,7 @@ There are two locations you can place config files. 1. In the root of your project 2. In your `$HOME/.ape` directory (global) -Project settings take precedent, but global settings allow you to configure preferences across all projects, such as your default mainnet provider (e.g. Alchemy versus running your own node). +Project settings take precedence, but global settings allow you to configure preferences across all projects, such as your default mainnet provider (e.g. Alchemy versus running your own node). This guide serves as an index of some settings you can include in any `ape-config.yaml` file. This guide is **PURPOSELY** alphabetized to facilitate easier look-up of keys. diff --git a/docs/userguides/proxy.md b/docs/userguides/proxy.md index a4a9d023fe..a919fdd7a3 100644 --- a/docs/userguides/proxy.md +++ b/docs/userguides/proxy.md @@ -1,7 +1,7 @@ # Proxy Contracts Ape is able to detect proxy contracts so that it uses the target interface when interacting with a contract. -The following proxies are supporting in `ape-ethereum`: +The following proxies are supported in `ape-ethereum`: | Proxy Type | Short Description | | ------------ | --------------------------------- | @@ -27,7 +27,7 @@ from ape import Contract my_contract = Contract("0x...") ``` -Ape will check the address you give it and detect if hosts a proxy contract. +Ape will check the address you give it and detect if it hosts a proxy contract. In the case where it determines the address is a proxy contract, it resolves the address of the implementation (every proxy is different) and returns the interface for the implementation contract. This allows you to still call methods as you normally do on proxy contracts. diff --git a/tests/README.md b/tests/README.md index df075ba914..4834f8a3e9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,7 +4,7 @@ Due to its plugin-based architecture and registration as a pytest plugin, testin ### MyPy Type Annotations -This codebase uses MyPy extensively, not only to help aide in finding typing issues within the codebase, but also as a means to help plugin writers avoid integration issues with the library. +This codebase uses MyPy extensively, not only to help aid in finding typing issues within the codebase, but also as a means to help plugin writers avoid integration issues with the library. Please try to use MyPy Type Annotations as much as possible in the Core codebase, except where there exists an issue that would hugely complicate its use from a lack of available machinery. ### Functional Tests From 92abc8626649f04b093b594eb0256abd4e9cf24c Mon Sep 17 00:00:00 2001 From: slush Date: Tue, 26 Nov 2024 15:29:27 -0600 Subject: [PATCH 25/28] fix: updates setuptools to version compatible with PEP-625 (#2390) --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46ced0756d..5a5e27f308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=51.1.1", "wheel", "setuptools_scm[toml]>=5.0,<8"] +requires = ["setuptools>=75.0.0", "wheel", "setuptools_scm[toml]>=5.0"] [tool.mypy] exclude = ["build/", "dist/", "docs/", "tests/integration/cli/projects/"] diff --git a/setup.py b/setup.py index 37ae1886f4..dafa63191b 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ ], "doc": ["sphinx-ape"], "release": [ # `release` GitHub Action job uses this - "setuptools", # Installation tool + "setuptools>=75", # Installation tool "wheel", # Packaging tool "twine==3.8.0", # Package upload tool ], From 6b12573d8d1f2324c8fe44a750d8b4fac6b2b62c Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 3 Dec 2024 00:57:30 +0700 Subject: [PATCH 26/28] perf: getting `ape console` to launch faster (#2379) --- setup.py | 3 +- src/ape/api/networks.py | 131 ++++++++++++----------- src/ape/cli/choices.py | 79 ++++---------- src/ape/cli/options.py | 17 ++- src/ape/managers/networks.py | 118 ++++++++++++++------ src/ape/managers/plugins.py | 12 ++- src/ape/managers/project.py | 16 ++- src/ape_console/_cli.py | 166 ++++++++++++++++++++--------- src/ape_test/provider.py | 121 ++++++++++++++++----- tests/functional/test_cli.py | 4 +- tests/functional/test_console.py | 63 +++++++---- tests/functional/test_ecosystem.py | 5 +- 12 files changed, 463 insertions(+), 272 deletions(-) diff --git a/setup.py b/setup.py index dafa63191b..2a3a65f98d 100644 --- a/setup.py +++ b/setup.py @@ -98,6 +98,7 @@ "ijson>=3.1.4,<4", "ipython>=8.18.1,<9", "lazyasd>=0.1.4", + "asttokens>=2.4.1,<3", # Peer dependency; w/o pin container build fails. # Pandas peer-dep: Numpy 2.0 causes issues for some users. "numpy<2", "packaging>=23.0,<24", @@ -120,7 +121,7 @@ # All version pins dependent on web3[tester] "eth-abi", "eth-account", - "eth-typing", + "eth-typing>=3.5.2,<4", "eth-utils", "hexbytes", "py-geth>=5.1.0,<6", diff --git a/src/ape/api/networks.py b/src/ape/api/networks.py index 601b2633a7..e6f7dd2611 100644 --- a/src/ape/api/networks.py +++ b/src/ape/api/networks.py @@ -250,9 +250,53 @@ def networks(self) -> dict[str, "NetworkAPI"]: Returns: dict[str, :class:`~ape.api.networks.NetworkAPI`] """ - networks = {**self._networks_from_plugins} + return { + **self._networks_from_evmchains, + **self._networks_from_plugins, + **self._custom_networks, + } + + @cached_property + def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]: + return { + network_name: network_class(name=network_name, ecosystem=self) + for _, (ecosystem_name, network_name, network_class) in self.plugin_manager.networks + if ecosystem_name == self.name + } + + @cached_property + def _networks_from_evmchains(self) -> dict[str, "NetworkAPI"]: + # NOTE: Purposely exclude plugins here so we also prefer plugins. + networks = { + network_name: create_network_type(data["chainId"], data["chainId"])( + name=network_name, ecosystem=self + ) + for network_name, data in PUBLIC_CHAIN_META.get(self.name, {}).items() + if network_name not in self._networks_from_plugins + } + forked_networks: dict[str, ForkedNetworkAPI] = {} + for network_name, network in networks.items(): + if network_name.endswith("-fork"): + # Already a fork. + continue + + fork_network_name = f"{network_name}-fork" + if any(x == fork_network_name for x in networks): + # The forked version of this network is already known. + continue + + forked_networks[fork_network_name] = ForkedNetworkAPI( + name=fork_network_name, ecosystem=self + ) + + return {**networks, **forked_networks} - # Include configured custom networks. + @property + def _custom_networks(self) -> dict[str, "NetworkAPI"]: + """ + Networks from config. + """ + networks: dict[str, "NetworkAPI"] = {} custom_networks: list[dict] = [ n for n in self.network_manager.custom_networks @@ -300,48 +344,8 @@ def networks(self) -> dict[str, "NetworkAPI"]: network_api._is_custom = True networks[net_name] = network_api - # Add any remaining networks from EVM chains here (but don't override). - # NOTE: Only applicable to EVM-based ecosystems, of course. - # Otherwise, this is a no-op. - networks = {**self._networks_from_evmchains, **networks} - return networks - @cached_property - def _networks_from_plugins(self) -> dict[str, "NetworkAPI"]: - return { - network_name: network_class(name=network_name, ecosystem=self) - for _, (ecosystem_name, network_name, network_class) in self.plugin_manager.networks - if ecosystem_name == self.name - } - - @cached_property - def _networks_from_evmchains(self) -> dict[str, "NetworkAPI"]: - # NOTE: Purposely exclude plugins here so we also prefer plugins. - networks = { - network_name: create_network_type(data["chainId"], data["chainId"])( - name=network_name, ecosystem=self - ) - for network_name, data in PUBLIC_CHAIN_META.get(self.name, {}).items() - if network_name not in self._networks_from_plugins - } - forked_networks: dict[str, ForkedNetworkAPI] = {} - for network_name, network in networks.items(): - if network_name.endswith("-fork"): - # Already a fork. - continue - - fork_network_name = f"{network_name}-fork" - if any(x == fork_network_name for x in networks): - # The forked version of this network is already known. - continue - - forked_networks[fork_network_name] = ForkedNetworkAPI( - name=fork_network_name, ecosystem=self - ) - - return {**networks, **forked_networks} - def __post_init__(self): if len(self.networks) == 0: raise NetworkError("Must define at least one network in ecosystem") @@ -1087,12 +1091,13 @@ def providers(self): # -> dict[str, Partial[ProviderAPI]] Returns: dict[str, partial[:class:`~ape.api.providers.ProviderAPI`]] """ - from ape.plugins._utils import clean_plugin_name - providers = {} for _, plugin_tuple in self._get_plugin_providers(): ecosystem_name, network_name, provider_class = plugin_tuple - provider_name = clean_plugin_name(provider_class.__module__.split(".")[0]) + provider_name = ( + provider_class.__module__.split(".")[0].replace("_", "-").replace("ape-", "") + ) + is_custom_with_config = self._is_custom and self.default_provider_name == provider_name # NOTE: Custom networks that are NOT from config must work with any provider. # Also, ensure we are only adding forked providers for forked networks and @@ -1101,8 +1106,8 @@ def providers(self): # -> dict[str, Partial[ProviderAPI]] # TODO: In 0.9, add a better way for class-level ForkedProviders to define # themselves as "Fork" providers. if ( - self.is_adhoc - or (self.ecosystem.name == ecosystem_name and self.name == network_name) + (self.ecosystem.name == ecosystem_name and self.name == network_name) + or self.is_adhoc or ( is_custom_with_config and ( @@ -1130,6 +1135,11 @@ def _get_plugin_providers(self): # NOTE: Abstracted for testing purposes. return self.plugin_manager.providers + def _get_plugin_provider_names(self) -> Iterator[str]: + for _, plugin_tuple in self._get_plugin_providers(): + ecosystem_name, network_name, provider_class = plugin_tuple + yield provider_class.__module__.split(".")[0].replace("_", "-").replace("ape-", "") + def get_provider( self, provider_name: Optional[str] = None, @@ -1147,20 +1157,16 @@ def get_provider( Returns: :class:`~ape.api.providers.ProviderAPI` """ - provider_name = provider_name or self.default_provider_name - if not provider_name: - from ape.managers.config import CONFIG_FILE_NAME - + if not (provider_name := provider_name or self.default_provider_name): raise NetworkError( f"No default provider for network '{self.name}'. " - f"Set one in your {CONFIG_FILE_NAME}:\n" + "Set one in your pyproject.toml/ape-config.yaml file:\n" f"\n{self.ecosystem.name}:" f"\n {self.name}:" "\n default_provider: " ) provider_settings = provider_settings or {} - if ":" in provider_name: # NOTE: Shortcut that allows `--network ecosystem:network:http://...` to work provider_settings["uri"] = provider_name @@ -1170,19 +1176,20 @@ def get_provider( provider_settings["ipc_path"] = provider_name provider_name = "node" - # If it can fork Ethereum (and we are asking for it) assume it can fork this one. - # TODO: Refactor this approach to work for custom-forked non-EVM networks. - common_forking_providers = self.network_manager.ethereum.mainnet_fork.providers if provider_name in self.providers: provider = self.providers[provider_name](provider_settings=provider_settings) return _set_provider(provider) - elif self.is_fork and provider_name in common_forking_providers: - provider = common_forking_providers[provider_name]( - provider_settings=provider_settings, - network=self, - ) - return _set_provider(provider) + elif self.is_fork: + # If it can fork Ethereum (and we are asking for it) assume it can fork this one. + # TODO: Refactor this approach to work for custom-forked non-EVM networks. + common_forking_providers = self.network_manager.ethereum.mainnet_fork.providers + if provider_name in common_forking_providers: + provider = common_forking_providers[provider_name]( + provider_settings=provider_settings, + network=self, + ) + return _set_provider(provider) raise ProviderNotFoundError( provider_name, diff --git a/src/ape/cli/choices.py b/src/ape/cli/choices.py index dd3685ab67..0962a4f1e2 100644 --- a/src/ape/cli/choices.py +++ b/src/ape/cli/choices.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union import click -from click import BadParameter, Choice, Context, Parameter +from click import Choice, Context, Parameter from ape.exceptions import ( AccountsError, @@ -360,7 +360,7 @@ def __init__( ecosystem: _NETWORK_FILTER = None, network: _NETWORK_FILTER = None, provider: _NETWORK_FILTER = None, - base_type: Optional[type] = None, + base_type: Optional[Union[type, str]] = None, callback: Optional[Callable] = None, ): self._base_type = base_type @@ -372,15 +372,14 @@ def __init__( # NOTE: Purposely avoid super().init for performance reasons. @property - def base_type(self) -> type["ProviderAPI"]: + def base_type(self) -> Union[type["ProviderAPI"], str]: if self._base_type is not None: return self._base_type - # perf: property exists to delay import ProviderAPI at init time. - from ape.api.providers import ProviderAPI - - self._base_type = ProviderAPI - return ProviderAPI + # perf: Keep base-type as a forward-ref when only using the default. + # so things load faster. + self._base_type = "ProviderAPI" + return self._base_type @base_type.setter def base_type(self, value): @@ -394,62 +393,20 @@ def get_metavar(self, param): return "[ecosystem-name][:[network-name][:[provider-name]]]" def convert(self, value: Any, param: Optional[Parameter], ctx: Optional[Context]) -> Any: - from ape.utils.basemodel import ManagerAccessMixin as access - - choice: Optional[Union[str, "ProviderAPI"]] - networks = access.network_manager - if not value: - choice = None - - elif value.lower() in ("none", "null"): - choice = _NONE_NETWORK + if not value or value.lower() in ("none", "null"): + return self.callback(ctx, param, _NONE_NETWORK) if self.callback else _NONE_NETWORK - elif self.is_custom_value(value): - # By-pass choice constraints when using custom network. - choice = value + if self.base_type == "ProviderAPI" or isinstance(self.base_type, type): + # Return the provider. + from ape.utils.basemodel import ManagerAccessMixin as access - else: - # Regular conditions. + networks = access.network_manager try: - # Validate result. - choice = super().convert(value, param, ctx) - except BadParameter: - # Attempt to get the provider anyway. - # Sometimes, depending on the provider, it'll still work. - # (as-is the case for custom-forked networks). - try: - choice = networks.get_provider_from_choice(network_choice=value) - - except (EcosystemNotFoundError, NetworkNotFoundError, ProviderNotFoundError) as err: - # This error makes more sense, as it has attempted parsing. - # Show this message as the BadParameter message. - raise click.BadParameter(str(err)) from err - - except Exception as err: - # If an error was not raised for some reason, raise a simpler error. - # NOTE: Still avoid showing the massive network options list. - raise click.BadParameter( - "Invalid network choice. Use `ape networks list` to see options." - ) from err - - if choice not in (None, _NONE_NETWORK) and isinstance(choice, str): - from ape.api.providers import ProviderAPI - - if issubclass(self.base_type, ProviderAPI): - # Return the provider. - choice = networks.get_provider_from_choice(network_choice=value) - - return self.callback(ctx, param, choice) if self.callback else choice - - @classmethod - def is_custom_value(cls, value) -> bool: - return ( - value is not None - and isinstance(value, str) - and cls.CUSTOM_NETWORK_PATTERN.match(value) is not None - or str(value).startswith("http://") - or str(value).startswith("https://") - ) + value = networks.get_provider_from_choice(network_choice=value) + except (EcosystemNotFoundError, NetworkNotFoundError, ProviderNotFoundError) as err: + self.fail(str(err)) + + return self.callback(ctx, param, value) if self.callback else value class OutputFormat(Enum): diff --git a/src/ape/cli/options.py b/src/ape/cli/options.py index 88c715c040..1c492e113f 100644 --- a/src/ape/cli/options.py +++ b/src/ape/cli/options.py @@ -264,7 +264,10 @@ def decorator(f): def callback(ctx, param, value): keep_as_choice_str = param.type.base_type is str - provider_obj = _get_provider(value, default, keep_as_choice_str) + try: + provider_obj = _get_provider(value, default, keep_as_choice_str) + except Exception as err: + raise click.BadOptionUsage("--network", str(err), ctx) if provider_obj: _update_context_with_network(ctx, provider_obj, requested_network_objects) @@ -528,6 +531,10 @@ def handle_parse_result(self, ctx, opts, args): return IncompatibleOption +def _project_path_callback(ctx, param, val): + return Path(val) if val else Path.cwd() + + def _project_callback(ctx, param, val): if "--help" in sys.argv or "-h" in sys.argv: # Perf: project option is eager; have to check sys.argv to @@ -560,10 +567,16 @@ def _project_callback(ctx, param, val): def project_option(**kwargs): + _type = kwargs.pop("type", None) + callback = ( + _project_path_callback + if (isinstance(_type, type) and issubclass(_type, Path)) + else _project_callback + ) return click.option( "--project", help="The path to a local project or manifest", - callback=_project_callback, + callback=callback, metavar="PATH", is_eager=True, **kwargs, diff --git a/src/ape/managers/networks.py b/src/ape/managers/networks.py index 06fb23003f..e3d1f81a2a 100644 --- a/src/ape/managers/networks.py +++ b/src/ape/managers/networks.py @@ -4,7 +4,7 @@ from evmchains import PUBLIC_CHAIN_META -from ape.api.networks import EcosystemAPI, NetworkAPI, ProviderContextManager +from ape.api.networks import ProviderContextManager from ape.exceptions import EcosystemNotFoundError, NetworkError, NetworkNotFoundError from ape.managers.base import BaseManager from ape.utils.basemodel import ( @@ -17,6 +17,7 @@ from ape_ethereum.provider import EthereumNodeProvider if TYPE_CHECKING: + from ape.api.networks import EcosystemAPI, NetworkAPI from ape.api.providers import ProviderAPI from ape.utils.rpc import RPCHeaders @@ -62,7 +63,7 @@ def active_provider(self, new_value: "ProviderAPI"): self._active_provider = new_value @property - def network(self) -> NetworkAPI: + def network(self) -> "NetworkAPI": """ The current network if connected to one. @@ -76,7 +77,7 @@ def network(self) -> NetworkAPI: return self.provider.network @property - def ecosystem(self) -> EcosystemAPI: + def ecosystem(self) -> "EcosystemAPI": """ The current ecosystem if connected to one. @@ -194,28 +195,45 @@ def custom_networks(self) -> list[dict]: Custom network data defined in various ape-config files or added adhoc to the network manager. """ + return [*self._custom_networks_from_config, *self._custom_networks] + + @cached_property + def _custom_networks_from_config(self) -> list[dict]: return [ - *[ - n.model_dump(by_alias=True) - for n in self.config_manager.get_config("networks").get("custom", []) - ], - *self._custom_networks, + n.model_dump(by_alias=True) + for n in self.config_manager.get_config("networks").get("custom", []) ] @property - def ecosystems(self) -> dict[str, EcosystemAPI]: + def ecosystems(self) -> dict[str, "EcosystemAPI"]: """ All the registered ecosystems in ``ape``, such as ``ethereum``. """ - plugin_ecosystems = self._plugin_ecosystems + return { + **self._evmchains_ecosystems, + **self._plugin_ecosystems, + **self._custom_ecosystems, + } - # Load config-based custom ecosystems. - # NOTE: Non-local projects will automatically add their custom networks - # to `self.custom_networks`. + @cached_property + def _plugin_ecosystems(self) -> dict[str, "EcosystemAPI"]: + # Load plugins (possibly for first time). + plugins: list[tuple] = self.plugin_manager.ecosystems + return {n: cls(name=n) for n, cls in plugins} + + @cached_property + def _custom_ecosystems(self) -> dict[str, "EcosystemAPI"]: custom_networks: list = self.custom_networks + plugin_ecosystems = self._plugin_ecosystems + evm_chains = self._evmchains_ecosystems + custom_ecosystems: dict[str, "EcosystemAPI"] = {} for custom_network in custom_networks: ecosystem_name = custom_network["ecosystem"] - if ecosystem_name in plugin_ecosystems: + if ( + ecosystem_name in plugin_ecosystems + or ecosystem_name in evm_chains + or ecosystem_name in custom_ecosystems + ): # Already included in a prior network. continue @@ -239,19 +257,13 @@ def ecosystems(self) -> dict[str, EcosystemAPI]: update={"name": ecosystem_name}, cache_clear=("_networks_from_plugins", "_networks_from_evmchains"), ) - plugin_ecosystems[ecosystem_name] = ecosystem_cls + custom_ecosystems[ecosystem_name] = ecosystem_cls - return {**self._evmchains_ecosystems, **plugin_ecosystems} + return custom_ecosystems @cached_property - def _plugin_ecosystems(self) -> dict[str, EcosystemAPI]: - # Load plugins. - plugins = self.plugin_manager.ecosystems - return {n: cls(name=n) for n, cls in plugins} # type: ignore[operator] - - @cached_property - def _evmchains_ecosystems(self) -> dict[str, EcosystemAPI]: - ecosystems: dict[str, EcosystemAPI] = {} + def _evmchains_ecosystems(self) -> dict[str, "EcosystemAPI"]: + ecosystems: dict[str, "EcosystemAPI"] = {} for name in PUBLIC_CHAIN_META: ecosystem_name = name.lower().replace(" ", "-") symbol = None @@ -347,7 +359,7 @@ def __ape_extra_attributes__(self) -> Iterator[ExtraModelAttributes]: ) @only_raise_attribute_error - def __getattr__(self, attr_name: str) -> EcosystemAPI: + def __getattr__(self, attr_name: str) -> "EcosystemAPI": """ Get an ecosystem via ``.`` access. @@ -448,7 +460,7 @@ def get_network_choices( if ecosystem_has_providers: yield ecosystem_name - def get_ecosystem(self, ecosystem_name: str) -> EcosystemAPI: + def get_ecosystem(self, ecosystem_name: str) -> "EcosystemAPI": """ Get the ecosystem for the given name. @@ -461,11 +473,46 @@ def get_ecosystem(self, ecosystem_name: str) -> EcosystemAPI: Returns: :class:`~ape.api.networks.EcosystemAPI` """ - if ecosystem_name in self.ecosystem_names: - return self.ecosystems[ecosystem_name] + # NOTE: This method purposely avoids "just checking self.ecosystems" + # for performance reasons and exiting the search as early as possible. + ecosystem_name = ecosystem_name.lower().replace(" ", "-") + try: + return self._plugin_ecosystems[ecosystem_name] + except KeyError: + pass + + # Check if custom. + try: + return self._custom_ecosystems[ecosystem_name] + except KeyError: + pass + + if ecosystem := self._get_ecosystem_from_evmchains(ecosystem_name): + return ecosystem raise EcosystemNotFoundError(ecosystem_name, options=self.ecosystem_names) + def _get_ecosystem_from_evmchains(self, ecosystem_name: str) -> Optional["EcosystemAPI"]: + if ecosystem_name not in PUBLIC_CHAIN_META: + return None + + symbol = None + for net in PUBLIC_CHAIN_META[ecosystem_name].values(): + if not (native_currency := net.get("nativeCurrency")): + continue + + if "symbol" not in native_currency: + continue + + symbol = native_currency["symbol"] + break + + symbol = symbol or "ETH" + + # Is an EVM chain, can automatically make a class using evm-chains. + evm_class = self._plugin_ecosystems["ethereum"].__class__ + return evm_class(name=ecosystem_name, fee_token_symbol=symbol) + def get_provider_from_choice( self, network_choice: Optional[str] = None, @@ -491,7 +538,6 @@ def get_provider_from_choice( Returns: :class:`~ape.api.providers.ProviderAPI` """ - if network_choice is None: default_network = self.default_ecosystem.default_network return default_network.get_provider(provider_settings=provider_settings) @@ -586,10 +632,10 @@ def default_ecosystem_name(self) -> str: if name := self._default_ecosystem_name: return name - return self.config_manager.default_ecosystem or "ethereum" + return self.local_project.config.default_ecosystem or "ethereum" - @property - def default_ecosystem(self) -> EcosystemAPI: + @cached_property + def default_ecosystem(self) -> "EcosystemAPI": """ The default ecosystem. Call :meth:`~ape.managers.networks.NetworkManager.set_default_ecosystem` to @@ -597,7 +643,7 @@ def default_ecosystem(self) -> EcosystemAPI: only a single ecosystem installed, such as Ethereum, then get that ecosystem. """ - return self.ecosystems[self.default_ecosystem_name] + return self.get_ecosystem(self.default_ecosystem_name) def set_default_ecosystem(self, ecosystem_name: str): """ @@ -673,6 +719,12 @@ def _get_ecosystem_data( return ecosystem_data + def _invalidate_cache(self): + # NOTE: Called when changing config programmatically. + self.__dict__.pop("_custom_ecosystems", None) + self.__dict__.pop("_custom_networks_from_config", None) + self._custom_networks = [] + def _validate_filter(arg: Optional[Union[list[str], str]], options: set[str]): filters = arg or [] diff --git a/src/ape/managers/plugins.py b/src/ape/managers/plugins.py index c20383f040..ebf3a5a76f 100644 --- a/src/ape/managers/plugins.py +++ b/src/ape/managers/plugins.py @@ -1,6 +1,7 @@ from collections.abc import Generator, Iterable, Iterator from functools import cached_property from importlib import import_module +from itertools import chain from typing import Any, Optional from ape.exceptions import ApeAttributeError @@ -123,13 +124,16 @@ def _register_plugins(self): if self.__registered: return - plugins = list({n.replace("-", "_") for n in get_plugin_dists()}) - plugin_modules = tuple([*plugins, *CORE_PLUGINS]) + handled = set() + plugins = (n.replace("-", "_") for n in get_plugin_dists()) + for module_name in chain(plugins, iter(CORE_PLUGINS)): + if module_name in handled: + continue - for module_name in plugin_modules: + handled.add(module_name) try: module = import_module(module_name) - pluggy_manager.register(module) + pluggy_manager.register(module, name=module_name) except Exception as err: if module_name in CORE_PLUGINS or module_name == "ape": # Always raise core plugin registration errors. diff --git a/src/ape/managers/project.py b/src/ape/managers/project.py index 9fd69439db..203619d752 100644 --- a/src/ape/managers/project.py +++ b/src/ape/managers/project.py @@ -1631,9 +1631,15 @@ def from_python_library( def create_temporary_project( cls, config_override: Optional[dict] = None ) -> Iterator["LocalProject"]: + cls._invalidate_project_dependent_caches() with create_tempdir() as path: yield LocalProject(path, config_override=config_override) + @classmethod + def _invalidate_project_dependent_caches(cls): + cls.account_manager.test_accounts.reset() + cls.network_manager._invalidate_cache() + class Project(ProjectManager): """ @@ -1940,7 +1946,7 @@ def reconfigure(self, **overrides): self._config_override = overrides _ = self.config - self.account_manager.test_accounts.reset() + self._invalidate_project_dependent_caches() def extract_manifest(self) -> PackageManifest: # Attempt to compile, if needed. @@ -2281,10 +2287,10 @@ def project_api(self) -> ProjectAPI: return default_project # ape-config.yaml does no exist. Check for another ProjectAPI type. - project_classes: list[type[ProjectAPI]] = [ - t[1] for t in list(self.plugin_manager.projects) # type: ignore - ] - plugins = [t for t in project_classes if not issubclass(t, ApeProject)] + project_classes: Iterator[type[ProjectAPI]] = ( + t[1] for t in self.plugin_manager.projects # type: ignore + ) + plugins = (t for t in project_classes if not issubclass(t, ApeProject)) for api in plugins: if instance := api.attempt_validate(path=self._base_path): return instance diff --git a/src/ape_console/_cli.py b/src/ape_console/_cli.py index a3d3431cac..f04310580f 100644 --- a/src/ape_console/_cli.py +++ b/src/ape_console/_cli.py @@ -2,11 +2,14 @@ import inspect import logging import sys +from functools import cached_property +from importlib import import_module from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader from os import environ +from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, Optional, Union, cast import click @@ -39,7 +42,7 @@ def _code_callback(ctx, param, value) -> list[str]: context_settings=dict(ignore_unknown_options=True), ) @ape_cli_context() -@project_option(hidden=True) # Hidden as mostly used for test purposes. +@project_option(hidden=True, type=Path) # Hidden as mostly used for test purposes. @click.option("-c", "--code", help="Program passed in as a string", callback=_code_callback) def cli(cli_ctx, project, code): """Opens a console for the local project.""" @@ -60,51 +63,111 @@ def import_extras_file(file_path) -> ModuleType: return module -def load_console_extras(**namespace: Any) -> dict[str, Any]: - """load and return namespace updates from ape_console_extras.py files if - they exist""" - from ape.utils.basemodel import ManagerAccessMixin as access - - pm = namespace.get("project", access.local_project) - global_extras = pm.config_manager.DATA_FOLDER.joinpath(CONSOLE_EXTRAS_FILENAME) - project_extras = pm.path.joinpath(CONSOLE_EXTRAS_FILENAME) - - for extras_file in [global_extras, project_extras]: +class ApeConsoleNamespace(dict): + def __init__(self, **kwargs): + # Initialize the dictionary with provided keyword arguments + project = kwargs.get("project", self._ape.project) + kwargs["project"] = self._ape.Project(project) if isinstance(project, Path) else project + super().__init__(**kwargs) + + def __getitem__(self, key: str): + # First, attempt to retrieve the key from the dictionary itself + if super().__contains__(key): + return super().__getitem__(key) + + # Custom behavior for "ape" key + if key == "ape": + res = self._ape + self[key] = res # Cache the result + return res + + # Attempt to get the key from extras + try: + res = self._get_extra(key) + except KeyError: + pass + else: + self[key] = res # Cache the result + return res + + # Attempt to retrieve the key from the Ape module. + try: + res = self._get_from_ape(key) + except AttributeError: + raise KeyError(key) + + # Cache the result and return + self[key] = res + return res + + def __setitem__(self, key, value): + # Override to set items directly into the dictionary + super().__setitem__(key, value) + + def __contains__(self, item: str) -> bool: # type: ignore + return self.get(item) is not None + + def update(self, mapping, **kwargs) -> None: # type: ignore + # Override to update the dictionary directly + super().update(mapping, **kwargs) + + @property + def _ape(self) -> ModuleType: + return import_module("ape") + + @cached_property + def _local_path(self) -> Path: + return self["project"].path.joinpath(CONSOLE_EXTRAS_FILENAME) + + @cached_property + def _global_path(self) -> Path: + return self._ape.config.DATA_FOLDER.joinpath(CONSOLE_EXTRAS_FILENAME) + + @cached_property + def _local_extras(self) -> dict: + return self._load_extras_file(self._local_path) + + @cached_property + def _global_extras(self) -> dict: + return self._load_extras_file(self._global_path) + + def get(self, key: str, default: Optional[Any] = None): + try: + return self.__getitem__(key) + except KeyError: + return default + + def _get_extra(self, key: str): + try: + return self._local_extras[key] + except KeyError: + return self._global_extras[key] + + def _get_from_ape(self, key: str): + return getattr(self._ape, key) + + def _load_extras_file(self, extras_file: Path) -> dict: if not extras_file.is_file(): - continue + return {} module = import_extras_file(extras_file) ape_init_extras = getattr(module, "ape_init_extras", None) + all_extras: dict = {} - # If found, execute ape_init_extras() function. if ape_init_extras is not None: - # Figure out the kwargs the func is looking for and assemble - # from the original namespace func_spec = inspect.getfullargspec(ape_init_extras) - init_kwargs: dict[str, Any] = {k: namespace.get(k) for k in func_spec.args} - - # Execute functionality with existing console namespace as - # kwargs. + init_kwargs: dict[str, Any] = {k: self._get_from_ape(k) for k in func_spec.args} extras = ape_init_extras(**init_kwargs) - # If ape_init_extras returned a dict expect it to be new symbols if isinstance(extras, dict): - namespace.update(extras) - - # Add any public symbols from the module into the console namespace - for k in dir(module): - if k != "ape_init_extras" and not k.startswith("_"): - # Prevent override of existing namespace symbols - if k in namespace: - continue + all_extras.update(extras) - namespace[k] = getattr(module, k) - - return namespace + all_extras.update({k: getattr(module, k) for k in dir(module) if k not in all_extras}) + return all_extras def console( - project: Optional["ProjectManager"] = None, + project: Optional[Union["ProjectManager", Path]] = None, verbose: bool = False, extra_locals: Optional[dict] = None, embed: bool = False, @@ -113,11 +176,19 @@ def console( import IPython from IPython.terminal.ipapp import Config as IPythonConfig - import ape from ape.utils.misc import _python_version from ape.version import version as ape_version - project = project or ape.project + extra_locals = extra_locals or {} + if project is None: + from ape.utils.basemodel import ManagerAccessMixin + + project = ManagerAccessMixin.local_project + + else: + extra_locals["project"] = project + + project_path: Path = project if isinstance(project, Path) else project.path banner = "" if verbose: banner = """ @@ -131,29 +202,14 @@ def console( python_version=_python_version, ipython_version=IPython.__version__, ape_version=ape_version, - project_path=project.path, + project_path=project_path, ) if not environ.get("APE_TESTING"): faulthandler.enable() # NOTE: In case we segfault - namespace = {component: getattr(ape, component) for component in ape.__all__} - namespace["project"] = project # Use the given project. - namespace["ape"] = ape - # Allows modules relative to the project. - sys.path.insert(0, f"{project.path}") - - # NOTE: `ape_console_extras` only is meant to work with default namespace. - # Load extras before local namespace to avoid console extras receiving - # the wrong values for its arguments. - console_extras = load_console_extras(**namespace) - - if extra_locals: - namespace.update(extra_locals) - - if console_extras: - namespace.update(console_extras) + sys.path.insert(0, f"{project_path}") ipy_config = IPythonConfig() ape_testing = environ.get("APE_TESTING") @@ -163,9 +219,15 @@ def console( # Required for click.testing.CliRunner support. embed = True + namespace = _create_namespace(**extra_locals) _launch_console(namespace, ipy_config, embed, banner, code=code) +def _create_namespace(**values) -> dict: + # Abstracted for testing purposes. + return ApeConsoleNamespace(**values) + + def _launch_console( namespace: dict, ipy_config: "IPythonConfig", diff --git a/src/ape_test/provider.py b/src/ape_test/provider.py index f6c63e8060..2bd44fd13d 100644 --- a/src/ape_test/provider.py +++ b/src/ape_test/provider.py @@ -7,7 +7,9 @@ from eth.exceptions import HeaderNotFound from eth_pydantic_types import HexBytes +from eth_tester import EthereumTester # type: ignore from eth_tester.backends import PyEVMBackend # type: ignore +from eth_tester.backends.pyevm.main import setup_tester_chain # type: ignore from eth_tester.exceptions import TransactionFailed # type: ignore from eth_utils import is_0x_prefixed, to_hex from eth_utils.exceptions import ValidationError @@ -41,6 +43,89 @@ from ape.api.transactions import ReceiptAPI, TransactionAPI from ape.types.events import ContractLog, LogFilter from ape.types.vm import BlockID, SnapshotID + from ape_test.config import ApeTestConfig + + +class ApeEVMBackend(PyEVMBackend): + """ + A lazier version of PyEVMBackend for the Ape framework. + """ + + def __init__(self, config: "ApeTestConfig"): + self.config = config + # Lazily set. + self._chain = None + + @property + def hd_path(self) -> str: + return (self.config.hd_path or DEFAULT_TEST_HD_PATH).rstrip("/") + + @property + def balance(self) -> int: + return self.config.balance + + @property + def genesis_state(self) -> dict: + return self.generate_genesis_state( + mnemonic=self.config.mnemonic, + overrides={"balance": self.balance}, + num_accounts=self.config.number_of_accounts, + hd_path=self.hd_path, + ) + + @cached_property + def _setup_tester_chain(self): + return setup_tester_chain( + None, + self.genesis_state, + self.config.number_of_accounts, + None, + self.config.mnemonic, + self.config.hd_path, + ) + + @property + def account_keys(self): + return self._setup_tester_chain[0] + + @property + def chain(self): + if self._chain is None: + # Initial chain. + self._chain = self._setup_tester_chain[1] + + return self._chain + + @chain.setter + def chain(self, value): + self._chain = value # Changes during snapshot reverting. + + +class ApeTester(EthereumTesterProvider): + def __init__( + self, config: "ApeTestConfig", chain_id: int, backend: Optional[ApeEVMBackend] = None + ): + self.config = config + self.chain_id = chain_id + self._ethereum_tester = None if backend is None else EthereumTester(backend) + + @property + def ethereum_tester(self) -> EthereumTester: + if self._ethereum_tester is None: + backend = ApeEVMBackend(self.config) + self._ethereum_tester = EthereumTester(backend) + + return self._ethereum_tester + + @ethereum_tester.setter + def ethereum_tester(self, value): + self._ethereum_tester = value + + @cached_property + def api_endpoints(self) -> dict: # type: ignore + endpoints = {**API_ENDPOINTS} + endpoints["eth"] = merge(endpoints["eth"], {"chainId": static_return(self.chain_id)}) + return endpoints class LocalProvider(TestProviderAPI, Web3Provider): @@ -53,32 +138,16 @@ class LocalProvider(TestProviderAPI, Web3Provider): ) @property - def evm_backend(self) -> PyEVMBackend: - if self._evm_backend is None: - raise ProviderNotConnectedError() + def config(self) -> "ApeTestConfig": # type: ignore + return super().config # type: ignore - return self._evm_backend + @property + def evm_backend(self) -> ApeEVMBackend: + return self.tester.ethereum_tester.backend @cached_property - def tester(self): - chain_id = self.settings.chain_id - if self._web3 is not None: - connected_chain_id = self.make_request("eth_chainId") - if connected_chain_id == chain_id: - # Is already connected and settings have not changed. - return - - hd_path = (self.config.hd_path or DEFAULT_TEST_HD_PATH).rstrip("/") - state_overrides = {"balance": self.test_config.balance} - self._evm_backend = PyEVMBackend.from_mnemonic( - genesis_state_overrides=state_overrides, - hd_path=hd_path, - mnemonic=self.config.mnemonic, - num_accounts=self.config.number_of_accounts, - ) - endpoints = {**API_ENDPOINTS} - endpoints["eth"] = merge(endpoints["eth"], {"chainId": static_return(chain_id)}) - return EthereumTesterProvider(ethereum_tester=self._evm_backend, api_endpoints=endpoints) + def tester(self) -> ApeTester: + return ApeTester(self.config, self.settings.chain_id) @property def auto_mine(self) -> bool: @@ -98,12 +167,10 @@ def max_gas(self) -> int: return self.evm_backend.get_block_by_number("latest")["gas_limit"] def connect(self): - if "tester" in self.__dict__: - del self.__dict__["tester"] - + self.__dict__.pop("tester", None) self._web3 = Web3(self.tester) # Handle disabling auto-mine if the user requested via config. - if self.config.provider.auto_mine is False: + if self.settings.auto_mine is False: self.auto_mine = False # type: ignore[misc] def disconnect(self): diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index 35ea0dc489..e7148683b9 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -314,7 +314,9 @@ def test_network_option_specify_custom_network( def cmd(network): click.echo(f"Value is '{getattr(network, 'name', network)}'") - result = runner.invoke(cmd, ("--network", f"ethereum:{network_name}:node")) + result = runner.invoke( + cmd, ("--network", f"ethereum:{network_name}:node"), catch_exceptions=False + ) assert result.exit_code == 0 assert f"Value is '{network_name}'" in result.output diff --git a/tests/functional/test_console.py b/tests/functional/test_console.py index a55650fea3..c81be5da9e 100644 --- a/tests/functional/test_console.py +++ b/tests/functional/test_console.py @@ -3,8 +3,10 @@ import pytest from ape import Project +from ape.managers.accounts import AccountManager +from ape.managers.project import LocalProject from ape.utils import ManagerAccessMixin, create_tempdir -from ape_console._cli import console +from ape_console._cli import CONSOLE_EXTRAS_FILENAME, ApeConsoleNamespace, console from ape_console.plugin import custom_exception_handler @@ -14,41 +16,30 @@ def mock_console(mocker): return mocker.patch("ape_console._cli._launch_console") -@pytest.fixture(autouse=True) -def mock_ape_console_extras(mocker): - """Prevent actually loading console extras files.""" - return mocker.patch("ape_console._cli.load_console_extras") - - -def test_console_extras_uses_ape_namespace(mocker, mock_console, mock_ape_console_extras): +def test_console_extras_uses_ape_namespace(mocker, mock_console): """ Test that if console is given extras, those are included in the console but not as args to the extras files, as those files expect items from the default ape namespace. """ + namespace_patch = mocker.patch("ape_console._cli._create_namespace") accounts_custom = mocker.MagicMock() extras = {"accounts": accounts_custom} console(extra_locals=extras) + actual = namespace_patch.call_args[1] + assert actual["accounts"] == accounts_custom - # Show extras file still load using Ape namespace. - actual = mock_ape_console_extras.call_args[1] - assert actual["accounts"] != accounts_custom - - # Show the custom accounts do get used in console. - assert mock_console.call_args[0][0]["accounts"] == accounts_custom - -def test_console_custom_project(mock_console, mock_ape_console_extras): +def test_console_custom_project(mock_console): with create_tempdir() as path: project = Project(path) + extras_file = path / CONSOLE_EXTRAS_FILENAME + extras_file.touch() console(project=project) - actuals = ( - mock_console.call_args[0][0]["project"], # Launch namespace - mock_ape_console_extras.call_args[1]["project"], # extras-load namespace - ) + extras = mock_console.call_args[0][0] + actual = extras["project"] - for actual in actuals: - assert actual == project + assert actual == project # Ensure sys.path was updated correctly. assert sys.path[0] == str(project.path) @@ -73,3 +64,31 @@ def test_custom_exception_handler_handles_non_ape_project(mocker): # We are expecting the local project's path in the handler. expected_path = ManagerAccessMixin.local_project.path handler_patch.assert_called_once_with(err, [expected_path]) + + +class TestApeConsoleNamespace: + def test_accounts(self): + extras = ApeConsoleNamespace() + assert isinstance(extras["accounts"], AccountManager) + + @pytest.mark.parametrize("scope", ("local", "global")) + def test_extras(self, scope): + extras = ApeConsoleNamespace() + _ = getattr(extras, f"_{scope}_extras") + extras.__dict__[f"_{scope}_extras"] = {"foo": "123"} + assert extras["foo"] == "123" + + @pytest.mark.parametrize("scope", ("local", "global")) + def test_extras_load_using_ape_namespace(self, scope): + extras = ApeConsoleNamespace() + _ = getattr(extras, f"_{scope}_path") + extras_content = """ +def ape_init_extras(project): + return {"foo": type(project)} +""" + with create_tempdir() as temp: + extras_file = temp / CONSOLE_EXTRAS_FILENAME + extras_file.write_text(extras_content) + extras.__dict__[f"_{scope}_path"] = extras_file + extras.__dict__.pop(f"_{scope}_extras", None) + assert extras["foo"] is LocalProject diff --git a/tests/functional/test_ecosystem.py b/tests/functional/test_ecosystem.py index 981afecd80..f93d27a464 100644 --- a/tests/functional/test_ecosystem.py +++ b/tests/functional/test_ecosystem.py @@ -978,8 +978,9 @@ def test_networks_when_custom_ecosystem( def test_networks_multiple_networks_with_same_name(custom_networks_config_dict, ethereum, project): data = copy.deepcopy(custom_networks_config_dict) - data["networks"]["custom"][0]["name"] = "mainnet" # There already is a mainnet in "ethereum". - expected = ".*More than one network named 'mainnet' in ecosystem 'ethereum'.*" + data["networks"]["custom"][0]["name"] = "foonet" + data["networks"]["custom"][1]["name"] = "foonet" + expected = ".*More than one network named 'foonet' in ecosystem 'ethereum'.*" with project.temp_config(**data): with pytest.raises(NetworkError, match=expected): _ = ethereum.networks From 9ea28c83d6822b8dbfc1ab62376f9c4f94678abc Mon Sep 17 00:00:00 2001 From: antazoey Date: Wed, 4 Dec 2024 22:30:21 +0700 Subject: [PATCH 27/28] docs(fix): fix core plugin method auto-doc (#2395) --- docs/methoddocs/ape_accounts.md | 2 +- docs/methoddocs/ape_compile.md | 2 +- docs/methoddocs/ape_ethereum.md | 22 +++++++++++++++++++++- docs/methoddocs/ape_node.md | 2 +- docs/methoddocs/ape_pm.md | 12 +++++++++++- docs/methoddocs/ape_test.md | 12 +++++++++++- 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/docs/methoddocs/ape_accounts.md b/docs/methoddocs/ape_accounts.md index c767fdda8d..4b4a60aa0d 100644 --- a/docs/methoddocs/ape_accounts.md +++ b/docs/methoddocs/ape_accounts.md @@ -1,6 +1,6 @@ # ape-accounts ```{eval-rst} -.. automodule:: ape_accounts +.. automodule:: ape_accounts.accounts :members: ``` diff --git a/docs/methoddocs/ape_compile.md b/docs/methoddocs/ape_compile.md index 7e03a36234..7078f3adc8 100644 --- a/docs/methoddocs/ape_compile.md +++ b/docs/methoddocs/ape_compile.md @@ -1,6 +1,6 @@ # ape-compile ```{eval-rst} -.. automodule:: ape_compile +.. automodule:: ape_compile.config :members: ``` diff --git a/docs/methoddocs/ape_ethereum.md b/docs/methoddocs/ape_ethereum.md index bbd95a504c..137d0f4d57 100644 --- a/docs/methoddocs/ape_ethereum.md +++ b/docs/methoddocs/ape_ethereum.md @@ -1,7 +1,27 @@ # ape-ethereum ```{eval-rst} -.. automodule:: ape_ethereum +.. automodule:: ape_ethereum.ecosystem + :members: +``` + +```{eval-rst} +.. automodule:: ape_ethereum.provider + :members: +``` + +```{eval-rst} +.. automodule:: ape_ethereum.proxies + :members: +``` + +```{eval-rst} +.. automodule:: ape_ethereum.trace + :members: +``` + +```{eval-rst} +.. automodule:: ape_ethereum.transactions :members: ``` diff --git a/docs/methoddocs/ape_node.md b/docs/methoddocs/ape_node.md index af5df2262b..af96ee985f 100644 --- a/docs/methoddocs/ape_node.md +++ b/docs/methoddocs/ape_node.md @@ -1,6 +1,6 @@ # ape-node ```{eval-rst} -.. automodule:: ape_node +.. automodule:: ape_node.provider :members: ``` diff --git a/docs/methoddocs/ape_pm.md b/docs/methoddocs/ape_pm.md index 75469d4dca..a60f9e93df 100644 --- a/docs/methoddocs/ape_pm.md +++ b/docs/methoddocs/ape_pm.md @@ -1,6 +1,16 @@ # ape-pm ```{eval-rst} -.. automodule:: ape_pm +.. automodule:: ape_pm.compiler + :members: +``` + +```{eval-rst} +.. automodule:: ape_pm.dependency + :members: +``` + +```{eval-rst} +.. automodule:: ape_pm.project :members: ``` diff --git a/docs/methoddocs/ape_test.md b/docs/methoddocs/ape_test.md index 312210351e..104f978eb7 100644 --- a/docs/methoddocs/ape_test.md +++ b/docs/methoddocs/ape_test.md @@ -1,6 +1,16 @@ # ape-test ```{eval-rst} -.. automodule:: ape_test +.. automodule:: ape_test.accounts + :members: +``` + +```{eval-rst} +.. automodule:: ape_test.config + :members: +``` + +```{eval-rst} +.. automodule:: ape_test.provider :members: ``` From ae256d2ee79e9e3a58e645a65aac805ed6bc34ca Mon Sep 17 00:00:00 2001 From: slush Date: Wed, 4 Dec 2024 18:32:44 -0600 Subject: [PATCH 28/28] feat: support python 3.13 (#2393) Co-authored-by: antazoey --- .github/workflows/test.yaml | 2 +- pyproject.toml | 2 +- setup.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eed3dd1541..5313a554cf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -63,7 +63,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] # eventually add `windows-latest` - python-version: [3.9, "3.10", "3.11", "3.12"] + python-version: [3.9, "3.10", "3.11", "3.12", "3.13"] env: GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 5a5e27f308..711634407f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ write_to = "src/ape/version.py" # character. [tool.black] line-length = 100 -target-version = ['py39', 'py310', 'py311', 'py312'] +target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] include = '\.pyi?$' [tool.pytest.ini_options] diff --git a/setup.py b/setup.py index 2a3a65f98d..2c8b7a851d 100644 --- a/setup.py +++ b/setup.py @@ -171,5 +171,6 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], )