diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index 55269835972ba..faf5c29744522 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -24,6 +24,7 @@ from ruamel.yaml import YAML from ..utils.command import Command, default_bin +from ..utils.logger import running_in_ci from ..utils.source import arrow_path from ..compat import _ensure_path @@ -168,6 +169,9 @@ def get(self, service_name): def __getitem__(self, service_name): return self.get(service_name) + def verbosity_args(self): + return ['--quiet'] if running_in_ci() else [] + class Docker(Command): @@ -233,7 +237,7 @@ def _execute_docker(self, *args, **kwargs): def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True): def _pull(service): - args = ['pull', '--quiet'] + args = ['pull'] + self.config.verbosity_args() if service['image'] in self.pull_memory: return @@ -427,10 +431,11 @@ def run(self, service_name, command=None, *, env=None, volumes=None, def push(self, service_name, user=None, password=None): def _push(service): + args = ['push'] + self.config.verbosity_args() if self.config.using_docker: - return self._execute_docker('push', '--quiet', service['image']) + return self._execute_docker(*args, service['image']) else: - return self._execute_compose('push', '--quiet', service['name']) + return self._execute_compose(*args, service['name']) if user is not None: try: diff --git a/dev/archery/archery/docker/tests/test_docker.py b/dev/archery/archery/docker/tests/test_docker.py index 5dd4b1bccecbe..432d1c0a35202 100644 --- a/dev/archery/archery/docker/tests/test_docker.py +++ b/dev/archery/archery/docker/tests/test_docker.py @@ -217,6 +217,14 @@ def arrow_compose_path(tmpdir): return create_config(tmpdir, arrow_compose_yml, arrow_compose_env) +@pytest.fixture(autouse=True) +def no_ci_env_variables(monkeypatch): + """Make sure that the tests behave the same on CI as when run locally""" + monkeypatch.delenv("APPVEYOR", raising=False) + monkeypatch.delenv("BUILD_BUILDURI", raising=False) + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + + def test_config_validation(tmpdir): config_path = create_config(tmpdir, missing_service_compose_yml) msg = "`sub-foo` is defined in `x-hierarchy` bot not in `services`" @@ -270,7 +278,7 @@ def test_compose_default_params_and_env(arrow_compose_path): def test_forwarding_env_variables(arrow_compose_path): expected_calls = [ - "pull --quiet --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-cpp", "build conda-cpp", ] expected_env = PartialEnv( @@ -286,38 +294,48 @@ def test_forwarding_env_variables(arrow_compose_path): compose.build('conda-cpp') -def test_compose_pull(arrow_compose_path): +def test_compose_pull(arrow_compose_path, monkeypatch): compose = DockerCompose(arrow_compose_path) expected_calls = [ - "pull --quiet --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-cpp", ] with assert_compose_calls(compose, expected_calls): compose.clear_pull_memory() compose.pull('conda-cpp') expected_calls = [ - "pull --quiet --ignore-pull-failures conda-cpp", - "pull --quiet --ignore-pull-failures conda-python", - "pull --quiet --ignore-pull-failures conda-python-pandas" + "pull --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-python", + "pull --ignore-pull-failures conda-python-pandas" ] with assert_compose_calls(compose, expected_calls): compose.clear_pull_memory() compose.pull('conda-python-pandas') expected_calls = [ - "pull --quiet --ignore-pull-failures conda-cpp", - "pull --quiet --ignore-pull-failures conda-python", + "pull --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-python", ] with assert_compose_calls(compose, expected_calls): compose.clear_pull_memory() compose.pull('conda-python-pandas', pull_leaf=False) + with monkeypatch.context() as m: + # `--quiet` is passed to `docker` on CI + m.setenv("GITHUB_ACTIONS", "true") + expected_calls = [ + "pull --quiet --ignore-pull-failures conda-cpp", + ] + with assert_compose_calls(compose, expected_calls): + compose.clear_pull_memory() + compose.pull('conda-cpp') + def test_compose_pull_params(arrow_compose_path): expected_calls = [ - "pull --quiet --ignore-pull-failures conda-cpp", - "pull --quiet --ignore-pull-failures conda-python", + "pull --ignore-pull-failures conda-cpp", + "pull --ignore-pull-failures conda-python", ] compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) expected_env = PartialEnv(PYTHON='3.8', PANDAS='latest') @@ -483,7 +501,7 @@ def test_compose_push(arrow_compose_path): for image in ["conda-cpp", "conda-python", "conda-python-pandas"]: expected_calls.append( mock.call(["docker", "compose", f"--file={compose.config.path}", - "push", "--quiet", image], check=True, env=expected_env) + "push", image], check=True, env=expected_env) ) with assert_subprocess_calls(expected_calls): compose.push('conda-python-pandas', user='user', password='pass') diff --git a/dev/archery/archery/utils/logger.py b/dev/archery/archery/utils/logger.py index b315a52b7a000..4ab119ea7d951 100644 --- a/dev/archery/archery/utils/logger.py +++ b/dev/archery/archery/utils/logger.py @@ -30,7 +30,23 @@ def __init__(self, quiet=False): ctx = LoggingContext() -in_github_actions = (os.environ.get("GITHUB_ACTIONS") == "true") + +# Note: detection routines for many CI services can be found +# in https://github.com/semantic-release/env-ci +def in_appveyor(): + return os.environ.get("APPVEYOR", "").lower() == "true" + + +def in_azure_pipelines(): + return os.environ.get("BUILD_BUILDURI", "") != "" + + +def in_github_actions(): + return os.environ.get("GITHUB_ACTIONS") == "true" + + +def running_in_ci(): + return in_appveyor() or in_azure_pipelines() or in_github_actions() @contextlib.contextmanager @@ -43,10 +59,10 @@ def group(name, output=None): if output is None: def output(message): print(message, flush=True) - if in_github_actions: + if in_github_actions(): output(f"::group::{name}") try: yield finally: - if in_github_actions: + if in_github_actions(): output("::endgroup::")