From b870a9c0cc6e8ff27a6d05f1df23728794d61cc3 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Thu, 20 Jul 2023 18:28:49 +0200 Subject: [PATCH 01/10] Upgrade PyYAML (#3654) this version adds an upper-bound for cython at build time (ideally we'd switch to using wheels in GHA as we do internally, but our existing pip-custom-platform setup will take some time to untangle) --- paasta_tools/cli/cmds/spark_run.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paasta_tools/cli/cmds/spark_run.py b/paasta_tools/cli/cmds/spark_run.py index 568cbd45ba..9638cd0467 100644 --- a/paasta_tools/cli/cmds/spark_run.py +++ b/paasta_tools/cli/cmds/spark_run.py @@ -1203,7 +1203,7 @@ def paasta_spark_run(args): document = POD_TEMPLATE.format( spark_pod_label=limit_size_with_hash(f"exec-{app_base_name}"), ) - parsed_pod_template = yaml.load(document) + parsed_pod_template = yaml.safe_load(document) with open(pod_template_path, "w") as f: yaml.dump(parsed_pod_template, f) diff --git a/requirements.txt b/requirements.txt index 89da4ec80b..30e16b3f68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -79,7 +79,7 @@ python-iptables==1.0.1 python-utils==2.0.1 pytimeparse==1.1.5 pytz==2016.10 -pyyaml==5.4.1 +pyyaml==6.0.1 repoze.lru==0.6 requests==2.25.0 requests-cache==0.6.3 From a6a19b3cf7a35a0dfe7da0d3c705e14b0f8786ad Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Thu, 20 Jul 2023 18:37:03 +0200 Subject: [PATCH 02/10] Fixing setup_kubernetes_cr exception (#3658) * Fixing setup_kubernetes_cr exception * Fixing tests and logic --- paasta_tools/setup_kubernetes_cr.py | 20 ++++++++++++++------ tests/test_setup_kubernetes_cr.py | 4 ++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/paasta_tools/setup_kubernetes_cr.py b/paasta_tools/setup_kubernetes_cr.py index ac9037a746..3d17369653 100644 --- a/paasta_tools/setup_kubernetes_cr.py +++ b/paasta_tools/setup_kubernetes_cr.py @@ -29,6 +29,7 @@ from typing import Sequence import yaml +from kubernetes.client.exceptions import ApiException from paasta_tools.cli.utils import LONG_RUNNING_INSTANCE_TYPE_HANDLERS from paasta_tools.flink_tools import get_flink_ingress_url_root @@ -154,6 +155,7 @@ def setup_all_custom_resources( ) -> bool: got_results = False + succeeded = False # We support two versions due to our upgrade to 1.22 # this functions runs succefully when any of the two apiextensions # succeed to update the CRDs as the cluster could be in any version @@ -162,12 +164,18 @@ def setup_all_custom_resources( kube_client.apiextensions, kube_client.apiextensions_v1_beta1, ]: - cluster_crds = { - crd.spec.names.kind - for crd in apiextension.list_custom_resource_definition( + + try: + crds_list = apiextension.list_custom_resource_definition( label_selector=paasta_prefixed("service") ).items - } + except ApiException: + log.debug( + "Listing CRDs with apiextensions/v1 not supported on this cluster, falling back to v1beta1" + ) + crds_list = [] + + cluster_crds = {crd.spec.names.kind for crd in crds_list} log.debug(f"CRDs found: {cluster_crds}") results = [] for crd in custom_resource_definitions: @@ -202,11 +210,11 @@ def setup_all_custom_resources( if results: got_results = True if any(results): - return True + succeeded = True # we want to return True if we never called `setup_custom_resources` # (i.e., we noop'd) or if any call to `setup_custom_resources` # succeed (handled above) - otherwise, we want to return False - return not got_results + return succeeded or not got_results def setup_custom_resources( diff --git a/tests/test_setup_kubernetes_cr.py b/tests/test_setup_kubernetes_cr.py index 6626a14829..2e3512125a 100644 --- a/tests/test_setup_kubernetes_cr.py +++ b/tests/test_setup_kubernetes_cr.py @@ -147,6 +147,10 @@ def test_setup_all_custom_resources_flink(): mock.Mock(items=[flink_crd]) ) + mock_client.apiextensions_v1_beta1.list_custom_resource_definition.return_value = mock.Mock( + items=[] + ) + custom_resource_definitions = [ mock.Mock( kube_kind=mock.Mock(plural="flinks", singular="flink", kind="Flink") From 4086401b04e5f4b31ed601405352734934543153 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Thu, 20 Jul 2023 09:50:20 -0700 Subject: [PATCH 03/10] Released 0.190.0 via make release --- debian/changelog | 8 ++++++++ paasta_tools/__init__.py | 2 +- yelp_package/Makefile | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0630df3a6c..8d92e1f69b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +paasta-tools (0.190.0) xenial; urgency=medium + + * 0.190.0 tagged with 'make release' + Commit: Fixing setup_kubernetes_cr exception (#3658) * Fixing + setup_kubernetes_cr exception * Fixing tests and logic + + -- Wilmer Bandres Thu, 20 Jul 2023 09:49:46 -0700 + paasta-tools (0.189.0) xenial; urgency=medium * 0.189.0 tagged with 'make release' diff --git a/paasta_tools/__init__.py b/paasta_tools/__init__.py index e7b99b31aa..886908f66b 100644 --- a/paasta_tools/__init__.py +++ b/paasta_tools/__init__.py @@ -17,4 +17,4 @@ # setup phase, the dependencies may not exist on disk yet. # # Don't bump version manually. See `make release` docs in ./Makefile -__version__ = "0.189.0" +__version__ = "0.190.0" diff --git a/yelp_package/Makefile b/yelp_package/Makefile index 7b6d5c3bad..72c780bb02 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -13,7 +13,7 @@ # limitations under the License. # Edit this release and run "make release" -RELEASE=0.189.0 +RELEASE=0.190.0 SHELL=/bin/bash From 5577b6df0b3f4bbb0cc58858db64f0aca24765f6 Mon Sep 17 00:00:00 2001 From: Jen Patague Date: Fri, 21 Jul 2023 13:44:14 -0700 Subject: [PATCH 04/10] TRON-1968: Catch exceptions when updaitng MASTER config --- paasta_tools/setup_tron_namespace.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/paasta_tools/setup_tron_namespace.py b/paasta_tools/setup_tron_namespace.py index 0af494a64e..93aa990f3e 100755 --- a/paasta_tools/setup_tron_namespace.py +++ b/paasta_tools/setup_tron_namespace.py @@ -105,12 +105,16 @@ def main(): log.info(f"{master_config}") updated.append(MASTER_NAMESPACE) else: - if client.update_namespace(MASTER_NAMESPACE, master_config): - updated.append(MASTER_NAMESPACE) - log.debug(f"Updated {MASTER_NAMESPACE}") - else: - skipped.append(MASTER_NAMESPACE) - log.debug(f"Skipped {MASTER_NAMESPACE}") + try: + if client.update_namespace(MASTER_NAMESPACE, master_config): + updated.append(MASTER_NAMESPACE) + log.debug(f"Updated {MASTER_NAMESPACE}") + else: + skipped.append(MASTER_NAMESPACE) + log.debug(f"Skipped {MASTER_NAMESPACE}") + except Exception: + failed.append(MASTER_NAMESPACE) + log.exception(f"Error while updating {MASTER_NAMESPACE}:") k8s_enabled_for_cluster = ( yaml.safe_load(master_config).get("k8s_options", {}).get("enabled", False) From d4686fc81ed120f5995752782e5606dfdb78182e Mon Sep 17 00:00:00 2001 From: Jen Patague Date: Tue, 25 Jul 2023 10:32:13 -0700 Subject: [PATCH 05/10] Released 0.190.1 via make release --- debian/changelog | 8 ++++++++ paasta_tools/__init__.py | 2 +- yelp_package/Makefile | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 8d92e1f69b..4d81afa864 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +paasta-tools (0.190.1) xenial; urgency=medium + + * 0.190.1 tagged with 'make release' + Commit: Merge pull request #3660 from Yelp/u/jfong/TRON-1968-master- + config TRON-1968: Catch exceptions when updaitng MASTER config + + -- Jen Patague Tue, 25 Jul 2023 10:31:24 -0700 + paasta-tools (0.190.0) xenial; urgency=medium * 0.190.0 tagged with 'make release' diff --git a/paasta_tools/__init__.py b/paasta_tools/__init__.py index 886908f66b..e3d96ac8cb 100644 --- a/paasta_tools/__init__.py +++ b/paasta_tools/__init__.py @@ -17,4 +17,4 @@ # setup phase, the dependencies may not exist on disk yet. # # Don't bump version manually. See `make release` docs in ./Makefile -__version__ = "0.190.0" +__version__ = "0.190.1" diff --git a/yelp_package/Makefile b/yelp_package/Makefile index 72c780bb02..293e82f455 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -13,7 +13,7 @@ # limitations under the License. # Edit this release and run "make release" -RELEASE=0.190.0 +RELEASE=0.190.1 SHELL=/bin/bash From 1b057c61139183ea425f3d11564bd8506e7693af Mon Sep 17 00:00:00 2001 From: Vincent Thibault Date: Thu, 27 Jul 2023 13:05:07 -0700 Subject: [PATCH 06/10] TRON-1636: Setup tron secret_volumes in setup_tron_namespace --- .gitignore | 1 + .../cli/schemas/kubernetes_schema.json | 1 + paasta_tools/cli/schemas/tron_schema.json | 45 ++++++ paasta_tools/secret_tools.py | 8 ++ paasta_tools/tron_tools.py | 33 +++++ paasta_tools/utils.py | 4 + tests/test_tron_tools.py | 136 +++++++++++++++++- 7 files changed, 226 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3001631ecb..cb0388c093 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ example_cluster/paasta/docker_registry.json general_itests/fake_etc_paasta/clusters.json pip-wheel-metadata debian/debhelper-build-stamp +unique-run # Coverage artifacts .coverage diff --git a/paasta_tools/cli/schemas/kubernetes_schema.json b/paasta_tools/cli/schemas/kubernetes_schema.json index 6b088c5d93..dc9eef5473 100644 --- a/paasta_tools/cli/schemas/kubernetes_schema.json +++ b/paasta_tools/cli/schemas/kubernetes_schema.json @@ -557,6 +557,7 @@ }, "items": { "type": "array", + "maxItems": 1, "items": { "type": "object", "properties": { diff --git a/paasta_tools/cli/schemas/tron_schema.json b/paasta_tools/cli/schemas/tron_schema.json index 1e0b872baf..9134bfc934 100644 --- a/paasta_tools/cli/schemas/tron_schema.json +++ b/paasta_tools/cli/schemas/tron_schema.json @@ -236,6 +236,51 @@ }, "uniqueItems": true }, + "secret_volumes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "container_path": { + "type": "string" + }, + "secret_name": { + "type": "string" + }, + "default_mode": { + "type": "string" + }, + "items": { + "type": "array", + "maxItems": 1, + "items": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "path": { + "type": "string" + }, + "mode": { + "type": "string" + } + }, + "required": [ + "key", + "path" + ] + }, + "uniqueItems": true + } + }, + "required": [ + "container_path", + "secret_name" + ] + }, + "uniqueItems": true + }, "cluster": { "type": "string" }, diff --git a/paasta_tools/secret_tools.py b/paasta_tools/secret_tools.py index 4dc35f5cf9..bfbefd00e6 100644 --- a/paasta_tools/secret_tools.py +++ b/paasta_tools/secret_tools.py @@ -42,6 +42,14 @@ def is_shared_secret(env_var_val: str) -> bool: return env_var_val.startswith("SHARED_") +def is_shared_secret_from_secret_name(soa_dir: str, secret_name: str) -> bool: + """Alternative way of figuring if a secret is shared, directly from the secret_name.""" + secret_path = os.path.join( + soa_dir, SHARED_SECRET_SERVICE, "secrets", f"{secret_name}.json" + ) + return os.path.isfile(secret_path) + + def get_hmac_for_secret( env_var_val: str, service: str, soa_dir: str, secret_environment: str ) -> Optional[str]: diff --git a/paasta_tools/tron_tools.py b/paasta_tools/tron_tools.py index 9115f8711e..2f40a4e650 100644 --- a/paasta_tools/tron_tools.py +++ b/paasta_tools/tron_tools.py @@ -52,6 +52,7 @@ from paasta_tools.utils import NoDeploymentsAvailable from paasta_tools.utils import time_cache from paasta_tools.utils import filter_templates_from_config +from paasta_tools.utils import TronSecretVolume from paasta_tools.kubernetes_tools import ( allowlist_denylist_to_requirements, create_or_find_service_account_name, @@ -68,6 +69,7 @@ ) from paasta_tools.secret_tools import is_secret_ref from paasta_tools.secret_tools import is_shared_secret +from paasta_tools.secret_tools import is_shared_secret_from_secret_name from paasta_tools.secret_tools import get_secret_name_from_ref from paasta_tools.kubernetes_tools import get_paasta_secret_name from paasta_tools.secret_tools import SHARED_SECRET_SERVICE @@ -413,10 +415,40 @@ def get_job_name(self): def get_action_name(self): return self.action + def get_secret_volumes(self) -> List[TronSecretVolume]: # type: ignore + """Adds the secret_volume_name to the objet so tron/task_processing can load it downstream without replicating code.""" + secret_volumes = super().get_secret_volumes() + return [ + TronSecretVolume( + secret_volume_name=self.get_secret_volume_name( + secret_volume["secret_name"] + ), + secret_name=secret_volume["secret_name"], + container_path=secret_volume["container_path"], + default_mode=secret_volume["default_mode"], + items=secret_volume["items"], + ) + for secret_volume in secret_volumes + ] + def get_namespace(self) -> str: """Get namespace from config, default to 'paasta'""" return self.config_dict.get("namespace", KUBERNETES_NAMESPACE) + def get_secret_volume_name(self, secret_name: str) -> str: + service = ( + self.service + if not is_shared_secret_from_secret_name( + soa_dir=self.soa_dir, secret_name=secret_name + ) + else SHARED_SECRET_SERVICE + ) + return get_paasta_secret_name( + self.get_namespace(), + service, + secret_name, + ) + def get_deploy_group(self) -> Optional[str]: return self.config_dict.get("deploy_group", None) @@ -869,6 +901,7 @@ def format_tron_action_dict(action_config: TronActionConfig, use_k8s: bool = Fal "node": action_config.get_node(), "retries": action_config.get_retries(), "retries_delay": action_config.get_retries_delay(), + "secret_volumes": action_config.get_secret_volumes(), "expected_runtime": action_config.get_expected_runtime(), "trigger_downstreams": action_config.get_trigger_downstreams(), "triggered_by": action_config.get_triggered_by(), diff --git a/paasta_tools/utils.py b/paasta_tools/utils.py index 1c33ecf139..29cc57e59b 100644 --- a/paasta_tools/utils.py +++ b/paasta_tools/utils.py @@ -273,6 +273,10 @@ class SecretVolume(TypedDict, total=False): items: List[SecretVolumeItem] +class TronSecretVolume(SecretVolume, total=False): + secret_volume_name: str + + class MonitoringDict(TypedDict, total=False): alert_after: Union[str, float] check_every: str diff --git a/tests/test_tron_tools.py b/tests/test_tron_tools.py index cfbc62b494..bdfaafe2b0 100644 --- a/tests/test_tron_tools.py +++ b/tests/test_tron_tools.py @@ -1,5 +1,7 @@ import datetime import hashlib +import os +import tempfile import mock import pytest @@ -7,6 +9,7 @@ from paasta_tools import tron_tools from paasta_tools import utils +from paasta_tools.secret_tools import SHARED_SECRET_SERVICE from paasta_tools.tron_tools import MASTER_NAMESPACE from paasta_tools.tron_tools import MESOS_EXECUTOR_NAMES from paasta_tools.utils import CAPS_DROP @@ -142,6 +145,63 @@ def test_get_secret_env(self, action_config, test_env, expected_env): secret_env = action_config.get_secret_env() assert secret_env == expected_env + @pytest.mark.parametrize( + ("test_secret_volumes", "expected_secret_volumes"), + ( + ( + [ + { + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], + [ + { + "secret_volume_name": "tron-secret-my--service-secret1", + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], + ), + ), + ) + def test_get_secret_volumes( + self, action_config, test_secret_volumes, expected_secret_volumes + ): + action_config.config_dict["secret_volumes"] = test_secret_volumes + secret_volumes = action_config.get_secret_volumes() + assert secret_volumes == expected_secret_volumes + + @pytest.mark.parametrize( + ("is_shared, secret_name, expected_secret_volume_name"), + ( + (False, "secret1", "tron-secret-my--service-secret1"), + (True, "secret1", "tron-secret-underscore-shared-secret1"), + ), + ) + def test_get_secret_volume_name( + self, action_config, is_shared, secret_name, expected_secret_volume_name + ): + + with tempfile.TemporaryDirectory() as dir_path: + service = action_config.service if not is_shared else SHARED_SECRET_SERVICE + secret_path = os.path.join( + dir_path, service, "secrets", f"{secret_name}.json" + ) + os.makedirs(os.path.dirname(secret_path), exist_ok=True) + with open(secret_path, "w") as f: + f.write("FOOBAR") + + with mock.patch.object(action_config, "soa_dir", dir_path): + assert ( + action_config.get_secret_volume_name(secret_name) + == expected_secret_volume_name + ) + def test_get_executor_default(self, action_config): assert action_config.get_executor() == "paasta" @@ -787,6 +847,14 @@ def test_format_tron_action_dict_paasta(self): "disk": 42, "pool": "special_pool", "env": {"SHELL": "/bin/bash"}, + "secret_volumes": [ + { + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"} ], @@ -835,6 +903,15 @@ def test_format_tron_action_dict_paasta(self): "mem": 1200, "disk": 42, "env": mock.ANY, + "secret_volumes": [ + { + "secret_volume_name": "tron-secret-my--service-secret1", + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"} ], @@ -875,6 +952,14 @@ def test_format_tron_action_dict_spark(self): "disk": 42, "pool": "special_pool", "env": {"SHELL": "/bin/bash"}, + "secret_volumes": [ + { + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"} ], @@ -1003,6 +1088,15 @@ def test_format_tron_action_dict_spark(self): "mem": 1200, "disk": 42, "env": mock.ANY, + "secret_volumes": [ + { + "secret_volume_name": "tron-secret-my--service-secret1", + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"} ], @@ -1094,6 +1188,7 @@ def test_format_tron_action_dict_paasta_k8s_service_account(self): "env": mock.ANY, "secret_env": {}, "field_selector_env": {"PAASTA_POD_IP": {"field_path": "status.podIP"}}, + "secret_volumes": [], "extra_volumes": [], "service_account_name": "a-magic-sa", } @@ -1140,6 +1235,14 @@ def test_format_tron_action_dict_paasta_k8s( "disk": 42, "pool": "special_pool", "env": {"SHELL": "/bin/bash", "SOME_SECRET": "SECRET(secret_name)"}, + "secret_volumes": [ + { + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"} ], @@ -1178,6 +1281,10 @@ def test_format_tron_action_dict_paasta_k8s( ), mock.patch( "paasta_tools.tron_tools.load_system_paasta_config", autospec=True, + ), mock.patch( + "paasta_tools.secret_tools.is_shared_secret_from_secret_name", + autospec=True, + return_value=False, ): result = tron_tools.format_tron_action_dict(action_config, use_k8s=True) @@ -1218,6 +1325,15 @@ def test_format_tron_action_dict_paasta_k8s( "key": "secret_name", } }, + "secret_volumes": [ + { + "secret_volume_name": "tron-secret-my--service-secret1", + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "field_selector_env": {"PAASTA_POD_IP": {"field_path": "status.podIP"}}, "extra_volumes": [ {"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"} @@ -1248,6 +1364,14 @@ def test_format_tron_action_dict_paasta_no_branch_dict(self): "disk": 42, "pool": "special_pool", "env": {"SHELL": "/bin/bash"}, + "secret_volumes": [ + { + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"containerPath": "/nail/tmp", "hostPath": "/nail/tmp", "mode": "RW"} ], @@ -1273,7 +1397,6 @@ def test_format_tron_action_dict_paasta_no_branch_dict(self): autospec=True, ): result = tron_tools.format_tron_action_dict(action_config) - assert result == { "command": "echo something", "requires": ["required_action"], @@ -1284,6 +1407,15 @@ def test_format_tron_action_dict_paasta_no_branch_dict(self): "mem": 1200, "disk": 42, "env": mock.ANY, + "secret_volumes": [ + { + "secret_volume_name": "tron-secret-my--service-secret1", + "secret_name": "secret1", + "container_path": "/b/c", + "default_mode": "0644", + "items": [{"key": "secret1", "path": "abc"}], + } + ], "extra_volumes": [ {"container_path": "/nail/tmp", "host_path": "/nail/tmp", "mode": "RW"} ], @@ -1430,7 +1562,7 @@ def test_create_complete_config_e2e(self, tmpdir): # that are not static, this will cause continuous reconfiguration, which # will add significant load to the Tron API, which happened in DAR-1461. # but if this is intended, just change the hash. - assert hasher.hexdigest() == "f740410f7ae2794f9924121c1115e15d" + assert hasher.hexdigest() == "35972651618a848ac6bf7947245dbaea" def test_override_default_pool_override(self, tmpdir): soa_dir = tmpdir.mkdir("test_create_complete_config_soa") From b9972463ea6ceb523fdfc11644ff05f372f5e564 Mon Sep 17 00:00:00 2001 From: Jen Patague Date: Fri, 28 Jul 2023 11:48:55 -0700 Subject: [PATCH 07/10] COMPINFRA-2938: Support security-check in parallel steps --- paasta_tools/cli/cmds/check.py | 8 +++++++- tests/cli/test_cmds_check.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/paasta_tools/cli/cmds/check.py b/paasta_tools/cli/cmds/check.py index 65767cd7d8..fa23c10035 100644 --- a/paasta_tools/cli/cmds/check.py +++ b/paasta_tools/cli/cmds/check.py @@ -77,7 +77,13 @@ def deploy_check(service_path): def deploy_has_security_check(service, soa_dir): pipeline = get_pipeline_config(service=service, soa_dir=soa_dir) - steps = [step["step"] for step in pipeline] + steps = [step["step"] for step in pipeline if not step.get("parallel")] + steps += [ + substep["step"] + for step in pipeline + if step.get("parallel") + for substep in step.get("parallel") + ] if "security-check" in steps: print(PaastaCheckMessages.DEPLOY_SECURITY_FOUND) return True diff --git a/tests/cli/test_cmds_check.py b/tests/cli/test_cmds_check.py index 4b30b845e3..b5c971db5e 100644 --- a/tests/cli/test_cmds_check.py +++ b/tests/cli/test_cmds_check.py @@ -417,6 +417,24 @@ def test_deploy_has_security_check_true(mock_pipeline_config, capfd): assert actual is True +@patch("paasta_tools.cli.cmds.check.get_pipeline_config", autospec=True) +def test_deploy_has_parallel_security_check_true(mock_pipeline_config, capfd): + mock_pipeline_config.return_value = [ + { + "step": "initial", + "parallel": [ + {"step": "test"}, + {"step": "security-check"}, + ], + }, + {"step": "push-to-registry"}, + {"step": "hab.canary", "trigger_next_step_manually": True}, + {"step": "hab.main"}, + ] + actual = deploy_has_security_check(service="fake_service", soa_dir="/fake/path") + assert actual is True + + @patch("paasta_tools.cli.cmds.check.get_instance_config", autospec=True) @patch("paasta_tools.cli.cmds.check.list_clusters", autospec=True) @patch("paasta_tools.cli.cmds.check.get_service_instance_list", autospec=True) From 4dad92ff2f51aae878e92c9e1523301ad0763450 Mon Sep 17 00:00:00 2001 From: Jen Patague Date: Fri, 28 Jul 2023 12:52:32 -0700 Subject: [PATCH 08/10] Released 0.190.2 via make release --- debian/changelog | 9 +++++++++ paasta_tools/__init__.py | 2 +- yelp_package/Makefile | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 4d81afa864..43ff3a6b06 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +paasta-tools (0.190.2) xenial; urgency=medium + + * 0.190.2 tagged with 'make release' + Commit: Merge pull request #3663 from + Yelp/u/jfong/check_support_parallel_step COMPINFRA-2938: Support + security-check in parallel steps + + -- Jen Patague Fri, 28 Jul 2023 12:51:55 -0700 + paasta-tools (0.190.1) xenial; urgency=medium * 0.190.1 tagged with 'make release' diff --git a/paasta_tools/__init__.py b/paasta_tools/__init__.py index e3d96ac8cb..cb9278b186 100644 --- a/paasta_tools/__init__.py +++ b/paasta_tools/__init__.py @@ -17,4 +17,4 @@ # setup phase, the dependencies may not exist on disk yet. # # Don't bump version manually. See `make release` docs in ./Makefile -__version__ = "0.190.1" +__version__ = "0.190.2" diff --git a/yelp_package/Makefile b/yelp_package/Makefile index 293e82f455..7a82696149 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -13,7 +13,7 @@ # limitations under the License. # Edit this release and run "make release" -RELEASE=0.190.1 +RELEASE=0.190.2 SHELL=/bin/bash From 0c532e688a46a1490328143bd7f2569a3020110e Mon Sep 17 00:00:00 2001 From: Vincent Thibault Date: Mon, 31 Jul 2023 10:28:33 -0700 Subject: [PATCH 09/10] Released 0.191.0 via make release --- debian/changelog | 9 +++++++++ paasta_tools/__init__.py | 2 +- yelp_package/Makefile | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 43ff3a6b06..734941aeec 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +paasta-tools (0.191.0) xenial; urgency=medium + + * 0.191.0 tagged with 'make release' + Commit: Merge pull request #3653 from Yelp/revert-3652-revert-3615- + u/vit/tron-1636-add-secret-volume TRON-1636: Setup tron + secret_volumes in setup_tron_namespace + + -- Vincent Thibault Mon, 31 Jul 2023 10:27:13 -0700 + paasta-tools (0.190.2) xenial; urgency=medium * 0.190.2 tagged with 'make release' diff --git a/paasta_tools/__init__.py b/paasta_tools/__init__.py index cb9278b186..e5fe25912c 100644 --- a/paasta_tools/__init__.py +++ b/paasta_tools/__init__.py @@ -17,4 +17,4 @@ # setup phase, the dependencies may not exist on disk yet. # # Don't bump version manually. See `make release` docs in ./Makefile -__version__ = "0.190.2" +__version__ = "0.191.0" diff --git a/yelp_package/Makefile b/yelp_package/Makefile index 7a82696149..7ce492138a 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -13,7 +13,7 @@ # limitations under the License. # Edit this release and run "make release" -RELEASE=0.190.2 +RELEASE=0.191.0 SHELL=/bin/bash From a0a7514d4e4bdc23d70039be5cc43fea15c49e2f Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Mon, 31 Jul 2023 10:54:07 -0700 Subject: [PATCH 10/10] Don't hard-code yelpcorp PyPI registry Internal ticket: CORESERV-12777 Same idea as https://github.com/Yelp/Tron/pull/921 --- .github/workflows/ci.yml | 3 +-- .github/workflows/pypi.yml | 4 +--- Makefile | 23 +++++++++---------- debian/rules | 7 ++---- general_itests/fake_simple_service/Dockerfile | 3 --- general_itests/fake_simple_service/Makefile | 4 +--- requirements-bootstrap.txt | 1 - tox.ini | 14 ++++------- yelp_package/Makefile | 5 ++-- yelp_package/dockerfiles/bionic/Dockerfile | 2 +- yelp_package/dockerfiles/gitremote/Dockerfile | 2 -- .../dockerfiles/hacheck-sidecar/Dockerfile | 3 --- yelp_package/dockerfiles/itest/api/Dockerfile | 2 +- .../dockerfiles/itest/hacheck/Dockerfile | 2 +- .../dockerfiles/itest/httpdrain/Dockerfile | 2 +- yelp_package/dockerfiles/itest/k8s/Dockerfile | 2 +- .../dockerfiles/itest/marathon/Dockerfile | 3 --- .../dockerfiles/itest/mesos/Dockerfile | 3 --- .../dockerfiles/itest/zookeeper/Dockerfile | 3 --- yelp_package/dockerfiles/jammy/Dockerfile | 2 +- .../dockerfiles/mesos-paasta/Dockerfile | 2 +- .../dockerfiles/playground/Dockerfile | 2 +- yelp_package/dockerfiles/xenial/Dockerfile | 2 +- yelp_package/extra_requirements_yelp.txt | 1 - 24 files changed, 33 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f63af1de0..2bd51d6e02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,6 @@ jobs: - py37-linux,docs,mypy,tests - general_itests env: - PIP_INDEX_URL: https://pypi.python.org/simple DOCKER_REGISTRY: "" steps: - uses: actions/checkout@v2 @@ -29,7 +28,7 @@ jobs: python-version: 3.7 - run: python -m pip install --upgrade pip - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve - - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }} + - run: tox -e ${{ matrix.toxenv }} k8s_itests: runs-on: ubuntu-20.04 env: diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 788966e87b..1e5c33b110 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,7 +16,6 @@ jobs: - py37-linux,docs,mypy,tests - general_itests env: - PIP_INDEX_URL: https://pypi.python.org/simple DOCKER_REGISTRY: "" steps: - uses: actions/checkout@v2 @@ -25,13 +24,12 @@ jobs: python-version: 3.7 - run: python -m pip install --upgrade pip - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve - - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }} + - run: tox -e ${{ matrix.toxenv }} pypi: # lets run tests before we push anything to pypi, much like we do internally needs: tox runs-on: ubuntu-20.04 env: - PIP_INDEX_URL: https://pypi.python.org/simple DOCKER_REGISTRY: "" steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 1f1a5a7539..252379a84b 100644 --- a/Makefile +++ b/Makefile @@ -22,42 +22,41 @@ else endif ifeq ($(PAASTA_ENV),YELP) - export PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple export DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/ else - export PIP_INDEX_URL ?= https://pypi.python.org/simple export DOCKER_REGISTRY ?= "" + export INDEX_URL_BUILD_ARG ?= PIP_INDEX_URL endif .PHONY: all docs test itest k8s_itests quick-test dev: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) + .paasta/bin/tox docs: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e docs + .paasta/bin/tox -e docs test: .paasta/bin/activate if [ "$(PAASTA_ENV)" != "YELP" ]; then \ - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests; \ + .paasta/bin/tox -e tests; \ else \ - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy; \ + .paasta/bin/tox -e tests-yelpy; \ fi test-yelpy: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy + .paasta/bin/tox -e tests-yelpy test-not-yelpy: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e tests + .paasta/bin/tox -e tests quick-test: .tox/py37-linux TZ=UTC .tox/py37-linux/bin/py.test --last-failed -x -- tests .tox/py37-linux: .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) + .paasta/bin/tox dev-api: .tox/py37-linux - .paasta/bin/tox -i $(PIP_INDEX_URL) -e dev-api + .paasta/bin/tox -e dev-api .paasta/bin/activate: requirements.txt requirements-dev.txt test -d .paasta/bin/activate || virtualenv -p python3.7 .paasta @@ -69,7 +68,7 @@ dev-api: .tox/py37-linux touch .paasta/bin/activate itest: test .paasta/bin/activate - .paasta/bin/tox -i $(PIP_INDEX_URL) -e general_itests + .paasta/bin/tox -e general_itests itest_%: # See the makefile in yelp_package/Makefile for packaging stuff @@ -150,7 +149,7 @@ generate_deployments_for_service: | soa_config_playground .tox/py37-linux .PHONY: playground-api playground-api: .tox/py37-linux | soa_config_playground - .paasta/bin/tox -i $(PIP_INDEX_URL) -e playground-api + .paasta/bin/tox -e playground-api .PHONY: setup-kubernetes-job setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service diff --git a/debian/rules b/debian/rules index 6b37c72bd9..d0b72bb95d 100755 --- a/debian/rules +++ b/debian/rules @@ -1,8 +1,6 @@ #!/usr/bin/make -f # -*- makefile -*- -PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple - %: dh $@ --with python-virtualenv @@ -21,9 +19,8 @@ PACKAGE=$(shell dh_listpackages) DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs DH_VENV_DIR=debian/$(PACKAGE)$(DH_VIRTUALENV_INSTALL_ROOT)/$(PACKAGE) override_dh_virtualenv: - dh_virtualenv -i $(PIP_INDEX_URL) \ + dh_virtualenv \ --python=/usr/bin/python3.7 \ --preinstall no-manylinux1 \ - --preinstall=-rrequirements-bootstrap.txt \ - --pip-tool pip-custom-platform + --preinstall=-rrequirements-bootstrap.txt cp yelp_package/gopath/paasta_go $(DH_VENV_DIR)/bin/paasta_go diff --git a/general_itests/fake_simple_service/Dockerfile b/general_itests/fake_simple_service/Dockerfile index 3b7196ae25..2249026bef 100644 --- a/general_itests/fake_simple_service/Dockerfile +++ b/general_itests/fake_simple_service/Dockerfile @@ -14,6 +14,3 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial - -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=${PIP_INDEX_URL} diff --git a/general_itests/fake_simple_service/Makefile b/general_itests/fake_simple_service/Makefile index c723c5c479..dace326e85 100644 --- a/general_itests/fake_simple_service/Makefile +++ b/general_itests/fake_simple_service/Makefile @@ -16,13 +16,11 @@ DOCKER_TAG ?= fake_simple_service-$(USER)-dev ifeq ($(findstring .yelpcorp.com,$(shell hostname -f)), .yelpcorp.com) DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/ - PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple else DOCKER_REGISTRY ?= "" - PIP_INDEX_URL ?= https://pypi.python.org/simple endif .PHONY: cook-image cook-image: - docker build --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) . + docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) . diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt index dc06bbadb8..b24f00fb9b 100644 --- a/requirements-bootstrap.txt +++ b/requirements-bootstrap.txt @@ -1,5 +1,4 @@ pip==18.1 -pip-custom-platform==0.5.0 setuptools==39.0.1 venv-update==3.2.4 wheel==0.32.3 diff --git a/tox.ini b/tox.ini index 09bf738423..df9a404684 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,14 @@ [tox] skipsdist=True envlist=py37-linux -tox_pip_extensions_ext_pip_custom_platform = true tox_pip_extensions_ext_venv_update = true docker_compose_version = 1.26.2 [testenv] -# The Makefile and override the indexserver to the public one when -# running outside of Yelp. -indexserver = https://pypi.yelpcorp.com/simple basepython = python3.7 passenv = SSH_AUTH_SOCK setenv = TZ = UTC - PIP_INDEX_URL = {env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} deps = --requirement={toxinidir}/requirements.txt --requirement={toxinidir}/requirements-dev.txt @@ -24,7 +19,7 @@ commands = # that said, most of the time people will run make test which will use tox to install these in a # faster way (using venv-update) - so this is really just here for anyone that like to just invoke # `tox` directly and with no explicit env - -pip-custom-platform install -i https://pypi.yelpcorp.com/simple -r yelp_package/extra_requirements_yelp.txt + -pip install -r yelp_package/extra_requirements_yelp.txt [testenv:dev-api] envdir = .tox/py37-linux/ @@ -105,24 +100,25 @@ passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH + INDEX_URL_BUILD_ARG changedir=k8s_itests/ commands = # Build /etc/paasta used by docker-compose {toxinidir}/k8s_itests/scripts/setup.sh # Run paasta-tools k8s_itests in docker docker-compose down - docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} + docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple docker-compose up \ --abort-on-container-exit [testenv:example_cluster] changedir=example_cluster/ -passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH +passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG deps = docker-compose=={[tox]docker_compose_version} commands = docker-compose down - docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple} + docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple # Fire up the marathon cluster in background # Run the paastatools container in foreground to catch the output # the `docker-compose run` vs `docker-compose up` is important here, as docker-compose run will diff --git a/yelp_package/Makefile b/yelp_package/Makefile index 7ce492138a..c900b069ea 100644 --- a/yelp_package/Makefile +++ b/yelp_package/Makefile @@ -19,7 +19,7 @@ SHELL=/bin/bash UID:=`id -u` GID:=`id -g` -DOCKER_RUN=docker run -t -v $(CURDIR)/../:/work:rw -e PIP_INDEX_URL=$(PIP_INDEX_URL) yelp/paastatools_$*_container +DOCKER_RUN=docker run -t -v $(CURDIR)/../:/work:rw yelp/paastatools_$*_container NOOP = true ifeq ($(PAASTA_ENV),YELP) @@ -36,7 +36,8 @@ build_%_docker: [ -d ../dist ] || mkdir ../dist docker pull "yelp/paastatools_$*_container" || true cd dockerfiles/$*/ && docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) \ - --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) -t "yelp/paastatools_$*_container" . + $(if $(filter-out $(PAASTA_ENV),YELP), --build-arg PIP_INDEX_URL=https://pypi.org/simple,) \ + -t "yelp/paastatools_$*_container" . .SECONDEXPANSION: itest_%: package_$$* diff --git a/yelp_package/dockerfiles/bionic/Dockerfile b/yelp_package/dockerfiles/bionic/Dockerfile index add3aa8ba5..be04501e19 100644 --- a/yelp_package/dockerfiles/bionic/Dockerfile +++ b/yelp_package/dockerfiles/bionic/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN rm /etc/dpkg/dpkg.cfg.d/excludes RUN apt-get update && apt-get install -yq gnupg2 diff --git a/yelp_package/dockerfiles/gitremote/Dockerfile b/yelp_package/dockerfiles/gitremote/Dockerfile index 173fcfada5..a9ef3d7bbc 100644 --- a/yelp_package/dockerfiles/gitremote/Dockerfile +++ b/yelp_package/dockerfiles/gitremote/Dockerfile @@ -1,8 +1,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ git \ diff --git a/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile b/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile index 2c95294e93..b69a133e34 100644 --- a/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile +++ b/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile @@ -1,9 +1,6 @@ FROM docker-dev.yelpcorp.com/bionic_yelp ARG HACHECK_VERSION=0.18.2-yelp1 -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - RUN apt-get update && \ apt-get install -y hacheck=${HACHECK_VERSION} paasta-tools python3-distutils && \ mkdir -p /etc/paasta diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile index a80924b9b4..4c30ff9466 100644 --- a/yelp_package/dockerfiles/itest/api/Dockerfile +++ b/yelp_package/dockerfiles/itest/api/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/hacheck/Dockerfile b/yelp_package/dockerfiles/itest/hacheck/Dockerfile index 2c1aadf91d..9533ac3121 100644 --- a/yelp_package/dockerfiles/itest/hacheck/Dockerfile +++ b/yelp_package/dockerfiles/itest/hacheck/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/httpdrain/Dockerfile b/yelp_package/dockerfiles/itest/httpdrain/Dockerfile index b425f7aaa6..0bb1aef38f 100644 --- a/yelp_package/dockerfiles/itest/httpdrain/Dockerfile +++ b/yelp_package/dockerfiles/itest/httpdrain/Dockerfile @@ -1,7 +1,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile index 0c79a789bc..2184c2699d 100644 --- a/yelp_package/dockerfiles/itest/k8s/Dockerfile +++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:bionic -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL # Need Python 3.7 diff --git a/yelp_package/dockerfiles/itest/marathon/Dockerfile b/yelp_package/dockerfiles/itest/marathon/Dockerfile index 69d0308660..32318c06de 100644 --- a/yelp_package/dockerfiles/itest/marathon/Dockerfile +++ b/yelp_package/dockerfiles/itest/marathon/Dockerfile @@ -15,9 +15,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - RUN apt-get update > /dev/null && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ apt-transport-https \ diff --git a/yelp_package/dockerfiles/itest/mesos/Dockerfile b/yelp_package/dockerfiles/itest/mesos/Dockerfile index f808cac49c..d14f3ba96a 100644 --- a/yelp_package/dockerfiles/itest/mesos/Dockerfile +++ b/yelp_package/dockerfiles/itest/mesos/Dockerfile @@ -15,9 +15,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}xenial_pkgbuild -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - # Install packages to allow apt to use a repository over HTTPS # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#xenial-1604 RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/itest/zookeeper/Dockerfile b/yelp_package/dockerfiles/itest/zookeeper/Dockerfile index 3768984d44..c91b541f06 100644 --- a/yelp_package/dockerfiles/itest/zookeeper/Dockerfile +++ b/yelp_package/dockerfiles/itest/zookeeper/Dockerfile @@ -15,9 +15,6 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple -ENV PIP_INDEX_URL=$PIP_INDEX_URL - RUN apt-get update > /dev/null && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ zookeeper > /dev/null && \ diff --git a/yelp_package/dockerfiles/jammy/Dockerfile b/yelp_package/dockerfiles/jammy/Dockerfile index 98ebee7de8..a5bb90b118 100644 --- a/yelp_package/dockerfiles/jammy/Dockerfile +++ b/yelp_package/dockerfiles/jammy/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}ubuntu:jammy -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN rm /etc/dpkg/dpkg.cfg.d/excludes RUN apt-get update && apt-get install -yq gnupg2 diff --git a/yelp_package/dockerfiles/mesos-paasta/Dockerfile b/yelp_package/dockerfiles/mesos-paasta/Dockerfile index 1b13e466c5..7a97968860 100644 --- a/yelp_package/dockerfiles/mesos-paasta/Dockerfile +++ b/yelp_package/dockerfiles/mesos-paasta/Dockerfile @@ -1,6 +1,6 @@ FROM example_cluster_mesosbase -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/playground/Dockerfile b/yelp_package/dockerfiles/playground/Dockerfile index 7a4a1f76bd..b0318f9bb5 100644 --- a/yelp_package/dockerfiles/playground/Dockerfile +++ b/yelp_package/dockerfiles/playground/Dockerfile @@ -1,6 +1,6 @@ FROM example_cluster_itest_xenial -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN apt-get update > /dev/null && \ diff --git a/yelp_package/dockerfiles/xenial/Dockerfile b/yelp_package/dockerfiles/xenial/Dockerfile index 8846bf00b9..2062eeff5e 100644 --- a/yelp_package/dockerfiles/xenial/Dockerfile +++ b/yelp_package/dockerfiles/xenial/Dockerfile @@ -15,7 +15,7 @@ ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/ FROM ${DOCKER_REGISTRY}xenial_pkgbuild -ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple +ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple ENV PIP_INDEX_URL=$PIP_INDEX_URL RUN echo "deb http://repos.mesosphere.com/ubuntu xenial main" > /etc/apt/sources.list.d/mesosphere.list && \ diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt index 9e6521c0bd..959d7b80f9 100644 --- a/yelp_package/extra_requirements_yelp.txt +++ b/yelp_package/extra_requirements_yelp.txt @@ -1,4 +1,3 @@ ---index-url=https://pypi.yelpcorp.com/simple atomicfile==1.0 cached-property==1.3.1 cffi==1.15.0