Merge remote-tracking branch 'origin/master' into COMPINFRA-2833

Yelp · Aug 1, 2023 · 7e2d256 · 7e2d256
2 parents 48577bf + a76edf5
commit 7e2d256
Show file tree

Hide file tree

Showing 40 changed files with 350 additions and 83 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,6 @@ jobs:
           - py37-linux,docs,mypy,tests
           - general_itests
     env:
-      PIP_INDEX_URL: https://pypi.python.org/simple
       DOCKER_REGISTRY: ""
     steps:
       - uses: actions/checkout@v2
@@ -29,7 +28,7 @@ jobs:
           python-version: 3.7
       - run: python -m pip install --upgrade pip
       - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve
-      - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }}
+      - run: tox -e ${{ matrix.toxenv }}
   k8s_itests:
     runs-on: ubuntu-20.04
     env:

diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
@@ -16,7 +16,6 @@ jobs:
           - py37-linux,docs,mypy,tests
           - general_itests
     env:
-      PIP_INDEX_URL: https://pypi.python.org/simple
       DOCKER_REGISTRY: ""
     steps:
       - uses: actions/checkout@v2
@@ -25,13 +24,12 @@ jobs:
           python-version: 3.7
       - run: python -m pip install --upgrade pip
       - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve
-      - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }}
+      - run: tox -e ${{ matrix.toxenv }}
   pypi:
     # lets run tests before we push anything to pypi, much like we do internally
     needs: tox
     runs-on: ubuntu-20.04
     env:
-      PIP_INDEX_URL: https://pypi.python.org/simple
       DOCKER_REGISTRY: ""
     steps:
       - uses: actions/checkout@v2

diff --git a/.gitignore b/.gitignore
@@ -48,6 +48,7 @@ example_cluster/paasta/docker_registry.json
 general_itests/fake_etc_paasta/clusters.json
 pip-wheel-metadata
 debian/debhelper-build-stamp
+unique-run
 
 # Coverage artifacts
 .coverage

diff --git a/Makefile b/Makefile
@@ -22,42 +22,41 @@ else
 endif
 
 ifeq ($(PAASTA_ENV),YELP)
-	export PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple
 	export DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/
 else
-	export PIP_INDEX_URL ?= https://pypi.python.org/simple
 	export DOCKER_REGISTRY ?= ""
+	export INDEX_URL_BUILD_ARG ?= PIP_INDEX_URL
 endif
 
 .PHONY: all docs test itest k8s_itests quick-test
 
 dev: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL)
+	.paasta/bin/tox
 
 docs: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e docs
+	.paasta/bin/tox -e docs
 
 test: .paasta/bin/activate
 	if [ "$(PAASTA_ENV)" != "YELP" ]; then \
-		.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests; \
+		.paasta/bin/tox -e tests; \
 	else \
-		.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy; \
+		.paasta/bin/tox -e tests-yelpy; \
 	fi
 
 test-yelpy: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy
+	.paasta/bin/tox -e tests-yelpy
 
 test-not-yelpy: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests
+	.paasta/bin/tox -e tests
 
 quick-test: .tox/py37-linux
 	TZ=UTC .tox/py37-linux/bin/py.test --last-failed -x -- tests
 
 .tox/py37-linux: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL)
+	.paasta/bin/tox
 
 dev-api: .tox/py37-linux
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e dev-api
+	.paasta/bin/tox -e dev-api
 
 .paasta/bin/activate: requirements.txt requirements-dev.txt
 	test -d .paasta/bin/activate || virtualenv -p python3.7 .paasta
@@ -69,7 +68,7 @@ dev-api: .tox/py37-linux
 	touch .paasta/bin/activate
 
 itest: test .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e general_itests
+	.paasta/bin/tox -e general_itests
 
 itest_%:
 	# See the makefile in yelp_package/Makefile for packaging stuff
@@ -150,7 +149,7 @@ generate_deployments_for_service: | soa_config_playground .tox/py37-linux
 
 .PHONY: playground-api
 playground-api: .tox/py37-linux | soa_config_playground
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e playground-api
+	.paasta/bin/tox -e playground-api
 
 .PHONY: setup-kubernetes-job
 setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service

diff --git a/debian/changelog b/debian/changelog
@@ -1,3 +1,37 @@
+paasta-tools (0.191.0) xenial; urgency=medium
+
+  * 0.191.0 tagged with 'make release'
+    Commit: Merge pull request #3653 from Yelp/revert-3652-revert-3615-
+    u/vit/tron-1636-add-secret-volume  TRON-1636: Setup tron
+    secret_volumes in setup_tron_namespace
+
+ -- Vincent Thibault <[email protected]>  Mon, 31 Jul 2023 10:27:13 -0700
+
+paasta-tools (0.190.2) xenial; urgency=medium
+
+  * 0.190.2 tagged with 'make release'
+    Commit: Merge pull request #3663 from
+    Yelp/u/jfong/check_support_parallel_step  COMPINFRA-2938: Support
+    security-check in parallel steps
+
+ -- Jen Patague <[email protected]>  Fri, 28 Jul 2023 12:51:55 -0700
+
+paasta-tools (0.190.1) xenial; urgency=medium
+
+  * 0.190.1 tagged with 'make release'
+    Commit: Merge pull request #3660 from Yelp/u/jfong/TRON-1968-master-
+    config  TRON-1968: Catch exceptions when updaitng MASTER config
+
+ -- Jen Patague <[email protected]>  Tue, 25 Jul 2023 10:31:24 -0700
+
+paasta-tools (0.190.0) xenial; urgency=medium
+
+  * 0.190.0 tagged with 'make release'
+    Commit: Fixing setup_kubernetes_cr exception (#3658)  * Fixing
+    setup_kubernetes_cr exception  * Fixing tests and logic
+
+ -- Wilmer Bandres <[email protected]>  Thu, 20 Jul 2023 09:49:46 -0700
+
 paasta-tools (0.189.0) xenial; urgency=medium
 
   * 0.189.0 tagged with 'make release'

diff --git a/debian/rules b/debian/rules
@@ -1,8 +1,6 @@
 #!/usr/bin/make -f
 # -*- makefile -*-
 
-PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple
-
 %:
 	dh $@ --with python-virtualenv
 
@@ -21,9 +19,8 @@ PACKAGE=$(shell dh_listpackages)
 DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
 DH_VENV_DIR=debian/$(PACKAGE)$(DH_VIRTUALENV_INSTALL_ROOT)/$(PACKAGE)
 override_dh_virtualenv:
-	dh_virtualenv -i $(PIP_INDEX_URL) \
+	dh_virtualenv \
 		--python=/usr/bin/python3.7 \
 		--preinstall no-manylinux1 \
-		--preinstall=-rrequirements-bootstrap.txt \
-		--pip-tool pip-custom-platform
+		--preinstall=-rrequirements-bootstrap.txt
 	cp yelp_package/gopath/paasta_go $(DH_VENV_DIR)/bin/paasta_go
diff --git a/general_itests/fake_simple_service/Dockerfile b/general_itests/fake_simple_service/Dockerfile
@@ -14,6 +14,3 @@
 
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:xenial
-
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=${PIP_INDEX_URL}
diff --git a/general_itests/fake_simple_service/Makefile b/general_itests/fake_simple_service/Makefile
@@ -16,13 +16,11 @@ DOCKER_TAG ?= fake_simple_service-$(USER)-dev
 
 ifeq ($(findstring .yelpcorp.com,$(shell hostname -f)), .yelpcorp.com)
 	DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/
-	PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple
 else
 	DOCKER_REGISTRY ?= ""
-	PIP_INDEX_URL ?= https://pypi.python.org/simple
 endif
 
 .PHONY: cook-image
 
 cook-image:
-	docker build --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) .
+	docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) .
diff --git a/paasta_tools/__init__.py b/paasta_tools/__init__.py
@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "0.189.0"
+__version__ = "0.191.0"
diff --git a/paasta_tools/cli/cmds/check.py b/paasta_tools/cli/cmds/check.py
@@ -77,7 +77,13 @@ def deploy_check(service_path):
 
 def deploy_has_security_check(service, soa_dir):
     pipeline = get_pipeline_config(service=service, soa_dir=soa_dir)
-    steps = [step["step"] for step in pipeline]
+    steps = [step["step"] for step in pipeline if not step.get("parallel")]
+    steps += [
+        substep["step"]
+        for step in pipeline
+        if step.get("parallel")
+        for substep in step.get("parallel")
+    ]
     if "security-check" in steps:
         print(PaastaCheckMessages.DEPLOY_SECURITY_FOUND)
         return True

diff --git a/paasta_tools/cli/cmds/spark_run.py b/paasta_tools/cli/cmds/spark_run.py
@@ -1203,7 +1203,7 @@ def paasta_spark_run(args):
         document = POD_TEMPLATE.format(
             spark_pod_label=limit_size_with_hash(f"exec-{app_base_name}"),
         )
-        parsed_pod_template = yaml.load(document)
+        parsed_pod_template = yaml.safe_load(document)
         with open(pod_template_path, "w") as f:
             yaml.dump(parsed_pod_template, f)
 

diff --git a/paasta_tools/cli/schemas/kubernetes_schema.json b/paasta_tools/cli/schemas/kubernetes_schema.json
@@ -557,6 +557,7 @@
                             },
                             "items": {
                                 "type": "array",
+                                "maxItems": 1,
                                 "items": {
                                     "type": "object",
                                     "properties": {

diff --git a/paasta_tools/cli/schemas/tron_schema.json b/paasta_tools/cli/schemas/tron_schema.json
@@ -236,6 +236,51 @@
                     },
                     "uniqueItems": true
                 },
+                "secret_volumes": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "container_path": {
+                                "type": "string"
+                            },
+                            "secret_name": {
+                                "type": "string"
+                            },
+                            "default_mode": {
+                                "type": "string"
+                            },
+                            "items": {
+                                "type": "array",
+                                "maxItems": 1,
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "key": {
+                                            "type": "string"
+                                        },
+                                        "path": {
+                                            "type": "string"
+                                        },
+                                        "mode": {
+                                            "type": "string"
+                                        }
+                                    },
+                                    "required": [
+                                        "key",
+                                        "path"
+                                    ]
+                                },
+                                "uniqueItems": true
+                            }
+                        },
+                        "required": [
+                            "container_path",
+                            "secret_name"
+                        ]
+                    },
+                    "uniqueItems": true
+                },
                 "cluster": {
                     "type": "string"
                 },

diff --git a/paasta_tools/secret_tools.py b/paasta_tools/secret_tools.py
@@ -42,6 +42,14 @@ def is_shared_secret(env_var_val: str) -> bool:
     return env_var_val.startswith("SHARED_")
 
 
+def is_shared_secret_from_secret_name(soa_dir: str, secret_name: str) -> bool:
+    """Alternative way of figuring if a secret is shared, directly from the secret_name."""
+    secret_path = os.path.join(
+        soa_dir, SHARED_SECRET_SERVICE, "secrets", f"{secret_name}.json"
+    )
+    return os.path.isfile(secret_path)
+
+
 def get_hmac_for_secret(
     env_var_val: str, service: str, soa_dir: str, secret_environment: str
 ) -> Optional[str]:

diff --git a/paasta_tools/setup_kubernetes_cr.py b/paasta_tools/setup_kubernetes_cr.py
@@ -29,6 +29,7 @@
 from typing import Sequence
 
 import yaml
+from kubernetes.client.exceptions import ApiException
 
 from paasta_tools.cli.utils import LONG_RUNNING_INSTANCE_TYPE_HANDLERS
 from paasta_tools.flink_tools import get_flink_ingress_url_root
@@ -154,6 +155,7 @@ def setup_all_custom_resources(
 ) -> bool:
 
     got_results = False
+    succeeded = False
     # We support two versions due to our upgrade to 1.22
     # this functions runs succefully when any of the two apiextensions
     # succeed to update the CRDs as the cluster could be in any version
@@ -162,12 +164,18 @@ def setup_all_custom_resources(
         kube_client.apiextensions,
         kube_client.apiextensions_v1_beta1,
     ]:
-        cluster_crds = {
-            crd.spec.names.kind
-            for crd in apiextension.list_custom_resource_definition(
+
+        try:
+            crds_list = apiextension.list_custom_resource_definition(
                 label_selector=paasta_prefixed("service")
             ).items
-        }
+        except ApiException:
+            log.debug(
+                "Listing CRDs with apiextensions/v1 not supported on this cluster, falling back to v1beta1"
+            )
+            crds_list = []
+
+        cluster_crds = {crd.spec.names.kind for crd in crds_list}
         log.debug(f"CRDs found: {cluster_crds}")
         results = []
         for crd in custom_resource_definitions:
@@ -202,11 +210,11 @@ def setup_all_custom_resources(
         if results:
             got_results = True
             if any(results):
-                return True
+                succeeded = True
     # we want to return True if we never called `setup_custom_resources`
     # (i.e., we noop'd) or if any call to `setup_custom_resources`
     # succeed (handled above) - otherwise, we want to return False
-    return not got_results
+    return succeeded or not got_results
 
 
 def setup_custom_resources(

diff --git a/paasta_tools/setup_tron_namespace.py b/paasta_tools/setup_tron_namespace.py
@@ -105,12 +105,16 @@ def main():
         log.info(f"{master_config}")
         updated.append(MASTER_NAMESPACE)
     else:
-        if client.update_namespace(MASTER_NAMESPACE, master_config):
-            updated.append(MASTER_NAMESPACE)
-            log.debug(f"Updated {MASTER_NAMESPACE}")
-        else:
-            skipped.append(MASTER_NAMESPACE)
-            log.debug(f"Skipped {MASTER_NAMESPACE}")
+        try:
+            if client.update_namespace(MASTER_NAMESPACE, master_config):
+                updated.append(MASTER_NAMESPACE)
+                log.debug(f"Updated {MASTER_NAMESPACE}")
+            else:
+                skipped.append(MASTER_NAMESPACE)
+                log.debug(f"Skipped {MASTER_NAMESPACE}")
+        except Exception:
+            failed.append(MASTER_NAMESPACE)
+            log.exception(f"Error while updating {MASTER_NAMESPACE}:")
 
     k8s_enabled_for_cluster = (
         yaml.safe_load(master_config).get("k8s_options", {}).get("enabled", False)