From 7817a99c15043cc20c7675cb5977f2bc96d9ee4d Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 7 Nov 2023 14:11:42 -0500 Subject: [PATCH 01/14] Add GHCR build --- .github/workflows/build_container_image.yaml | 62 +++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_container_image.yaml b/.github/workflows/build_container_image.yaml index 53a4a285a73d..dbff3546e9ae 100644 --- a/.github/workflows/build_container_image.yaml +++ b/.github/workflows/build_container_image.yaml @@ -10,8 +10,67 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + ghcrbuild: + name: Build container image for GHCR + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # https://stackoverflow.com/questions/59810838/how-to-get-the-short-sha-for-the-github-workflow + - name: Set outputs + id: commit + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Set branch name + id: branch + run: | + if [[ "$GITHUB_REF" == "refs/tags/"* ]]; then + echo "name=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT + elif [[ "$GITHUB_REF" == "refs/heads/dev" ]]; then + echo "name=dev" >> $GITHUB_OUTPUT + elif [[ "$GITHUB_REF" == "refs/heads/release_"* ]]; then + echo "name=${GITHUB_REF#refs/heads/release_}-auto" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Extract metadata for container image + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=raw,value=${{steps.branch.outputs.name}} + - name: Build args + id: buildargs + run: | + echo "gitcommit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "builddate=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + platforms: linux/amd64 + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push container image to ghcr + uses: docker/build-push-action@v4 + with: + build-args: | + GIT_COMMIT=${{ steps.buildargs.outputs.gitcommit }} + BUILD_DATE=${{ steps.buildargs.outputs.builddate }} + IMAGE_TAG=${{ steps.branch.outputs.name }} + file: .k8s_ci.Dockerfile + push: true + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + build: - name: Build container image + name: Build container image for Galaxy repos runs-on: ubuntu-latest if: github.repository_owner == 'galaxyproject' steps: @@ -58,3 +117,4 @@ jobs: uses: actions-hub/docker@master with: args: push galaxy/galaxy-min:${{ steps.branch.outputs.name }} + From ffc9faa600c49099e1b90c002a157787673cbe90 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Fri, 5 Jul 2024 17:23:43 -0400 Subject: [PATCH 02/14] Update pykube version --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index f35fa4fc6cc1..46a381cac1c5 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -36,7 +36,7 @@ custos-sdk chronos-python==1.2.1 # Kubernetes job runner -pykube-ng==21.3.0 +pykube-ng==23.6.0 # Synnefo / Pithos+ object store client kamaki From 83eca9f81a944715159b5d6a0b01e6f8866369f4 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Thu, 7 Sep 2023 16:43:01 -0400 Subject: [PATCH 03/14] Add tls secret override --- lib/galaxy/jobs/runners/kubernetes.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 56fef20a0ecb..917bb5e54009 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -109,6 +109,7 @@ def __init__(self, app, nworkers, **kwargs): k8s_interactivetools_use_ssl=dict(map=bool, default=False), k8s_interactivetools_ingress_annotations=dict(map=str), k8s_interactivetools_ingress_class=dict(map=str, default=None), + k8s_interactivetools_tls_secret=dict(map=str, default=None), ) if "runner_param_specs" not in kwargs: @@ -496,9 +497,18 @@ def __get_k8s_ingress_spec(self, ajs): k8s_spec_template["spec"]["ingressClassName"] = default_ingress_class if self.runner_params.get("k8s_interactivetools_use_ssl"): domains = list({e["domain"] for e in entry_points}) - k8s_spec_template["spec"]["tls"] = [ - {"hosts": [domain], "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains - ] + override_secret = self.runner_params.get("k8s_interactivetools_tls_secret") + if override_secret: + k8s_spec_template["spec"]["tls"] = [ + {"hosts": [domain], "secretName": override_secret} for domain in domains + ] + else: + k8s_spec_template["spec"]["tls"] = [ + { + "hosts": [domain], + "secretName": re.sub("[^a-z0-9-]", "-", domain) + } for domain in domains + ] if self.runner_params.get("k8s_interactivetools_ingress_annotations"): new_ann = yaml.safe_load(self.runner_params.get("k8s_interactivetools_ingress_annotations")) k8s_spec_template["metadata"]["annotations"].update(new_ann) From d387970ce2cce47ebd05d78f5705d75a0230e051 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Fri, 15 Sep 2023 16:51:55 -0400 Subject: [PATCH 04/14] Add support for multiple ingress versions --- lib/galaxy/jobs/runners/kubernetes.py | 76 ++++++++++++++++++--------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 917bb5e54009..79a1b13a0c05 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -20,6 +20,7 @@ ) from galaxy.jobs.runners.util.pykube_util import ( deduplicate_entries, + DEFAULT_INGRESS_API_VERSION, DEFAULT_JOB_API_VERSION, delete_ingress, delete_job, @@ -110,6 +111,7 @@ def __init__(self, app, nworkers, **kwargs): k8s_interactivetools_ingress_annotations=dict(map=str), k8s_interactivetools_ingress_class=dict(map=str, default=None), k8s_interactivetools_tls_secret=dict(map=str, default=None), + k8s_ingress_api_version=dict(map=str, default=DEFAULT_INGRESS_API_VERSION), ) if "runner_param_specs" not in kwargs: @@ -250,6 +252,7 @@ def __configure_port_routing(self, ajs): service = Service(self._pykube_api, k8s_service_obj) service.create() ingress = Ingress(self._pykube_api, k8s_ingress_obj) + ingress.version = self.runner_params.get("k8s_ingress_api_version") ingress.create() def __get_overridable_params(self, job_wrapper, param_key): @@ -430,6 +433,54 @@ def __get_k8s_service_spec(self, ajs): } return k8s_spec_template + def __get_k8s_ingress_rules_spec(self, ajs, entry_points): + """This represents the template for the "rules" portion of the Ingress spec.""" + if "v1beta1" in self.runner_params.get("k8s_ingress_api_version"): + rules_spec = [ + { + "host": ep["domain"], + "http": { + "paths": [ + { + "backend": { + "serviceName": self.__get_k8s_job_name( + self.__produce_k8s_job_prefix(), ajs.job_wrapper + ), + "servicePort": int(ep["tool_port"]), + }, + "path": ep.get("entry_path", "/"), + "pathType": "Prefix", + } + ] + }, + } + for ep in entry_points + ] + else: + rules_spec = [ + { + "host": ep["domain"], + "http": { + "paths": [ + { + "backend": { + "service": { + "name": self.__get_k8s_job_name( + self.__produce_k8s_job_prefix(), ajs.job_wrapper + ), + "port": {"number": int(ep["tool_port"])}, + } + }, + "path": ep.get("entry_path", "/"), + "pathType": "ImplementationSpecific", + } + ] + }, + } + for ep in entry_points + ] + return rules_spec + def __get_k8s_ingress_spec(self, ajs): """The k8s spec template is nothing but a Ingress spec, except that it is nested and does not have an apiversion nor kind.""" @@ -467,30 +518,7 @@ def __get_k8s_ingress_spec(self, ajs): }, "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id}, }, - "spec": { - "rules": [ - { - "host": ep["domain"], - "http": { - "paths": [ - { - "backend": { - "service": { - "name": self.__get_k8s_job_name( - self.__produce_k8s_job_prefix(), ajs.job_wrapper - ), - "port": {"number": int(ep["tool_port"])}, - } - }, - "path": ep.get("entry_path", "/"), - "pathType": "Prefix", - } - ] - }, - } - for ep in entry_points - ], - }, + "spec": {"rules": self.__get_k8s_ingress_rules_spec(ajs, entry_points)}, } default_ingress_class = self.runner_params.get("k8s_interactivetools_ingress_class") if default_ingress_class: From 8274d7d5ff36837a5f0a9b843ebcec14bbc12b08 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Mon, 8 Jul 2024 12:25:39 -0400 Subject: [PATCH 05/14] Linting indentation fix --- lib/galaxy/jobs/runners/kubernetes.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 79a1b13a0c05..32c3169f7e40 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -532,10 +532,8 @@ def __get_k8s_ingress_spec(self, ajs): ] else: k8s_spec_template["spec"]["tls"] = [ - { - "hosts": [domain], - "secretName": re.sub("[^a-z0-9-]", "-", domain) - } for domain in domains + {"hosts": [domain], + "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains ] if self.runner_params.get("k8s_interactivetools_ingress_annotations"): new_ann = yaml.safe_load(self.runner_params.get("k8s_interactivetools_ingress_annotations")) From d3717adfe23461f4ecf4dca314f91b3dd8836e68 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Mon, 8 Jul 2024 13:18:19 -0400 Subject: [PATCH 06/14] Linting via black --- lib/galaxy/jobs/runners/kubernetes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 32c3169f7e40..f01bbbe4cf8b 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -532,8 +532,7 @@ def __get_k8s_ingress_spec(self, ajs): ] else: k8s_spec_template["spec"]["tls"] = [ - {"hosts": [domain], - "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains + {"hosts": [domain], "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains ] if self.runner_params.get("k8s_interactivetools_ingress_annotations"): new_ann = yaml.safe_load(self.runner_params.get("k8s_interactivetools_ingress_annotations")) From 4bc02aa66b7b543c0d72ec754cd6ce9e65dc4ba7 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Mon, 15 Jul 2024 15:42:23 -0400 Subject: [PATCH 07/14] Update lib/galaxy/jobs/runners/kubernetes.py Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- lib/galaxy/jobs/runners/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index f01bbbe4cf8b..d7daf9e86da3 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -252,7 +252,7 @@ def __configure_port_routing(self, ajs): service = Service(self._pykube_api, k8s_service_obj) service.create() ingress = Ingress(self._pykube_api, k8s_ingress_obj) - ingress.version = self.runner_params.get("k8s_ingress_api_version") + ingress.version = self.runner_params["k8s_ingress_api_version"] ingress.create() def __get_overridable_params(self, job_wrapper, param_key): From 89c03974d9d98fe8f5576517d3ac1c5ea440ea78 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Mon, 15 Jul 2024 15:47:39 -0400 Subject: [PATCH 08/14] Apply fix to other get --- lib/galaxy/jobs/runners/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index d7daf9e86da3..62518ae14acf 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -435,7 +435,7 @@ def __get_k8s_service_spec(self, ajs): def __get_k8s_ingress_rules_spec(self, ajs, entry_points): """This represents the template for the "rules" portion of the Ingress spec.""" - if "v1beta1" in self.runner_params.get("k8s_ingress_api_version"): + if "v1beta1" in self.runner_params["k8s_ingress_api_version"]: rules_spec = [ { "host": ep["domain"], From b42b416a27ef1f5b77d2fa3e3167dd796d0d66bd Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Fri, 5 Jul 2024 18:04:10 -0400 Subject: [PATCH 09/14] Pod status check for k8s runner jobs Change active check for ITs --- lib/galaxy/jobs/runners/kubernetes.py | 16 +++++++++++++++- lib/galaxy/jobs/runners/util/pykube_util.py | 12 ++++++++++++ lib/galaxy/model/__init__.py | 3 +-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 62518ae14acf..f7e61c20e7d2 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -35,6 +35,7 @@ HTTPError, Ingress, ingress_object_dict, + is_pod_running, is_pod_unschedulable, Job, job_object_dict, @@ -802,10 +803,12 @@ def check_watched_item(self, job_state): pass else: pass - else: + elif self.__check_job_pod_running(job_state): log.debug("Job set to running...") job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) + else: + pass return job_state elif job_persisted_state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has not been deleted, @@ -956,6 +959,17 @@ def __job_failed_due_to_low_memory(self, job_state): return False + def __check_job_pod_running(self, job_state): + """ + checks the state of the pod to see if it is unschedulable. + """ + pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) + if not pods.response["items"]: + return False + + pod = Pod(self._pykube_api, pods.response["items"][0]) + return is_pod_running(self._pykube_api, pod, self.runner_params["k8s_namespace"]) + def __job_pending_due_to_unschedulable_pod(self, job_state): """ checks the state of the pod to see if it is unschedulable. diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index 6fc2c039f741..19fda9d68a02 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -94,6 +94,17 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None): return Pod.objects(pykube_api).filter(selector=f"job-name={job_name}", namespace=namespace) +def is_pod_running(pykube_api, pod, namespace=None): + is_running = not any( + c.get("state", {}).get("running", {}).get("startedAt", False) == False + for c in pod.obj["status"].get("containerStatuses", []) + ) + if pod.obj["status"].get("phase") == "Running" and is_running: + return True + + return False + + def is_pod_unschedulable(pykube_api, pod, namespace=None): is_unschedulable = any(c.get("reason") == "Unschedulable" for c in pod.obj["status"].get("conditions", [])) if pod.obj["status"].get("phase") == "Pending" and is_unschedulable: @@ -311,6 +322,7 @@ def galaxy_instance_id(params): "find_pod_object_by_name", "galaxy_instance_id", "HTTPError", + "is_pod_running", "is_pod_unschedulable", "Job", "Service", diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 586543f0f9c4..72f29af77791 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2912,8 +2912,7 @@ def __init__( @property def active(self): if self.configured and not self.deleted: - # FIXME: don't included queued? - return not self.job.finished + return self.job.running return False @property From 18d3dfc37819aeb019acd68ac1dd19399f79ee5d Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 16 Jul 2024 16:33:26 -0400 Subject: [PATCH 10/14] Fix comment --- lib/galaxy/jobs/runners/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index f7e61c20e7d2..4094f2863e90 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -961,7 +961,7 @@ def __job_failed_due_to_low_memory(self, job_state): def __check_job_pod_running(self, job_state): """ - checks the state of the pod to see if it is unschedulable. + checks the state of the pod to see if it is running. """ pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) if not pods.response["items"]: From 521b85319231b984e7950e36bd9d9a440fcffcb0 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 30 Jul 2024 13:30:27 -0400 Subject: [PATCH 11/14] Update lib/galaxy/jobs/runners/kubernetes.py Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- lib/galaxy/jobs/runners/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 4094f2863e90..33e503270b78 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -808,7 +808,7 @@ def check_watched_item(self, job_state): job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) else: - pass + log.debug(f"Job id: {job_state.job_id} with k8s id: {job.name} scheduled and is waiting to start...") return job_state elif job_persisted_state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has not been deleted, From 090b29fb11abe5a6e307882e0dc9c8846584ffc3 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 30 Jul 2024 13:37:26 -0400 Subject: [PATCH 12/14] Linting fix --- lib/galaxy/jobs/runners/kubernetes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 33e503270b78..ce94e12e5673 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -808,7 +808,9 @@ def check_watched_item(self, job_state): job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) else: - log.debug(f"Job id: {job_state.job_id} with k8s id: {job.name} scheduled and is waiting to start...") + log.debug( + f"Job id: {job_state.job_id} with k8s id: {job.name} scheduled and is waiting to start..." + ) return job_state elif job_persisted_state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has not been deleted, From e83aa79bc9f9b840d1ba59b267da7ae2f3eac970 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 30 Jul 2024 13:53:37 -0400 Subject: [PATCH 13/14] not any not -> all --- lib/galaxy/jobs/runners/util/pykube_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index 19fda9d68a02..0c4861bf22ae 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -95,8 +95,8 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None): def is_pod_running(pykube_api, pod, namespace=None): - is_running = not any( - c.get("state", {}).get("running", {}).get("startedAt", False) == False + is_running = all( + c.get("state", {}).get("running", {}).get("startedAt", False) for c in pod.obj["status"].get("containerStatuses", []) ) if pod.obj["status"].get("phase") == "Running" and is_running: From ceec2f075efa178ee1a43dc8414d5f9b8de30664 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 30 Jul 2024 13:59:29 -0400 Subject: [PATCH 14/14] Check only phase --- lib/galaxy/jobs/runners/util/pykube_util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index 0c4861bf22ae..b6122dd8aa00 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -95,11 +95,7 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None): def is_pod_running(pykube_api, pod, namespace=None): - is_running = all( - c.get("state", {}).get("running", {}).get("startedAt", False) - for c in pod.obj["status"].get("containerStatuses", []) - ) - if pod.obj["status"].get("phase") == "Running" and is_running: + if pod.obj["status"].get("phase") == "Running": return True return False