From e53ead0468b373f5ff22dc44607e05ae26e9d312 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Mon, 12 Feb 2024 15:51:25 +0100 Subject: [PATCH] Configure dummy pipeline to emit warnings/errors (#984) * feat: Add --dry-run to deploy.py * Use --with-warnings/errors in server deployment * Make warnings/errors random in dummy processing * Add flag to enable random errors in dummy pipeline * Refactor deployment template to remove server env check --- deploy.py | 37 +++++++++++++------ .../loculus-preprocessing-deployment.yaml | 18 +++++---- kubernetes/loculus/values.yaml | 3 ++ preprocessing/dummy/main.py | 8 +++- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/deploy.py b/deploy.py index d0048572e..cc4bac09d 100755 --- a/deploy.py +++ b/deploy.py @@ -4,7 +4,6 @@ import subprocess import time from pathlib import Path -import json import os import yaml @@ -28,6 +27,7 @@ parser = argparse.ArgumentParser(description='Manage k3d cluster and helm installations.') subparsers = parser.add_subparsers(dest='subcommand', required=True, help='Subcommands') +parser.add_argument('--dry-run', action='store_true', help='Print commands instead of executing them') cluster_parser = subparsers.add_parser('cluster', help='Start the k3d cluster') cluster_parser.add_argument('--dev', action='store_true', @@ -52,6 +52,16 @@ args = parser.parse_args() +def run_command(command: list[str], **kwargs): + if args.dry_run: + if isinstance(command, str): + print(command) + else: + print(" ".join(map(str,command))) + return subprocess.CompletedProcess(args=command, returncode=0, stdout="", stderr="") + else: + return subprocess.run(command, **kwargs) + def main(): if args.subcommand == 'cluster': @@ -70,13 +80,13 @@ def handle_cluster(): remove_port(BACKEND_PORT_MAPPING) if args.delete: print(f"Deleting cluster '{CLUSTER_NAME}'.") - subprocess.run(['k3d', 'cluster', 'delete', CLUSTER_NAME]) + run_command(['k3d', 'cluster', 'delete', CLUSTER_NAME]) return if cluster_exists(CLUSTER_NAME): print(f"Cluster '{CLUSTER_NAME}' already exists.") else: - subprocess.run(f"k3d cluster create {CLUSTER_NAME} {' '.join(PORTS)} --agents 2", + run_command(f"k3d cluster create {CLUSTER_NAME} {' '.join(PORTS)} --agents 2", shell=True, check=True) while not is_traefik_running(): @@ -87,13 +97,13 @@ def handle_cluster(): def is_traefik_running(namespace='kube-system', label='app.kubernetes.io/name=traefik'): try: - result = subprocess.run(['kubectl', 'get', 'pods', '-n', namespace, '-l', label], + result = run_command(['kubectl', 'get', 'pods', '-n', namespace, '-l', label], capture_output=True, text=True) if result.returncode != 0: print(f"Error executing kubectl: {result.stderr}") return False - if 'Running' in result.stdout: + if 'Running' in result.stdout or args.dry_run: return True except subprocess.SubprocessError as e: print(f"Error checking Traefik status: {e}") @@ -106,13 +116,13 @@ def remove_port(port_mapping): def cluster_exists(cluster_name): - result = subprocess.run(['k3d', 'cluster', 'list'], capture_output=True, text=True) + result = run_command(['k3d', 'cluster', 'list'], capture_output=True, text=True) return cluster_name in result.stdout def handle_helm(): if args.uninstall: - subprocess.run(['helm', 'uninstall', HELM_RELEASE_NAME], check=True) + run_command(['helm', 'uninstall', HELM_RELEASE_NAME], check=True) return @@ -141,14 +151,14 @@ def handle_helm(): if get_codespace_name(): parameters += ['--set', "codespaceName="+get_codespace_name()] - subprocess.run(parameters, check=True) + run_command(parameters, check=True) def get_docker_config_json(): if args.dockerconfigjson: return args.dockerconfigjson else: - command = subprocess.run('base64 ~/.docker/config.json', capture_output=True, text=True, shell=True) + command = run_command('base64 ~/.docker/config.json', capture_output=True, text=True, shell=True) return command.stdout.replace('\n', '') @@ -156,7 +166,7 @@ def handle_helm_upgrade(): parameters = [ 'helm', 'upgrade', HELM_RELEASE_NAME, HELM_CHART_DIR, ] - subprocess.run(parameters, check=True) + run_command(parameters, check=True) def get_codespace_name(): return os.environ.get('CODESPACE_NAME', None) @@ -178,7 +188,7 @@ def generate_configs(): runtime_config_path = TEMP_DIR / 'runtime_config.json' generate_config(helm_chart, 'templates/loculus-website-config.yaml', runtime_config_path, codespace_name) - subprocess.run(['python', 'kubernetes/config-processor/config-processor.py', TEMP_DIR, output_dir], check=True) + run_command(['python', 'kubernetes/config-processor/config-processor.py', TEMP_DIR, output_dir], check=True) def generate_config(helm_chart, template, output_path, codespace_name=None): helm_template_cmd = ['helm', 'template', 'name-does-not-matter', helm_chart, '--show-only', template] @@ -189,7 +199,10 @@ def generate_config(helm_chart, template, output_path, codespace_name=None): helm_template_cmd.extend(['--set', 'environment=local']) helm_template_cmd.extend(['--set', 'disableWebsite=true']) helm_template_cmd.extend(['--set', 'disableBackend=true']) - helm_output = subprocess.run(helm_template_cmd, capture_output=True, text=True, check=True).stdout + helm_output = run_command(helm_template_cmd, capture_output=True, text=True, check=True).stdout + if args.dry_run: + return + parsed_yaml = yaml.safe_load(helm_output) config_data = parsed_yaml['data'][output_path.name] diff --git a/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml b/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml index b14041651..a8bbb6ed6 100644 --- a/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml +++ b/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml @@ -3,8 +3,7 @@ "http://host.k3d.internal:8079" "http://loculus-backend-service:8079" }} -{{- $keycloakHost := printf "authentication-%s" .Values.host }} - +{{- $keycloakHost := printf "https://authentication-%s" $.Values.host }} {{- if not .Values.disablePreprocessing }} {{- range $key, $value := .Values.instances }} --- @@ -35,11 +34,16 @@ spec: - "{{ $arg }}" {{- end }} - "--backend-host={{ $backendHost }}/{{ $key }}" - {{- if eq $.Values.environment "server" }} - - "--keycloak-host=https://{{ $keycloakHost}}" - {{- else }} - - "--keycloak-host=http://loculus-keycloak-service:8083" - {{- end -}} + - "--keycloak-host={{ $keycloakHost }}" + {{- if $value.preprocessing.warnings }} + - "--withWarnings" + {{- end }} + {{- if $value.preprocessing.errors }} + - "--withErrors" + {{- end }} + {{- if $value.preprocessing.randomWarnError }} + - "--randomWarnError" + {{- end }} {{- if $value.preprocessing.configFile }} - "--config=/etc/config/preprocessing-config.yaml" volumeMounts: diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 3a998a710..192a3f103 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -43,6 +43,9 @@ instances: image: ghcr.io/loculus-project/preprocessing-dummy args: - "--watch" + warnings: true + errors: true + randomWarnError: true referenceGenomes: nucleotideSequences: - name: "main" diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py index 386df4b6c..716e3b830 100644 --- a/preprocessing/dummy/main.py +++ b/preprocessing/dummy/main.py @@ -14,6 +14,7 @@ parser.add_argument("--watch", action="store_true", help="Watch and keep running. Fetches new data every 10 seconds.") parser.add_argument("--withErrors", action="store_true", help="Add errors to processed data.") parser.add_argument("--withWarnings", action="store_true", help="Add warnings to processed data.") +parser.add_argument("--randomWarnError", action="store_true", help="Make errors and warnings occur stochastically") parser.add_argument("--maxSequences", type=int, help="Max number of sequence entry versions to process.") parser.add_argument("--keycloak-host", type=str, default="http://172.0.0.1:8083", help="Host address of Keycloak") parser.add_argument("--keycloak-user", type=str, default="dummy_preprocessing_pipeline", @@ -27,6 +28,7 @@ watch_mode = args.watch addErrors = args.withErrors addWarnings = args.withWarnings +randomWarnError = args.randomWarnError keycloakHost = args.keycloak_host keycloakUser = args.keycloak_user keycloakPassword = args.keycloak_password @@ -90,7 +92,8 @@ def process(unprocessed: List[Sequence]) -> List[Sequence]: {"metadata": metadata, **mock_sequences}, ) - if addErrors: + disable_randomly = randomWarnError and random.choice([True, True, False]) + if addErrors and not disable_randomly: updated_sequence.errors = [ ProcessingAnnotation( [AnnotationSource(list(metadata.keys())[0], "Metadata")], @@ -107,7 +110,8 @@ def process(unprocessed: List[Sequence]) -> List[Sequence]: ), ] - if addWarnings: + disable_randomly = randomWarnError and random.choice([True, False]) + if addWarnings and not disable_randomly: updated_sequence.warnings = [ ProcessingAnnotation( [AnnotationSource(list(metadata.keys())[0], "Metadata")],