From 428ee5fb8453f3de549aab531883b1b8ad7ecdfd Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Fri, 19 Jul 2024 10:51:10 -0700 Subject: [PATCH 1/3] propagate kubernetes custom annotations to the loadbalancer --- .gitignore | 3 +++ sky/provision/kubernetes/network_utils.py | 3 +++ sky/templates/kubernetes-loadbalancer.yml.j2 | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index efa74dd744b..31b37d1eab0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ sky/clouds/service_catalog/data_fetchers/*.csv .vscode/ .idea/ .env + +# For editor files +*.swp diff --git a/sky/provision/kubernetes/network_utils.py b/sky/provision/kubernetes/network_utils.py index 844f84a04f5..b2de231005c 100644 --- a/sky/provision/kubernetes/network_utils.py +++ b/sky/provision/kubernetes/network_utils.py @@ -75,6 +75,8 @@ def fill_loadbalancer_template(namespace: str, service_name: str, with open(template_path, 'r', encoding='utf-8') as fin: template = fin.read() + annotations = skypilot_config.get_nested( + ('kubernetes', 'custom_metadata', 'annotations'), {}) j2_template = jinja2.Template(template) cont = j2_template.render( namespace=namespace, @@ -82,6 +84,7 @@ def fill_loadbalancer_template(namespace: str, service_name: str, ports=ports, selector_key=selector_key, selector_value=selector_value, + annotations=annotations, ) content = yaml.safe_load(cont) return content diff --git a/sky/templates/kubernetes-loadbalancer.yml.j2 b/sky/templates/kubernetes-loadbalancer.yml.j2 index 08d8b0cc64c..256f36236d4 100644 --- a/sky/templates/kubernetes-loadbalancer.yml.j2 +++ b/sky/templates/kubernetes-loadbalancer.yml.j2 @@ -5,6 +5,10 @@ service_spec: name: {{ service_name }} labels: parent: skypilot + annotations: + {%- for key, value in annotations.items() %} + {{ key }}: {{ value|tojson }} + {%- endfor %} spec: type: LoadBalancer selector: From 9ef566fc36091085cd53e9926923812aceb0faa0 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Fri, 19 Jul 2024 11:31:23 -0700 Subject: [PATCH 2/3] Adding annotation propagation for ingress kubernetes services --- sky/provision/kubernetes/network_utils.py | 3 +++ sky/templates/kubernetes-ingress.yml.j2 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/sky/provision/kubernetes/network_utils.py b/sky/provision/kubernetes/network_utils.py index b2de231005c..1a5056ce9f3 100644 --- a/sky/provision/kubernetes/network_utils.py +++ b/sky/provision/kubernetes/network_utils.py @@ -101,6 +101,8 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int, f'Template "{_INGRESS_TEMPLATE_NAME}" does not exist.') with open(template_path, 'r', encoding='utf-8') as fin: template = fin.read() + annotations = skypilot_config.get_nested( + ('kubernetes', 'custom_metadata', 'annotations'), {}) j2_template = jinja2.Template(template) cont = j2_template.render( namespace=namespace, @@ -112,6 +114,7 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int, ingress_name=ingress_name, selector_key=selector_key, selector_value=selector_value, + annotations=annotations, ) content = yaml.safe_load(cont) diff --git a/sky/templates/kubernetes-ingress.yml.j2 b/sky/templates/kubernetes-ingress.yml.j2 index 84b3e9d6998..def19da9aa7 100644 --- a/sky/templates/kubernetes-ingress.yml.j2 +++ b/sky/templates/kubernetes-ingress.yml.j2 @@ -5,6 +5,9 @@ ingress_spec: annotations: nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 + {%- for key, value in annotations.items() %} + {{ key }}: {{ value|tojson }} + {%- endfor %} name: {{ ingress_name }} namespace: {{ namespace }} spec: From 58e4d2b48f944427a6a6d99446cfd009d985f4e9 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Fri, 19 Jul 2024 11:34:48 -0700 Subject: [PATCH 3/3] propagating labels for kubernetes --- sky/provision/kubernetes/network_utils.py | 6 ++++++ sky/templates/kubernetes-ingress.yml.j2 | 4 ++++ sky/templates/kubernetes-loadbalancer.yml.j2 | 3 +++ 3 files changed, 13 insertions(+) diff --git a/sky/provision/kubernetes/network_utils.py b/sky/provision/kubernetes/network_utils.py index 1a5056ce9f3..5486214b794 100644 --- a/sky/provision/kubernetes/network_utils.py +++ b/sky/provision/kubernetes/network_utils.py @@ -77,6 +77,8 @@ def fill_loadbalancer_template(namespace: str, service_name: str, template = fin.read() annotations = skypilot_config.get_nested( ('kubernetes', 'custom_metadata', 'annotations'), {}) + labels = skypilot_config.get_nested( + ('kubernetes', 'custom_metadata', 'labels'), {}) j2_template = jinja2.Template(template) cont = j2_template.render( namespace=namespace, @@ -85,6 +87,7 @@ def fill_loadbalancer_template(namespace: str, service_name: str, selector_key=selector_key, selector_value=selector_value, annotations=annotations, + labels=labels, ) content = yaml.safe_load(cont) return content @@ -103,6 +106,8 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int, template = fin.read() annotations = skypilot_config.get_nested( ('kubernetes', 'custom_metadata', 'annotations'), {}) + labels = skypilot_config.get_nested( + ('kubernetes', 'custom_metadata', 'labels'), {}) j2_template = jinja2.Template(template) cont = j2_template.render( namespace=namespace, @@ -115,6 +120,7 @@ def fill_ingress_template(namespace: str, service_details: List[Tuple[str, int, selector_key=selector_key, selector_value=selector_value, annotations=annotations, + labels=labels, ) content = yaml.safe_load(cont) diff --git a/sky/templates/kubernetes-ingress.yml.j2 b/sky/templates/kubernetes-ingress.yml.j2 index def19da9aa7..d419b6b2893 100644 --- a/sky/templates/kubernetes-ingress.yml.j2 +++ b/sky/templates/kubernetes-ingress.yml.j2 @@ -2,6 +2,10 @@ ingress_spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: + labels: + {%- for label_key, label_value in labels.items() %} + {{ label_key }}: {{ label_value|tojson }} + {%- endfor %} annotations: nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 diff --git a/sky/templates/kubernetes-loadbalancer.yml.j2 b/sky/templates/kubernetes-loadbalancer.yml.j2 index 256f36236d4..7afc35ab334 100644 --- a/sky/templates/kubernetes-loadbalancer.yml.j2 +++ b/sky/templates/kubernetes-loadbalancer.yml.j2 @@ -5,6 +5,9 @@ service_spec: name: {{ service_name }} labels: parent: skypilot + {%- for label_key, label_value in labels.items() %} + {{ label_key }}: {{ label_value|tojson }} + {%- endfor %} annotations: {%- for key, value in annotations.items() %} {{ key }}: {{ value|tojson }}