(Dev Testing): feat: (IAC-1211) Add optional helm authentication for dark site OCI Container Registries

Dockerfile

@@ -9,17 +9,17 @@ RUN apt-get update && apt-get upgrade -y \
   && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
 
 FROM baseline as tool_builder
-ARG kubectl_version=1.27.9
+ARG kubectl_version=1.27.11
 
 WORKDIR /build
 
 RUN curl -sLO https://storage.googleapis.com/kubernetes-release/release/v$kubectl_version/bin/linux/amd64/kubectl && chmod 755 ./kubectl
 
 # Installation
 FROM baseline
-ARG helm_version=3.14.0
-ARG aws_cli_version=2.13.33
-ARG gcp_cli_version=460.0.0-0
+ARG helm_version=3.14.2
+ARG aws_cli_version=2.15.22
+ARG gcp_cli_version=464.0.0
 
 # Add extra packages
 RUN apt-get update && apt-get install --no-install-recommends -y gzip wget git jq ssh sshpass skopeo rsync \

docs/CONFIG-VARS.md

@@ -148,12 +148,13 @@ When V4_CFG_MANAGE_STORAGE is set to `true`, the `sas` and `pg-storage` storage
 | V4_CFG_CR_PASSWORD | Container registry password | string | | false | By default, credentials are included in the downloaded deployment assets. | viya |
 | V4_CFG_CR_URL | Container registry server | string | https://cr.sas.com | false | | viya |
 
+
 ## Ingress
 
 | Name | Description | Type | Default | Required | Notes | Tasks |
 | :--- | ---: | ---: | ---: | ---: | ---: | ---: |
 | V4_CFG_INGRESS_TYPE | The ingress controller to deploy | string | "ingress" | true | Possible values: "ingress" | baseline, viya |
-| V4_CFG_INGRESS_FQDN | FQDN to the ingress for SAS Vya installation | string | | true | | viya |
+| V4_CFG_INGRESS_FQDN | FQDN to the ingress for SAS Viya installation | string | | true | | viya |
 | V4_CFG_INGRESS_MODE | Whether to create a public or private Loadbalancer endpoint | string | "public" | false | Possible values: "public", "private". Setting this option to "private" adds options to the ingress controller that create a LoadBalancer with private IP address(es) only. | baseline |
 
 ## Load Balancer
@@ -342,16 +343,16 @@ V4_CFG_POSTGRES_SERVERS:
 | :--- |------------:| ---: | ---: | ---: | ---: | ---: |
 | V4_WORKLOAD_ORCHESTRATOR_ENABLED | Enables the [SAS Workload Orchestrator](https://documentation.sas.com/?cdcId=itopscdc&cdcVersion=default&docsetId=dplyml0phy0dkr&docsetTarget=n08u2yg8tdkb4jn18u8zsi6yfv3d.htm#p1vo217m7ffso5n11vxwsyycw4tg) service and configures the required ClusterRole and ClusterRoleBinding used by the daemon. Setting this to false will disable SAS Workload Orchestrator service entirely | bool | true | false | This flag is only applicable for cadences 2023.08 and newer, this flag will perform no action on older cadences. | viya |
 
-The SAS Workload Orchestrator Service is used to manage workload started on demand through the launcher service. As of cadence 2023.08 this feature is now deployed by default. The SAS Workload Orchestrator daemons require information about resources on the nodes that can be used to run jobs. In order to obtain accurate resource information, it requires a ClusterRole and a ClusterRoleBinding to the SAS Workload Orchestrator service account which will be automatically configured by this project if you set `V4_WORKLOAD_ORCHESTRATOR_ENABLED` to true. 
+The SAS Workload Orchestrator Service is used to manage workload started on demand through the launcher service. As of cadence 2023.08 this feature is now deployed by default. The SAS Workload Orchestrator daemons require information about resources on the nodes that can be used to run jobs. In order to obtain accurate resource information, it requires a ClusterRole and a ClusterRoleBinding to the SAS Workload Orchestrator service account which will be automatically configured by this project if you set `V4_WORKLOAD_ORCHESTRATOR_ENABLED` to true.
 
-Additional documentation for the SAS Workload Orchestrator Service can be found here in the [SAS Viya Platform Operations documentation](https://documentation.sas.com/?cdcId=itopscdc&cdcVersion=default&docsetId=dplyml0phy0dkr&docsetTarget=n08u2yg8tdkb4jn18u8zsi6yfv3d.htm#p1vo217m7ffso5n11vxwsyycw4tg). 
+Additional documentation for the SAS Workload Orchestrator Service can be found here in the [SAS Viya Platform Operations documentation](https://documentation.sas.com/?cdcId=itopscdc&cdcVersion=default&docsetId=dplyml0phy0dkr&docsetTarget=n08u2yg8tdkb4jn18u8zsi6yfv3d.htm#p1vo217m7ffso5n11vxwsyycw4tg).
 
 ## Miscellaneous
 
 | Name | Description | Type | Default | Required | Notes | Tasks |
 | :--- | ---: | ---: | ---: | ---: | ---: | ---: |
 | V4_CFG_CLUSTER_NODE_POOL_MODE | The mode of cluster node pool to use | string | "standard" | false | [standard, minimal] | viya |
-| V4_CFG_EMBEDDED_LDAP_ENABLE | Deploy OpenLDAP in the namespace for authentication | bool | false | false | [Openldap Config](../roles/vdm/templates/generators/openldap-bootstrap-config.yaml) | viya |
+| V4_CFG_EMBEDDED_LDAP_ENABLE | Deploy OpenLDAP in the namespace for authentication | bool | false | false | [Openldap Config](../roles/vdm/templates/generators/openldap-bootstrap-config.yaml). If you do not set this value to true, you must set `V4_CFG_SITEDEFAULT` to point to a sitedefault file which contains values applicable for your authentication configuration. | viya |
 | V4_CFG_CONSUL_ENABLE_LOADBALANCER | Set up LoadBalancer to access the Consul user interface | bool | false | false | Consul UI port is 8500. | viya |
 | V4_CFG_ELASTICSEARCH_ENABLE | Enable search with Open Distro for ElasticSearch | bool | true | false | When deploying LTS earlier than 2020.1 or Stable earlier than 2020.1.2, set to false. | viya |
 | V4_CFG_VIYA_START_SCHEDULE | Configure your SAS Viya platform deployment to start on specific schedules | string |  | false | This variable accepts [CronJob schedule expressions](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#cron-schedule-syntax) to create your Viya start job schedule. See note below. | viya |

docs/user/Dependencies.md

@@ -6,28 +6,29 @@ The following list details our dependencies and versions (~ indicates multiple p
 
 | SOURCE         | NAME             | VERSION     |
 |----------------|------------------|-------------|
-| ~              | python           | >=3.10       |
+| ~              | python           | >=3.10      |
 | ~              | pip              | 3.x         |
 | ~              | unzip            | any         |
 | ~              | tar              | any         |
-| ~              | docker           | >=20.10.10  |
+| ~              | docker           | >=25.0.3    |
 | ~              | git              | any         |
 | ~              | rsync            | any         |
 | ~              | kubectl          | 1.26 - 1.28 |
-| ~              | Helm             | 3.14.0      |
-| pip3           | ansible          | 9.1.0       |
-| pip3           | openshift        | 0.13.1      |
-| pip3           | kubernetes       | 26.1.0      |
-| pip3           | dnspython        | 2.3.0       |
-| pip3           | docker           | 5.0.3       |
-| ansible-galaxy | community.docker | 2.7.8       |
-| ansible-galaxy | ansible.utils    | 2.3.0       |
-| ansible-galaxy | kubernetes.core  | 2.3.2       |
+| ~              | Helm             | 3.14.2      |
+| pip3           | ansible          | 9.2.0       |
+| pip3           | openshift        | 0.13.2      |
+| pip3           | kubernetes       | 27.2.0      |
+| pip3           | dnspython        | 2.6.1       |
+| pip3           | docker           | 7.0.0       |
+| pip3           | urllib3          | 1.26.18     |
+| ansible-galaxy | community.docker | 3.8.0       |
+| ansible-galaxy | ansible.utils    | 3.1.0       |
+| ansible-galaxy | kubernetes.core  | 3.0.1       |
 
 If you are using a provider based kubeconfig file created by viya4-iac-gcp:4.5.0 or newer, install these dependencies:
 | SOURCE         | NAME                    | VERSION     |
 |----------------|-------------------------|-------------|
-| ~              | gcloud                  | 460.0.0     |
+| ~              | gcloud                  | 464.0.0     |
 | ~              | gcloud-gke-auth-plugin  | >= 0.5.2    |
 
 Required project dependencies are generally pinned to known working or stable versions to ensure users have a smooth initial experience. In some cases it may be required to change the default version of a dependency. In such cases users are welcome to experiment with alternate versions, however compatibility may not be guaranteed.
@@ -48,7 +49,7 @@ As described in the [Docker Installation](./DockerUsage.md) section add addition
 ```bash
 # Override kubectl version
 docker build \
-	--build-arg kubectl_version=1.27.9 \
+	--build-arg kubectl_version=1.27.11 \
 	-t viya4-deployment .
 ```
 

requirements.txt

@@ -1,6 +1,6 @@
-ansible==9.1.0 # 8.6.0 # 2.10.7
-openshift==0.13.1 # 0.12.0
-kubernetes==26.1.0 # 12.0.1
-dnspython==2.3.0 # 2.1.0
-docker==5.0.3
+ansible==9.2.0 # 9.1.0 # 8.6.0 # 2.10.7
+openshift==0.13.2 # 0.13.1 # 0.12.0
+kubernetes==27.2.0 # 26.1.0 # 12.0.1
+dnspython==2.6.1 # 2.3.0 # 2.1.0
+docker==7.0.0 # 5.0.3
 urllib3==1.26.18

requirements.yaml

@@ -1,8 +1,8 @@
 ---
 collections:
   - name: ansible.utils
-    version: 2.3.0
+    version: 3.1.0 # 2.3.0
   - name: community.docker
-    version: 2.7.8
+    version: 3.8.0 # 2.7.8
   - name: kubernetes.core
-    version: 2.3.2
+    version: 3.0.0 # 2.3.2

roles/baseline/defaults/main.yml

@@ -8,6 +8,7 @@ V4_CFG_INGRESS_TYPE: ingress
 V4_CFG_INGRESS_MODE: public
 V4_CFG_MANAGE_STORAGE: true
 V4_CFG_AWS_LB_SUBNETS: ""
+V4_CFG_DARK_SITE_ENABLED: false
 
 ## Cert-manager
 CERT_MANAGER_NAME: cert-manager

roles/baseline/tasks/main.yaml

@@ -3,6 +3,22 @@
 
 
 ---
+- name: Helm authenticate to private repository
+  when:
+    - V4_CFG_DARK_SITE_ENABLED is defined
+    - V4_CFG_DARK_SITE_ENABLED
+    - V4_CFG_CR_USER is defined
+    - V4_CFG_CR_USER is not none
+    - V4_CFG_CR_PASSWORD is defined
+    - V4_CFG_CR_PASSWORD is not none
+  command:
+    cmd: |
+      helm registry login {{ V4_CFG_CR_URL }} -u {{ V4_CFG_CR_USER }} --password-stdin
+  args:
+    stdin: "{{ V4_CFG_CR_PASSWORD }}"
+  tags:
+    - baseline
+
 - name: Include nfs-subdir-external-provisioner
   include_tasks:
     file: nfs-subdir-external-provisioner.yaml

roles/monitoring/templates/host-based/user-values-prom-operator.yaml

@@ -17,6 +17,12 @@ prometheus:
     - {{ V4M_PROMETHEUS_FQDN }}
   prometheusSpec:
     externalUrl: "https://{{ V4M_PROMETHEUS_FQDN }}"
+    alertingEndpoints:
+      - name: v4m-alertmanager
+        port: http-web
+        scheme: https
+        tlsConfig:
+          insecureSkipVerify: true
     storageSpec:
       volumeClaimTemplate:
         spec:

roles/monitoring/templates/path-based/user-values-prom-operator.yaml

@@ -24,7 +24,7 @@ grafana:
 
 # Note that Prometheus and Alertmanager do not have any
 # authentication configured by default, exposing an
-# unauthenticated applicaton without other restrictions
+# unauthenticated application without other restrictions
 # in place is insecure.
 
 prometheus:
@@ -49,6 +49,13 @@ prometheus:
   prometheusSpec:
     routePrefix: /prometheus
     externalUrl: "https://{{ V4M_BASE_DOMAIN }}/prometheus"
+    alertingEndpoints:
+      - name: v4m-alertmanager
+        port: http-web
+        pathPrefix: "/alertmanager"
+        scheme: https
+        tlsConfig:
+          insecureSkipVerify: true
 
 alertmanager:
   # Disable default configuration of NodePort

roles/vdm/defaults/main.yaml

@@ -19,6 +19,8 @@ V4_CFG_CR_PASSWORD: null
 V4_CFG_CR_URL: https://cr.sas.com
 V4_CFG_CR_HOST: '{{ V4_CFG_CR_URL | regex_replace("^https?:\/\/(.*)\/?", "\1") }}'
 
+V4_CFG_DARK_SITE_ENABLED: false
+
 V4_CFG_SAS_API_KEY: null
 V4_CFG_SAS_API_SECRET: null
 

roles/vdm/tasks/main.yaml

@@ -63,6 +63,24 @@
     - uninstall
     - update
 
+- name: Helm authenticate to private repository
+  when:
+    - V4_CFG_DARK_SITE_ENABLED is defined
+    - V4_CFG_DARK_SITE_ENABLED
+    - V4_CFG_CR_USER is defined
+    - V4_CFG_CR_USER is not none
+    - V4_CFG_CR_PASSWORD is defined
+    - V4_CFG_CR_PASSWORD is not none
+  command:
+    cmd: |
+      helm registry login {{ V4_CFG_CR_URL }} -u {{ V4_CFG_CR_USER }} --password-stdin
+  args:
+    stdin: "{{ V4_CFG_CR_PASSWORD }}"
+  tags:
+    - install
+    - uninstall
+    - update
+
 - name: CR access
   when:
     - V4_CFG_CR_USER is not none