SPLAT-2171: Added AWS dedicated host scripts and job (#70274)

* Added provision/deprovision logic for dedicated hosts

* Added AWS dedicated hosts job

Author: vr4manta

ci-operator/config/openshift/release/openshift-release-master__nightly-4.21.yaml

@@ -833,6 +833,25 @@ tests:
       FEATURE_SET: TechPreviewNoUpgrade
     workflow: openshift-e2e-vsphere
   timeout: 6h0m0s
+- as: e2e-aws-ovn-dedicated
+  cron: '@yearly'
+  steps:
+    cluster_profile: aws
+    env:
+      DEDICATED_HOST: "yes"
+    leases:
+    - env: LEASED_RESOURCE
+      resource_type: aws-edge-zones-quota-slice
+    observers:
+      enable:
+      - observers-resource-watch
+    post:
+    - chain: gather-network
+    - chain: gather-core-dump
+    - chain: ipi-deprovision
+    - ref: ipi-deprovision-aws-dedicated-hosts
+    workflow: openshift-e2e-aws-ovn
+  timeout: 6h0m0s
 - as: e2e-aws-ovn-fips
   interval: 168h
   steps:

ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml

@@ -162722,6 +162722,83 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build11
+  cron: '@yearly'
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+    timeout: 6h0m0s
+  extra_refs:
+  - base_ref: master
+    org: openshift
+    repo: release
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws
+    ci-operator.openshift.io/variant: nightly-4.21
+    ci.openshift.io/generator: prowgen
+    ci.openshift.io/no-builds: "true"
+    job-release: "4.21"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-release-master-nightly-4.21-e2e-aws-ovn-dedicated
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=e2e-aws-ovn-dedicated
+      - --variant=nightly-4.21
+      command:
+      - ci-operator
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build11
   cron: '@weekly'

ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-commands.sh

@@ -386,3 +386,43 @@ platform:
 EOF
   yq-go m -a -x -i "${CONFIG}" "${patch_user_provisioned_dns}"
 fi
+
+# Add config for dedicated hosts to compute nodes if job is configured
+if [[ "${DEDICATED_HOST}" == "yes" ]]; then
+  echo "Detected dedicated host configured.  Starting install-config patching."
+  patch_dedicated_host="${SHARED_DIR}/install-config-dedicated-host.yaml.patch"
+
+  # Create Host for each zone.  If no zones configured, error out.  Zones can exist before script execution so we'll pull zone listing out for workers.
+  WORKER_ZONES=$(cat "${CONFIG}" | yq-v4 '.compute[] | select(.name == "worker") | .platform.aws.zones'[] )
+  if [[ "${WORKER_ZONES}" == "" ]]; then
+    echo "No zones configured,  Unable to determine where to create dedicated hosts."
+    exit
+  fi
+
+  cat > "${patch_dedicated_host}" << EOF
+compute:
+- name: worker
+  platform:
+    aws:
+      dedicatedHosts:
+        hostAffinity: Host
+        hosts: []
+EOF
+
+  for zone in ${WORKER_ZONES}; do
+    HOST_TYPE=$(echo "${COMPUTE_NODE_TYPE}" | cut -d'.' -f1)
+    echo "Creating dedicated host.  Region='${aws_source_region}' Zone='${zone}' InstanceFamily='${HOST_TYPE}'"
+
+    EXPIRATION_DATE=$(date -d '6 hours' --iso=minutes --utc)
+    HOST_SPECS='{"ResourceType":"dedicated-host","Tags":[{"Key":"Name","Value":"'${JOB_NAME_SAFE}'-'${zone}'"},{"Key":"CI-JOB","Value":"'${JOB_NAME_SAFE}'"},{"Key":"expirationDate","Value":"'${EXPIRATION_DATE}'"},{"Key":"ci-build-info","Value":"'${BUILD_ID}_${JOB_NAME}'"}]}'
+    HOST_ID=$(aws ec2 allocate-hosts --instance-type "${HOST_TYPE}.4xlarge" --auto-placement 'off' --host-recovery 'off' --tag-specifications "${HOST_SPECS}" --host-maintenance 'off' --quantity '1' --availability-zone "${zone}" --region "${aws_source_region}" | jq -r '.HostIds[0]')
+
+    # We need to pass in the vars since YQ doesnt see the loop variables
+    ZONE_NAME="${zone}" HOST_ID="${HOST_ID}" yq-v4 -i '.compute[] |= (select(.name == "worker") | .platform.aws.dedicatedHosts.hosts += [ { "id": strenv(HOST_ID), "zone": strenv(ZONE_NAME) } ])' "${patch_dedicated_host}"
+  done
+
+  # Update config with host ID
+  echo "Patching install-config.yaml for dedicated hosts."
+  yq-go m -x -i ${CONFIG} ${patch_dedicated_host}
+  cp "${patch_dedicated_host}" "${ARTIFACT_DIR}/"
+fi

ci-operator/step-registry/ipi/conf/aws/ipi-conf-aws-ref.yaml

@@ -106,5 +106,10 @@ ref:
     documentation: |-
       Experimental feature allowing jobs to use NAT instances instead of NAT gateways, in certain accounts, for cost
       reduction purposes.
+  - name: DEDICATED_HOST
+    default: "no"
+    documentation: |-
+      Allows users to enable configuration of dedicated hosts for compute nodes.  Valid options are "yes" and "no".  When "yes", the
+      configuration will create a dedicated host for each zone the "worker" compute pool has configured.
   documentation: |-
     The IPI AWS configure step generates the AWS-specific install-config.yaml contents based on the cluster profile and optional input files.

ci-operator/step-registry/ipi/deprovision/aws/dedicated-hosts/OWNERS

@@ -0,0 +1 @@
+../OWNERS

ci-operator/step-registry/ipi/deprovision/aws/dedicated-hosts/ipi-deprovision-aws-dedicated-hosts-commands.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -o nounset
+set -o errexit
+set -o pipefail
+
+export AWS_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/.awscred"
+CONFIG="${SHARED_DIR}/install-config.yaml"
+patch_dedicated_host="${SHARED_DIR}/install-config-dedicated-host.yaml.patch"
+
+if test ! -f "${patch_dedicated_host}"
+then
+  echo "No dedicated hosts patch file found, so assuming patch never occurred."
+  exit 0
+fi
+
+echo "Deprovisioning dedicated hosts..."
+
+# We get the region information from the install-config.yaml.  For the dedicated hosts, we are pulling from the patch file in
+# the event that an error occurred during creation of the dedicated host.
+REGION=$(yq-v4 -r '.platform.aws.region' ${CONFIG})
+for HOST in $(yq-v4 -r '.compute[] | select(.name == "worker") | .platform.aws.dedicatedHosts.hosts[] | .id' "${patch_dedicated_host}"); do
+  echo "Release host ${HOST}"
+  aws ec2 release-hosts --region "${REGION}" --host-ids "${HOST}"
+done

ci-operator/step-registry/ipi/deprovision/aws/dedicated-hosts/ipi-deprovision-aws-dedicated-hosts-ref.metadata.json

@@ -0,0 +1,10 @@
+{
+	"path": "ipi/deprovision/aws/dedicated-hosts/ipi-deprovision-aws-dedicated-hosts-ref.yaml",
+	"owners": {
+		"approvers": [
+			"jhixson74",
+			"patrickdillon",
+			"barbacbd"
+		]
+	}
+}

ci-operator/step-registry/ipi/deprovision/aws/dedicated-hosts/ipi-deprovision-aws-dedicated-hosts-ref.yaml

@@ -0,0 +1,11 @@
+ref:
+  as: ipi-deprovision-aws-dedicated-hosts
+  from: upi-installer
+  grace_period: 10m
+  commands: ipi-deprovision-aws-dedicated-hosts-commands.sh
+  resources:
+    requests:
+      cpu: 300m
+      memory: 300Mi
+  documentation: |-
+    This deprovision step tears down any dedicated hosts that were provisioned for AWS IPI dedicated host feature.