solve the label issue of singlenode through affinity

Signed-off-by: hancheng wang <[email protected]> solve the label issue of singlenode through affinity Signed-off-by: hancheng wang <[email protected]> solve the label issue of singlenode through affinity Signed-off-by: hancheng wang <[email protected]>
vhive-serverless · Jun 13, 2024 · f1d48e8 · f1d48e8
1 parent 857f4fa
commit f1d48e8
Show file tree

Hide file tree

Showing 7 changed files with 91 additions and 48 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -55,7 +55,7 @@ jobs:
           KNATIVE_DOMAIN=$INGRESS_HOST.sslip.io
           kubectl patch configmap -n knative-serving config-domain -p "{\"data\": {\"$KNATIVE_DOMAIN\": \"\"}}"
           kubectl patch configmap -n knative-serving config-autoscaler -p "{\"data\": {\"allow-zero-initial-scale\": \"true\"}}"
-          kubectl patch configmap -n knative-serving config-features -p "{\"data\": {\"kubernetes.podspec-nodeselector\": \"enabled\"}}"
+          kubectl patch configmap -n knative-serving config-features -p "{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}"
           kubectl label node knative-control-plane loader-nodetype=worker
 
       - name: Build and run loader

diff --git a/config/prometh_values_kn.yaml b/config/prometh_values_kn.yaml
@@ -7,12 +7,26 @@ alertmanager:
   serviceMonitor:
     interval: "15s"
   alertmanagerSpec:
-    nodeSelector:
-      loader-nodetype: monitoring
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: loader-nodetype
+            operator: In
+            values:
+            - monitoring
+            - singlenode
 
 grafana:
-  nodeSelector:
-    loader-nodetype: monitoring
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: loader-nodetype
+          operator: In
+          values:
+          - monitoring
+          - singlenode
   serviceMonitor:
     interval: "15s"
     scrapeTimeout: "15s" # Cannot be larger than Prometheus scrape intervals
@@ -59,8 +73,15 @@ kubeProxy:
     interval: "15s"
 
 kube-state-metrics:
-  nodeSelector:
-      loader-nodetype: monitoring
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: loader-nodetype
+          operator: In
+          values:
+          - monitoring
+          - singlenode
   metricLabelsAllowlist:
     - pods=[*]
     - deployments=[app.kubernetes.io/name,app.kubernetes.io/component,app.kubernetes.io/instance]
@@ -76,17 +97,31 @@ prometheus-node-exporter:
 prometheusOperator:
   admissionWebhooks:
     deployment:
-      nodeSelector:
-        loader-nodetype: monitoring
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: loader-nodetype
+              operator: In
+              values:
+              - monitoring
+              - singlenode
     patch:
       resources: 
         limits:
           cpu: 1
           memory: 5Gi
         requests:
           memory: 150Mi 
-      nodeSelector:
-        loader-nodetype: monitoring
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+            - key: loader-nodetype
+              operator: In
+              values:
+              - monitoring
+              - singlenode
 
   serviceMonitor:
     interval: "15s"
@@ -97,8 +132,15 @@ prometheusOperator:
     requests:
       cpu: 100m
       memory: 100Mi
-  nodeSelector:
-    loader-nodetype: monitoring
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: loader-nodetype
+          operator: In
+          values:
+          - monitoring
+          - singlenode
 
   prometheusConfigReloader:
     resources:
@@ -115,8 +157,15 @@ prometheus:
 
   prometheusSpec:
     scrapeInterval: "15s"
-    nodeSelector:
-      loader-nodetype: monitoring
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: loader-nodetype
+            operator: In
+            values:
+            - monitoring
+            - singlenode
     serviceMonitorSelectorNilUsesHelmValues: false
     podMonitorSelectorNilUsesHelmValues: false
     walCompression: false

diff --git a/scripts/setup/create_multinode.sh b/scripts/setup/create_multinode.sh
@@ -254,8 +254,8 @@ function copy_k8s_certificates() {
     # Force placement of metrics collectors and instrumentation on the loader node and control plane on master
     label_nodes $MASTER_NODE $1 # loader node is second on the list, becoming first after arg shift
 
-    # patch knative to accept nodeselector
-    server_exec $MASTER_NODE "cd loader; kubectl patch configmap config-features -n knative-serving -p '{\"data\": {\"kubernetes.podspec-nodeselector\": \"enabled\"}}'"
+    server_exec $MASTER_NODE "kubectl patch configmap -n knative-serving config-features -p '{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}'"
+
 
     if [[ "$DEPLOY_PROMETHEUS" == true ]]; then
         $DIR/expose_infra_metrics.sh $MASTER_NODE

diff --git a/scripts/setup/create_singlenode_container.sh b/scripts/setup/create_singlenode_container.sh
@@ -28,7 +28,6 @@ SERVER=$1
 DIR="$(pwd)/scripts/setup/"
 
 source "$(pwd)/scripts/setup/setup.cfg"
-source "$(pwd)/scripts/setup/versions.cfg"
 
 server_exec() { 
     ssh -oStrictHostKeyChecking=no -p 22 "$SERVER" $1; 
@@ -54,20 +53,22 @@ server_exec() {
     server_exec 'tmux new -d -s cluster'
     server_exec 'tmux send-keys -t containerd "sudo containerd" ENTER'
     sleep 3
-    server_exec 'pushd ~/vhive/scripts > /dev/null && ./setup_tool create_one_node_cluster stock-only && popd > /dev/null'
-    server_exec 'tmux send-keys -t cluster "watch -n 0.5 kubectl get pods -A" ENTER'
 
-    # Setup github authentication.
-    ACCESS_TOKEH="$(cat $GITHUB_TOKEN)"
+    server_exec 'pushd ~/vhive/scripts > /dev/null && ./setup_tool prepare_one_node_cluster stock-only && popd > /dev/null'
+    server_exec 'kubectl label nodes --all loader-nodetype=singlenode'
+    server_exec 'pushd ~/vhive/scripts > /dev/null && ./setup_tool setup_master_node stock-only && popd > /dev/null'
+
+    server_exec 'tmux send-keys -t cluster "watch -n 0.5 kubectl get pods -A" ENTER'
 
-    server_exec 'echo -en "\n\n" | ssh-keygen -t rsa'
-    server_exec 'ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts'
-    # server_exec 'RSA=$(cat ~/.ssh/id_rsa.pub)'
+    # Enable affinity
+    server_exec "kubectl patch configmap -n knative-serving config-features -p '{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}'"
 
-    server_exec 'curl -H "Authorization: token '"$ACCESS_TOKEH"'" --data "{\"title\":\"'"key:\$(hostname)"'\",\"key\":\"'"\$(cat ~/.ssh/id_rsa.pub)"'\"}" https://api.github.com/user/keys'
-    # server_exec 'sleep 5'
+    # Patch init scale
+    server_exec 'cd loader; bash scripts/setup/patch_init_scale.sh'
 
-    $DIR/expose_infra_metrics.sh $SERVER
+    if [[ "$DEPLOY_PROMETHEUS" == true ]]; then
+        $DIR/expose_infra_metrics.sh $SERVER
+    fi
 
     # Stabilize the node
     server_exec '~/loader/scripts/setup/stabilize.sh'

diff --git a/scripts/setup/rewrite_yaml_files.sh b/scripts/setup/rewrite_yaml_files.sh
@@ -56,23 +56,8 @@ cat serving-core.yaml |
             or .spec.template.metadata.labels.app == "domain-mapping"
             or .spec.template.metadata.labels.app == "domainmapping-webhook"
             or .spec.template.metadata.labels.app == "webhook"
-        ) | .spec.template.spec 
-    ) += {"nodeSelector": {"loader-nodetype": "master"}}' |
-    yq '
-    (
-        del
-        (
-            select
-            (
-                    .spec.template.metadata.labels.app == "activator"
-                or .spec.template.metadata.labels.app == "autoscaler"
-                or .spec.template.metadata.labels.app == "controller"
-                or .spec.template.metadata.labels.app == "domain-mapping"
-                or .spec.template.metadata.labels.app == "domainmapping-webhook"
-                or .spec.template.metadata.labels.app == "webhook"
-            ) | .spec.template.spec.affinity
-        )
-    )' |
+        ) | .spec.template.spec.affinity
+    ) = {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": {"nodeSelectorTerms": [{"matchExpressions": [{"key": "loader-nodetype", "operator": "In", "values": ["master", "singlenode"]}]}]}}}' |
     yq '
     (
         select

diff --git a/scripts/setup/setup.cfg b/scripts/setup/setup.cfg
@@ -1,4 +1,4 @@
-VHIVE_BRANCH='v1.7'
+VHIVE_BRANCH='v1.7.1'
 LOADER_BRANCH='main'
 CLUSTER_MODE='container' # choose from {container, firecracker, firecracker_snapshots}
 PODS_PER_NODE=240

diff --git a/workloads/container/trace_func_go.yaml b/workloads/container/trace_func_go.yaml
@@ -18,8 +18,16 @@ spec:
         autoscaling.knative.dev/target: $AUTOSCALING_TARGET
     spec:
       containerConcurrency: 1
-      nodeSelector:
-        loader-nodetype: worker
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: loader-nodetype
+                operator: In
+                values:
+                - worker
+                - singlenode
       containers:
         - image: ghcr.io/vhive-serverless/invitro_trace_function:latest
           # imagePullPolicy: Always  # No need if the tag is `latest`.