From 5ebfa70d035fbb5a409bc243cb5319e136fe0a77 Mon Sep 17 00:00:00 2001
From: Petr Horacek <p.horacek94@gmail.com>
Date: Mon, 14 Jan 2019 15:20:20 +0100
Subject: [PATCH] distribute sr-iov configuration using initContainer (#539)

In order to make network playbook operator friendly, we need to move
node configuration to daemon sets. With this patch, node sr-iov DP
configuration is stored in cluster-level config map and saved on a
node using SR-IOV DP init container.

The config map contains SR-IOV configuration per node (data key is
name of the node) and 'fallback-config' in case specific node configuration
is not provided.

In order to reconfigure node's SR-IOV conf, config map must be updated
and respective pod restarted.
---
 playbooks/network.yml                         |  8 ---
 .../network-multus/templates/cni-plugins.yml  | 59 +++++++++++++++++++
 .../templates/sriovdp-config.json             | 11 ----
 3 files changed, 59 insertions(+), 19 deletions(-)
 delete mode 100644 roles/network-multus/templates/sriovdp-config.json

diff --git a/playbooks/network.yml b/playbooks/network.yml
index 59188bf41..f4e982e64 100644
--- a/playbooks/network.yml
+++ b/playbooks/network.yml
@@ -28,14 +28,6 @@
 
     - name: configure SR-IOV device plugin
       block:
-        - name: Create /etc/pcidp
-          file: path=/etc/pcidp state=directory
-
-        - name: Configure SR-IOV DP allocation pool
-          template:
-            src: roles/network-multus/templates/sriovdp-config.json
-            dest: /etc/pcidp/config.json
-
         - name: Fix SELinux labels for /var/lib/kubelet/device-plugins/
           command: restorecon -Rv /var/lib/kubelet/device-plugins/
           ignore_errors: True
diff --git a/roles/network-multus/templates/cni-plugins.yml b/roles/network-multus/templates/cni-plugins.yml
index 9d6f7db68..6547b4801 100644
--- a/roles/network-multus/templates/cni-plugins.yml
+++ b/roles/network-multus/templates/cni-plugins.yml
@@ -1,4 +1,23 @@
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sriov-nodes-config
+  namespace: {{ network_namespace }}
+data:
+  fallback-config: |
+    {
+        "resourceList":
+        [
+            {
+                "resourceName": "sriov",
+                "rootDevices": [{{ sriov_pci_ids }}],
+                "sriovMode": true,
+                "deviceType": "vfio"
+            }
+        ]
+    }
+---
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
@@ -21,6 +40,39 @@ spec:
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
+      initContainers:
+      - name: config-container
+        image: fedora:29
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        command:
+        - /bin/bash
+        - -c
+        args:
+        - |
+          if [ -f /etc/sriov-nodes-config/$NODE_NAME ]; then
+            config_path=/etc/sriov-nodes-config/$NODE_NAME
+          else
+            config_path=/etc/sriov-nodes-config/fallback-config
+          fi
+          cp $config_path /etc/pcidp/config.json
+        volumeMounts:
+        - name: etc-pcidp
+          mountPath: /etc/pcidp
+        - name: sriov-nodes-config
+          mountPath: /etc/sriov-nodes-config
+        securityContext:
+          privileged: true
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
       containers:
       - name: cni-plugins
         image: {{ cni_provisioner_repo }}:{{ cni_provisioner_release }}
@@ -40,3 +92,10 @@ spec:
       - name: cnibin
         hostPath:
           path: /opt/cni/bin
+      - name: etc-pcidp
+        hostPath:
+          path: /etc/pcidp
+          type: DirectoryOrCreate
+      - name: sriov-nodes-config
+        configMap:
+          name: sriov-nodes-config
diff --git a/roles/network-multus/templates/sriovdp-config.json b/roles/network-multus/templates/sriovdp-config.json
deleted file mode 100644
index 622f46ce1..000000000
--- a/roles/network-multus/templates/sriovdp-config.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-    "resourceList":
-    [
-        {
-            "resourceName": "sriov",
-            "rootDevices": [{{ sriov_pci_ids }}],
-            "sriovMode": true,
-            "deviceType": "vfio"
-        }
-    ]
-}