From ddc98eb5ba73afdb99d69010fc62f9e83895c639 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 5 Jan 2024 01:22:11 +1100 Subject: [PATCH 01/21] feat: support cilium cin --- hack/run-integration-tests.sh | 3 +- hack/stack.sh | 2 +- .../manifests/cilium/cilium.yaml | 1154 +++++++++++++++++ magnum_cluster_api/resources.py | 56 +- 4 files changed, 1199 insertions(+), 16 deletions(-) create mode 100644 magnum_cluster_api/manifests/cilium/cilium.yaml diff --git a/hack/run-integration-tests.sh b/hack/run-integration-tests.sh index f385197c..2a50585f 100755 --- a/hack/run-integration-tests.sh +++ b/hack/run-integration-tests.sh @@ -23,6 +23,7 @@ source /opt/stack/openrc OS_DISTRO=${OS_DISTRO:-ubuntu} IMAGE_OS=${IMAGE_OS:-ubuntu-2204} NODE_COUNT=${NODE_COUNT:-2} +NETWORK_DRIVER=${NETWORK_DRIVER:-calico} SONOBUOY_VERSION=${SONOBUOY_VERSION:-0.56.16} SONOBUOY_ARCH=${SONOBUOY_ARCH:-amd64} DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} @@ -52,7 +53,7 @@ openstack coe cluster template create \ --master-lb-enabled \ --master-flavor m1.large \ --flavor m1.large \ - --network-driver calico \ + --network-driver ${NETWORK_DRIVER} \ --docker-storage-driver overlay2 \ --coe kubernetes \ --label kube_tag=${KUBE_TAG} \ diff --git a/hack/stack.sh b/hack/stack.sh index a3c00071..c5ad0411 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -85,7 +85,7 @@ MANILA_USE_SERVICE_INSTANCE_PASSWORD=True [[post-config|/etc/magnum/magnum.conf]] [cluster_template] -kubernetes_allowed_network_drivers = calico +kubernetes_allowed_network_drivers = calico,cilium kubernetes_default_network_driver = calico EOF diff --git a/magnum_cluster_api/manifests/cilium/cilium.yaml b/magnum_cluster_api/manifests/cilium/cilium.yaml new file mode 100644 index 00000000..3392f477 --- /dev/null +++ b/magnum_cluster_api/manifests/cilium/cilium.yaml @@ -0,0 +1,1154 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-ca-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: cilium-ca + namespace: kube-system +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtyZ2ViYTlKMi9EWU1ONFAyYk5oZ293RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkwTURFd05ERTBNVE0xTmxvWERUSTNNREV3TXpFMApNVE0xTmxvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFvdjFtYnZteVgySmVkOFByeXBFdUlCV3pEdERCQ0FIc3I4UVE2amVHL0R1eFM5TkIKbnZRK0hVWGlHeUFscHdPNHdtUHdvQUZPN0NuK254bnNjVDhlUjFtOXZTMzZvNEk3SUFKQnh5c0JVTmR1TzQrMwpwVzZqL3YwNU1QK2dQcXJ6c3VMdm9GWXB4aElWWU1BbWpza2JTZFRJcUV1ZDdwc1U3cXIrb2J5Qm9IMEs3U3I2CkV0VXI5SHJxTlI5UlZjSG1JRGp5eDV6bnZiOEgwZU1sUFg2NXNXVk5vbVMyamF2b2p3Yk9sRnVFRHoycVlMNmEKS2EwaGxCMEVaR3JqVGFEM2wxd1RFSXVaUkQ5SFdQNjZDV2doRHoyNnBEZWh3MjFZemx5V0tsL2F2V1o1Zm5FVQpsbDlWQmNHaWVJcEdmQWpPbmZJclpDNlVpWUZtcFJHNGhma09xUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGSTc0YkpVMm4yY2VWc0F0bnBZL1pZQWV1NGpETUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQnpkck91UHJpY3NBWWhzRVFwQU8zTGRnM0tjaUdpWXJiUFc5cFkyN2Ruc0NZZlI4dERySXkrCmxFVnpURmNlUVg1cExQMDZqOHJSWmljMmR5NFVqYVpGWnlhSGtwNElCaDhLU2Q0T3g0d1hkbmJkVFVZcUMvWGsKMXdZOHJkbzBuMkdnZXhFd0VmOURyY21Qb04zOWRXUkdnY3EvTUg1U0JoemJTUlRJZnJaSjZaM1B6YitvMUdydgpuUTVpT1k1RjJnNmlrZWFZUVJZSTJ1SStzaytndGJIOW5lVTBZZmFSOFZVYjl2L3JxTnl1czZyZlIvbDFuUTdHCmRXUW1ETG9wdEdGdGI3ZXFtYmN4RGtUOGE3THFzblE5V1puUTRNVzgyN1dqM09RT01tbFYxQ3hXcEY5ZWpJOXkKUjBadVY5SXF3TXBCRU1FU3kwdlFkUVdvcmUwcmo5M0cKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBb3YxbWJ2bXlYMkplZDhQcnlwRXVJQld6RHREQkNBSHNyOFFRNmplRy9EdXhTOU5CCm52UStIVVhpR3lBbHB3TzR3bVB3b0FGTzdDbitueG5zY1Q4ZVIxbTl2UzM2bzRJN0lBSkJ4eXNCVU5kdU80KzMKcFc2ai92MDVNUCtnUHFyenN1THZvRllweGhJVllNQW1qc2tiU2RUSXFFdWQ3cHNVN3FyK29ieUJvSDBLN1NyNgpFdFVyOUhycU5SOVJWY0htSURqeXg1em52YjhIMGVNbFBYNjVzV1ZOb21TMmphdm9qd2JPbEZ1RUR6MnFZTDZhCkthMGhsQjBFWkdyalRhRDNsMXdURUl1WlJEOUhXUDY2Q1dnaER6MjZwRGVodzIxWXpseVdLbC9hdldaNWZuRVUKbGw5VkJjR2llSXBHZkFqT25mSXJaQzZVaVlGbXBSRzRoZmtPcVFJREFRQUJBb0lCQUFFZ2FVWFpHZitJeVloZwpaVS9ZSXc1N3Qrb2hUVVJGNXlIcjVBWUtiamR1SGpLYytqWWtFQTNadnM5Yk1JZHdBaVh0MU9yMk4xcHdpOThECnFkdTZpUFlMUzhGMEFPaUNiUEF5czh1eG8yZXdwWVZwMytZWDFyQmJoYnF0K2FpVk8rdi9uNUpuSmhERmJUcG0KK1oxQkRKdzlDVW9JZzhYemNuaVgyL3k0TmRrcklVVnk0cUxTNEtLOXFkK0tMU1VPQTk0UWVNVGdQQkxBWVpTOQpVNDBFUFhxTGc1VjRBTitORTltWDh4ZHFIRWJJV05HaG96ZytjT3hqaEtHbkdEZ0RpcjZmSjRkU29ISVBVYkZ6CnhXUG44NWNrdE0vOUhyTGdaUmY2UVhrNm51bnZxc1l6d1Vkdnh5U2o1bDN5VktmcmZ0MGgyK3FYVEYwdGM0algKUzlpSkx4VUNnWUVBMWxrbjdEMk1WZDFrbC9FUlZCTHczNno4aEF5U3I5eWM0d2JHb05rYzVQeTVrZ0pISVdkQgpIeDBFbGJPLzJqL1g2ZnRETkoxb0JCL1VZT1BFTlRvU2IrK2hqSEovNDVkdGhwc3hrRFArZnlvdzZSQ3NiRCtPCkVIMGJiYi8yVmcvTnI1L1NyS0tQRXJnYnA0dHBGZWlGZ05icGdldE1XYU9ZNzBhcXlYb1RHUHNDZ1lFQXdxbGwKNXdFUzQ5NW42cnZWRnlCQTlwVHpvRVBNRldvemVYNU9waHNJMDdKcUd5UUZwM3hYNlVjc09CdEVQbjljNzFCUQppQlpaN2hTSFd4SUt4M0lSTlo2K1paSVhKSjJ4cmppOFRIMFBmMTVpTFV4Z2I3bUJmVHB6dzR6VXo0WkdjY2pZClYzU2hCcnF4UjAzdnJ3V1U4ZDk5blZtdVE2dk9FWHRSb3RtVjdhc0NnWUJmMlhwcXY1TVRpbTdES3Y5dXN4R1gKVkFtUEd2ZzFQOWZBQlhXbzA2MkNFQ2tEUzVWM01HOEVkTHRxSGZqNnBYWHRvWGdKL1NGWVJzdDNrQ0piRXQ3UAp3ejVnU1lzNndLb3ZlUktPdzFGWW1BdExEVE90TERaZzFURUF3TWhzNzZBd21xMWdtVWlWUEZMUmR4dkV1amVJCng1R2ZrUktST3dEV2JxbE1jeFE3SlFLQmdRQ1J6cEx1bWdaTk9tejZEU0dLeGZVQy9nVVNPVlluZ0RuNE9keXQKWExpYTRSdkxJb3M5cmN5bmhsRWY0WVNYajR6TUVJdVJzN1ErdVkrOWJtRTJqVUZJdVpRQnZwQ3BNNWxBVjJ6cgptZzArcVRLK25XUHFxN2ZMVmpvWjhtWUEzNkhxWHJkMW9uMXU3ZTRWNjlHcGRuUGZURGpwNEovQVBuTUc4eWp0CjR5OERLd0tCZ0RneWdBMlhYM3JnZm1jWXpOTDZ0cldBQU9HK2pmck9FZEVCNWtZYnM4T0JONWY3L05GbHhSYnMKQ2RIWnJSaWtFbUlZVnV1NjlHeVNPdUhKOXFVaUpYditWK1UzQkNKMmxaV2UwU3ZlQmI4U3BVZkhWdlNxYXpPcgphWjhnc1ROcXAvTUltaVNEcXZFM1ZxL3JveDRnSDdoVnBEdmtBRitUZTRTWXd5N1p1TWltCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== +--- +# Source: cilium/templates/hubble/tls-helm/ca-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: hubble-ca-secret + namespace: kube-system +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtyZ2ViYTlKMi9EWU1ONFAyYk5oZ293RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkwTURFd05ERTBNVE0xTmxvWERUSTNNREV3TXpFMApNVE0xTmxvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFvdjFtYnZteVgySmVkOFByeXBFdUlCV3pEdERCQ0FIc3I4UVE2amVHL0R1eFM5TkIKbnZRK0hVWGlHeUFscHdPNHdtUHdvQUZPN0NuK254bnNjVDhlUjFtOXZTMzZvNEk3SUFKQnh5c0JVTmR1TzQrMwpwVzZqL3YwNU1QK2dQcXJ6c3VMdm9GWXB4aElWWU1BbWpza2JTZFRJcUV1ZDdwc1U3cXIrb2J5Qm9IMEs3U3I2CkV0VXI5SHJxTlI5UlZjSG1JRGp5eDV6bnZiOEgwZU1sUFg2NXNXVk5vbVMyamF2b2p3Yk9sRnVFRHoycVlMNmEKS2EwaGxCMEVaR3JqVGFEM2wxd1RFSXVaUkQ5SFdQNjZDV2doRHoyNnBEZWh3MjFZemx5V0tsL2F2V1o1Zm5FVQpsbDlWQmNHaWVJcEdmQWpPbmZJclpDNlVpWUZtcFJHNGhma09xUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGSTc0YkpVMm4yY2VWc0F0bnBZL1pZQWV1NGpETUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQnpkck91UHJpY3NBWWhzRVFwQU8zTGRnM0tjaUdpWXJiUFc5cFkyN2Ruc0NZZlI4dERySXkrCmxFVnpURmNlUVg1cExQMDZqOHJSWmljMmR5NFVqYVpGWnlhSGtwNElCaDhLU2Q0T3g0d1hkbmJkVFVZcUMvWGsKMXdZOHJkbzBuMkdnZXhFd0VmOURyY21Qb04zOWRXUkdnY3EvTUg1U0JoemJTUlRJZnJaSjZaM1B6YitvMUdydgpuUTVpT1k1RjJnNmlrZWFZUVJZSTJ1SStzaytndGJIOW5lVTBZZmFSOFZVYjl2L3JxTnl1czZyZlIvbDFuUTdHCmRXUW1ETG9wdEdGdGI3ZXFtYmN4RGtUOGE3THFzblE5V1puUTRNVzgyN1dqM09RT01tbFYxQ3hXcEY5ZWpJOXkKUjBadVY5SXF3TXBCRU1FU3kwdlFkUVdvcmUwcmo5M0cKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBb3YxbWJ2bXlYMkplZDhQcnlwRXVJQld6RHREQkNBSHNyOFFRNmplRy9EdXhTOU5CCm52UStIVVhpR3lBbHB3TzR3bVB3b0FGTzdDbitueG5zY1Q4ZVIxbTl2UzM2bzRJN0lBSkJ4eXNCVU5kdU80KzMKcFc2ai92MDVNUCtnUHFyenN1THZvRllweGhJVllNQW1qc2tiU2RUSXFFdWQ3cHNVN3FyK29ieUJvSDBLN1NyNgpFdFVyOUhycU5SOVJWY0htSURqeXg1em52YjhIMGVNbFBYNjVzV1ZOb21TMmphdm9qd2JPbEZ1RUR6MnFZTDZhCkthMGhsQjBFWkdyalRhRDNsMXdURUl1WlJEOUhXUDY2Q1dnaER6MjZwRGVodzIxWXpseVdLbC9hdldaNWZuRVUKbGw5VkJjR2llSXBHZkFqT25mSXJaQzZVaVlGbXBSRzRoZmtPcVFJREFRQUJBb0lCQUFFZ2FVWFpHZitJeVloZwpaVS9ZSXc1N3Qrb2hUVVJGNXlIcjVBWUtiamR1SGpLYytqWWtFQTNadnM5Yk1JZHdBaVh0MU9yMk4xcHdpOThECnFkdTZpUFlMUzhGMEFPaUNiUEF5czh1eG8yZXdwWVZwMytZWDFyQmJoYnF0K2FpVk8rdi9uNUpuSmhERmJUcG0KK1oxQkRKdzlDVW9JZzhYemNuaVgyL3k0TmRrcklVVnk0cUxTNEtLOXFkK0tMU1VPQTk0UWVNVGdQQkxBWVpTOQpVNDBFUFhxTGc1VjRBTitORTltWDh4ZHFIRWJJV05HaG96ZytjT3hqaEtHbkdEZ0RpcjZmSjRkU29ISVBVYkZ6CnhXUG44NWNrdE0vOUhyTGdaUmY2UVhrNm51bnZxc1l6d1Vkdnh5U2o1bDN5VktmcmZ0MGgyK3FYVEYwdGM0algKUzlpSkx4VUNnWUVBMWxrbjdEMk1WZDFrbC9FUlZCTHczNno4aEF5U3I5eWM0d2JHb05rYzVQeTVrZ0pISVdkQgpIeDBFbGJPLzJqL1g2ZnRETkoxb0JCL1VZT1BFTlRvU2IrK2hqSEovNDVkdGhwc3hrRFArZnlvdzZSQ3NiRCtPCkVIMGJiYi8yVmcvTnI1L1NyS0tQRXJnYnA0dHBGZWlGZ05icGdldE1XYU9ZNzBhcXlYb1RHUHNDZ1lFQXdxbGwKNXdFUzQ5NW42cnZWRnlCQTlwVHpvRVBNRldvemVYNU9waHNJMDdKcUd5UUZwM3hYNlVjc09CdEVQbjljNzFCUQppQlpaN2hTSFd4SUt4M0lSTlo2K1paSVhKSjJ4cmppOFRIMFBmMTVpTFV4Z2I3bUJmVHB6dzR6VXo0WkdjY2pZClYzU2hCcnF4UjAzdnJ3V1U4ZDk5blZtdVE2dk9FWHRSb3RtVjdhc0NnWUJmMlhwcXY1TVRpbTdES3Y5dXN4R1gKVkFtUEd2ZzFQOWZBQlhXbzA2MkNFQ2tEUzVWM01HOEVkTHRxSGZqNnBYWHRvWGdKL1NGWVJzdDNrQ0piRXQ3UAp3ejVnU1lzNndLb3ZlUktPdzFGWW1BdExEVE90TERaZzFURUF3TWhzNzZBd21xMWdtVWlWUEZMUmR4dkV1amVJCng1R2ZrUktST3dEV2JxbE1jeFE3SlFLQmdRQ1J6cEx1bWdaTk9tejZEU0dLeGZVQy9nVVNPVlluZ0RuNE9keXQKWExpYTRSdkxJb3M5cmN5bmhsRWY0WVNYajR6TUVJdVJzN1ErdVkrOWJtRTJqVUZJdVpRQnZwQ3BNNWxBVjJ6cgptZzArcVRLK25XUHFxN2ZMVmpvWjhtWUEzNkhxWHJkMW9uMXU3ZTRWNjlHcGRuUGZURGpwNEovQVBuTUc4eWp0CjR5OERLd0tCZ0RneWdBMlhYM3JnZm1jWXpOTDZ0cldBQU9HK2pmck9FZEVCNWtZYnM4T0JONWY3L05GbHhSYnMKQ2RIWnJSaWtFbUlZVnV1NjlHeVNPdUhKOXFVaUpYditWK1UzQkNKMmxaV2UwU3ZlQmI4U3BVZkhWdlNxYXpPcgphWjhnc1ROcXAvTUltaVNEcXZFM1ZxL3JveDRnSDdoVnBEdmtBRitUZTRTWXd5N1p1TWltCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== +--- +# Source: cilium/templates/hubble/tls-helm/server-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: kube-system +type: kubernetes.io/tls +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtyZ2ViYTlKMi9EWU1ONFAyYk5oZ293RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkwTURFd05ERTBNVE0xTmxvWERUSTNNREV3TXpFMApNVE0xTmxvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFvdjFtYnZteVgySmVkOFByeXBFdUlCV3pEdERCQ0FIc3I4UVE2amVHL0R1eFM5TkIKbnZRK0hVWGlHeUFscHdPNHdtUHdvQUZPN0NuK254bnNjVDhlUjFtOXZTMzZvNEk3SUFKQnh5c0JVTmR1TzQrMwpwVzZqL3YwNU1QK2dQcXJ6c3VMdm9GWXB4aElWWU1BbWpza2JTZFRJcUV1ZDdwc1U3cXIrb2J5Qm9IMEs3U3I2CkV0VXI5SHJxTlI5UlZjSG1JRGp5eDV6bnZiOEgwZU1sUFg2NXNXVk5vbVMyamF2b2p3Yk9sRnVFRHoycVlMNmEKS2EwaGxCMEVaR3JqVGFEM2wxd1RFSXVaUkQ5SFdQNjZDV2doRHoyNnBEZWh3MjFZemx5V0tsL2F2V1o1Zm5FVQpsbDlWQmNHaWVJcEdmQWpPbmZJclpDNlVpWUZtcFJHNGhma09xUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGSTc0YkpVMm4yY2VWc0F0bnBZL1pZQWV1NGpETUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQnpkck91UHJpY3NBWWhzRVFwQU8zTGRnM0tjaUdpWXJiUFc5cFkyN2Ruc0NZZlI4dERySXkrCmxFVnpURmNlUVg1cExQMDZqOHJSWmljMmR5NFVqYVpGWnlhSGtwNElCaDhLU2Q0T3g0d1hkbmJkVFVZcUMvWGsKMXdZOHJkbzBuMkdnZXhFd0VmOURyY21Qb04zOWRXUkdnY3EvTUg1U0JoemJTUlRJZnJaSjZaM1B6YitvMUdydgpuUTVpT1k1RjJnNmlrZWFZUVJZSTJ1SStzaytndGJIOW5lVTBZZmFSOFZVYjl2L3JxTnl1czZyZlIvbDFuUTdHCmRXUW1ETG9wdEdGdGI3ZXFtYmN4RGtUOGE3THFzblE5V1puUTRNVzgyN1dqM09RT01tbFYxQ3hXcEY5ZWpJOXkKUjBadVY5SXF3TXBCRU1FU3kwdlFkUVdvcmUwcmo5M0cKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRS1lqRTJkUmZnK3hDVDBld01hRjB3REFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalF3TVRBME1UUXhNelUzV2hjTk1qY3dNVEF6TVRReApNelUzV2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBd3NwWUVpSUhiTzlLLzY0YzQ1cmkKeTJHVkZYVldBZ3F6VlQ5aVMyQWFhQTFTeGRhV1JDWlhxekZ1Q3dCcmFSR3FBMmR6WjFVYlJ0WFVzYytPVUYxNgprbkNaZ1RTbCt2YlhmR2RlL05JT0JIVlZIWmoxUTYyTHptelN0ZmVNMjVDWE5aZVpiSVo1VTM1VG1OVDRrRzhFCkZvZjcraXNGeldvcGZCc1RQdERvUXd2eFF2NW8rV0wvTVFmOXR1YTBIN0cvZTJzQlFRWnRjeGJhVEhuRUlRVzgKUXJKcGFQajhZN0Y1Y1U3ZFJzR2pJem9WODFMMk51QjRlT0pvVmxVY3ZTZzdrQTRnTXlNa0w2UW9sQkN2YkZVdQpDYjN5aXNUNytFeFVVSkszS2VoNWUzTm9oOStIWmdLcHdhamR3eWV1ZFlXdjlIenB5Rjd6WUZpZ1VvcGFuaHhECmR3SURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JTTytHeVZOcDluSGxiQQpMWjZXUDJXQUhydUl3ekFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCUUlvSnhIR3p3eVplakRWaXk1YnlJdXR6K1hubzMKVGtxdHZDWHRrRS9yb0lvYUhBL0JLZzBQSEIxYS9PcVRrT0VOQjZVUGc0RDczU3QzeWhkdGlhYjh2YmJSd2w1MApGaWxBUUh1OE5YVGpIZVJERVB0U2NldXdPRy9XQnM1WmcxeDNIT3FaWWJzaXBtNFNWRFlPa0FHOUxyZGFUY0p6CmhpbHpGSFlCeVprR1MwenNkUkRvTHJSVjVSRUVicEdwSmxBM2ZScGYrNC9MdklOaWhTbFRQWnlESmYrVkRnYm0KeklBTnJ3QUJEcWVoeXMrVmVhQ3B3NFk2Rm1DcWVlemZ0cThoTjFMbmNJbWhxckI2YjdpUGxXTy8xU2k1WC9pUwpIVi9uRHZsM0E4K3JjOU1maTEydzAvTkxleTVjbHBmRklGTktqeXhuNXRTUXlzenBveXo3RFF4RAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBd3NwWUVpSUhiTzlLLzY0YzQ1cml5MkdWRlhWV0FncXpWVDlpUzJBYWFBMVN4ZGFXClJDWlhxekZ1Q3dCcmFSR3FBMmR6WjFVYlJ0WFVzYytPVUYxNmtuQ1pnVFNsK3ZiWGZHZGUvTklPQkhWVkhaajEKUTYyTHptelN0ZmVNMjVDWE5aZVpiSVo1VTM1VG1OVDRrRzhFRm9mNytpc0Z6V29wZkJzVFB0RG9Rd3Z4UXY1bworV0wvTVFmOXR1YTBIN0cvZTJzQlFRWnRjeGJhVEhuRUlRVzhRckpwYVBqOFk3RjVjVTdkUnNHakl6b1Y4MUwyCk51QjRlT0pvVmxVY3ZTZzdrQTRnTXlNa0w2UW9sQkN2YkZVdUNiM3lpc1Q3K0V4VVVKSzNLZWg1ZTNOb2g5K0gKWmdLcHdhamR3eWV1ZFlXdjlIenB5Rjd6WUZpZ1VvcGFuaHhEZHdJREFRQUJBb0lCQVFDUC9IcDFqT25vUUVvMApEVVFWaXB3eXNvTTlSV3E2UDZhb1VSNGJrWUp6ZjhXN3NBK0xlTHc5TTJDZ2lVTGNlODRzamYva1ZpaS9oN2lkClowdFBaSW9DVFhGcTdMdjBjeHArUHZzYjd6bWxXZ1g4eHVBbGt0ZFJVMmUrVXFxd2NvVVRmR2hRckdacXJXcW0KQW1kcWZHV05NaCswb0xDOVVzZ3ZrT1A4L3ZkdzlzeHdpdlRLd0IvQkEvTHRzZkRIVmJUSkRUNnpqd2xFdmt1dQpxdHpjK1h4bWhzOTQxVjB6TWFxV2NGSnZubU5sWGgvWE83T3hNUHpBTTJoMjRDOXlIWEZ3cGZ1RVhHeGJ0a0FlClUxdUhLTjFBczQ3dVZWMm5kQkdhWGhiWkhSRy9sNUt6L2dnbmNMblJSRDEzcUtJY202K3NCbWN4QjJLNENOUVUKb012amxXN2hBb0dCQU5UaE10SmJVTnlyWWpWQkF0eS85MnVFaCtoNWJmM2ZVZE5ITGxJdWllUjR3bSt2NDYwMApkSk5qdm5kazRSMUI1L3hxWFVlTGdpUWtTKytwNDhSdFJzWDRtQkkxV3BycXg2SG1FcjZGOXJKS1gyRDlJcEJ1CmdjR2JvYUZkWEJCYU1qNjhTb0pRdjRmTzExVklaVDBnTFdKaTk1emZsaW9rNjNZK3VpM1BHQ3MxQW9HQkFPby8KSWVoNXBVTU1FSUlWaEtVSnYwdHowbXVkVDRiMDhVUkFkNVZWTDFhQ2liNFM3NFo2WTRkMThzVEttWmIxOXR6Ugo3Z2s5ZlpUOEZUczJ3K0EzU3ZCNXpzKzZaWFdobWJCeDVtSTZQZWhvb3Y1Y1Z1cDNhZGREMWI5ZTZEZmFxeDF1CmttWGk3YS9iYzBsTWxWNFhadEs4K0JVMjFTY3I3MDhxb3l0L0FKMTdBb0dCQUtITGJ5YjJCR1ZUVk5jSXpEWWYKS050czVTYkpjOUdxTTdTU3V4LzNBeExFdlhMSmlnK1FlTEcwODRRbkJmbUlwOXJ1dEdxMkczbUFwTDVJcnpZUgpuK0xEYUVFZWovaUNUS3crQXkwUEFFck52K1FsUUducHMzRUU2am1NalhaWU55djdTOTN6cFQ2QTlLUjYyNGN6CnQ0cTVrVnlIaXNnSk8zU3BoRE5nM1F1cEFvR0FCNVBUUFBueVpxRFZSOS82VW54bU84S2JZKy9LeXNyUm95dVMKc3JuMnRnMnk5dnNWMUh2UkRHaThXYkZVRnBZT1RrWmNOT0JRVjZsdjVtSjNJNkdtUXZFdHgzcDRIYjRFZ3JpcwpnekR2SDcyQ3lFZWthQ0ZFd3pubWlESkJ3TXpLREVHdmFxeFVsQU85VHZtek96LzJpNHFoVzg1U1J0b2MrNnFkCmJLTEhHbEVDZ1lBdGRQOFBZRkpsY1B1THVZMkxoM2tpRHZ4UjF3NFQxeXliMit1MXVpRU9LRTBRS20rNDdYRnUKN2U3TjZLR1dIMUZseVFuVVR4QUptcmVDcjY4bFp4QUZhQWVOdXdmZTRqZHdYWm9aMHpJajVlU0FHeTg5MmZ4YwpVRnVaTmxhVkdaMm4rSXh3bWFHRlpRU2NXOGlKK005enRrZTJKTmxBT2JZSkYwZ0tHaE94ZkE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + skip-cnp-status-startup-clean: "false" + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "false" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: "vxlan" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + + enable-ipv4-masquerade: "true" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + enable-local-redirect-policy: "false" + + kube-proxy-replacement: "disabled" + bpf-lb-sock: "false" + enable-health-check-nodeport: "true" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + cni-uninstall: "true" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-remote-node-identity: "true" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + ipam: "cluster-pool" + cluster-pool-ipv4-cidr: "10.0.0.0/8" + cluster-pool-ipv4-mask-size: "24" + disable-cnp-status-updates: "true" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "false" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + enable-k8s-terminating-endpoint: "true" + enable-sctp: "false" + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + - ciliumendpoints/status + - ciliumendpoints + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + spec: + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + /cni-install.sh --enable-debug=false --cni-exclusive=true --log-file=/var/run/cilium/cilium-cni.log + preStop: + exec: + command: + - /cni-uninstall.sh + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + - name: tmp + mountPath: /tmp + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + command: + - cilium + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.13.10" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: "cilium" + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: ca.crt + path: client-ca.crt + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.13.10" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +# Only create the namespace if it's different from Ingress secret namespace or Ingress is not enabled. + +# Only create the namespace if it's different from Ingress and Gateway API secret namespaces (if enabled). diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 35db1727..0a45b697 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -18,6 +18,7 @@ import os import textwrap import types +from collections import Counter import certifi import pkg_resources @@ -45,6 +46,7 @@ CONF = cfg.CONF CALICO_TAG = "v3.24.2" +CILIUM_TAG = "v1.13.10" CLUSTER_CLASS_VERSION = pkg_resources.require("magnum_cluster_api")[0].version CLUSTER_CLASS_NAME = f"magnum-v{CLUSTER_CLASS_VERSION}" @@ -166,10 +168,14 @@ def __init__( self.cluster = cluster def get_object(self) -> pykube.ConfigMap: + # NOTE(mnaser): We have to assert that the only CNIs we support are Calico and Cilium. + assert Counter(CONF.cluster_template.kubernetes_allowed_network_drivers) == Counter(["calico", "cilium"]) + manifests_path = pkg_resources.resource_filename( "magnum_cluster_api", "manifests" ) calico_version = self.cluster.labels.get("calico_tag", CALICO_TAG) + cilium_version = self.cluster.labels.get("cilium_tag", CILIUM_TAG) repository = utils.get_cluster_container_infra_prefix(self.cluster) @@ -190,15 +196,32 @@ def get_object(self) -> pykube.ConfigMap: ) for manifest in glob.glob(os.path.join(manifests_path, "ccm/*.yaml")) }, - **{ - "calico.yml": image_utils.update_manifest_images( - self.cluster.uuid, - os.path.join(manifests_path, f"calico/{calico_version}.yaml"), - repository=repository, - ) - }, } + if self.cluster.cluster_template.network_driver == "cilium": + data = { + **data, + **{ + "cilium.yml": image_utils.update_manifest_images( + self.cluster.uuid, + os.path.join(manifests_path, f"cilium/{cilium_version}.yaml"), + repository=repository, + ) + }, + } + + if self.cluster.cluster_template.network_driver == "calico": + data = { + **data, + **{ + "calico.yml": image_utils.update_manifest_images( + self.cluster.uuid, + os.path.join(manifests_path, f"calico/{calico_version}.yaml"), + repository=repository, + ) + }, + } + if cinder.is_enabled(self.cluster): volume_types = osc.cinder().volume_types.list() default_volume_type = osc.cinder().volume_types.default() @@ -2316,9 +2339,12 @@ def __init__( @property def labels(self) -> dict: - cni_version = self.cluster.labels.get("calico_tag", CALICO_TAG) + if self.cluster.cluster_template.network_driver == "calico": + cni = f"calico-{self.cluster.labels.get("calico_tag", CALICO_TAG)}" + if self.cluster.cluster_template.network_driver == "cilium": + cni = f"cilium-{self.cluster.labels.get("cilium_tag", CILIUM_TAG)}" labels = { - "cni": f"calico-{cni_version}", + "cni": cni, } return {**super().labels, **labels} @@ -2331,6 +2357,12 @@ def get_or_none(self) -> objects.Cluster: def get_object(self) -> objects.Cluster: osc = clients.get_openstack_api(self.context) default_volume_type = osc.cinder().volume_types.default() + + if self.cluster.cluster_template.network_driver == "calico": + pod_cidr = self.cluster.labels.get("calico_ipv4pool", "10.100.0.0/16") + if self.cluster.cluster_template.network_driver == "cilium": + pod_cidr = self.cluster.labels.get("cilium_ipv4pool", "10.100.0.0/16") + return objects.Cluster( self.api, { @@ -2347,11 +2379,7 @@ def get_object(self) -> objects.Cluster: "dns_cluster_domain", "cluster.local" ), "pods": { - "cidrBlocks": [ - self.cluster.labels.get( - "calico_ipv4pool", "10.100.0.0/16" - ) - ], + "cidrBlocks": [pod_cidr], }, "services": { "cidrBlocks": [ From 5b8cd40923885f30dc5e170a16a7b529d9187a42 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 5 Jan 2024 01:25:39 +1100 Subject: [PATCH 02/21] fix lint error --- magnum_cluster_api/resources.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 0a45b697..f5102273 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -169,7 +169,9 @@ def __init__( def get_object(self) -> pykube.ConfigMap: # NOTE(mnaser): We have to assert that the only CNIs we support are Calico and Cilium. - assert Counter(CONF.cluster_template.kubernetes_allowed_network_drivers) == Counter(["calico", "cilium"]) + assert Counter( + CONF.cluster_template.kubernetes_allowed_network_drivers + ) == Counter(["calico", "cilium"]) manifests_path = pkg_resources.resource_filename( "magnum_cluster_api", "manifests" @@ -2340,12 +2342,19 @@ def __init__( @property def labels(self) -> dict: if self.cluster.cluster_template.network_driver == "calico": - cni = f"calico-{self.cluster.labels.get("calico_tag", CALICO_TAG)}" + cni_version = self.cluster.labels.get( + "calico_tag", CALICO_TAG + ) + labels = { + "cni": f"calico-{cni_version}", + } if self.cluster.cluster_template.network_driver == "cilium": - cni = f"cilium-{self.cluster.labels.get("cilium_tag", CILIUM_TAG)}" - labels = { - "cni": cni, - } + cni_version = self.cluster.labels.get( + "cilium_tag", CILIUM_TAG + ) + labels = { + "cni": f"cilium-{cni_version}", + } return {**super().labels, **labels} @@ -2359,9 +2368,13 @@ def get_object(self) -> objects.Cluster: default_volume_type = osc.cinder().volume_types.default() if self.cluster.cluster_template.network_driver == "calico": - pod_cidr = self.cluster.labels.get("calico_ipv4pool", "10.100.0.0/16") + pod_cidr = self.cluster.labels.get( + "calico_ipv4pool", "10.100.0.0/16" + ) if self.cluster.cluster_template.network_driver == "cilium": - pod_cidr = self.cluster.labels.get("cilium_ipv4pool", "10.100.0.0/16") + pod_cidr = self.cluster.labels.get( + "cilium_ipv4pool", "10.100.0.0/16" + ) return objects.Cluster( self.api, From 274c14a7bebe12c962768cd8c73557afe59ae204 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 28 Mar 2024 02:31:18 +1100 Subject: [PATCH 03/21] set version in manifest file --- .../manifests/cilium/{cilium.yaml => v1.13.10.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename magnum_cluster_api/manifests/cilium/{cilium.yaml => v1.13.10.yaml} (100%) diff --git a/magnum_cluster_api/manifests/cilium/cilium.yaml b/magnum_cluster_api/manifests/cilium/v1.13.10.yaml similarity index 100% rename from magnum_cluster_api/manifests/cilium/cilium.yaml rename to magnum_cluster_api/manifests/cilium/v1.13.10.yaml From 685bf266d9fc3ab60273a4a983894b769fe99197 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 28 Mar 2024 04:52:03 +1100 Subject: [PATCH 04/21] use helm for cilium --- .gitignore | 4 +- Earthfile | 17 +- Makefile | 6 +- .../manifests/cilium/v1.13.10.yaml | 1154 ----------------- magnum_cluster_api/resources.py | 28 +- vendir.lock.yml | 13 + vendir.yml | 18 + 7 files changed, 68 insertions(+), 1172 deletions(-) delete mode 100644 magnum_cluster_api/manifests/cilium/v1.13.10.yaml create mode 100644 vendir.lock.yml create mode 100644 vendir.yml diff --git a/.gitignore b/.gitignore index d490d5e1..05db8a13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ .direnv __pycache__ dist -magnum_cluster_api/charts/* -!magnum_cluster_api/charts/.gitkeep -!magnum_cluster_api/charts/k8s-keystone-auth +magnum_cluster_api/charts/vendor/* site \ No newline at end of file diff --git a/Earthfile b/Earthfile index 3154af27..eb1b14c3 100644 --- a/Earthfile +++ b/Earthfile @@ -1,15 +1,22 @@ VERSION 0.7 +vendir: + FROM github.com/vexxhost/atmosphere/images/curl+image + ARG TARGETOS + ARG TARGETARCH + ARG VERSION=v0.40.0 + RUN curl -Lo vendir https://github.com/carvel-dev/vendir/releases/download/${VERSION}/vendir-${TARGETOS}-${TARGETARCH} + RUN chmod +x vendir && ./vendir version + SAVE ARTIFACT vendir + build: FROM github.com/vexxhost/atmosphere/images/magnum+build - COPY github.com/vexxhost/atmosphere/images/helm+binary/helm /usr/local/bin/helm - RUN helm repo add autoscaler https://kubernetes.github.io/autoscaler - RUN helm repo update + COPY +vendir/vendir /usr/local/bin/vendir COPY --dir magnum_cluster_api/ pyproject.toml README.md /src WORKDIR /src - RUN helm fetch autoscaler/cluster-autoscaler --version 9.29.1 --untar --untardir magnum_cluster_api/charts + RUN vendir sync COPY hack/add-omt-to-clusterrole.patch /hack/ - RUN patch -p0 magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrole.yaml < /hack/add-omt-to-clusterrole.patch + RUN patch -p0 magnum_cluster_api/charts/vendor/cluster-autoscaler/templates/clusterrole.yaml < /hack/add-omt-to-clusterrole.patch DO github.com/vexxhost/atmosphere/images/openstack-service+PIP_INSTALL --PACKAGES /src SAVE ARTIFACT /var/lib/openstack venv diff --git a/Makefile b/Makefile index b6f2c723..d73c655f 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,8 @@ clean: rm -rfv magnum_cluster_api/charts/cluster-autoscaler vendor: clean - helm repo add autoscaler https://kubernetes.github.io/autoscaler - helm repo update - helm fetch autoscaler/cluster-autoscaler --version 9.29.1 --untar --untardir magnum_cluster_api/charts - patch -p0 magnum_cluster_api/charts/cluster-autoscaler/templates/clusterrole.yaml < hack/add-omt-to-clusterrole.patch + vendir sync + patch -p0 magnum_cluster_api/charts/vendor/cluster-autoscaler/templates/clusterrole.yaml < hack/add-omt-to-clusterrole.patch poetry: pipx install poetry diff --git a/magnum_cluster_api/manifests/cilium/v1.13.10.yaml b/magnum_cluster_api/manifests/cilium/v1.13.10.yaml deleted file mode 100644 index 3392f477..00000000 --- a/magnum_cluster_api/manifests/cilium/v1.13.10.yaml +++ /dev/null @@ -1,1154 +0,0 @@ ---- -# Source: cilium/templates/cilium-agent/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-ca-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: cilium-ca - namespace: kube-system -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtyZ2ViYTlKMi9EWU1ONFAyYk5oZ293RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkwTURFd05ERTBNVE0xTmxvWERUSTNNREV3TXpFMApNVE0xTmxvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFvdjFtYnZteVgySmVkOFByeXBFdUlCV3pEdERCQ0FIc3I4UVE2amVHL0R1eFM5TkIKbnZRK0hVWGlHeUFscHdPNHdtUHdvQUZPN0NuK254bnNjVDhlUjFtOXZTMzZvNEk3SUFKQnh5c0JVTmR1TzQrMwpwVzZqL3YwNU1QK2dQcXJ6c3VMdm9GWXB4aElWWU1BbWpza2JTZFRJcUV1ZDdwc1U3cXIrb2J5Qm9IMEs3U3I2CkV0VXI5SHJxTlI5UlZjSG1JRGp5eDV6bnZiOEgwZU1sUFg2NXNXVk5vbVMyamF2b2p3Yk9sRnVFRHoycVlMNmEKS2EwaGxCMEVaR3JqVGFEM2wxd1RFSXVaUkQ5SFdQNjZDV2doRHoyNnBEZWh3MjFZemx5V0tsL2F2V1o1Zm5FVQpsbDlWQmNHaWVJcEdmQWpPbmZJclpDNlVpWUZtcFJHNGhma09xUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGSTc0YkpVMm4yY2VWc0F0bnBZL1pZQWV1NGpETUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQnpkck91UHJpY3NBWWhzRVFwQU8zTGRnM0tjaUdpWXJiUFc5cFkyN2Ruc0NZZlI4dERySXkrCmxFVnpURmNlUVg1cExQMDZqOHJSWmljMmR5NFVqYVpGWnlhSGtwNElCaDhLU2Q0T3g0d1hkbmJkVFVZcUMvWGsKMXdZOHJkbzBuMkdnZXhFd0VmOURyY21Qb04zOWRXUkdnY3EvTUg1U0JoemJTUlRJZnJaSjZaM1B6YitvMUdydgpuUTVpT1k1RjJnNmlrZWFZUVJZSTJ1SStzaytndGJIOW5lVTBZZmFSOFZVYjl2L3JxTnl1czZyZlIvbDFuUTdHCmRXUW1ETG9wdEdGdGI3ZXFtYmN4RGtUOGE3THFzblE5V1puUTRNVzgyN1dqM09RT01tbFYxQ3hXcEY5ZWpJOXkKUjBadVY5SXF3TXBCRU1FU3kwdlFkUVdvcmUwcmo5M0cKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBb3YxbWJ2bXlYMkplZDhQcnlwRXVJQld6RHREQkNBSHNyOFFRNmplRy9EdXhTOU5CCm52UStIVVhpR3lBbHB3TzR3bVB3b0FGTzdDbitueG5zY1Q4ZVIxbTl2UzM2bzRJN0lBSkJ4eXNCVU5kdU80KzMKcFc2ai92MDVNUCtnUHFyenN1THZvRllweGhJVllNQW1qc2tiU2RUSXFFdWQ3cHNVN3FyK29ieUJvSDBLN1NyNgpFdFVyOUhycU5SOVJWY0htSURqeXg1em52YjhIMGVNbFBYNjVzV1ZOb21TMmphdm9qd2JPbEZ1RUR6MnFZTDZhCkthMGhsQjBFWkdyalRhRDNsMXdURUl1WlJEOUhXUDY2Q1dnaER6MjZwRGVodzIxWXpseVdLbC9hdldaNWZuRVUKbGw5VkJjR2llSXBHZkFqT25mSXJaQzZVaVlGbXBSRzRoZmtPcVFJREFRQUJBb0lCQUFFZ2FVWFpHZitJeVloZwpaVS9ZSXc1N3Qrb2hUVVJGNXlIcjVBWUtiamR1SGpLYytqWWtFQTNadnM5Yk1JZHdBaVh0MU9yMk4xcHdpOThECnFkdTZpUFlMUzhGMEFPaUNiUEF5czh1eG8yZXdwWVZwMytZWDFyQmJoYnF0K2FpVk8rdi9uNUpuSmhERmJUcG0KK1oxQkRKdzlDVW9JZzhYemNuaVgyL3k0TmRrcklVVnk0cUxTNEtLOXFkK0tMU1VPQTk0UWVNVGdQQkxBWVpTOQpVNDBFUFhxTGc1VjRBTitORTltWDh4ZHFIRWJJV05HaG96ZytjT3hqaEtHbkdEZ0RpcjZmSjRkU29ISVBVYkZ6CnhXUG44NWNrdE0vOUhyTGdaUmY2UVhrNm51bnZxc1l6d1Vkdnh5U2o1bDN5VktmcmZ0MGgyK3FYVEYwdGM0algKUzlpSkx4VUNnWUVBMWxrbjdEMk1WZDFrbC9FUlZCTHczNno4aEF5U3I5eWM0d2JHb05rYzVQeTVrZ0pISVdkQgpIeDBFbGJPLzJqL1g2ZnRETkoxb0JCL1VZT1BFTlRvU2IrK2hqSEovNDVkdGhwc3hrRFArZnlvdzZSQ3NiRCtPCkVIMGJiYi8yVmcvTnI1L1NyS0tQRXJnYnA0dHBGZWlGZ05icGdldE1XYU9ZNzBhcXlYb1RHUHNDZ1lFQXdxbGwKNXdFUzQ5NW42cnZWRnlCQTlwVHpvRVBNRldvemVYNU9waHNJMDdKcUd5UUZwM3hYNlVjc09CdEVQbjljNzFCUQppQlpaN2hTSFd4SUt4M0lSTlo2K1paSVhKSjJ4cmppOFRIMFBmMTVpTFV4Z2I3bUJmVHB6dzR6VXo0WkdjY2pZClYzU2hCcnF4UjAzdnJ3V1U4ZDk5blZtdVE2dk9FWHRSb3RtVjdhc0NnWUJmMlhwcXY1TVRpbTdES3Y5dXN4R1gKVkFtUEd2ZzFQOWZBQlhXbzA2MkNFQ2tEUzVWM01HOEVkTHRxSGZqNnBYWHRvWGdKL1NGWVJzdDNrQ0piRXQ3UAp3ejVnU1lzNndLb3ZlUktPdzFGWW1BdExEVE90TERaZzFURUF3TWhzNzZBd21xMWdtVWlWUEZMUmR4dkV1amVJCng1R2ZrUktST3dEV2JxbE1jeFE3SlFLQmdRQ1J6cEx1bWdaTk9tejZEU0dLeGZVQy9nVVNPVlluZ0RuNE9keXQKWExpYTRSdkxJb3M5cmN5bmhsRWY0WVNYajR6TUVJdVJzN1ErdVkrOWJtRTJqVUZJdVpRQnZwQ3BNNWxBVjJ6cgptZzArcVRLK25XUHFxN2ZMVmpvWjhtWUEzNkhxWHJkMW9uMXU3ZTRWNjlHcGRuUGZURGpwNEovQVBuTUc4eWp0CjR5OERLd0tCZ0RneWdBMlhYM3JnZm1jWXpOTDZ0cldBQU9HK2pmck9FZEVCNWtZYnM4T0JONWY3L05GbHhSYnMKQ2RIWnJSaWtFbUlZVnV1NjlHeVNPdUhKOXFVaUpYditWK1UzQkNKMmxaV2UwU3ZlQmI4U3BVZkhWdlNxYXpPcgphWjhnc1ROcXAvTUltaVNEcXZFM1ZxL3JveDRnSDdoVnBEdmtBRitUZTRTWXd5N1p1TWltCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== ---- -# Source: cilium/templates/hubble/tls-helm/ca-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: hubble-ca-secret - namespace: kube-system -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtyZ2ViYTlKMi9EWU1ONFAyYk5oZ293RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkwTURFd05ERTBNVE0xTmxvWERUSTNNREV3TXpFMApNVE0xTmxvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFvdjFtYnZteVgySmVkOFByeXBFdUlCV3pEdERCQ0FIc3I4UVE2amVHL0R1eFM5TkIKbnZRK0hVWGlHeUFscHdPNHdtUHdvQUZPN0NuK254bnNjVDhlUjFtOXZTMzZvNEk3SUFKQnh5c0JVTmR1TzQrMwpwVzZqL3YwNU1QK2dQcXJ6c3VMdm9GWXB4aElWWU1BbWpza2JTZFRJcUV1ZDdwc1U3cXIrb2J5Qm9IMEs3U3I2CkV0VXI5SHJxTlI5UlZjSG1JRGp5eDV6bnZiOEgwZU1sUFg2NXNXVk5vbVMyamF2b2p3Yk9sRnVFRHoycVlMNmEKS2EwaGxCMEVaR3JqVGFEM2wxd1RFSXVaUkQ5SFdQNjZDV2doRHoyNnBEZWh3MjFZemx5V0tsL2F2V1o1Zm5FVQpsbDlWQmNHaWVJcEdmQWpPbmZJclpDNlVpWUZtcFJHNGhma09xUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGSTc0YkpVMm4yY2VWc0F0bnBZL1pZQWV1NGpETUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQnpkck91UHJpY3NBWWhzRVFwQU8zTGRnM0tjaUdpWXJiUFc5cFkyN2Ruc0NZZlI4dERySXkrCmxFVnpURmNlUVg1cExQMDZqOHJSWmljMmR5NFVqYVpGWnlhSGtwNElCaDhLU2Q0T3g0d1hkbmJkVFVZcUMvWGsKMXdZOHJkbzBuMkdnZXhFd0VmOURyY21Qb04zOWRXUkdnY3EvTUg1U0JoemJTUlRJZnJaSjZaM1B6YitvMUdydgpuUTVpT1k1RjJnNmlrZWFZUVJZSTJ1SStzaytndGJIOW5lVTBZZmFSOFZVYjl2L3JxTnl1czZyZlIvbDFuUTdHCmRXUW1ETG9wdEdGdGI3ZXFtYmN4RGtUOGE3THFzblE5V1puUTRNVzgyN1dqM09RT01tbFYxQ3hXcEY5ZWpJOXkKUjBadVY5SXF3TXBCRU1FU3kwdlFkUVdvcmUwcmo5M0cKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBb3YxbWJ2bXlYMkplZDhQcnlwRXVJQld6RHREQkNBSHNyOFFRNmplRy9EdXhTOU5CCm52UStIVVhpR3lBbHB3TzR3bVB3b0FGTzdDbitueG5zY1Q4ZVIxbTl2UzM2bzRJN0lBSkJ4eXNCVU5kdU80KzMKcFc2ai92MDVNUCtnUHFyenN1THZvRllweGhJVllNQW1qc2tiU2RUSXFFdWQ3cHNVN3FyK29ieUJvSDBLN1NyNgpFdFVyOUhycU5SOVJWY0htSURqeXg1em52YjhIMGVNbFBYNjVzV1ZOb21TMmphdm9qd2JPbEZ1RUR6MnFZTDZhCkthMGhsQjBFWkdyalRhRDNsMXdURUl1WlJEOUhXUDY2Q1dnaER6MjZwRGVodzIxWXpseVdLbC9hdldaNWZuRVUKbGw5VkJjR2llSXBHZkFqT25mSXJaQzZVaVlGbXBSRzRoZmtPcVFJREFRQUJBb0lCQUFFZ2FVWFpHZitJeVloZwpaVS9ZSXc1N3Qrb2hUVVJGNXlIcjVBWUtiamR1SGpLYytqWWtFQTNadnM5Yk1JZHdBaVh0MU9yMk4xcHdpOThECnFkdTZpUFlMUzhGMEFPaUNiUEF5czh1eG8yZXdwWVZwMytZWDFyQmJoYnF0K2FpVk8rdi9uNUpuSmhERmJUcG0KK1oxQkRKdzlDVW9JZzhYemNuaVgyL3k0TmRrcklVVnk0cUxTNEtLOXFkK0tMU1VPQTk0UWVNVGdQQkxBWVpTOQpVNDBFUFhxTGc1VjRBTitORTltWDh4ZHFIRWJJV05HaG96ZytjT3hqaEtHbkdEZ0RpcjZmSjRkU29ISVBVYkZ6CnhXUG44NWNrdE0vOUhyTGdaUmY2UVhrNm51bnZxc1l6d1Vkdnh5U2o1bDN5VktmcmZ0MGgyK3FYVEYwdGM0algKUzlpSkx4VUNnWUVBMWxrbjdEMk1WZDFrbC9FUlZCTHczNno4aEF5U3I5eWM0d2JHb05rYzVQeTVrZ0pISVdkQgpIeDBFbGJPLzJqL1g2ZnRETkoxb0JCL1VZT1BFTlRvU2IrK2hqSEovNDVkdGhwc3hrRFArZnlvdzZSQ3NiRCtPCkVIMGJiYi8yVmcvTnI1L1NyS0tQRXJnYnA0dHBGZWlGZ05icGdldE1XYU9ZNzBhcXlYb1RHUHNDZ1lFQXdxbGwKNXdFUzQ5NW42cnZWRnlCQTlwVHpvRVBNRldvemVYNU9waHNJMDdKcUd5UUZwM3hYNlVjc09CdEVQbjljNzFCUQppQlpaN2hTSFd4SUt4M0lSTlo2K1paSVhKSjJ4cmppOFRIMFBmMTVpTFV4Z2I3bUJmVHB6dzR6VXo0WkdjY2pZClYzU2hCcnF4UjAzdnJ3V1U4ZDk5blZtdVE2dk9FWHRSb3RtVjdhc0NnWUJmMlhwcXY1TVRpbTdES3Y5dXN4R1gKVkFtUEd2ZzFQOWZBQlhXbzA2MkNFQ2tEUzVWM01HOEVkTHRxSGZqNnBYWHRvWGdKL1NGWVJzdDNrQ0piRXQ3UAp3ejVnU1lzNndLb3ZlUktPdzFGWW1BdExEVE90TERaZzFURUF3TWhzNzZBd21xMWdtVWlWUEZMUmR4dkV1amVJCng1R2ZrUktST3dEV2JxbE1jeFE3SlFLQmdRQ1J6cEx1bWdaTk9tejZEU0dLeGZVQy9nVVNPVlluZ0RuNE9keXQKWExpYTRSdkxJb3M5cmN5bmhsRWY0WVNYajR6TUVJdVJzN1ErdVkrOWJtRTJqVUZJdVpRQnZwQ3BNNWxBVjJ6cgptZzArcVRLK25XUHFxN2ZMVmpvWjhtWUEzNkhxWHJkMW9uMXU3ZTRWNjlHcGRuUGZURGpwNEovQVBuTUc4eWp0CjR5OERLd0tCZ0RneWdBMlhYM3JnZm1jWXpOTDZ0cldBQU9HK2pmck9FZEVCNWtZYnM4T0JONWY3L05GbHhSYnMKQ2RIWnJSaWtFbUlZVnV1NjlHeVNPdUhKOXFVaUpYditWK1UzQkNKMmxaV2UwU3ZlQmI4U3BVZkhWdlNxYXpPcgphWjhnc1ROcXAvTUltaVNEcXZFM1ZxL3JveDRnSDdoVnBEdmtBRitUZTRTWXd5N1p1TWltCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== ---- -# Source: cilium/templates/hubble/tls-helm/server-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: hubble-server-certs - namespace: kube-system -type: kubernetes.io/tls -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUtyZ2ViYTlKMi9EWU1ONFAyYk5oZ293RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkwTURFd05ERTBNVE0xTmxvWERUSTNNREV3TXpFMApNVE0xTmxvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFvdjFtYnZteVgySmVkOFByeXBFdUlCV3pEdERCQ0FIc3I4UVE2amVHL0R1eFM5TkIKbnZRK0hVWGlHeUFscHdPNHdtUHdvQUZPN0NuK254bnNjVDhlUjFtOXZTMzZvNEk3SUFKQnh5c0JVTmR1TzQrMwpwVzZqL3YwNU1QK2dQcXJ6c3VMdm9GWXB4aElWWU1BbWpza2JTZFRJcUV1ZDdwc1U3cXIrb2J5Qm9IMEs3U3I2CkV0VXI5SHJxTlI5UlZjSG1JRGp5eDV6bnZiOEgwZU1sUFg2NXNXVk5vbVMyamF2b2p3Yk9sRnVFRHoycVlMNmEKS2EwaGxCMEVaR3JqVGFEM2wxd1RFSXVaUkQ5SFdQNjZDV2doRHoyNnBEZWh3MjFZemx5V0tsL2F2V1o1Zm5FVQpsbDlWQmNHaWVJcEdmQWpPbmZJclpDNlVpWUZtcFJHNGhma09xUUlEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGSTc0YkpVMm4yY2VWc0F0bnBZL1pZQWV1NGpETUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQnpkck91UHJpY3NBWWhzRVFwQU8zTGRnM0tjaUdpWXJiUFc5cFkyN2Ruc0NZZlI4dERySXkrCmxFVnpURmNlUVg1cExQMDZqOHJSWmljMmR5NFVqYVpGWnlhSGtwNElCaDhLU2Q0T3g0d1hkbmJkVFVZcUMvWGsKMXdZOHJkbzBuMkdnZXhFd0VmOURyY21Qb04zOWRXUkdnY3EvTUg1U0JoemJTUlRJZnJaSjZaM1B6YitvMUdydgpuUTVpT1k1RjJnNmlrZWFZUVJZSTJ1SStzaytndGJIOW5lVTBZZmFSOFZVYjl2L3JxTnl1czZyZlIvbDFuUTdHCmRXUW1ETG9wdEdGdGI3ZXFtYmN4RGtUOGE3THFzblE5V1puUTRNVzgyN1dqM09RT01tbFYxQ3hXcEY5ZWpJOXkKUjBadVY5SXF3TXBCRU1FU3kwdlFkUVdvcmUwcmo5M0cKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lRS1lqRTJkUmZnK3hDVDBld01hRjB3REFOQmdrcWhraUc5dzBCQVFzRkFEQVUKTVJJd0VBWURWUVFERXdsRGFXeHBkVzBnUTBFd0hoY05NalF3TVRBME1UUXhNelUzV2hjTk1qY3dNVEF6TVRReApNelUzV2pBcU1TZ3dKZ1lEVlFRRERCOHFMbVJsWm1GMWJIUXVhSFZpWW14bExXZHljR011WTJsc2FYVnRMbWx2Ck1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBd3NwWUVpSUhiTzlLLzY0YzQ1cmkKeTJHVkZYVldBZ3F6VlQ5aVMyQWFhQTFTeGRhV1JDWlhxekZ1Q3dCcmFSR3FBMmR6WjFVYlJ0WFVzYytPVUYxNgprbkNaZ1RTbCt2YlhmR2RlL05JT0JIVlZIWmoxUTYyTHptelN0ZmVNMjVDWE5aZVpiSVo1VTM1VG1OVDRrRzhFCkZvZjcraXNGeldvcGZCc1RQdERvUXd2eFF2NW8rV0wvTVFmOXR1YTBIN0cvZTJzQlFRWnRjeGJhVEhuRUlRVzgKUXJKcGFQajhZN0Y1Y1U3ZFJzR2pJem9WODFMMk51QjRlT0pvVmxVY3ZTZzdrQTRnTXlNa0w2UW9sQkN2YkZVdQpDYjN5aXNUNytFeFVVSkszS2VoNWUzTm9oOStIWmdLcHdhamR3eWV1ZFlXdjlIenB5Rjd6WUZpZ1VvcGFuaHhECmR3SURBUUFCbzRHTk1JR0tNQTRHQTFVZER3RUIvd1FFQXdJRm9EQWRCZ05WSFNVRUZqQVVCZ2dyQmdFRkJRY0QKQVFZSUt3WUJCUVVIQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JTTytHeVZOcDluSGxiQQpMWjZXUDJXQUhydUl3ekFxQmdOVkhSRUVJekFoZ2g4cUxtUmxabUYxYkhRdWFIVmlZbXhsTFdkeWNHTXVZMmxzCmFYVnRMbWx2TUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCUUlvSnhIR3p3eVplakRWaXk1YnlJdXR6K1hubzMKVGtxdHZDWHRrRS9yb0lvYUhBL0JLZzBQSEIxYS9PcVRrT0VOQjZVUGc0RDczU3QzeWhkdGlhYjh2YmJSd2w1MApGaWxBUUh1OE5YVGpIZVJERVB0U2NldXdPRy9XQnM1WmcxeDNIT3FaWWJzaXBtNFNWRFlPa0FHOUxyZGFUY0p6CmhpbHpGSFlCeVprR1MwenNkUkRvTHJSVjVSRUVicEdwSmxBM2ZScGYrNC9MdklOaWhTbFRQWnlESmYrVkRnYm0KeklBTnJ3QUJEcWVoeXMrVmVhQ3B3NFk2Rm1DcWVlemZ0cThoTjFMbmNJbWhxckI2YjdpUGxXTy8xU2k1WC9pUwpIVi9uRHZsM0E4K3JjOU1maTEydzAvTkxleTVjbHBmRklGTktqeXhuNXRTUXlzenBveXo3RFF4RAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBd3NwWUVpSUhiTzlLLzY0YzQ1cml5MkdWRlhWV0FncXpWVDlpUzJBYWFBMVN4ZGFXClJDWlhxekZ1Q3dCcmFSR3FBMmR6WjFVYlJ0WFVzYytPVUYxNmtuQ1pnVFNsK3ZiWGZHZGUvTklPQkhWVkhaajEKUTYyTHptelN0ZmVNMjVDWE5aZVpiSVo1VTM1VG1OVDRrRzhFRm9mNytpc0Z6V29wZkJzVFB0RG9Rd3Z4UXY1bworV0wvTVFmOXR1YTBIN0cvZTJzQlFRWnRjeGJhVEhuRUlRVzhRckpwYVBqOFk3RjVjVTdkUnNHakl6b1Y4MUwyCk51QjRlT0pvVmxVY3ZTZzdrQTRnTXlNa0w2UW9sQkN2YkZVdUNiM3lpc1Q3K0V4VVVKSzNLZWg1ZTNOb2g5K0gKWmdLcHdhamR3eWV1ZFlXdjlIenB5Rjd6WUZpZ1VvcGFuaHhEZHdJREFRQUJBb0lCQVFDUC9IcDFqT25vUUVvMApEVVFWaXB3eXNvTTlSV3E2UDZhb1VSNGJrWUp6ZjhXN3NBK0xlTHc5TTJDZ2lVTGNlODRzamYva1ZpaS9oN2lkClowdFBaSW9DVFhGcTdMdjBjeHArUHZzYjd6bWxXZ1g4eHVBbGt0ZFJVMmUrVXFxd2NvVVRmR2hRckdacXJXcW0KQW1kcWZHV05NaCswb0xDOVVzZ3ZrT1A4L3ZkdzlzeHdpdlRLd0IvQkEvTHRzZkRIVmJUSkRUNnpqd2xFdmt1dQpxdHpjK1h4bWhzOTQxVjB6TWFxV2NGSnZubU5sWGgvWE83T3hNUHpBTTJoMjRDOXlIWEZ3cGZ1RVhHeGJ0a0FlClUxdUhLTjFBczQ3dVZWMm5kQkdhWGhiWkhSRy9sNUt6L2dnbmNMblJSRDEzcUtJY202K3NCbWN4QjJLNENOUVUKb012amxXN2hBb0dCQU5UaE10SmJVTnlyWWpWQkF0eS85MnVFaCtoNWJmM2ZVZE5ITGxJdWllUjR3bSt2NDYwMApkSk5qdm5kazRSMUI1L3hxWFVlTGdpUWtTKytwNDhSdFJzWDRtQkkxV3BycXg2SG1FcjZGOXJKS1gyRDlJcEJ1CmdjR2JvYUZkWEJCYU1qNjhTb0pRdjRmTzExVklaVDBnTFdKaTk1emZsaW9rNjNZK3VpM1BHQ3MxQW9HQkFPby8KSWVoNXBVTU1FSUlWaEtVSnYwdHowbXVkVDRiMDhVUkFkNVZWTDFhQ2liNFM3NFo2WTRkMThzVEttWmIxOXR6Ugo3Z2s5ZlpUOEZUczJ3K0EzU3ZCNXpzKzZaWFdobWJCeDVtSTZQZWhvb3Y1Y1Z1cDNhZGREMWI5ZTZEZmFxeDF1CmttWGk3YS9iYzBsTWxWNFhadEs4K0JVMjFTY3I3MDhxb3l0L0FKMTdBb0dCQUtITGJ5YjJCR1ZUVk5jSXpEWWYKS050czVTYkpjOUdxTTdTU3V4LzNBeExFdlhMSmlnK1FlTEcwODRRbkJmbUlwOXJ1dEdxMkczbUFwTDVJcnpZUgpuK0xEYUVFZWovaUNUS3crQXkwUEFFck52K1FsUUducHMzRUU2am1NalhaWU55djdTOTN6cFQ2QTlLUjYyNGN6CnQ0cTVrVnlIaXNnSk8zU3BoRE5nM1F1cEFvR0FCNVBUUFBueVpxRFZSOS82VW54bU84S2JZKy9LeXNyUm95dVMKc3JuMnRnMnk5dnNWMUh2UkRHaThXYkZVRnBZT1RrWmNOT0JRVjZsdjVtSjNJNkdtUXZFdHgzcDRIYjRFZ3JpcwpnekR2SDcyQ3lFZWthQ0ZFd3pubWlESkJ3TXpLREVHdmFxeFVsQU85VHZtek96LzJpNHFoVzg1U1J0b2MrNnFkCmJLTEhHbEVDZ1lBdGRQOFBZRkpsY1B1THVZMkxoM2tpRHZ4UjF3NFQxeXliMit1MXVpRU9LRTBRS20rNDdYRnUKN2U3TjZLR1dIMUZseVFuVVR4QUptcmVDcjY4bFp4QUZhQWVOdXdmZTRqZHdYWm9aMHpJajVlU0FHeTg5MmZ4YwpVRnVaTmxhVkdaMm4rSXh3bWFHRlpRU2NXOGlKK005enRrZTJKTmxBT2JZSkYwZ0tHaE94ZkE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= ---- -# Source: cilium/templates/cilium-configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-config - namespace: kube-system -data: - - # Identity allocation mode selects how identities are shared between cilium - # nodes by setting how they are stored. The options are "crd" or "kvstore". - # - "crd" stores identities in kubernetes as CRDs (custom resource definition). - # These can be queried with: - # kubectl get ciliumid - # - "kvstore" stores identities in an etcd kvstore, that is - # configured below. Cilium versions before 1.6 supported only the kvstore - # backend. Upgrades from these older cilium versions should continue using - # the kvstore by commenting out the identity-allocation-mode below, or - # setting it to "kvstore". - identity-allocation-mode: crd - identity-heartbeat-timeout: "30m0s" - identity-gc-interval: "15m0s" - cilium-endpoint-gc-interval: "5m0s" - nodes-gc-interval: "5m0s" - skip-cnp-status-startup-clean: "false" - # Disable the usage of CiliumEndpoint CRD - disable-endpoint-crd: "false" - - # If you want to run cilium in debug mode change this value to true - debug: "false" - debug-verbose: "" - # The agent can be put into the following three policy enforcement modes - # default, always and never. - # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes - enable-policy: "default" - - # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 - # address. - enable-ipv4: "true" - - # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 - # address. - enable-ipv6: "false" - # Users who wish to specify their own custom CNI configuration file must set - # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. - custom-cni-conf: "false" - enable-bpf-clock-probe: "false" - # If you want cilium monitor to aggregate tracing for packets, set this level - # to "low", "medium", or "maximum". The higher the level, the less packets - # that will be seen in monitor output. - monitor-aggregation: medium - - # The monitor aggregation interval governs the typical time between monitor - # notification events for each allowed connection. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-interval: "5s" - - # The monitor aggregation flags determine which TCP flags which, upon the - # first observation, cause monitor notifications to be generated. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-flags: all - # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic - # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. - bpf-map-dynamic-size-ratio: "0.0025" - # bpf-policy-map-max specifies the maximum number of entries in endpoint - # policy map (per endpoint) - bpf-policy-map-max: "16384" - # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, - # backend and affinity maps. - bpf-lb-map-max: "65536" - bpf-lb-external-clusterip: "false" - - # Pre-allocation of map entries allows per-packet latency to be reduced, at - # the expense of up-front memory allocation for the entries in the maps. The - # default value below will minimize memory usage in the default installation; - # users who are sensitive to latency may consider setting this to "true". - # - # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore - # this option and behave as though it is set to "true". - # - # If this value is modified, then during the next Cilium startup the restore - # of existing endpoints and tracking of ongoing connections may be disrupted. - # As a result, reply packets may be dropped and the load-balancing decisions - # for established connections may change. - # - # If this option is set to "false" during an upgrade from 1.3 or earlier to - # 1.4 or later, then it may cause one-time disruptions during the upgrade. - preallocate-bpf-maps: "false" - - # Regular expression matching compatible Istio sidecar istio-proxy - # container image names - sidecar-istio-proxy-image: "cilium/istio_proxy" - - # Name of the cluster. Only relevant when building a mesh of clusters. - cluster-name: default - # Unique ID of the cluster. Must be unique across all conneted clusters and - # in the range of 1 and 255. Only relevant when building a mesh of clusters. - cluster-id: "0" - - # Encapsulation mode for communication between nodes - # Possible values: - # - disabled - # - vxlan (default) - # - geneve - tunnel: "vxlan" - - - # Enables L7 proxy for L7 policy enforcement and visibility - enable-l7-proxy: "true" - - enable-ipv4-masquerade: "true" - enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "true" - - enable-xt-socket-fallback: "true" - install-no-conntrack-iptables-rules: "false" - - auto-direct-node-routes: "false" - enable-local-redirect-policy: "false" - - kube-proxy-replacement: "disabled" - bpf-lb-sock: "false" - enable-health-check-nodeport: "true" - node-port-bind-protection: "true" - enable-auto-protect-node-port-range: "true" - enable-svc-source-range-check: "true" - enable-l2-neigh-discovery: "true" - arping-refresh-period: "30s" - cni-uninstall: "true" - enable-endpoint-health-checking: "true" - enable-health-checking: "true" - enable-well-known-identities: "false" - enable-remote-node-identity: "true" - synchronize-k8s-nodes: "true" - operator-api-serve-addr: "127.0.0.1:9234" - # Enable Hubble gRPC service. - enable-hubble: "true" - # UNIX domain socket for Hubble server to listen to. - hubble-socket-path: "/var/run/cilium/hubble.sock" - # An additional address for Hubble server to listen to (e.g. ":4244"). - hubble-listen-address: ":4244" - hubble-disable-tls: "false" - hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt - hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key - hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt - ipam: "cluster-pool" - cluster-pool-ipv4-cidr: "10.0.0.0/8" - cluster-pool-ipv4-mask-size: "24" - disable-cnp-status-updates: "true" - enable-vtep: "false" - vtep-endpoint: "" - vtep-cidr: "" - vtep-mask: "" - vtep-mac: "" - enable-bgp-control-plane: "false" - procfs: "/host/proc" - bpf-root: "/sys/fs/bpf" - cgroup-root: "/run/cilium/cgroupv2" - enable-k8s-terminating-endpoint: "true" - enable-sctp: "false" - remove-cilium-node-taints: "true" - set-cilium-is-up-condition: "true" - unmanaged-pod-watcher-interval: "15" - tofqdns-dns-reject-response-code: "refused" - tofqdns-enable-dns-compression: "true" - tofqdns-endpoint-max-ip-per-hostname: "50" - tofqdns-idle-connection-grace-period: "0s" - tofqdns-max-deferred-connection-deletes: "10000" - tofqdns-min-ttl: "3600" - tofqdns-proxy-response-max-delay: "100ms" - agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" ---- -# Source: cilium/templates/cilium-agent/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - pods - - endpoints - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list - - watch - # This is used when validating policies in preflight. This will need to stay - # until we figure out how to avoid "get" inside the preflight, and then - # should be removed ideally. - - get -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumbgppeeringpolicies - - ciliumclusterwideenvoyconfigs - - ciliumclusterwidenetworkpolicies - - ciliumegressgatewaypolicies - - ciliumendpoints - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumidentities - - ciliumlocalredirectpolicies - - ciliumnetworkpolicies - - ciliumnodes - - ciliumnodeconfigs - verbs: - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - - ciliumendpoints - - ciliumnodes - verbs: - - create -- apiGroups: - - cilium.io - # To synchronize garbage collection of such resources - resources: - - ciliumidentities - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - verbs: - - delete - - get -- apiGroups: - - cilium.io - resources: - - ciliumnodes - - ciliumnodes/status - verbs: - - get - - update -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - - ciliumendpoints/status - - ciliumendpoints - verbs: - - patch ---- -# Source: cilium/templates/cilium-operator/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - - delete -- apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch -- apiGroups: - - "" - resources: - # To remove node taints - - nodes - # To set NetworkUnavailable false on startup - - nodes/status - verbs: - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform LB IP allocation for BGP - - services/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - # to check apiserver connectivity - - namespaces - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform the translation of a CNP that contains `ToGroup` to its endpoints - - services - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumclusterwidenetworkpolicies - verbs: - # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' - - create - - update - - deletecollection - # To update the status of the CNPs and CCNPs - - patch - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - verbs: - # Update the auto-generated CNPs and CCNPs status. - - patch - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - - ciliumidentities - verbs: - # To perform garbage collection of such resources - - delete - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - verbs: - # To synchronize garbage collection of such resources - - update -- apiGroups: - - cilium.io - resources: - - ciliumnodes - verbs: - - create - - update - - get - - list - - watch - # To perform CiliumNode garbage collector - - delete -- apiGroups: - - cilium.io - resources: - - ciliumnodes/status - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpointslices - - ciliumenvoyconfigs - verbs: - - create - - update - - get - - list - - watch - - delete - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - update - resourceNames: - - ciliumloadbalancerippools.cilium.io - - ciliumbgppeeringpolicies.cilium.io - - ciliumclusterwideenvoyconfigs.cilium.io - - ciliumclusterwidenetworkpolicies.cilium.io - - ciliumegressgatewaypolicies.cilium.io - - ciliumendpoints.cilium.io - - ciliumendpointslices.cilium.io - - ciliumenvoyconfigs.cilium.io - - ciliumexternalworkloads.cilium.io - - ciliumidentities.cilium.io - - ciliumlocalredirectpolicies.cilium.io - - ciliumnetworkpolicies.cilium.io - - ciliumnodes.cilium.io - - ciliumnodeconfigs.cilium.io -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools/status - verbs: - - patch -# For cilium-operator running in HA mode. -# -# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election -# between multiple running instances. -# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less -# common and fewer objects in the cluster watch "all Leases". -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update ---- -# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium-operator -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-config-agent -subjects: - - kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/hubble/peer-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: hubble-peer - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: hubble-peer -spec: - selector: - k8s-app: cilium - ports: - - name: peer-service - port: 443 - protocol: TCP - targetPort: 4244 ---- -# Source: cilium/templates/cilium-agent/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-agent -spec: - selector: - matchLabels: - k8s-app: cilium - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - # Set app AppArmor's profile to "unconfined". The value of this annotation - # can be modified as long users know which profiles they have available - # in AppArmor. - container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" - container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" - container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" - container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" - labels: - k8s-app: cilium - app.kubernetes.io/name: cilium-agent - app.kubernetes.io/part-of: cilium - spec: - containers: - - name: cilium-agent - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - command: - - cilium-agent - args: - - --config-dir=/tmp/cilium/config-map - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 105 - periodSeconds: 2 - successThreshold: 1 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ - - name: CILIUM_CNI_CHAINING_MODE - valueFrom: - configMapKeyRef: - name: cilium-config - key: cni-chaining-mode - optional: true - - name: CILIUM_CUSTOM_CNI_CONF - valueFrom: - configMapKeyRef: - name: cilium-config - key: custom-cni-conf - optional: true - lifecycle: - postStart: - exec: - command: - - "bash" - - "-c" - - | - /cni-install.sh --enable-debug=false --cni-exclusive=true --log-file=/var/run/cilium/cilium-cni.log - preStop: - exec: - command: - - /cni-uninstall.sh - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - CHOWN - - KILL - - NET_ADMIN - - NET_RAW - - IPC_LOCK - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - - DAC_OVERRIDE - - FOWNER - - SETGID - - SETUID - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - # Unprivileged containers need to mount /proc/sys/net from the host - # to have write access - - mountPath: /host/proc/sys/net - name: host-proc-sys-net - # Unprivileged containers need to mount /proc/sys/kernel from the host - # to have write access - - mountPath: /host/proc/sys/kernel - name: host-proc-sys-kernel - - name: bpf-maps - mountPath: /sys/fs/bpf - # Unprivileged containers can't set mount propagation to bidirectional - # in this case we will mount the bpf fs from an init container that - # is privileged and set the mount propagation from host to container - # in Cilium. - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium - - name: etc-cni-netd - mountPath: /host/etc/cni/net.d - - name: clustermesh-secrets - mountPath: /var/lib/cilium/clustermesh - readOnly: true - # Needed to be able to load kernel modules - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: xtables-lock - mountPath: /run/xtables.lock - - name: hubble-tls - mountPath: /var/lib/cilium/tls/hubble - readOnly: true - - name: tmp - mountPath: /tmp - initContainers: - - name: config - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - command: - - cilium - - build-config - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumeMounts: - - name: tmp - mountPath: /tmp - terminationMessagePolicy: FallbackToLogsOnError - # Required to mount cgroup2 filesystem on the underlying Kubernetes node. - # We use nsenter command with host's cgroup and mount namespaces enabled. - - name: mount-cgroup - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - env: - - name: CGROUP_ROOT - value: /run/cilium/cgroupv2 - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh and mount that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-mount /hostbin/cilium-mount; - nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; - rm /hostbin/cilium-mount - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - - name: apply-sysctl-overwrites - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - env: - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; - nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; - rm /hostbin/cilium-sysctlfix - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - # Mount the bpf fs if it is not mounted. We will perform this task - # from a privileged container because the mount propagation bidirectional - # only works from privileged containers. - - name: mount-bpf-fs - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - args: - - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' - command: - - /bin/bash - - -c - - -- - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - privileged: true - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional - - name: clean-cilium-state - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - command: - - /init-container.sh - env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-state - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-bpf-state - optional: true - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - drop: - - ALL - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - # Required to mount cgroup filesystem from the host to cilium agent pod - - name: cilium-cgroup - mountPath: /run/cilium/cgroupv2 - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium # wait-for-kube-proxy - # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - - name: install-cni-binaries - image: "quay.io/cilium/cilium:v1.13.10" - imagePullPolicy: IfNotPresent - command: - - "/install-plugin.sh" - resources: - requests: - cpu: 100m - memory: 10Mi - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: cni-path - mountPath: /host/opt/cni/bin # .Values.cni.install - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccount: "cilium" - serviceAccountName: "cilium" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - # For sharing configuration between the "config" initContainer and the agent - - name: tmp - emptyDir: {} - # To keep state between restarts / upgrades - - name: cilium-run - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate - # To mount cgroup2 filesystem on the host - - name: hostproc - hostPath: - path: /proc - type: Directory - # To keep state between restarts / upgrades for cgroup2 filesystem - - name: cilium-cgroup - hostPath: - path: /run/cilium/cgroupv2 - type: DirectoryOrCreate - # To install cilium cni plugin in the host - - name: cni-path - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - # To install cilium cni configuration in the host - - name: etc-cni-netd - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - # To be able to load kernel modules - - name: lib-modules - hostPath: - path: /lib/modules - # To access iptables concurrently with other processes (e.g. kube-proxy) - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # To read the clustermesh configuration - - name: clustermesh-secrets - secret: - secretName: cilium-clustermesh - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - optional: true - - name: host-proc-sys-net - hostPath: - path: /proc/sys/net - type: Directory - - name: host-proc-sys-kernel - hostPath: - path: /proc/sys/kernel - type: Directory - - name: hubble-tls - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: hubble-server-certs - optional: true - items: - - key: ca.crt - path: client-ca.crt - - key: tls.crt - path: server.crt - - key: tls.key - path: server.key ---- -# Source: cilium/templates/cilium-operator/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cilium-operator - namespace: kube-system - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator -spec: - # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go - # for more details. - replicas: 2 - selector: - matchLabels: - io.cilium/app: operator - name: cilium-operator - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - annotations: - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator - spec: - containers: - - name: cilium-operator - image: "quay.io/cilium/operator-generic:v1.13.10" - imagePullPolicy: IfNotPresent - command: - - cilium-operator-generic - args: - - --config-dir=/tmp/cilium/config-map - - --debug=$(CILIUM_DEBUG) - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_DEBUG - valueFrom: - configMapKeyRef: - key: debug - name: cilium-config - optional: true - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 3 - volumeMounts: - - name: cilium-config-path - mountPath: /tmp/cilium/config-map - readOnly: true - terminationMessagePolicy: FallbackToLogsOnError - hostNetwork: true - restartPolicy: Always - priorityClassName: system-cluster-critical - serviceAccount: "cilium-operator" - serviceAccountName: "cilium-operator" - automountServiceAccountToken: true - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - io.cilium/app: operator - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - # To read the configuration from the config map - - name: cilium-config-path - configMap: - name: cilium-config ---- -# Source: cilium/templates/cilium-secrets-namespace.yaml -# Only create the namespace if it's different from Ingress secret namespace or Ingress is not enabled. - -# Only create the namespace if it's different from Ingress and Gateway API secret namespaces (if enabled). diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index f5102273..e1b5e0cb 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -74,7 +74,7 @@ def apply(self): release_name=self.cluster.stack_id, chart_ref=os.path.join( pkg_resources.resource_filename("magnum_cluster_api", "charts"), - "cluster-autoscaler/", + "vendor/cluster-autoscaler/", ), values={ "fullnameOverride": f"{self.cluster.stack_id}-autoscaler", @@ -204,11 +204,27 @@ def get_object(self) -> pykube.ConfigMap: data = { **data, **{ - "cilium.yml": image_utils.update_manifest_images( - self.cluster.uuid, - os.path.join(manifests_path, f"cilium/{cilium_version}.yaml"), - repository=repository, - ) + "cilium.yml": helm.TemplateReleaseCommand( + namespace="kube-system", + release_name="cilium", + chart_ref=os.path.join( + pkg_resources.resource_filename( + "magnum_cluster_api", "charts" + ), + "vendor/cilium/", + ), + values={ + "ipam": { + "operator": { + "clusterPoolIPv4PodCIDRList": [ + utils.get_cluster_label( + self.cluster, "cilium_ipv4pool", "10.100.0.0/16", + ), + ], + }, + }, + }, + )(repository=repository) }, } diff --git a/vendir.lock.yml b/vendir.lock.yml new file mode 100644 index 00000000..767fc7ce --- /dev/null +++ b/vendir.lock.yml @@ -0,0 +1,13 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - helmChart: + appVersion: 1.27.2 + version: 9.29.1 + path: cluster-autoscaler + - helmChart: + appVersion: 1.15.3 + version: 1.15.3 + path: cilium + path: magnum_cluster_api/charts/vendor +kind: LockConfig diff --git a/vendir.yml b/vendir.yml new file mode 100644 index 00000000..f824dbc5 --- /dev/null +++ b/vendir.yml @@ -0,0 +1,18 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: magnum_cluster_api/charts/vendor + excludePaths: k8s-keystone-auth + contents: + - path: cluster-autoscaler + helmChart: + name: cluster-autoscaler + version: 9.29.1 + repository: + url: https://kubernetes.github.io/autoscaler + - path: cilium + helmChart: + name: cilium + version: 1.15.3 + repository: + url: https://helm.cilium.io/ From f3190e2a48b8c02c50f72b8419b84dd01d098cbf Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 28 Mar 2024 05:00:27 +1100 Subject: [PATCH 05/21] fix vendir --- Earthfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index eb1b14c3..3e0199c6 100644 --- a/Earthfile +++ b/Earthfile @@ -12,7 +12,8 @@ vendir: build: FROM github.com/vexxhost/atmosphere/images/magnum+build COPY +vendir/vendir /usr/local/bin/vendir - COPY --dir magnum_cluster_api/ pyproject.toml README.md /src + COPY github.com/vexxhost/atmosphere/images/helm+binary/helm /usr/local/bin/helm + COPY --dir magnum_cluster_api/ pyproject.toml README.md vendir.yml /src WORKDIR /src RUN vendir sync COPY hack/add-omt-to-clusterrole.patch /hack/ From 75329fa88342aae60891e45791219b48e62247fe Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 28 Mar 2024 05:10:49 +1100 Subject: [PATCH 06/21] add vendir in hack --- hack/setup-vendir.sh | 23 +++++++++++++++++++++++ hack/stack.sh | 3 +++ 2 files changed, 26 insertions(+) create mode 100644 hack/setup-vendir.sh diff --git a/hack/setup-vendir.sh b/hack/setup-vendir.sh new file mode 100644 index 00000000..27ebacb2 --- /dev/null +++ b/hack/setup-vendir.sh @@ -0,0 +1,23 @@ +#!/bin/bash -xe + +# Copyright (c) 2024 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Versions to test +VENDIR_VERSION=${VENDIR_VERSION:-v0.40.0} + +# Install `vendir` CLI +curl -Lo /tmp/vendir https://github.com/carvel-dev/vendir/releases/download/${VENDIR_VERSION}/vendir-linux-amd64 +chmod +x /tmp/vendir +sudo mv /tmp/vendir /usr/local/bin/vendir diff --git a/hack/stack.sh b/hack/stack.sh index c5ad0411..438c9a1b 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -107,6 +107,9 @@ EOF # Install CAPI/CAPO ./hack/setup-capo.sh +# Install vendir +./hack/setup-vendir.sh + # Vendor the chart make vendor From c560f3ff0ed7131e5d0fe72ee34d331bed0401bd Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 28 Mar 2024 05:29:05 +1100 Subject: [PATCH 07/21] set cilium tag --- magnum_cluster_api/resources.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index e1b5e0cb..a2ebc8db 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -46,7 +46,7 @@ CONF = cfg.CONF CALICO_TAG = "v3.24.2" -CILIUM_TAG = "v1.13.10" +CILIUM_TAG = "v1.15.3" CLUSTER_CLASS_VERSION = pkg_resources.require("magnum_cluster_api")[0].version CLUSTER_CLASS_NAME = f"magnum-v{CLUSTER_CLASS_VERSION}" @@ -214,11 +214,21 @@ def get_object(self) -> pykube.ConfigMap: "vendor/cilium/", ), values={ + "image": { + "tag": cilium_version, + }, + "operator": { + "image": { + "tag": cilium_version, + }, + }, "ipam": { "operator": { "clusterPoolIPv4PodCIDRList": [ utils.get_cluster_label( - self.cluster, "cilium_ipv4pool", "10.100.0.0/16", + self.cluster, + "cilium_ipv4pool", + "10.100.0.0/16", ), ], }, From 04c8c2a8a294487ec9cd6d0a3af43beda854fcd6 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Thu, 28 Mar 2024 23:59:38 +1100 Subject: [PATCH 08/21] update makefile for vendir install --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d73c655f..733c272e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,12 @@ clean: - rm -rfv magnum_cluster_api/charts/cluster-autoscaler + rm -rfv magnum_cluster_api/charts/vendor -vendor: clean +vendir: + curl -Lo vendir https://github.com/carvel-dev/vendir/releases/download/v0.40.0/vendir-linux-amd64 + chmod +x vendir && ./vendir version + mv vendir /usr/local/bin/vendir + +vendor: clean vendir vendir sync patch -p0 magnum_cluster_api/charts/vendor/cluster-autoscaler/templates/clusterrole.yaml < hack/add-omt-to-clusterrole.patch From 305a9ca2785d7dbb308cf04d3ad1bafc8fe2221d Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 29 Mar 2024 00:18:15 +1100 Subject: [PATCH 09/21] remove cni assertion --- magnum_cluster_api/resources.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index a2ebc8db..705050a2 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -168,11 +168,6 @@ def __init__( self.cluster = cluster def get_object(self) -> pykube.ConfigMap: - # NOTE(mnaser): We have to assert that the only CNIs we support are Calico and Cilium. - assert Counter( - CONF.cluster_template.kubernetes_allowed_network_drivers - ) == Counter(["calico", "cilium"]) - manifests_path = pkg_resources.resource_filename( "magnum_cluster_api", "manifests" ) From 184702e3d621f3562b2fdda33a314e9f34d50673 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 29 Mar 2024 00:28:24 +1100 Subject: [PATCH 10/21] add validation step for network drivers --- magnum_cluster_api/exceptions.py | 4 ++++ magnum_cluster_api/utils.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/magnum_cluster_api/exceptions.py b/magnum_cluster_api/exceptions.py index 925ed40b..fffeae26 100644 --- a/magnum_cluster_api/exceptions.py +++ b/magnum_cluster_api/exceptions.py @@ -57,6 +57,10 @@ class ClusterMasterCountEven(Exception): pass +class UnsupportedCNI(Exception): + pass + + class OpenstackFlavorInvalidName(exception.InvalidName): message = _("Expected a flavor name but received flavor id %(flavor)s.") diff --git a/magnum_cluster_api/utils.py b/magnum_cluster_api/utils.py index 403d1632..881ce5c4 100644 --- a/magnum_cluster_api/utils.py +++ b/magnum_cluster_api/utils.py @@ -362,9 +362,14 @@ def validate_flavor_name(cli: clients.OpenStackClients, flavor: str): def validate_cluster(ctx: context.RequestContext, cluster: magnum_objects.Cluster): + # Check network driver + if cluster.cluster_template.network_driver not in ["cilium", "calico"]: + raise mcapi_exceptions.UnsupportedCNI + # Check master count if (cluster.master_count % 2) == 0: raise mcapi_exceptions.ClusterMasterCountEven + # Validate flavors osc = clients.get_openstack_api(ctx) validate_flavor_name(osc, cluster.master_flavor_id) From 0b36211d17a10d64e5b632af717204845eca2b70 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 29 Mar 2024 02:16:53 +1100 Subject: [PATCH 11/21] fix tests --- magnum_cluster_api/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 705050a2..a0da727a 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -18,7 +18,6 @@ import os import textwrap import types -from collections import Counter import certifi import pkg_resources @@ -2362,6 +2361,7 @@ def __init__( @property def labels(self) -> dict: + labels = {} if self.cluster.cluster_template.network_driver == "calico": cni_version = self.cluster.labels.get( "calico_tag", CALICO_TAG From 71c7d028fd3a39ee2d2a0acc4b2c8520258b3bbc Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 29 Mar 2024 02:19:08 +1100 Subject: [PATCH 12/21] add executable perm to vendir install script --- hack/setup-vendir.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 hack/setup-vendir.sh diff --git a/hack/setup-vendir.sh b/hack/setup-vendir.sh old mode 100644 new mode 100755 From 9280ac152e81bc3b1eb787a5c939962a47b33173 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 29 Mar 2024 02:59:22 +1100 Subject: [PATCH 13/21] fix functional test --- Makefile | 2 +- magnum_cluster_api/resources.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 733c272e..a25fcc84 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ clean: vendir: curl -Lo vendir https://github.com/carvel-dev/vendir/releases/download/v0.40.0/vendir-linux-amd64 chmod +x vendir && ./vendir version - mv vendir /usr/local/bin/vendir + sudo mv vendir /usr/local/bin/vendir vendor: clean vendir vendir sync diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index a0da727a..040f4672 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -56,6 +56,8 @@ AUTOSCALE_ANNOTATION_MIN = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size" AUTOSCALE_ANNOTATION_MAX = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size" +DEFAULT_POD_CIDR = "10.100.0.0/16" + class ClusterAutoscalerHelmRelease: def __init__(self, api, cluster) -> None: @@ -222,7 +224,7 @@ def get_object(self) -> pykube.ConfigMap: utils.get_cluster_label( self.cluster, "cilium_ipv4pool", - "10.100.0.0/16", + DEFAULT_POD_CIDR, ), ], }, @@ -2387,14 +2389,16 @@ def get_or_none(self) -> objects.Cluster: def get_object(self) -> objects.Cluster: osc = clients.get_openstack_api(self.context) default_volume_type = osc.cinder().volume_types.default() - + pod_cidr = DEFAULT_POD_CIDR if self.cluster.cluster_template.network_driver == "calico": pod_cidr = self.cluster.labels.get( - "calico_ipv4pool", "10.100.0.0/16" + "calico_ipv4pool", + DEFAULT_POD_CIDR, ) if self.cluster.cluster_template.network_driver == "cilium": pod_cidr = self.cluster.labels.get( - "cilium_ipv4pool", "10.100.0.0/16" + "cilium_ipv4pool", + DEFAULT_POD_CIDR, ) return objects.Cluster( From 769de60bf7b32d0bfbac246ed92b90cfd55bf5b3 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 30 Mar 2024 01:12:56 +1100 Subject: [PATCH 14/21] resolve conflict --- magnum_cluster_api/resources.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 040f4672..af3f3a6a 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -2365,16 +2365,12 @@ def __init__( def labels(self) -> dict: labels = {} if self.cluster.cluster_template.network_driver == "calico": - cni_version = self.cluster.labels.get( - "calico_tag", CALICO_TAG - ) + cni_version = self.cluster.labels.get("calico_tag", CALICO_TAG) labels = { "cni": f"calico-{cni_version}", } if self.cluster.cluster_template.network_driver == "cilium": - cni_version = self.cluster.labels.get( - "cilium_tag", CILIUM_TAG - ) + cni_version = self.cluster.labels.get("cilium_tag", CILIUM_TAG) labels = { "cni": f"cilium-{cni_version}", } From 223c38ab18dfba9a39912d732dd720fb02fe0c8d Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Sat, 30 Mar 2024 01:20:56 +1100 Subject: [PATCH 15/21] fix typo --- magnum_cluster_api/resources.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index af3f3a6a..468bd900 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -221,8 +221,7 @@ def get_object(self) -> pykube.ConfigMap: "ipam": { "operator": { "clusterPoolIPv4PodCIDRList": [ - utils.get_cluster_label( - self.cluster, + self.cluster.labels.get( "cilium_ipv4pool", DEFAULT_POD_CIDR, ), From d015e6f7d20d1d7c093165331201e2cf958e1f17 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Fri, 5 Apr 2024 20:22:23 +1100 Subject: [PATCH 16/21] add zuul ci jobs for cilium network driver --- zuul.d/jobs-flatcar.yaml | 15 ++++++++++++++- zuul.d/jobs-rockylinux-8.yaml | 15 ++++++++++++++- zuul.d/jobs-rockylinux-9.yaml | 15 ++++++++++++++- zuul.d/jobs-ubuntu-2204.yaml | 15 ++++++++++++++- zuul.d/playbooks/sonobuoy/run.yml | 1 + 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/zuul.d/jobs-flatcar.yaml b/zuul.d/jobs-flatcar.yaml index fd64538c..c7791546 100644 --- a/zuul.d/jobs-flatcar.yaml +++ b/zuul.d/jobs-flatcar.yaml @@ -28,9 +28,22 @@ vars: kube_tag: v1.27.8 +- job: + name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8-calico + parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.8 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-sonobuoy-flatcar-v1.27.8-cilium + parent: magnum-cluster-api-sonobuoy-flatcar-v1.27.8 + vars: + network_driver: cilium + - project-template: name: magnum-cluster-api-flatcar check: jobs: - magnum-cluster-api-image-build-flatcar-v1.27.8 - - magnum-cluster-api-sonobuoy-flatcar-v1.27.8 + - magnum-cluster-api-sonobuoy-flatcar-v1.27.8-calico + - magnum-cluster-api-sonobuoy-flatcar-v1.27.8-cilium diff --git a/zuul.d/jobs-rockylinux-8.yaml b/zuul.d/jobs-rockylinux-8.yaml index 5a78fd23..4ad048a5 100644 --- a/zuul.d/jobs-rockylinux-8.yaml +++ b/zuul.d/jobs-rockylinux-8.yaml @@ -28,9 +28,22 @@ vars: kube_tag: v1.27.8 +- job: + name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-calico + parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-cilium + parent: magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8 + vars: + network_driver: cilium + - project-template: name: magnum-cluster-api-rockylinux-8 check: jobs: - magnum-cluster-api-image-build-rockylinux-8-v1.27.8 - - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8 + - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-calico + - magnum-cluster-api-sonobuoy-rockylinux-8-v1.27.8-cilium diff --git a/zuul.d/jobs-rockylinux-9.yaml b/zuul.d/jobs-rockylinux-9.yaml index 73e5b6f8..835719d9 100644 --- a/zuul.d/jobs-rockylinux-9.yaml +++ b/zuul.d/jobs-rockylinux-9.yaml @@ -28,9 +28,22 @@ vars: kube_tag: v1.27.8 +- job: + name: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-calico + parent: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-cilium + parent: magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8 + vars: + network_driver: cilium + - project-template: name: magnum-cluster-api-rockylinux-9 check: jobs: - magnum-cluster-api-image-build-rockylinux-9-v1.27.8 - - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8 + - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-calico + - magnum-cluster-api-sonobuoy-rockylinux-9-v1.27.8-cilium diff --git a/zuul.d/jobs-ubuntu-2204.yaml b/zuul.d/jobs-ubuntu-2204.yaml index cd333e64..32973330 100644 --- a/zuul.d/jobs-ubuntu-2204.yaml +++ b/zuul.d/jobs-ubuntu-2204.yaml @@ -28,9 +28,22 @@ vars: kube_tag: v1.27.8 +- job: + name: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8-calico + parent: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8-cilium + parent: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8 + vars: + network_driver: cilium + - project-template: name: magnum-cluster-api-ubuntu-2204 check: jobs: - magnum-cluster-api-image-build-ubuntu-2204-v1.27.8 - - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8 + - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8-calico + - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.27.8-cilium diff --git a/zuul.d/playbooks/sonobuoy/run.yml b/zuul.d/playbooks/sonobuoy/run.yml index ac00ea89..014f524e 100644 --- a/zuul.d/playbooks/sonobuoy/run.yml +++ b/zuul.d/playbooks/sonobuoy/run.yml @@ -23,6 +23,7 @@ KUBE_TAG: "{{ kube_tag }}" NODE_COUNT: 2 BUILD_NEW_IMAGE: "{{ fetch_artifact.changed }}" + NETWORK_DRIVER: "{{ network_driver }}" - name: Copy Sonobuoy results to output folder copy: From 6df62c46737ca59e28738ba569ea05b89e8e79ed Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Wed, 10 Apr 2024 01:52:15 +1000 Subject: [PATCH 17/21] enable sessionAffinity in cilium For users who run with kube-proxy (i.e. with Cilium's kube-proxy replacement disabled), the ClusterIP service loadbalancing when a request is sent from a pod running in a non-host network namespace is still performed at the pod network interface (until https://github.com/cilium/cilium/issues/16197 is fixed). For this case the session affinity support is disabled by default. --- magnum_cluster_api/resources.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 468bd900..84fb81ed 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -218,6 +218,12 @@ def get_object(self) -> pykube.ConfigMap: "tag": cilium_version, }, }, + # NOTE(okozachenko1203): For users who run with kube-proxy (i.e. with Cilium's kube-proxy replacement + # disabled), the ClusterIP service loadbalancing when a request is sent from a pod + # running in a non-host network namespace is still performed at the pod network + # interface (until https://github.com/cilium/cilium/issues/16197 is fixed). + # For this case the session affinity support is disabled by default. + "sessionAffinity": "true", "ipam": { "operator": { "clusterPoolIPv4PodCIDRList": [ From 309577c318a65209289afa17fde4d32b8b21857b Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Wed, 10 Apr 2024 02:18:09 +1000 Subject: [PATCH 18/21] fix flake8 errors --- magnum_cluster_api/resources.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 84fb81ed..68b75557 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -218,11 +218,12 @@ def get_object(self) -> pykube.ConfigMap: "tag": cilium_version, }, }, - # NOTE(okozachenko1203): For users who run with kube-proxy (i.e. with Cilium's kube-proxy replacement - # disabled), the ClusterIP service loadbalancing when a request is sent from a pod - # running in a non-host network namespace is still performed at the pod network - # interface (until https://github.com/cilium/cilium/issues/16197 is fixed). - # For this case the session affinity support is disabled by default. + # NOTE(okozachenko1203): For users who run with kube-proxy (i.e. with Cilium's kube-proxy + # replacement disabled), the ClusterIP service loadbalancing when a + # request is sent from a pod running in a non-host network namespace + # is still performed at the pod network interface (until + # https://github.com/cilium/cilium/issues/16197 is fixed). For this + # case the session affinity support is disabled by default. "sessionAffinity": "true", "ipam": { "operator": { From 33f8edaeefd7fc96824e4b5a2cfaa00b1d9533e0 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Wed, 10 Apr 2024 15:32:55 +1000 Subject: [PATCH 19/21] ignore the below test failure until the upstream issue fixed Failure of `[sig-network] HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol [LinuxOnly] [Conformance]` is expected until https://github.com/cilium/cilium/issues/14287 is fixed --- hack/run-integration-tests.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hack/run-integration-tests.sh b/hack/run-integration-tests.sh index 2a50585f..ae22d158 100755 --- a/hack/run-integration-tests.sh +++ b/hack/run-integration-tests.sh @@ -114,8 +114,17 @@ RESULTS_FILE=$(./sonobuoy retrieve --filename sonobuoy-results.tar.gz) # Print results ./sonobuoy results ${RESULTS_FILE} + # Fail if the Sonobuoy tests failed if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -q "Status: passed"; then - echo "Sonobuoy tests failed" - exit 1 + if [[ ${NETWORK_DRIVER} == "cilium" ]]; then + # NOTE(okozachenko1203): One failure is expected until https://github.com/cilium/cilium/issues/14287 is fixed. + if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -Eq "Failed: 1$"; then + echo "Sonobuoy tests failed" + exit 1 + fi + else + echo "Sonobuoy tests failed" + exit 1 + fi fi From 2238c71cc2e9ecccc97e31ff509bedd1a0ad73d6 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Tue, 18 Jun 2024 23:19:40 +1000 Subject: [PATCH 20/21] use portmap chain mode --- hack/run-integration-tests.sh | 12 ++---------- magnum_cluster_api/resources.py | 3 +++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/hack/run-integration-tests.sh b/hack/run-integration-tests.sh index ae22d158..aa669199 100755 --- a/hack/run-integration-tests.sh +++ b/hack/run-integration-tests.sh @@ -117,14 +117,6 @@ RESULTS_FILE=$(./sonobuoy retrieve --filename sonobuoy-results.tar.gz) # Fail if the Sonobuoy tests failed if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -q "Status: passed"; then - if [[ ${NETWORK_DRIVER} == "cilium" ]]; then - # NOTE(okozachenko1203): One failure is expected until https://github.com/cilium/cilium/issues/14287 is fixed. - if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -Eq "Failed: 1$"; then - echo "Sonobuoy tests failed" - exit 1 - fi - else - echo "Sonobuoy tests failed" - exit 1 - fi + echo "Sonobuoy tests failed" + exit 1 fi diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 68b75557..72e0ef36 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -210,6 +210,9 @@ def get_object(self) -> pykube.ConfigMap: "vendor/cilium/", ), values={ + "cni": { + "chainingMode": "portmap" + }, "image": { "tag": cilium_version, }, From ea62d406d08720d82045b7ec1f4d2a62c9f79339 Mon Sep 17 00:00:00 2001 From: okozachenko1203 Date: Tue, 18 Jun 2024 23:23:04 +1000 Subject: [PATCH 21/21] fix lint error --- magnum_cluster_api/resources.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/magnum_cluster_api/resources.py b/magnum_cluster_api/resources.py index 72e0ef36..519c5854 100644 --- a/magnum_cluster_api/resources.py +++ b/magnum_cluster_api/resources.py @@ -210,9 +210,7 @@ def get_object(self) -> pykube.ConfigMap: "vendor/cilium/", ), values={ - "cni": { - "chainingMode": "portmap" - }, + "cni": {"chainingMode": "portmap"}, "image": { "tag": cilium_version, },