Merge branch 'master' into andyzhangx/upgrade-csi-sidecar

.github/README-RENOVATE.md

@@ -2,6 +2,7 @@
 
 # Table of Contents
 
+- [Table of Contents](#table-of-contents)
 - [TL;DR](#tldr)
 - [Renovate configurations](#renovate-configurations)
   - [Package rules](#package-rules)
@@ -15,12 +16,13 @@
     - [Auto update packages for OS ubuntu xx.xx in components.json](#auto-update-packages-for-os-ubuntu-xxxx-in-componentsjson)
     - [(Optional context) How to ensure a single component will not be updated by 2 multiple custom managers?](#optional-context-how-to-ensure-a-single-component-will-not-be-updated-by-2-multiple-custom-managers)
   - [Custom data sources](#custom-data-sources)
-    - [(Optional context) Please read this section if you are going to config your own transformTemplates](#optional-context-please-read-this-section-if-you-are-going-to-config-your-own-transformtemplates)
+    - [(Optional context) Please read this section if you are going to config your own transformTemplates.](#optional-context-please-read-this-section-if-you-are-going-to-config-your-own-transformtemplates)
 - [Hands-on guide and FAQ](#hands-on-guide-and-faq)
   - [Okay, I just have 5 minutes. Please just tell me how to onboard a new package/container now to Renovate.json for auto-update.](#okay-i-just-have-5-minutes-please-just-tell-me-how-to-onboard-a-new-packagecontainer-now-to-renovatejson-for-auto-update)
   - [What is the responsibility of a PR assignee?](#what-is-the-responsibility-of-a-pr-assignee)
   - [What components are onboarded to Renovate for auto-update and what are not yet?](#what-components-are-onboarded-to-renovate-for-auto-update-and-what-are-not-yet)
   - [Details on supporting the MAR OCI artifacts.](#details-on-supporting-the-mar-oci-artifacts)
+  - [How to enable auto-merge for a component's patch version update?](#how-to-enable-auto-merge-for-a-components-patch-version-update)
 # TL;DR
 This readme is mainly describing how the renovate.json is constructed and the reasoning behind. If you are adding a new component to be cached in VHD, please refer to this [Readme-components](../parts/linux/cloud-init/artifacts/README-COMPONENTS.md) for tutorial. If you are onboarding a newly added component to Renovate automatic updates, you can jump to the [Hands-on guide and FAQ](#hands-on-guide-and-faq).
 
@@ -404,4 +406,26 @@ And next you will see
 ```
 where 
 - `${version}` will be resolved at runtime with the `latestVersion` and `previousLatestVersion` defined above.
-- `${CPU_ARCH}` will be resolved at runtime depending on the CPU architecture of the Node (VM) under provisioning.
+- `${CPU_ARCH}` will be resolved at runtime depending on the CPU architecture of the Node (VM) under provisioning.
+
+## How to enable auto-merge for a component's patch version update?
+This is a common scenarior where we want the PR to be merged automatically when a PR is created for a patch version update. You can refer to `moby-runc` and `moby-containerd` in `AgentBaker/.github/renovate.json` as an example. 
+
+```
+   {
+      "matchPackageNames": ["moby-runc", "moby-containerd"],
+      "matchUpdateTypes": [
+        "patch"
+      ],
+      "automerge": true,
+      "enabled": true,
+      "assignees": ["devinwong", "anujmaheshwari1", "cameronmeissner", "AlisonB319", "lilypan26", "djsly", "jason1028kr", "UtheMan", "zachary-bailey", "ganeshkumarashok"],
+      "reviewers": ["devinwong", "anujmaheshwari1", "cameronmeissner", "AlisonB319", "lilypan26", "djsly", "jason1028kr", "UtheMan", "zachary-bailey", "ganeshkumarashok"]
+    },
+```
+The config includes:
+- `matchPackageNames`: The name of the component's renovateTag in `AgentBaker/parts/linux/cloud-init/artifacts/components.json`. For example `moby-containerd`, `oss/kubernetes/kube-proxy`, `oss/binaries/kubernetes/kubernetes-node`. Wildcard character (*) is supported too. For example, `"matchPackageNames": ["oss/kubernetes-csi/*"],`
+- `matchUpdateTypes`: The type of version updates (`patch`) to which this rule applies.
+- `automerge`: Set to `true` to automatically merge PRs created by this rule. Default is `false`.
+- `enabled`: Set to `true` to enable this rule.
+- `assignees` and `reviewers`: The same group of GitHub IDs who will be assigned to and can review and approve the automatically created PRs.

aks-node-controller/proto/aksnodeconfig/v1/gpu_config.proto

@@ -16,4 +16,7 @@ message GpuConfig {
 
   // Represents the GPU instance profile.
   string gpu_instance_profile = 4;
+
+  // Same as enable_nvidia, but for AMD GPUs.
+  optional bool enable_amd_gpu = 5;
 }

e2e/scenario_test.go

@@ -253,6 +253,32 @@ func Test_AzureLinuxV2_GPUAzureCNI(t *testing.T) {
 	})
 }
 
+func Test_AzureLinuxV2_GPUAzureCNI_Scriptless(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "AzureLinux V2 (CgroupV2) gpu scenario on cluster configured with Azure CNI",
+		Tags: Tags{
+			GPU: true,
+		},
+		Config: Config{
+			Cluster: ClusterAzureNetwork,
+			VHD:     config.VHDAzureLinuxV2Gen2,
+			AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) {
+				config.NetworkConfig.NetworkPlugin = aksnodeconfigv1.NetworkPlugin_NETWORK_PLUGIN_AZURE
+				config.VmSize = "Standard_NC6s_v3"
+				config.GpuConfig.ConfigGpuDriver = true
+				config.GpuConfig.GpuDevicePlugin = false
+				config.GpuConfig.EnableNvidia = to.Ptr(true)
+
+			},
+			VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
+				vmss.SKU.Name = to.Ptr("Standard_NC6s_v3")
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+			},
+		},
+	})
+}
+
 func Test_AzureLinuxV2_WASM(t *testing.T) {
 	RunScenario(t, &Scenario{
 		Description: "tests that a new AzureLinuxV2 (CgroupV2) node using krustlet can be properly bootstrapped",
@@ -1442,6 +1468,7 @@ func Test_Ubuntu2404Gen2(t *testing.T) {
 				runcVersions := getExpectedPackageVersions("runc", "ubuntu", "r2404")
 				ValidateContainerd2Properties(ctx, s, containerdVersions)
 				ValidateRunc12Properties(ctx, s, runcVersions)
+				ValidateContainerRuntimePlugins(ctx, s)
 			},
 		},
 	})

e2e/validators.go

@@ -367,13 +367,19 @@ func ValidateContainerd2Properties(ctx context.Context, s *Scenario, versions []
 	require.Truef(s.T, strings.HasPrefix(versions[0], "2."), "expected moby-containerd version to start with '2.', got %v", versions[0])
 
 	ValidateInstalledPackageVersion(ctx, s, "moby-containerd", versions[0])
-	// assert that /etc/containerd/config.toml exists and does not contain deprecated properties from 1.7
-	ValidateFileExcludesContent(ctx, s, "/etc/containerd/config.toml", "CriuPath")
 	// assert that containerd.server service file does not contain LimitNOFILE
 	// https://github.com/containerd/containerd/blob/main/docs/containerd-2.0.md#limitnofile-configuration-has-been-removed
 	ValidateFileExcludesContent(ctx, s, "/etc/systemd/system/containerd.service", "LimitNOFILE")
+}
+
+func ValidateContainerRuntimePlugins(ctx context.Context, s *Scenario) {
 	// nri plugin is enabled by default
 	ValidateDirectoryContent(ctx, s, "/var/run/nri", []string{"nri.sock"})
+	// cri plugin has deprecated properties
+	// assert that /etc/containerd/config.toml exists and does not contain deprecated properties from 1.7
+	ValidateFileExcludesContent(ctx, s, "/etc/containerd/config.toml", "CriuPath")
+	// level=warning msg="Ignoring unknown key in TOML for plugin" error="strict mode: fields in the document are missing in the target struct" key=sandbox_image plugin=io.containerd.grpc.v1.cri
+	ValidateFileExcludesContent(ctx, s, "/etc/containerd/config.toml", "sandbox_image")
 }
 
 func ValidateRunc12Properties(ctx context.Context, s *Scenario, versions []string) {

parts/linux/cloud-init/artifacts/aks-log-collector.sh

@@ -191,7 +191,7 @@ zip -DZ deflate "${ZIP}" /proc/@(cmdline|cpuinfo|filesystems|interrupts|loadavg|
 collectToZip collect/file_listings.txt find /dev /etc /var/lib/waagent /var/log -ls
 
 # Collect system information
-collectToZip collect/blkid.txt blkid
+collectToZip collect/blkid.txt blkid $(find /dev -type b ! -name 'sr*')
 collectToZip collect/du_bytes.txt df -al
 collectToZip collect/du_inodes.txt df -ail
 collectToZip collect/diskinfo.txt lsblk

parts/linux/cloud-init/artifacts/cis.sh

@@ -185,7 +185,7 @@ function maskNfsServer() {
     # Note that on ubuntu systems, it isn't installed but on mariner/azurelinux we need it
     # due to a dependency, but disable it by default.
     if systemctl list-unit-files nfs-server.service >/dev/null; then
-        systemctl --now mask nfs-server || $ERR_SYSTEMCTL_MASK_FAIL
+        systemctl --now mask nfs-server || exit $ERR_SYSTEMCTL_MASK_FAIL
     fi
 }
 

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -11,6 +11,15 @@ removeContainerd() {
 }
 
 installDeps() {
+    # The nftables package turns on a service by default that tries to load config files,
+    # but the stock config files in the package have no uncommented lines and make the service
+    # fail to start. Masking it as it's not used, and the stop action of "flush tables" can
+    # result in rules getting cleared unexpectedly. Azure Linux 3 fixes this, so we only need
+    # this in 2.0.
+    if [[ $OS_VERSION == "2.0" ]]; then
+      systemctl --now mask nftables.service || exit $ERR_SYSTEMCTL_MASK_FAIL
+    fi
+
     dnf_makecache || exit $ERR_APT_UPDATE_TIMEOUT
     dnf_update || exit $ERR_APT_DIST_UPGRADE_TIMEOUT
     for dnf_package in ca-certificates check-restart cifs-utils cloud-init-azure-kvp conntrack-tools cracklib dnf-automatic ebtables ethtool fuse git inotify-tools iotop iproute ipset iptables jq kernel-devel logrotate lsof nmap-ncat nfs-utils pam pigz psmisc rsyslog socat sysstat traceroute util-linux xz zip blobfuse2 nftables iscsi-initiator-utils; do