aks_checklist.en.json

{
  "items": [
    {
      "category": "BC and DR",
      "subcategory": "Requirements",
      "text": "Define non-functional requirements such as SLAs, RTO and RPO.",
      "guid": "bc14aea6-e65d-48d9-a3ad-c218e6436b06",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/cloud-adoption-framework/ready/enterprise-scale/business-continuity-and-disaster-recovery"
    },
    {
      "category": "BC and DR",
      "subcategory": "Disaster Recovery",
      "text": "Perform DR tests",
      "guid": "36cb45e5-7960-4332-9bdf-8cc23318da61",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/cloud-adoption-framework/ready/enterprise-scale/business-continuity-and-disaster-recovery"
    },
    {
      "category": "BC and DR",
      "subcategory": "High Availability",
      "text": "Use Traffic Manager or Front Door as a global load balancer for region failover",
      "guid": "170265f4-bb46-4a39-9af7-f317284797b1",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-multi-region"
    },
    {
      "category": "BC and DR",
      "subcategory": "High Availability",
      "text": "Use Availability Zones if supported in your region",
      "guid": "578a219a-46be-4b54-9350-24922634292b",
      "severity": "Medium",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(zones) | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(zones) | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/availability-zones"
    },
    {
      "category": "BC and DR",
      "subcategory": "High Availability",
      "text": "Use the SLA-backed AKS offering",
      "guid": "71d41e36-10cc-457b-9a4b-1410d4395898",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where sku.tier=='Paid' | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where sku.tier=='Free' | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/uptime-sla"
    },
    {
      "category": "BC and DR",
      "subcategory": "High Availability",
      "text": "Use Disruption Budgets in your pod specs",
      "guid": "c1288b3c-6a57-4cfc-9444-51e1a3d3453a",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-scheduler"
    },
    {
      "category": "BC and DR",
      "subcategory": "High Availability",
      "text": "If using a private registry, configure region replication to store images in multiple regions",
      "guid": "3c763963-7a55-42d5-a15e-401955387e5c",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/container-registry/container-registry-geo-replication"
    },
    {
      "category": "Identity and Access Management",
      "subcategory": "Identity",
      "text": "Use managed identities instead of Service Principals",
      "guid": "ed127dd1-42b0-46b2-8c69-99a646f3389a",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where properties.servicePrincipalProfile.clientId=='msi' | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where properties.servicePrincipalProfile.clientId!='msi' | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/use-managed-identity"
    },
    {
      "category": "Identity and Access Management",
      "subcategory": "Identity",
      "text": "Integrate authentication with AAD (using the managed integration)",
      "guid": "7e42c78e-78c0-46a6-8a21-94956e698dc4",
      "severity": "Medium",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.aadProfile) | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.aadProfile) | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/managed-aad"
    },
    {
      "category": "Identity and Access Management",
      "subcategory": "Identity",
      "text": "Limit access to admin kubeconfig (get-credentials --admin)",
      "guid": "a2fe27b2-e287-401a-8352-beedf79b488d",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/control-kubeconfig-access"
    },
    {
      "category": "Identity and Access Management",
      "subcategory": "Identity",
      "text": "Integrate authorization with AAD RBAC",
      "guid": "eec4962c-c3bd-421b-b77f-26e5e6b3bec3",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/manage-azure-rbac"
    },
    {
      "category": "Identity and Access Management",
      "subcategory": "Identity",
      "text": "Use namespaces for restricting RBAC privilege in Kubernetes",
      "guid": "d4f3537c-1346-4dc5-9027-a71ffe1bd05d",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-identity"
    },
    {
      "category": "Identity and Access Management",
      "subcategory": "Identity",
      "text": "Use Pod identity for workload identities (addon v2 in preview today)",
      "guid": "d2e0d5d7-71d4-41e3-910c-c57b4a4b1410",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-identity"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "IPAM",
      "text": "Choose the best CNI plugin for your requirements (Azure CNI recommended)",
      "guid": "a0f61565-9de5-458f-a372-49c831112dbd",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where properties.networkProfile.networkPlugin=='azure' | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where properties.networkProfile.networkPlugin=='kubenet' | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-network"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "IPAM",
      "text": "If using Azure CNI, size your subnet accordingly",
      "guid": "7faf12e7-0943-4f63-8472-2da29c2b1cd6",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/configure-azure-cni"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "IPAM",
      "text": "If using Azure CNI, check the maximum pods/node (default 30)",
      "guid": "22f54b29-bade-43aa-b1e8-c38ec9366673",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/configure-azure-cni"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "IPAM",
      "text": "If using private-IP LoadBalancer services, use a dedicated subnet (not the AKS subnet)",
      "guid": "13c00567-4b1e-4945-a459-c373e7ed6162",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/aks/internal-lb"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Filter egress traffic with AzFW/NVA if your security requirements mandate it",
      "guid": "3b365a91-7ecb-4e48-bbe5-4cd7df2e8bba",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where properties.networkProfile.outboundType=='userDefinedRouting' | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where properties.networkProfile.outboundType!='userDefinedRouting' | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/limit-egress-traffic"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "If using a public API endpoint, restrict the IP addresses that can access it",
      "guid": "c4581559-bb91-463e-a908-aed8c44ce3b2",
      "severity": "Medium",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where (isnull(properties.apiServerAccessProfile.enablePrivateCluster) or properties.apiServerAccessProfile.enablePrivateCluster==false) and isnotnull(properties.apiServerAccessProfile.authorizedIPRanges) | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where (isnull(properties.apiServerAccessProfile.enablePrivateCluster) or properties.apiServerAccessProfile.enablePrivateCluster==false) and isnull(properties.apiServerAccessProfile.authorizedIPRanges) | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/api-server-authorized-ip-ranges"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Scalability",
      "text": "Use an ingress controller to expose web-based apps",
      "guid": "b3808b9f-a1cf-4204-ad01-3a923ce474db",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/concepts-network"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Best practices",
      "text": "If using AGIC, do not share an AppGW across clusters",
      "guid": "cbd8ac2a-aebc-4a2a-94da-1dbf3dc99248",
      "severity": "Medium",
      "link": "https://azure.github.io/application-gateway-kubernetes-ingress/setup/install-existing/"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Cost",
      "text": "Use Private Endpoints (preferred) or VNet Service Endpoints to access PaaS from the cluster",
      "guid": "c3c39c98-6bb2-4c12-859a-114b5e3df584",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/private-link/private-link-overview"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Use private clusters if your requirements mandate it",
      "guid": "ecccd979-3b6b-4cda-9b50-eb2eb03dda6d",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.apiServerAccessProfile.enablePrivateCluster) and properties.apiServerAccessProfile.enablePrivateCluster==true | distinct name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.apiServerAccessProfile.enablePrivateCluster) or properties.apiServerAccessProfile.enablePrivateCluster==false | distinct name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/private-clusters"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Enable a Kubernetes Network Policy option (Calico/Azure)",
      "guid": "58d7c892-ddb1-407d-9769-ae669ca48e4a",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.networkProfile.networkPolicy) | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.networkProfile.networkPolicy) | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/use-network-policies"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Use Network policies to increase intra-cluster security",
      "guid": "85e2223e-ce8b-4b12-907c-a5f16f158e3e",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-network"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Use a WAF for web workloads (UIs or APIs)",
      "guid": "a3a92c2d-e7e2-4165-a3a8-7af4a7a1f893",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-network"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Use DDoS Standard in the AKS VNet",
      "guid": "9bda4776-8f24-4c11-9775-c2ea55b46a94",
      "severity": "Medium",
      "graph_success": "Resources | where type=~'microsoft.containerservice/managedclusters' | project resourceGroup,name,pools=properties.agentPoolProfiles | mv-expand pools | project subnetId=tostring(pools.vnetSubnetID) | where isnotempty(subnetId) | join (Resources | where type=='microsoft.network/virtualnetworks' | project id,resourceGroup,name,enableDdosProtection=tostring(properties.enableDdosProtection),subnets=properties.subnets | mv-expand subnets | project id,resourceGroup,name,enableDdosProtection,subnetId=tostring(subnets.id)) on subnetId | distinct id,resourceGroup,name,enableDdosProtection | where enableDdosProtection == 'true'",
      "graph_failure": "Resources | where type=~'microsoft.containerservice/managedclusters' | project resourceGroup,name,pools=properties.agentPoolProfiles | mv-expand pools | project subnetId=tostring(pools.vnetSubnetID) | where isnotempty(subnetId) | join (Resources | where type=='microsoft.network/virtualnetworks' | project id,resourceGroup,name,enableDdosProtection=tostring(properties.enableDdosProtection),subnets=properties.subnets | mv-expand subnets | project id,resourceGroup,name,enableDdosProtection,subnetId=tostring(subnets.id)) on subnetId | distinct id,resourceGroup,name,enableDdosProtection | where enableDdosProtection == 'false'",
      "link": "https://docs.microsoft.com/azure/virtual-network/ddos-protection-overview"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Best practices",
      "text": "Do not use AKS Application Routing Add-On",
      "guid": "8008ae7d-7e4b-4475-a6c8-bdbf59bce65d",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.addonProfiles.httpApplicationRouting) or properties.addonProfiles.httpApplicationRouting.enabled==false | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.addonProfiles.httpApplicationRouting) and properties.addonProfiles.httpApplicationRouting.enabled==true | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/http-application-routing"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Best practices",
      "text": "Use the standard ALB (as opposed to the basic one)",
      "guid": "ba7da7be-9952-4914-a384-5d997cb39132",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where properties.networkProfile.loadBalancerSku=='standard' | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where properties.networkProfile.loadBalancerSku=='basic' | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/load-balancer-standard"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "HA",
      "text": "If hybrid connectivity is required, use 2xER or ER+VPN for better availability",
      "guid": "e8a03f97-8794-468d-96a7-86d60f96c97b",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/expressroute/designing-for-disaster-recovery-with-expressroute-privatepeering"
    },
    {
      "category": "Network Topology and Connectivity",
      "subcategory": "Security",
      "text": "Consider using a service mesh for advanced microservice communication management",
      "guid": "e9855d04-c3c3-49c9-a6bb-2c12159a114b",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/servicemesh-about"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Secrets",
      "text": "Store your secrets in Azure Key Vault with the CSI Secrets Store driver",
      "guid": "5e3df584-eccc-4d97-a3b6-bcda3b50eb2e",
      "severity": "Medium",
      "link": "https://github.com/Azure/secrets-store-csi-driver-provider-azure"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Secrets",
      "text": "If user Service Principals for the cluster, refresh credentials periodically (like quarterly)",
      "guid": "b03dda6d-58d7-4c89-8ddb-107d5769ae66",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/update-credentials"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Use Azure Policy for Kubernetes to ensure cluster compliance",
      "guid": "9ca48e4a-85e2-4223-bce8-bb12307ca5f1",
      "severity": "Medium",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.addonProfiles.azurepolicy) and properties.addonProfiles.azurepolicy.enabled==true | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.addonProfiles.azurepolicy) or properties.addonProfiles.azurepolicy.enabled==false | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/governance/policy/concepts/policy-for-kubernetes"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Separate applications from the control plane with user/system nodepools",
      "guid": "6f158e3e-a3a9-42c2-be7e-2165c3a87af4",
      "severity": "Medium",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | project id,resourceGroup,name,pools=properties.agentPoolProfiles | project id,name,resourceGroup,poolcount=array_length(pools) | where poolcount > 1",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | project id,resourceGroup,name,pools=properties.agentPoolProfiles | project id,name,resourceGroup,poolcount=array_length(pools) | where poolcount == 1",
      "link": "https://docs.microsoft.com/azure/aks/use-system-pools"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Label your system nodepool to use it with node selectors",
      "guid": "a7a1f893-9bda-4477-98f2-4c116775c2ea",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/aks/use-multiple-node-pools"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Use a private registry for your images, such as ACR",
      "guid": "55b46a94-8008-4ae7-b7e4-b475b6c8bdbf",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/container-registry/"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Scan your images for vulnerabilities",
      "guid": "59bce65d-e8a0-43f9-9879-468d66a786d6",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/security-center/container-security"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Use Azure Security Center to detect security posture vulnerabiities",
      "guid": "cc639637-a652-42ac-89e8-06965388e9de",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/security-center/container-security"
    },
    {
      "category": "Governance and Security",
      "subcategory": "Compliance",
      "text": "Define app separation requirements (namespace/nodepool/cluster)",
      "guid": "d167dd18-2b0a-4c24-8b99-9a646f8389a7",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-cluster-isolation"
    },
    {
      "category": "Cost Governance",
      "subcategory": "Cost",
      "text": "Use an external application such as kubecost to allocate custs to different users",
      "guid": "f82cb8eb-8c0a-4a63-a25a-4956eaa8dc4a",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/cloud-adoption-framework/scenarios/aks/eslz-cost-governance-with-kubecost"
    },
    {
      "category": "Operations",
      "subcategory": "Storage",
      "text": "Use ephemeral OS disks",
      "guid": "24367b33-6971-45b1-952b-eee0b9b588de",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | project id,resourceGroup,name,pools=properties.agentPoolProfiles | mvexpand pools | where pools.osDiskType=='Ephemeral' | project id,name=strcat(name,'-',pools.name), resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | project id,resourceGroup,name,pools=properties.agentPoolProfiles | mvexpand pools | where pools.osDiskType=='Managed' | project id,name=strcat(name,'-',pools.name), resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/cluster-configuration"
    },
    {
      "category": "Operations",
      "subcategory": "Storage",
      "text": "If not using ephemeral disks, use larger OS disks for the nodes, especially if many pods/node",
      "guid": "fc4972cc-3cd2-45bf-a707-6e9eab4bed32",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/virtual-machines/disks-types"
    },
    {
      "category": "Operations",
      "subcategory": "Storage",
      "text": "Avoid keeping state in the cluster, and store data outside (AzStorage, AzSQL, Cosmos, etc)",
      "guid": "9f7547c1-747d-4c56-868a-714435bd19dd",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-multi-region"
    },
    {
      "category": "Operations",
      "subcategory": "Storage",
      "text": "If using AzFiles Standard, consider AzFiles Premium and/or ANF for performance reasons",
      "guid": "24429eb7-2281-4376-85cc-57b4a4b18142",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-storage"
    },
    {
      "category": "Operations",
      "subcategory": "Storage",
      "text": "If using Azure Disks and AZs, consider dedicating nodes per AZ and WaitForFirstConsumer",
      "guid": "83958a8c-2689-4b32-ab57-cfc64546135a",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/availability-zones"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Check regularly Azure Advisor for recommendations on your cluster",
      "guid": "337453a3-cc63-4963-9a65-22ac19e80696",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/advisor/advisor-get-started"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Develop your YAML manifests with intelligent text editors such as vscode+kubeadvisor",
      "guid": "5388e9de-d167-4dd1-a2b0-ac241b999a64",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-scheduler"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "Monitor your cluster metrics with Container Insights (or other tools like Prometheus)",
      "guid": "6f8389a7-f82c-4b8e-a8c0-aa63a25a4956",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-overview"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "Store and analyze your cluster logs with Container Insights (or other tools like Telegraf/ElasticSearch)",
      "guid": "eaa8dc4a-2436-47b3-9697-15b1752beee0",
      "severity": "High",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.addonProfiles.omsagent) and properties.addonProfiles.omsagent.enabled==true | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.addonProfiles.omsagent) or properties.addonProfiles.omsagent.enabled==false | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-overview"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "Configure Diagnostic Settins in your clusters to send API logs to Azure Monitor",
      "guid": "b9b588de-fc49-472c-a3cd-25bfb7076e9e",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/monitor-aks"
    },
    {
      "category": "Operations",
      "subcategory": "Alerting",
      "text": "Configure alerts on the most critical metrics (see Container Insights for recommendations)",
      "guid": "67f7a9ed-5b31-4f38-a3f3-9812b2463cff",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-metric-alerts"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Have a regular process to upgrade the cluster periodically (quarterly, for example)",
      "guid": "e189c599-df0d-45a7-9dd4-ce32c1881370",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/supported-kubernetes-versions"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Use kured for Linux node upgrades",
      "guid": "6f7c4c0d-4e51-4464-ad24-57ed67138b82",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/node-updates-kured"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Consider gitops to deploy applications or cluster configuration to multiple clusters",
      "guid": "0102ce16-ee30-41e6-b882-e52e4621dd68",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/architecture/example-scenario/bedrock/bedrock-automated-deployments"
    },
    {
      "category": "Operations",
      "subcategory": "Cost",
      "text": "Consider spot node pools for non time-sensitive workloads",
      "guid": "c5a5b252-1e44-4a59-a9d2-399c4d7b68d0",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/aks/spot-node-pool"
    },
    {
      "category": "Operations",
      "subcategory": "Cost",
      "text": "Consider AKS virtual node for quick bursting",
      "guid": "c755562f-2b4e-4456-9b4d-874a748b662e",
      "severity": "Low",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.addonProfiles.aciConnectorLinux) and properties.addonProfiles.aciConnectorLinux.enabled==true | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.addonProfiles.aciConnectorLinux) or properties.addonProfiles.aciConnectorLinux.enabled==false | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/concepts-scale"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Make sure no changes are performed by operators in the node RG (aka 'infra RG')",
      "guid": "31d7aaab-7571-4449-ab80-53d89e89d17b",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/faq"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Use custom Node RG (aka 'Infra RG') name",
      "guid": "73b32a5a-67f7-4a9e-b5b3-1f38c3f39812",
      "severity": "Low",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where properties.nodeResourceGroup !startswith 'MC_' | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where properties.nodeResourceGroup startswith 'MC_' | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/cluster-configuration"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Do not use deprecated Kubernetes APIs in your YAML manifests",
      "guid": "b2463cff-e189-4c59-adf0-d5a73dd4ce32",
      "severity": "Medium",
      "link": "https://kubernetes.io/docs/setup/release/notes/"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Taint Windows nodes",
      "guid": "c1881370-6f7c-44c0-b4e5-14648d2457ed",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure-stack/aks-hci/adapt-apps-mixed-os-clusters"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Keep windows containers patch level in sync with host patch level",
      "guid": "67138b82-0102-4ce1-9ee3-01e6e882e52e",
      "severity": "Low",
      "link": "https://docs.microsoft.com/virtualization/windowscontainers/deploy-containers/version-compatibility?tabs=windows-server-20H2%2Cwindows-10-20H2"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "Monitor CPU and memory utilization of the nodes",
      "guid": "4621dd68-c5a5-4be2-bdb1-1726769ef669",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/azure-monitor/containers/container-insights-analyze"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "If using Azure CNI, monitor % of pod IPs consumed per node",
      "guid": "1a4835ac-9422-423e-ae80-b123081a5417",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/configure-azure-cni"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "Monitor OS disk queue depth in nodes",
      "guid": "415833ea-3ad3-4c2d-b733-165c3acbe04b",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/virtual-machines/premium-storage-performance"
    },
    {
      "category": "Operations",
      "subcategory": "Monitoring",
      "text": "If not using egress filtering with AzFW/NVA, monitor standard ALB allocated SNAT ports",
      "guid": "be209d39-fda4-4777-a424-d116785c2fa5",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/load-balancer-standard"
    },
    {
      "category": "Operations",
      "subcategory": "Compliance",
      "text": "Send master logs (aka API logs) to Log Analytics",
      "guid": "5b56ad48-408f-4e72-934c-476ba280dcf5",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/aks/monitor-aks"
    },
    {
      "category": "Operations",
      "subcategory": "Scalability",
      "text": "Use the Cluster Autoscaler",
      "guid": "90ce65de-8e13-4f9c-abd4-69266abca264",
      "severity": "Medium",
      "graph_success": "where type=='microsoft.containerservice/managedclusters' | where isnotnull(properties.austoscalerProfile) | distinct id,name,resourceGroup",
      "graph_failure": "where type=='microsoft.containerservice/managedclusters' | where isnull(properties.austoscalerProfile) | distinct id,name,resourceGroup",
      "link": "https://docs.microsoft.com/azure/aks/concepts-scale"
    },
    {
      "category": "Operations",
      "subcategory": "Scalability",
      "text": "Use the Horizontal Pod Autoscaler when required",
      "guid": "faa19bfe-9d55-4d04-a3c4-919ca1b2d121",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/concepts-scale"
    },
    {
      "category": "Operations",
      "subcategory": "Scalability",
      "text": "Consider an appropriate node size, not too large or too small",
      "guid": "5ae124ba-34df-4585-bcdc-e9bd3bb0cdb3",
      "severity": "High",
      "link": "https://blog.cloudtrooper.net/2020/10/23/which-vm-size-should-i-choose-as-aks-node/"
    },
    {
      "category": "Operations",
      "subcategory": "Resources",
      "text": "Configure requests and limits in your pod specs",
      "guid": "b54eb2eb-03dd-4aa3-9927-18e2edb11726",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-scheduler"
    },
    {
      "category": "Operations",
      "subcategory": "Resources",
      "text": "Enforce resource quotas for namespaces",
      "guid": "769ef669-1a48-435a-a942-223ece80b123",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/aks/operator-best-practices-scheduler"
    },
    {
      "category": "Operations",
      "subcategory": "Resources",
      "text": "Ensure your subscription has enough quota to scale out your nodepools",
      "guid": "081a5417-4158-433e-a3ad-3c2de733165c",
      "severity": "High",
      "link": "https://docs.microsoft.com/azure/azure-resource-manager/management/azure-subscription-service-limits"
    },
    {
      "category": "Application Deployment",
      "subcategory": "Infrastructure as Code",
      "text": "Use automation through ARM/TF to create your Azure resources",
      "guid": "3acbe04b-be20-49d3-afda-47778424d116",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/developer/terraform/create-k8s-cluster-with-tf-and-aks"
    },
    {
      "category": "Application Deployment",
      "subcategory": "Development",
      "text": "Use canary or blue/green deployments",
      "guid": "785c2fa5-5b56-4ad4-a408-fe72734c476b",
      "severity": "Medium",
      "link": "https://docs.microsoft.com/azure/architecture/reference-architectures/containers/aks/secure-baseline-aks"
    },
    {
      "category": "Application Deployment",
      "subcategory": "Development",
      "text": "Use KEDA if running event-driven workloads",
      "guid": "a280dcf5-90ce-465d-b8e1-3f9ccbd46926",
      "severity": "Low",
      "link": "https://docs.microsoft.com/azure/azure-functions/functions-kubernetes-keda"
    },
    {
      "category": "Application Deployment",
      "subcategory": "Development",
      "text": "Use Dapr to ease microservice development",
      "guid": "26886d20-b66c-457b-a591-19bf8e8f5c58",
      "severity": "Low",
      "link": "https://dapr.io/"
    }
  ],
  "categories": [
    {
      "name": "Identity and Access Management"
    },
    {
      "name": "Network Topology and Connectivity"
    },
    {
      "name": "BC and DR"
    },
    {
      "name": "Governance and Security"
    },
    {
      "name": "Cost Governance"
    },
    {
      "name": "Operations"
    },
    {
      "name": "Application Deployment"
    }
  ],
  "status": [
    {
      "name": "Not verified"
    },
    {
      "name": "Open"
    },
    {
      "name": "Fulfilled"
    },
    {
      "name": "N/A"
    }
  ],
  "severities": [
    {
      "name": "High"
    },
    {
      "name": "Medium"
    },
    {
      "name": "Low"
    }
  ],
  "metadata": {
    "name": "Azure AKS Review"
  }
}