Add Datadog agent to Kube cluster

.github/workflows/deploy_environments.yaml

@@ -0,0 +1,27 @@
+name: Deploy Environments
+permissions: read-all
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    branches:
+      - main
+
+jobs:
+  deploy_dev3:
+    name: Deploy dev3
+    uses: ./.github/workflows/deploy_k8s_cluster.yaml
+    with:
+      environment: dev3
+    secrets: inherit
+    if: github.ref == 'refs/heads/main'
+
+  deploy_prod1:
+    name: Deploy prod1
+    uses: ./.github/workflows/deploy_k8s_cluster.yaml
+    with:
+      environment: prod1
+    secrets: inherit
+    if: github.ref == 'refs/heads/main'

.github/workflows/deploy_dev3.yaml → .github/workflows/deploy_k8s_cluster.yaml

@@ -1,14 +1,12 @@
-name: Deploy Dev 3
+name: Deploy K8s Cluster
+permissions: read-all
 
 on:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-    branches:
-      - main
-
-permissions: read-all
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
 
 env:
   # Secrets
@@ -17,6 +15,7 @@ env:
   TF_VAR_mesh_proxmox_token_secret: ${{ secrets.TF_VAR_MESHDB_PROXMOX_TOKEN_SECRET }}
   TF_VAR_mesh_local_password: ${{ secrets.TF_VAR_MESHDB_LOCAL_PASSWORD }}
   TF_VAR_k3s_token: ${{ secrets.TF_VAR_K3S_TOKEN }}
+  TF_VAR_DATADOG_API_KEY: ${{ secrets.TF_VAR_DATADOG_API_KEY }}
   # Credentials for deployment to AWS
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -27,7 +26,7 @@ env:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    environment: dev3
+    environment: ${{ inputs.environment }}
     steps:
       - name: Checkout
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # @v4
@@ -90,6 +89,7 @@ jobs:
         run: |
           echo "${{ secrets.SSH_PRIVATE_KEY }}" > mesh_cluster/mesh${{ vars.ENV_NAME }}
           echo "${{ secrets.SSH_PUBLIC_KEY }}" > mesh_cluster/mesh${{ vars.ENV_NAME }}.pub
+          echo "${{ secrets.DATADOG_API_KEY }}" > mesh_cluster/datadog_api_key
           chmod 600 mesh_cluster/mesh${{ vars.ENV_NAME }}
           chmod 600 mesh_cluster/mesh${{ vars.ENV_NAME }}.pub
           terraform apply -auto-approve -input=false -var-file=${{ vars.ENV_NAME }}.tfvars

ansible/roles/k8s-cluster-helm/files/datadog_agent.yaml

@@ -0,0 +1,16 @@
+apiVersion: datadoghq.com/v2alpha1
+kind: DatadogAgent
+metadata:
+  name: datadog
+  namespace: datadog
+spec:
+  global:
+    credentials:
+      apiSecret:
+        secretName: datadog-api-key
+        keyName: api-key
+  features:
+    apm:
+      enabled: true
+    logCollection:
+      enabled: true

ansible/roles/k8s-cluster-helm/tasks/main.yaml

@@ -1,6 +1,6 @@
 - name: Copy manifiest
-  ansible.builtin.copy:
-    src: ./files/cluster_manifest.yaml
+  ansible.builtin.template:
+    src: ./templates/cluster_manifest.yaml.j2
     dest: /root/cluster_manifest.yaml
     owner: root
     group: root
@@ -12,3 +12,18 @@
     chdir: /root/
     cmd: kubectl apply -f cluster_manifest.yaml
   become: true
+
+- name: Copy datadog agent manifiest
+  ansible.builtin.copy:
+    src: ./files/datadog_agent.yaml
+    dest: /root/datadog_agent.yaml
+    owner: root
+    group: root
+    mode: '0600'
+  become: true
+
+- name: Apply datadog agent manifest
+  ansible.builtin.command:
+    chdir: /root/
+    cmd: kubectl apply -f datadog_agent.yaml
+  become: true

ansible/roles/k8s-cluster-helm/templates/cluster_manifest.yaml.j2

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  repo: https://charts.longhorn.io
+  chart: longhorn
+  targetNamespace: longhorn-system
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: datadog
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: datadog-api-key
+  namespace: datadog
+type: Opaque
+data:
+  api-key: {{ DATADOG_API_KEY }}
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: datadog-operator
+  namespace: datadog
+spec:
+  repo: https://helm.datadoghq.com 
+  chart: datadog-operator
+  targetNamespace: datadog
+

terraform/cluster.tf

@@ -22,5 +22,6 @@ module "some_mesh_cluster" {
   meshdb_fqdn                   = var.meshdb_fqdn
   mesh_local_password           = var.mesh_local_password
   k3s_token                     = var.k3s_token
+  DATADOG_API_KEY               = var.DATADOG_API_KEY
   vm_nic                        = var.vm_nic
-}
+}

terraform/mesh_cluster/ansible.tf

@@ -6,6 +6,7 @@ resource "ansible_group" "mgrs" {
     ansible_ssh_private_key_file = "../terraform/${path.module}/mesh${var.mesh_env_name}"
     ansible_ssh_common_args      = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
     K3S_TOKEN                    = var.k3s_token
+    DATADOG_API_KEY              = var.DATADOG_API_KEY
   }
 }
 
@@ -58,4 +59,4 @@ resource "ansible_host" "meshworker" {
 resource "ansible_host" "k8slb" {
   name   = var.mesh_lb_ip
   groups = [ansible_group.lb.name]
-}
+}

terraform/mesh_cluster/vars.tf

@@ -81,6 +81,12 @@ variable "k3s_token" {
   sensitive   = true
 }
 
+variable "DATADOG_API_KEY" {
+  type        = string
+  description = "datadog API key"
+  sensitive   = true
+}
+
 variable "mesh_mgr_cores" {
   type        = list(number)
   description = "list of core counts for the manager nodes"

terraform/vars.tf

@@ -20,6 +20,12 @@ variable "k3s_token" {
   sensitive   = true
 }
 
+variable "DATADOG_API_KEY" {
+  type        = string
+  description = "datadog API key"
+  sensitive   = true
+}
+
 variable "mesh_proxmox_node" {
   type        = string
   description = "target node on the proxmox server"