From 51d2166b6af3a63adc9adc7eab8310e7f39d10c0 Mon Sep 17 00:00:00 2001
From: Neeraj Sharma <neeraj@last9.io>
Date: Thu, 14 Apr 2022 00:32:38 +0530
Subject: [PATCH 1/3] prometheus source k8s cluster module

---
 prometheus/k8s/k8s_resources.hcl | 49 ++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 prometheus/k8s/k8s_resources.hcl

diff --git a/prometheus/k8s/k8s_resources.hcl b/prometheus/k8s/k8s_resources.hcl
new file mode 100644
index 0000000..a783ba8
--- /dev/null
+++ b/prometheus/k8s/k8s_resources.hcl
@@ -0,0 +1,49 @@
+scraper prometheus_kube_cluster module {
+  frequency  = 120
+  lookback   = 600
+  timeout    = 90
+  resolution = 60
+  lag        = 60
+
+  gauge "available_nodes" {
+    source prometheus "available_nodes" {
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='Ready', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='Ready'}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+
+  gauge "disk_pressure_nodes" {
+    source prometheus "disk_pressure_nodes" {
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='DiskPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='DiskPressure'}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+
+  gauge "pid_pressure_nodes" {
+    source prometheus "pid_pressure_nodes" {
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='PIDPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='PIDPressure'}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+
+  gauge "memory_pressure_nodes" {
+    source prometheus "memory_pressure_nodes" {
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='MemoryPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='MemoryPressure'}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+
+  gauge "requested_memory" {
+    source prometheus "requested_memory" {
+      query = "label_replace(sum(kube_pod_container_resource_requests{resource='memory', unit='byte'})/sum by (cluster)(kube_node_status_allocatable{resource='memory', unit='byte'}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+
+  gauge "requested_cpu" {
+    source prometheus "requested_cpu" {
+      query = "label_replace(sum(kube_pod_container_resource_requests{resource='cpu', unit='core'})/sum by (cluster)(kube_node_status_allocatable{resource='cpu', unit='core'}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+
+  gauge "saturated_nodes" {
+    source prometheus "saturated_nodes" {
+      query = "label_replace(sum by (cluster) (kube_node_spec_unschedulable{}), 'cluster', '$input{cluster}', '', '')"
+    }
+  }
+}

From b6ef666998739fb4d256b8ab21d70958359df38c Mon Sep 17 00:00:00 2001
From: Neeraj Sharma <neeraj@last9.io>
Date: Thu, 14 Apr 2022 00:37:56 +0530
Subject: [PATCH 2/3] proetheus source name fixed to promql

---
 prometheus/k8s/k8s_resources.hcl | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/prometheus/k8s/k8s_resources.hcl b/prometheus/k8s/k8s_resources.hcl
index a783ba8..7f2926b 100644
--- a/prometheus/k8s/k8s_resources.hcl
+++ b/prometheus/k8s/k8s_resources.hcl
@@ -1,4 +1,4 @@
-scraper prometheus_kube_cluster module {
+scraper promql_kube_cluster module {
   frequency  = 120
   lookback   = 600
   timeout    = 90
@@ -6,43 +6,43 @@ scraper prometheus_kube_cluster module {
   lag        = 60
 
   gauge "available_nodes" {
-    source prometheus "available_nodes" {
+    source promql "available_nodes" {
       query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='Ready', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='Ready'}), 'cluster', '$input{cluster}', '', '')"
     }
   }
 
   gauge "disk_pressure_nodes" {
-    source prometheus "disk_pressure_nodes" {
+    source promql "disk_pressure_nodes" {
       query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='DiskPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='DiskPressure'}), 'cluster', '$input{cluster}', '', '')"
     }
   }
 
   gauge "pid_pressure_nodes" {
-    source prometheus "pid_pressure_nodes" {
+    source promql "pid_pressure_nodes" {
       query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='PIDPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='PIDPressure'}), 'cluster', '$input{cluster}', '', '')"
     }
   }
 
   gauge "memory_pressure_nodes" {
-    source prometheus "memory_pressure_nodes" {
+    source promql "memory_pressure_nodes" {
       query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='MemoryPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='MemoryPressure'}), 'cluster', '$input{cluster}', '', '')"
     }
   }
 
   gauge "requested_memory" {
-    source prometheus "requested_memory" {
+    source promql "requested_memory" {
       query = "label_replace(sum(kube_pod_container_resource_requests{resource='memory', unit='byte'})/sum by (cluster)(kube_node_status_allocatable{resource='memory', unit='byte'}), 'cluster', '$input{cluster}', '', '')"
     }
   }
 
   gauge "requested_cpu" {
-    source prometheus "requested_cpu" {
+    source promql "requested_cpu" {
       query = "label_replace(sum(kube_pod_container_resource_requests{resource='cpu', unit='core'})/sum by (cluster)(kube_node_status_allocatable{resource='cpu', unit='core'}), 'cluster', '$input{cluster}', '', '')"
     }
   }
 
   gauge "saturated_nodes" {
-    source prometheus "saturated_nodes" {
+    source promql "saturated_nodes" {
       query = "label_replace(sum by (cluster) (kube_node_spec_unschedulable{}), 'cluster', '$input{cluster}', '', '')"
     }
   }

From 3cdd667be5d47e70bc4da446e589330281fadc77 Mon Sep 17 00:00:00 2001
From: Neeraj Sharma <neeraj@last9.io>
Date: Thu, 14 Apr 2022 09:42:29 +0530
Subject: [PATCH 3/3] k8s node and cluster with namespace module added

---
 prometheus/k8s/k8s_resources.hcl | 91 +++++++++++++++++++++++++++++---
 1 file changed, 84 insertions(+), 7 deletions(-)

diff --git a/prometheus/k8s/k8s_resources.hcl b/prometheus/k8s/k8s_resources.hcl
index 7f2926b..eed0d2d 100644
--- a/prometheus/k8s/k8s_resources.hcl
+++ b/prometheus/k8s/k8s_resources.hcl
@@ -7,43 +7,120 @@ scraper promql_kube_cluster module {
 
   gauge "available_nodes" {
     source promql "available_nodes" {
-      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='Ready', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='Ready'}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='Ready', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='Ready'}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 
   gauge "disk_pressure_nodes" {
     source promql "disk_pressure_nodes" {
-      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='DiskPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='DiskPressure'}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='DiskPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='DiskPressure'}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 
   gauge "pid_pressure_nodes" {
     source promql "pid_pressure_nodes" {
-      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='PIDPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='PIDPressure'}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='PIDPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='PIDPressure'}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 
   gauge "memory_pressure_nodes" {
     source promql "memory_pressure_nodes" {
-      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='MemoryPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='MemoryPressure'}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum by (cluster)(kube_node_status_condition{condition='MemoryPressure', status='true'})/ sum by (cluster)(kube_node_status_condition{condition='MemoryPressure'}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 
   gauge "requested_memory" {
     source promql "requested_memory" {
-      query = "label_replace(sum(kube_pod_container_resource_requests{resource='memory', unit='byte'})/sum by (cluster)(kube_node_status_allocatable{resource='memory', unit='byte'}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum(kube_pod_container_resource_requests{resource='memory', unit='byte'})/sum by (cluster)(kube_node_status_allocatable{resource='memory', unit='byte'}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 
   gauge "requested_cpu" {
     source promql "requested_cpu" {
-      query = "label_replace(sum(kube_pod_container_resource_requests{resource='cpu', unit='core'})/sum by (cluster)(kube_node_status_allocatable{resource='cpu', unit='core'}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum(kube_pod_container_resource_requests{resource='cpu', unit='core'})/sum by (cluster)(kube_node_status_allocatable{resource='cpu', unit='core'}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 
   gauge "saturated_nodes" {
     source promql "saturated_nodes" {
-      query = "label_replace(sum by (cluster) (kube_node_spec_unschedulable{}), 'cluster', '$input{cluster}', '', '')"
+      query = "label_replace(sum by (cluster) (kube_node_spec_unschedulable{}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+}
+
+scraper promql_kube_cluster_with_namespace module {
+  frequency  = 120
+  lookback   = 600
+  timeout    = 90
+  resolution = 60
+  lag        = 60
+
+  gauge "unscheduled_pods" {
+    source promql "unscheduled_pods" {
+      query = "label_replace(sum by (cluster, namespace) (increase(kube_pod_status_unschedulable{}[1m])), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "desired_pods" {
+    source promql "desired_pods" {
+      query = "label_replace(sum by (cluster, namespace) (increase(kube_pod_status_phase{}[1m])), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "running_pods" {
+    source promql "running_pods" {
+      query = "label_replace(sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Running'}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "pending_pods" {
+    source promql "pending_pods" {
+      query = "label_replace(sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Pending'}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "failed_and_unknown_pods" {
+    source promql "failed_and_unknown_pods" {
+      query = "label_replace(sum by (cluster, namespace) (kube_pod_status_phase{phase=~'Failed|Unknown'}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "container_restarts" {
+    source promql "container_restarts" {
+      query = "label_replace(sum by (cluster, namespace) (rate(kube_pod_container_status_restarts_total{}[1m])*60), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+}
+
+scraper promql_kube_node module {
+  frequency  = 120
+  lookback   = 600
+  timeout    = 90
+  resolution = 60
+  lag        = 60
+
+  gauge "disk_pressure" {
+    source promql "disk_pressure" {
+      query = "label_replace(sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure', status='true'})/ sum by (cluster, node)(kube_node_status_condition{condition='DiskPressure'}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "pid_pressure" {
+    source promql "pid_pressure" {
+      query = "label_replace(sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure', status='true'})/ sum by (cluster, node)(kube_node_status_condition{condition='PIDPressure'}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "memory_pressure" {
+    source promql "memory_pressure" {
+      query = "label_replace(sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure', status='true'})/ sum by (cluster, node)(kube_node_status_condition{condition='MemoryPressure'}), 'cluster', '${resources.each.cluster}', '', '')"
+    }
+  }
+
+  gauge "saturated" {
+    source promql "saturated" {
+      query = "label_replace(sum by (cluster, node) (kube_node_spec_unschedulable{}), 'cluster', '${resources.each.cluster}', '', '')"
     }
   }
 }