Added definition, compose and ReadMe files

nerrang · Feb 1, 2020 · e3b4517 · e3b4517
commit e3b4517
Show file tree

Hide file tree

Showing 7 changed files with 386 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -0,0 +1,68 @@
+# ECS monitoring using Prometheus and Grafana
+Setup ECS monitoring using Prometheus and Grafana
+
+## Requirements:
+
+1. Modify 'init-container' definition to download the correct 'prometheus.yml' configuration file.
+2. Modify 'prometheus.yml' to make sure it queries and filters correct set of EC2 instance which needs to be monitored
+   In this case I used filter (in ec2_sd_configs section) with tag 'aws:autoscaling:groupName' and vlaue 'ECS_ASG-2' as all instances in my ECS clsuter was tagged with it.
+3. Use EC2 instance profile for ECS Cluster instance with permission to make describe calls for EC2 instances. This will be used by Prometheus to discover instance in your ECS cluster and add it in Prometheus targets (http://monitor_ec2_public_ip:9090/targets).
+4. Make sure instances in cluster can reach each other on ports 9090, 9100, and 9200 and open Grafana port 3000 for the IP range from where you need to access the Grafana Dashboard.
+
+## Usages Instructions:
+
+1. Create Task Definitions for cAdvisor, Node-Exporter, Prometheus and Grafana:
+
+```
+aws ecs register-task-definition --cli-input-json file://./cadvisor-node-exporter-definition.json grafana-definition.json
+aws ecs register-task-definition --cli-input-json file://./prometheus-grafana-definition.json
+```
+
+2. Create a DAEMON Service to run cAdvisor, Node-Exporter on every node in ECS Cluster:
+
+```
+aws ecs create-service --cluster MyWorkingCluster --service-name cadvisor-node-exporter --task-definition cadvisor-node-exporter-definition:1 --launch-type EC2 --scheduling-strategy DAEMON --region us-west-2
+```
+
+3. Run one task for Prometheus and Grafana in the clsuter:
+
+```
+aws ecs run-task --cluster MyWorkingCluster --task-definition prometheus-grafana-definition:3  --region us-west-2
+```
+
+4. Access Grafana Dashboard using URL: http://monitor_ec2_public_ip:3000
+   Use user:admin and password:admin to login and then the password.
+
+5. After logginig in, add datasource:
+
+<img src="https://miro.medium.com/max/2732/0*I8R5h2bM0DpgaoVs" title="Adding the Datasource" alt="Adding the Datasource" />
+
+* Fig 1: Monitoring results for Uptime, Containers, Load(Numerical Representation), DiskSpace, Memory, Filesystem Usage, CPU Usage, Memory Usage, Targets Online, Total Memory Usage, Alerts
+
+
+6. Import or Create a custom Grafana-Dashboard to monitor Docker containers running on the host:
+
+![Fig 2](https://miro.medium.com/max/2732/0*W0yXIT_P-1Gc_sY4)
+
+* Fig 2: Monitoring results for Uptime, Containers, Load(Numerical Representation), DiskSpace, Memory, Filesystem Usage, CPU Usage, Memory Usage, Targets Online, Total Memory Usage, Alerts.
+
+![Fig 3](https://miro.medium.com/max/2732/0*EO1JyVMHPkbFEYdk)
+
+* Fig 3: Monitoring results for CPU Usage, Network Traffic, Load, Available Memory, Node Network Traffic, Disk I/O, Node Memory, Filesystem Available, Container Network Input, Container Network Output.
+
+![Fig 4](https://miro.medium.com/max/2732/0*HeRmOCOHeHeJkkBB)
+
+* Fig 4: Monitoring results for System Load on Node and Cached Memory per Container(Stacked).
+
+![Fig 5](https://miro.medium.com/max/2732/0*Gmqz4PFyP5LjfRNn)
+
+* Fig 5: Monitoring Results for CPU Usage per container, Sent Traffic per Container, Received Network Traffic per Container, Memory Usage per Container.
+
+![Fig 6](https://miro.medium.com/max/2724/0*WjYd0f9n53689GUl)
+
+* Fig 6: Monitoring results for Network Rx, Network Tx, Tables for Usage Memory, Remaining Memory, Limit Memory.
+
+![Fig 7](https://miro.medium.com/max/2728/0*3jj0V42Ph67rZwFN)
+
+* Fig 7: Monitoring results for Running versions and Metrics.
+
diff --git a/cadvisor-node-exporter-compose.yml b/cadvisor-node-exporter-compose.yml
@@ -0,0 +1,36 @@
+version: '3'
+
+services:
+  node-exporter:
+    image: prom/node-exporter
+    ports:
+    - '9100:9100'
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: '100M'
+        reservations:
+          cpus: '0.1'
+          memory: '50M'
+  cadvisor-exporter:
+    image: google/cadvisor
+    ports:
+    - "9200:8080"
+    privileged: true
+    volumes:
+    - /:/rootfs:ro
+    - /cgroup:/cgroup:ro
+    - /var/run:/var/run:ro
+    - /var/lib/docker/:/var/lib/docker:ro
+    - /dev/disk/:/dev/disk:ro
+    - /cgroup:/sys/fs/cgroup:ro
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: '100M'
+        reservations:
+          cpus: '0.1'
+          memory: '50M'
+
diff --git a/cadvisor-node-exporter-definition.json b/cadvisor-node-exporter-definition.json
@@ -0,0 +1,107 @@
+{
+    "family": "cadvisor-node-exporter-definition",
+    "taskRoleArn": "arn:aws:iam::342241566140:role/AmazonECSTaskRole",
+        "containerDefinitions": [
+            {
+                "name": "node-exporter",
+                "image": "prom/node-exporter",
+                "cpu": 512,
+                "memory": 256,
+                "memoryReservation": 128,
+                "portMappings": [
+                    {
+                        "containerPort": 9100,
+                        "hostPort": 9100,
+                        "protocol": "tcp"
+                    }
+                ],
+                "essential": true
+            },
+            {
+                "name": "cadvisor-exporter",
+                "image": "google/cadvisor",
+                "cpu": 512,
+                "memory": 256,
+                "memoryReservation": 128,
+                "portMappings": [
+                    {
+                        "containerPort": 8080,
+                        "hostPort": 9200,
+                        "protocol": "tcp"
+                    }
+                ],
+                "essential": true,
+                "mountPoints": [
+                    {
+                        "sourceVolume": "root",
+                        "containerPath": "/rootfs",
+                        "readOnly": true
+                    },
+                    {
+                        "sourceVolume": "cgroup",
+                        "containerPath": "/cgroup",
+                        "readOnly": true
+                    },
+                    {
+                        "sourceVolume": "var_run",
+                        "containerPath": "/var/run",
+                        "readOnly": true
+                    },
+                    {
+                        "sourceVolume": "var_lib_docker",
+                        "containerPath": "/var/lib/docker",
+                        "readOnly": true
+                    },
+                    {
+                        "sourceVolume": "dev_disk",
+                        "containerPath": "/dev/disk",
+                        "readOnly": true
+                    },
+                    {
+                        "sourceVolume": "cgroup",
+                        "containerPath": "/sys/fs/cgroup",
+                        "readOnly": true
+                    }
+                ],
+                "privileged": true,
+                "readonlyRootFilesystem": false
+            }
+        ],
+        "volumes": [
+            {
+                "name": "root",
+                "host": {
+                    "sourcePath": "/"
+                }
+            },
+            {
+                "name": "cgroup",
+                "host": {
+                    "sourcePath": "/cgroup"
+                }
+            },
+            {
+                "name": "var_run",
+                "host": {
+                    "sourcePath": "/var/run"
+                }
+            },
+            {
+                "name": "var_lib_docker",
+                "host": {
+                    "sourcePath": "/var/lib/docker"
+                }
+            },
+            {
+                "name": "dev_disk",
+                "host": {
+                    "sourcePath": "/dev/disk"
+                }
+            }
+        ],
+    "requiresCompatibilities": [
+        "EC2"
+    ],
+    "cpu": "1024",
+    "memory": "512"
+}
diff --git a/grafana-definition.json b/grafana-definition.json
@@ -0,0 +1,27 @@
+{
+    "family": "grafana-definition",
+    "taskRoleArn": "arn:aws:iam::342241566140:role/AmazonECSTaskRole",
+        "containerDefinitions": [
+            {
+                "name": "grafana",
+                "image": "grafana/grafana",
+                "cpu": 512,
+                "memory": 512,
+                "memoryReservation": 128,
+                "portMappings": [
+                    {
+                        "containerPort": 3000,
+                        "hostPort": 3000,
+                        "protocol": "tcp"
+                    }
+                ],
+                "essential": true
+            }
+        ],
+
+    "requiresCompatibilities": [
+        "EC2"
+    ],
+    "cpu": "512",
+    "memory": "512"
+}
diff --git a/prometheus-grafana-compose.yml b/prometheus-grafana-compose.yml
@@ -0,0 +1,40 @@
+version: '3'
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.4.0
+    volumes:
+      - prometheus_config:/etc/prometheus/
+    ports:
+      - "9090:9090"
+    restart: always
+    depends_on:
+      - init-container
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: '100M'
+        reservations:
+          cpus: '0.1'
+          memory: '50M'
+  init-container:
+    image: alpine:latest
+    volumes:
+      - prometheus_config:/mnt/
+    command: wget https://raw.githubusercontent.com/santosh07bec/ecs-monitoring-with-prometheus-grafana/master/prometheus.yml -P /mnt/
+  grafana:
+    image: grafana/grafana
+    ports:
+      - "3000:3000"
+    restart: always
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: '100M'
+        reservations:
+          cpus: '0.1'
+          memory: '50M'
+volumes:
+  prometheus_config:
diff --git a/prometheus-grafana-definition.json b/prometheus-grafana-definition.json
@@ -0,0 +1,83 @@
+{
+    "family": "prometheus-grafana-definition",
+    "taskRoleArn": "arn:aws:iam::342241566140:role/AmazonECSTaskRole",
+        "containerDefinitions": [
+            {
+                "name": "init-container",
+                "image": "alpine:latest",
+                "cpu": 0,
+                "memory": 256,
+                "memoryReservation": 128,
+                "essential": false,
+                "command": [
+                    "wget",
+                    "https://raw.githubusercontent.com/santosh07bec/ecs-monitoring-with-prometheus-grafana/master/prometheus.yml",
+                    "-P",
+                    "/mnt"
+                ],
+                "mountPoints": [
+                    {
+                        "sourceVolume": "prometheus_config",
+                        "containerPath": "/mnt"
+                    }
+                ]
+            },
+            {
+                "name": "prometheus",
+                "image": "prom/prometheus:v2.4.0",
+                "cpu": 512,
+                "memory": 512,
+                "memoryReservation": 256,
+                "portMappings": [
+                    {
+                        "containerPort": 9090,
+                        "hostPort": 9090,
+                        "protocol": "tcp"
+                    }
+                ],
+                "essential": true,
+                "mountPoints": [
+                    {
+                        "sourceVolume": "prometheus_config",
+                        "containerPath": "/etc/prometheus"
+                    }
+                ],
+                "dependsOn": [
+                    {
+                        "containerName": "init-container",
+                        "condition": "COMPLETE"
+                    }
+                ]
+            },
+            {
+                "name": "grafana",
+                "image": "grafana/grafana",
+                "cpu": 512,
+                "memory": 512,
+                "memoryReservation": 128,
+                "portMappings": [
+                    {
+                        "containerPort": 3000,
+                        "hostPort": 3000,
+                        "protocol": "tcp"
+                    }
+                ],
+                "essential": true
+            }
+        ],
+        "volumes": [
+            {
+                "name": "prometheus_config",
+                "dockerVolumeConfiguration": {
+                    "scope": "task",
+                    "driver": "local"
+                }
+            }
+        ],
+
+    "requiresCompatibilities": [
+        "EC2"
+    ],
+    "cpu": "1024",
+    "memory": "512"
+}
diff --git a/prometheus.yml b/prometheus.yml
@@ -0,0 +1,25 @@
+global:
+  scrape_interval: 5s
+  external_labels:
+    monitor: 'devopsage-monitor'
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+    - targets: ['localhost:9090'] ## IP Address of the localhost
+  - job_name: 'ec2_aws_node_exporter_discovery'
+    ec2_sd_configs:
+      - port: 9100 # referred to nodeExporter exposed port
+        #role_arn: arn:aws:iam::342241566140:role/Ec2-Instance-Role
+        filters:
+        - name: tag:aws:autoscaling:groupName
+          values:
+          - ECS_ASG-2
+  - job_name: ecs_CAdvisor
+    ec2_sd_configs:
+      - port: 9200 # referred to cAdvisor exposed port
+        #role_arn: arn:aws:iam::342241566140:role/Ec2-Instance-Role
+        filters:
+        - name: tag:aws:autoscaling:groupName
+          values:
+          - ECS_ASG-2
+