diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bfba4b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +**/.DS_Store \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/Dockerfile b/04-solution-demos/compare-metrics/flink/Dockerfile new file mode 100644 index 0000000..7b2d158 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/Dockerfile @@ -0,0 +1,10 @@ +FROM flink:1.16.3-scala_2.12 + +# Download and copy both JAR files into the /opt/flink/lib directory +RUN wget -P /opt/flink/lib \ + https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.16.3/flink-sql-connector-kafka-1.16.3.jar && \ + wget -P /opt/flink/lib \ + https://repo.maven.apache.org/maven2/org/apache/flink/flink-metrics-prometheus/1.16.3/flink-metrics-prometheus-1.16.3.jar && \ + chown -R flink:flink /opt/flink/lib \ + RUN echo "metrics.reporters: prom" >> "$FLINK_HOME/conf/flink-conf.yaml"; \ + echo "metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory" >> "$FLINK_HOME/conf/flink-conf.yaml" \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/docker-compose.yml b/04-solution-demos/compare-metrics/flink/docker-compose.yml new file mode 100644 index 0000000..ff13c50 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/docker-compose.yml @@ -0,0 +1,125 @@ +version: '3.8' +services: + + # zookeeper + zookeeper: + #platform: linux/amd64 + image: bitnami/zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + networks: + - kafka-net + + # kafka + kafka: + image: confluentinc/cp-kafka:latest + container_name: kafka + depends_on: + - zookeeper + ports: + - "9093:9093" + - "29093:29093" + expose: + - "9093" + - "29093" + environment: + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:29093, LISTENER_DOCKER_EXTERNAL://localhost:9093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT, LISTENER_DOCKER_EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_AUTO_OFFSET_RESET: earliest + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - kafka-net + + # kafka producer + kafka-producer: + build: + context: . + dockerfile: flink_producer/Dockerfile + depends_on: + - kafka + - zookeeper + environment: + KAFKA_BOOTSTRAP_SERVERS: LISTENER_DOCKER_INTERNAL://kafka:29093 + restart: always + networks: + - kafka-net + + jobmanager: + build: . + depends_on: + - kafka-producer + ports: + - "8082:8081" + - "9249:9249" + command: jobmanager + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + checkpointing.mode: exactly_once + execution.checkpointing.interval: 60s + rest.flamegraph.enabled: true + networks: + - kafka-net + + taskmanager: + build: . + depends_on: + - jobmanager + command: taskmanager + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager.numberOfTaskSlots: 5 + networks: + - kafka-net + + sql-client: + build: . + command: bin/sql-client.sh + depends_on: + - jobmanager + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + rest.address: jobmanager + networks: + - kafka-net + + prometheus: + image: prom/prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + networks: + - kafka-net + + grafana: + image: grafana/grafana + ports: + - "3000:3000" + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + volumes: + - "./grafana_configs/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml" + - "./grafana_configs/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml" + - "./grafana_configs:/var/lib/grafana/dashboards" + networks: + - kafka-net + +networks: + kafka-net: + driver: bridge + name: kafka-net \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/flink-conf.yml b/04-solution-demos/compare-metrics/flink/flink-conf.yml new file mode 100644 index 0000000..7a1715d --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/flink-conf.yml @@ -0,0 +1,7 @@ +execution.checkpointing.interval: 60s +state.backend: filesystem +state.checkpoints.dir: file:///flink_checkpoint +state.savepoints.dir: file:///flink_checkpoint + +metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter +metrics.reporter.prom.port: 9249 # Port for Prometheus metrics \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/flink_producer/Dockerfile b/04-solution-demos/compare-metrics/flink/flink_producer/Dockerfile new file mode 100644 index 0000000..8349905 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/flink_producer/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8 + +RUN pip install kafka-python + +# Create a directory for the Python files +WORKDIR /app + +# Copy Python files into the container +COPY flink_producer/run_producers.py . +COPY flink_producer/constant_data.py . +COPY flink_producer/varying_data.py . + +# Run the Python script +CMD ["python", "run_producers.py"] \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/flink_producer/constant_data.py b/04-solution-demos/compare-metrics/flink/flink_producer/constant_data.py new file mode 100644 index 0000000..7105ba1 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/flink_producer/constant_data.py @@ -0,0 +1,77 @@ +import random +import json +import datetime +import time +import string +from kafka import KafkaProducer + +rate_per_second = 5 + +# Check if broker is available +def is_broker_available(): + global producer + try: + return True + except Exception as e: + print(f"Broker not available: {e}") + return False + +# Generate a random order ID +def generate_order_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=8)) + +# Generate a random customer ID +def generate_customer_id(): + return ''.join(random.choices(string.digits, k=3)) + +# Generate a random product ID +def generate_product_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=2)) + +# Generate random purchase event +def generate_purchase_event(): + order_id = generate_order_id() + customer_id = generate_customer_id() + product = generate_product_id() + quantity = random.randint(1,5) + timestamp = datetime.datetime.now().isoformat() + total_amount = round(random.uniform(10, 100) * quantity, 2) # Random total amount + return { + "order_id": order_id, + "customer_id": customer_id, + "prod": product, + "quant_in": quantity, + "ts": timestamp, + "tot_amnt_in": total_amount + } + +# Kafka topic to produce messages to +topic = 'purchase_constant' + +kafka_config = { + 'bootstrap_servers': ['kafka:29093'] +} +time.sleep(3) +# Kafka producer +producer = KafkaProducer(**kafka_config) + +if __name__ == "__main__": + + try: + # Produce messages to the Kafka topic + while is_broker_available(): + + message = generate_purchase_event() + message_str = json.dumps(message).encode('utf-8') + + producer.send(topic, message_str) + + time.sleep(1/rate_per_second) + + finally: + + print('Producer closed') + + # Wait for any outstanding messages to be delivered and delivery reports received + producer.flush() + producer.close() \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/flink_producer/run_producers.py b/04-solution-demos/compare-metrics/flink/flink_producer/run_producers.py new file mode 100644 index 0000000..75d8c13 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/flink_producer/run_producers.py @@ -0,0 +1,20 @@ +import subprocess +import os +import time + +# Get the current directory +current_directory = os.path.dirname(os.path.realpath(__file__)) + +# File paths for the Python files +file1_path = os.path.join(current_directory, 'constant_data.py') +file2_path = os.path.join(current_directory, 'varying_data.py') + +# Start file1.py in a separate process +process1 = subprocess.Popen(['python3', file1_path]) + +# Start file2.py in a separate process +process2 = subprocess.Popen(['python3', file2_path]) + +# Wait for both processes to finish (which they never will in this case) +process1.wait() +process2.wait() \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/flink_producer/varying_data.py b/04-solution-demos/compare-metrics/flink/flink_producer/varying_data.py new file mode 100644 index 0000000..87f0b55 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/flink_producer/varying_data.py @@ -0,0 +1,94 @@ +import random +import json +import datetime +import time +import string +from kafka import KafkaProducer + +rate_per_second = 5 + +# Check if broker is available +def is_broker_available(): + global producer + try: + return True + except Exception as e: + print(f"Broker not available: {e}") + return False + +# pause producer +def wait_until(): + current_time = datetime.datetime.now().second + + if current_time <= 30: + wait = 30 - current_time + elif current_time < 60: + wait = 60 - current_time + + print(current_time, wait) + # Otherwise, wait until the target time is reached + time.sleep(wait) + +# Generate a random order ID +def generate_order_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=8)) + +# Generate a random customer ID +def generate_customer_id(): + return ''.join(random.choices(string.digits, k=3)) + +# Generate a random product ID +def generate_product_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=2)) + +# Generate random purchase event +def generate_purchase_event(): + order_id = generate_order_id() + customer_id = generate_customer_id() + product = generate_product_id() + quantity = random.randint(1,5) + timestamp = datetime.datetime.now().isoformat() + total_amount = round(random.uniform(10, 100) * quantity, 2) # Random total amount + return { + "order_id": order_id, + "customer_id": customer_id, + "prod": product, + "quant_out": quantity, + "ts": timestamp, + "tot_amnt_out": total_amount + } + +# Kafka topic to produce messages to +topic = 'purchase_varying' + +kafka_config = { + 'bootstrap_servers': ['kafka:29093'] +} + +time.sleep(3) +# Kafka producer +producer = KafkaProducer(**kafka_config) + +if __name__ == "__main__": + + try: + # Produce messages to the Kafka topic + while is_broker_available(): + + if 0 <= datetime.datetime.now().second < 15 or 30 <= datetime.datetime.now().second < 45: + + message = generate_purchase_event() + message_str = json.dumps(message).encode('utf-8') + # Produce the message to the topic asynchronously + producer.send(topic, message_str) + time.sleep(1/rate_per_second) + + else: + wait_until() + + finally: + print('Producer closed') + + # Wait for any outstanding messages to be delivered and delivery reports received + producer.flush() + producer.close() diff --git a/04-solution-demos/compare-metrics/flink/grafana_configs/dashboard.yaml b/04-solution-demos/compare-metrics/flink/grafana_configs/dashboard.yaml new file mode 100644 index 0000000..d2b5afd --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/grafana_configs/dashboard.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'Flink Metrics' + orgId: 1 + folder: 'General' + type: file + disableDeletion: false + updateIntervalSeconds: 3 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/grafana_configs/datasource.yaml b/04-solution-demos/compare-metrics/flink/grafana_configs/datasource.yaml new file mode 100644 index 0000000..0b37dd4 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/grafana_configs/datasource.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 +deleteDatasources: + - name: flink-prometheus +datasources: + - name: flink-prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + withCredentials: false + isDefault: true + tlsAuth: false + tlsAuthWithCACert: false + version: 1 + editable: true \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/grafana_configs/flink-metrics-dashboard.json b/04-solution-demos/compare-metrics/flink/grafana_configs/flink-metrics-dashboard.json new file mode 100644 index 0000000..53250b0 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/grafana_configs/flink-metrics-dashboard.json @@ -0,0 +1,6277 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Job Manager and Task Manager", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 14911, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 159, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 160, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "flink_taskmanager_job_task_operator_KafkaSourceReader_KafkaConsumer_records_consumed_rate", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{operator_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Source Throughput (rows/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 161, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "flink_taskmanager_job_task_operator_KafkaSourceReader_KafkaConsumer_bytes_consumed_rate", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{operator_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Source Throughput (bits/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 163, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "flink_taskmanager_job_task_numRecordsInPerSecond", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Throughput per Task (rows/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 164, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "flink_taskmanager_job_task_numBytesInPerSecond", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Throughput per Task (bits/s)", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 113, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_taskmanager_Status_JVM_CPU_Load", + "interval": "", + "legendFormat": "taskmanager", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "flink_jobmanager_Status_JVM_CPU_Load", + "hide": false, + "instant": false, + "legendFormat": "jobmanager", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "sum(flink_taskmanager_Status_JVM_CPU_Load) + sum(flink_jobmanager_Status_JVM_CPU_Load)", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "C" + } + ], + "title": "Load Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Threads_Count{instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "jobmanager", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "flink_taskmanager_Status_JVM_Threads_Count", + "hide": false, + "instant": false, + "legendFormat": "taskmanager", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "sum(flink_taskmanager_Status_JVM_Threads_Count) + sum(flink_jobmanager_Status_JVM_Threads_Count)", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "C" + } + ], + "title": "Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_jobmanager_Status_JVM_CPU_Time{instance=~\"$jm_instance\"}[1m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "CPU Use Time Jobmanager", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_taskmanager_Status_JVM_CPU_Time{tm_id=~\"$tm_id\"}[1m])", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "CPU Use Time Taskmanager", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "JVM - CPU", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 122, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_Heap_Used{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "flink_taskmanager_Status_JVM_Memory_Heap_Used", + "hide": false, + "instant": false, + "legendFormat": "taskmanager", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "sum(flink_jobmanager_Status_JVM_Memory_Heap_Used) + sum(flink_taskmanager_Status_JVM_Memory_Heap_Used)", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "C" + } + ], + "title": "Heap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_jobmanager_Status_JVM_Memory_NonHeap_Used{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "flink_taskmanager_Status_JVM_Memory_NonHeap_Used", + "hide": false, + "instant": false, + "legendFormat": "taskmanager", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "expr": "sum(flink_jobmanager_Status_JVM_Memory_NonHeap_Used) + sum(flink_taskmanager_Status_JVM_Memory_NonHeap_Used)", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "C" + } + ], + "title": "Non Heap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 148, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(job) (flink_taskmanager_job_task_numBytesInPerSecond)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "In ", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (flink_taskmanager_job_task_numBytesOutPerSecond)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Out ", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Storage Remote I/O (per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (flink_taskmanager_job_task_numBytesIn)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "In ", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(job) (flink_taskmanager_job_task_numBytesOut)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Out ", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Storage Remote I/O Total ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 165, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(job_id) (flink_taskmanager_job_task_numBytesInPerSecond)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "I - {{job_id}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job_id) (flink_taskmanager_job_task_numBytesOutPerSecond)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "O - {{job_id}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Storage Remote I/O per Job (per second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 166, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job_id) (flink_taskmanager_job_task_numBytesIn)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "I - {{job_id}}", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(job_id) (flink_taskmanager_job_task_numBytesOut)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "O - {{job_id}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Storage Remote I/O Total per Job", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "JVM - Memory", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 55, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_UsedMemory", + "interval": "", + "legendFormat": "{{tm_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Used Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 28 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_AvailableMemory{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Available Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 28 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_UsedMemorySegments{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Used Memory Segments", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_Status_Shuffle_Netty_AvailableMemorySegments{tm_id=~\"$tm_id\"}", + "interval": "", + "legendFormat": "{{tm_id}}", + "refId": "A" + } + ], + "title": "Available Memory Segments", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Task Manager (Memory - Shuffle Netty)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 21 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_jobmanager_numRegisteredTaskManagers{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Task Managers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 21 + }, + "id": 32, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_jobmanager_taskSlotsAvailable{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Available Task Slots", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 21 + }, + "id": 33, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_jobmanager_numRunningJobs{instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Runnings Jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 21 + }, + "id": 42, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_jobmanager_job_fullRestarts{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Job Restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 3600000 + }, + { + "color": "dark-orange", + "value": 21600000 + }, + { + "color": "dark-purple", + "value": 86400000 + }, + { + "color": "dark-red", + "value": 259200000 + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 37, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_jobmanager_job_uptime{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job_id}} ({{instance}})", + "refId": "A" + } + ], + "title": "Job Uptime", + "type": "stat" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Job Manager (Slots & Jobs)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 106, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 108, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_jobmanager_job_totalNumberOfCheckpoints{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{job_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Checkpoints", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "flink_jobmanager_job_lastCheckpointFullSize{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Last Checkpoint Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_jobmanager_job_lastCheckpointDuration{job_name=~\"$job_name\", instance=~\"$jm_instance\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}}) on {{instance}}", + "refId": "A" + } + ], + "title": "Last Checkpoint Duration", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Job Manager (Checkpoints)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 53, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 47 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_isBackPressured{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Is Back Pressured", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 47 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_taskmanager_job_task_idleTimeMsPerSecond", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Idle Time Ms ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 47 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_taskmanager_job_task_checkpointStartDelayNanos{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Checkpoint Start Delay ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 47 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_taskmanager_job_task_checkpointAlignmentTime", + "interval": "", + "legendFormat": "{{task_name}} : {{job_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Checkpoint Alignment Time ", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Task Manager (Back Pressure)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 139, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 141, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_numBuffersInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Local", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBuffersInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Local (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_numBuffersInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers in Remote", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 144, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBuffersInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Number Buffers in Remote (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 145, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numBuffersOut{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 146, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numBuffersOutPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Number Buffers Out (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 155, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numRecordsIn{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Records In", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 156, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numRecordsInPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Records In (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 33 + }, + "id": 157, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_numRecordsOut{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Records Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 33 + }, + "id": 158, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_numRecordsOutPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Records Out (per Second)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Task Manager (Job Task - General)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 97, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 99, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "In Pool Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inputExclusiveBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Exclusive Buffers Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 102, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_outPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Out Pool Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_outputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Output Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_buffers_inputFloatingBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Floating Buffers Usage", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Task Manager (Job Task - Buffers)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 60, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "editorMode": "code", + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inputQueueLength", + "interval": "", + "legendFormat": "{{task_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Input Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Output_Buffers_outputQueueLength{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Output Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "In Pool Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Output_Buffers_outPoolUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Out Pool Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inputFloatingBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Floating Buffers Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_Buffers_inputExclusiveBuffersUsage{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Exclusive Buffers Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Buffers In Local", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "cps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Buffers In Local (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Buffers In Remote", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBuffersInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Num Buffers In Remote (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 45 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInLocal{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Num Bytes In Local", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 45 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInLocalPerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Bytes In Local (per Second)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 45 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "delta(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInRemote{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Bytes In Remote", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "binBps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 45 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "exemplar": true, + "expr": "rate(flink_taskmanager_job_task_Shuffle_Netty_Input_numBytesInRemotePerSecond{tm_id=~\"$tm_id\", job_name=~\"$job_name\", task_name=~\"$task_name\"}[1m])", + "interval": "", + "legendFormat": "{{job_name}} ({{job_id}} / {{task_id}}) {{tm_id}}", + "refId": "A" + } + ], + "title": "Input Number Bytes In Remote (per Second)", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "refId": "A" + } + ], + "title": "Task Manager (Job Task - Shuffle Netty)", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "flink", + "task manager", + "job manager", + "apache" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "flink-prometheus" + ], + "value": [ + "P3279612AFF85A07D" + ] + }, + "hide": 0, + "includeAll": false, + "multi": true, + "name": "Source", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "definition": "label_values(tm_id)", + "hide": 0, + "includeAll": true, + "label": "Task Manager", + "multi": true, + "name": "tm_id", + "options": [], + "query": { + "query": "label_values(tm_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "definition": "label_values(flink_jobmanager_numRegisteredTaskManagers,instance)", + "hide": 0, + "includeAll": true, + "label": "Job Manager", + "multi": true, + "name": "jm_instance", + "options": [], + "query": { + "query": "label_values(flink_jobmanager_numRegisteredTaskManagers,instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "definition": "label_values(flink_taskmanager_job_task_isBackPressured{tm_id=~\"$tm_id\"},job_name)", + "hide": 0, + "includeAll": true, + "label": "Job Name", + "multi": true, + "name": "job_name", + "options": [], + "query": { + "query": "label_values(flink_taskmanager_job_task_isBackPressured{tm_id=~\"$tm_id\"},job_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "definition": "label_values(flink_taskmanager_job_task_isBackPressured{job_name=~\"$job_name\"},task_name)", + "hide": 0, + "includeAll": true, + "label": "Task Name", + "multi": true, + "name": "task_name", + "options": [], + "query": { + "query": "label_values(flink_taskmanager_job_task_isBackPressured{job_name=~\"$job_name\"},task_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P3279612AFF85A07D" + }, + "filters": [], + "hide": 0, + "label": "Ad Hoc", + "name": "ad_hoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Apache Flink Dashboard", + "uid": "wKbnD5Gnk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/prometheus.yaml b/04-solution-demos/compare-metrics/flink/prometheus.yaml new file mode 100644 index 0000000..ea73476 --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/prometheus.yaml @@ -0,0 +1,17 @@ +global: + scrape_interval: 1s + evaluation_interval: 1s + +scrape_configs: + - job_name: "flink-task-manager" + dns_sd_configs: + - names: + - 'taskmanager' + refresh_interval: 1s + type: 'A' + port: 9249 + metrics_path: / + - job_name: "flink-job-manger" + static_configs: + - targets: [ "jobmanager:9249" ] + metrics_path: / \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/flink/readme.md b/04-solution-demos/compare-metrics/flink/readme.md new file mode 100644 index 0000000..2fdef1c --- /dev/null +++ b/04-solution-demos/compare-metrics/flink/readme.md @@ -0,0 +1,58 @@ +## Run Flink + +Within this `flink` directory, run `docker compose up -d` to start the project. + +Run `docker compose run sql-client` to start up the SQL client. Run the SQL queries included below line by line. + +```sql +SET 'execution.runtime-mode' = 'streaming'; +SET 'execution.checkpointing.mode' = 'EXACTLY_ONCE'; +SET 'execution.checkpointing.interval' = '1min'; + +CREATE TABLE constant ( + order_id varchar, + customer_id varchar, + prod varchar, + quant_in integer, + tot_amnt_in double precision, + ts varchar, + order_time as to_timestamp(ts, 'yyyy-MM-dd''T''HH:mm:ss.SSSSSS') + ) WITH ( + 'connector' = 'kafka', + 'topic' = 'purchase_constant', + 'properties.bootstrap.servers' = 'kafka:29093', + 'scan.startup.mode' = 'earliest-offset', + 'format' = 'json' +); + +CREATE TABLE varies ( + order_id varchar, + customer_id varchar, + prod varchar, + quant_out integer, + tot_amnt_out double precision, + ts varchar, + sell_time as to_timestamp(ts, 'yyyy-MM-dd''T''HH:mm:ss.SSSSSS') +) WITH ( + 'connector' = 'kafka', + 'topic' = 'purchase_varying', + 'properties.bootstrap.servers' = 'kafka:29093', + 'scan.startup.mode' = 'earliest-offset', + 'format' = 'json' +); + +CREATE VIEW j3 AS +SELECT + c.customer_id, + SUM(c.quant_in) - SUM(v.quant_out) AS quant_tot, + SUM(c.tot_amnt_in) - SUM(v.tot_amnt_out) AS amnt_tot +FROM + constant AS c +JOIN + varies AS v ON c.customer_id = v.customer_id +GROUP BY + c.customer_id; + +SELECT * FROM j3; +``` + diff --git a/04-solution-demos/compare-metrics/producer/constant_data.py b/04-solution-demos/compare-metrics/producer/constant_data.py new file mode 100644 index 0000000..afa70c5 --- /dev/null +++ b/04-solution-demos/compare-metrics/producer/constant_data.py @@ -0,0 +1,86 @@ +import random +import json +import datetime +import time +import string +from confluent_kafka.admin import AdminClient, NewTopic +from confluent_kafka import Producer + +rate_per_second = 5 + +kafka_config = { + 'bootstrap.servers': 'localhost:9092' +} + +# Kafka producer +producer = Producer(kafka_config) + +admin_client = AdminClient(kafka_config) + +# Kafka topic to produce messages to +topic = 'purchase_constant' +partitions = 1 +replication_factor = 1 + +# Create NewTopic object +new_topic = NewTopic(topic, num_partitions=partitions, replication_factor=replication_factor) + +# Create topic +admin_client.create_topics([new_topic]) + +# Check if broker is available +def is_broker_available(): + global producer + try: + return True + except Exception as e: + return False + +# Generate a random order ID +def generate_order_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=8)) + +# Generate a random customer ID +def generate_customer_id(): + return ''.join(random.choices(string.digits, k=3)) + +# Generate a random product ID +def generate_product_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=2)) + +# Generate random purchase event +def generate_purchase_event(): + order_id = generate_order_id() + customer_id = generate_customer_id() + product = generate_product_id() + quantity = random.randint(1,5) + timestamp = datetime.datetime.now().isoformat() + total_amount = round(random.uniform(10, 100) * quantity, 2) # Random total amount + return { + "order_id": order_id, + "customer_id": customer_id, + "prod": product, + "quant_in": quantity, + "ts": timestamp, + "tot_amnt_in": total_amount + } + +if __name__ == "__main__": + + try: + # Produce messages to the Kafka topic + while is_broker_available(): + + message = generate_purchase_event() + message_str = json.dumps(message) + + producer.produce(topic, message_str) + + time.sleep(1/rate_per_second) + + finally: + + print('Producer closed') + + # Wait for any outstanding messages to be delivered and delivery reports received + producer.flush() \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/producer/readme.md b/04-solution-demos/compare-metrics/producer/readme.md new file mode 100644 index 0000000..9443fef --- /dev/null +++ b/04-solution-demos/compare-metrics/producer/readme.md @@ -0,0 +1,75 @@ +# Run RisingWave + +Install the Confluent library for Python if it's not already installed on your device. + +```terminal +pip install confluent_kafka +``` + +The message producers also use the Python libraries random, json, datetime, time, and string, which are likely already downloaded. + +Run the `run_producers.py` python file to start producing messages. + +Start [RisingWave via docker-compose](https://docs.risingwave.com/docs/dev/risingwave-docker-compose/). + +Run the queries below to start consuming messages from the message queue. + +Open [Grafana](http://localhost:3001) and open the pre-build RisingWave dashboard. The main performance metrics are tracked. Change the timeframe to 5 minutes to get a better view of how the metrics change over time. + +The varying data source starts and stops every 15 seconds. Both message producers approximately produce messages at a rate of 5 messages/second. + +# Use the following queries to create sources and materialized views + +## Create sources that connect to the message producers + +```sql +create source constant ( + order_id varchar, + customer_id varchar, + prod_in varchar, + quant_in integer, + tot_amnt_in double precision, + ts varchar +) with ( + connector = 'kafka', + topic = 'purchase_constant', + properties.bootstrap.server = 'message_queue:29092' +) FORMAT PLAIN ENCODE JSON; + +create source varying ( + order_id varchar, + customer_id varchar, + prod_out varchar, + quant_out integer, + tot_amnt_out double precision, + ts varchar +) with ( + connector = 'kafka', + topic = 'purchase_varying', + properties.bootstrap.server = 'message_queue:29092' +) FORMAT PLAIN ENCODE JSON; + +create materialized view j3 as +SELECT c.customer_id, + sum(c.quant_in) - sum(v.quant_out) as quant_tot, + sum(c.tot_amnt_in) - sum(v.tot_amnt_out) as amnt_tot +FROM constant as c +JOIN varying as v on c.customer_id = v.customer_id +group by c.customer_id; + +select * from j3 limit 30; +``` + +A tumble function view (not included in flink) + +```sql +create materialized view v2 as select + order_id, customer_id, prod_out, quant_out, tot_amnt_out, ts::timestamptz +from varying; + +create materialized view out_1min as +select window_end, sum(quant_out) as quant_out, sum(tot_amnt_out) as tot_amnt_out +from tumble(v2, ts, interval '1' minute) +group by window_end; +``` + diff --git a/04-solution-demos/compare-metrics/producer/run.ipynb b/04-solution-demos/compare-metrics/producer/run.ipynb new file mode 100644 index 0000000..34f1834 --- /dev/null +++ b/04-solution-demos/compare-metrics/producer/run.ipynb @@ -0,0 +1,66 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "53 7\n", + "15 15\n", + "45 15\n" + ] + } + ], + "source": [ + "# Get the current directory\n", + "current_directory = os.getcwd()\n", + "\n", + "# File paths for the Python files\n", + "file1_path = os.path.join(current_directory, 'constant_data.py')\n", + "file2_path = os.path.join(current_directory, 'varying_data.py')\n", + "\n", + "# Start file1.py in a separate process\n", + "process1 = subprocess.Popen(['python3', file1_path])\n", + "\n", + "# Start file2.py in a separate process\n", + "process2 = subprocess.Popen(['python3', file2_path])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/04-solution-demos/compare-metrics/producer/run_producers.py b/04-solution-demos/compare-metrics/producer/run_producers.py new file mode 100644 index 0000000..f027af6 --- /dev/null +++ b/04-solution-demos/compare-metrics/producer/run_producers.py @@ -0,0 +1,19 @@ +import subprocess +import os + +# Get the current directory +current_directory = os.path.dirname(os.path.realpath(__file__)) + +# File paths for the Python files +file1_path = os.path.join(current_directory, 'constant_data.py') +file2_path = os.path.join(current_directory, 'varying_data.py') + +# Start file1.py in a separate process +process1 = subprocess.Popen(['python3', file1_path]) + +# Start file2.py in a separate process +process2 = subprocess.Popen(['python3', file2_path]) + +# Wait for both processes to finish (which they never will in this case) +process1.wait() +process2.wait() \ No newline at end of file diff --git a/04-solution-demos/compare-metrics/producer/varying_data.py b/04-solution-demos/compare-metrics/producer/varying_data.py new file mode 100644 index 0000000..ab97632 --- /dev/null +++ b/04-solution-demos/compare-metrics/producer/varying_data.py @@ -0,0 +1,102 @@ +import random +import json +import datetime +import time +import string +from confluent_kafka.admin import AdminClient, NewTopic +from confluent_kafka import Producer + +rate_per_second = 5 + +kafka_config = { + 'bootstrap.servers': 'localhost:9092' +} + +# Kafka producer +producer = Producer(kafka_config) + +admin_client = AdminClient(kafka_config) + +# Kafka topic to produce messages to +topic = 'purchase_varying' +partitions = 1 +replication_factor = 1 + +# Create NewTopic object +new_topic = NewTopic(topic, num_partitions=partitions, replication_factor=replication_factor) + +# Create topic +admin_client.create_topics([new_topic]) + +# Check if broker is available +def is_broker_available(): + global producer + try: + return True + except Exception as e: + return False + +# pause producer +def wait_until(): + current_time = datetime.datetime.now().second + + if current_time <= 30: + wait = 30 - current_time + elif current_time < 60: + wait = 60 - current_time + + print(current_time, wait) + # Otherwise, wait until the target time is reached + time.sleep(wait) + +# Generate a random order ID +def generate_order_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=8)) + +# Generate a random customer ID +def generate_customer_id(): + return ''.join(random.choices(string.digits, k=3)) + +# Generate a random product ID +def generate_product_id(): + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=2)) + +# Generate random purchase event +def generate_purchase_event(): + order_id = generate_order_id() + customer_id = generate_customer_id() + product = generate_product_id() + quantity = random.randint(1,5) + timestamp = datetime.datetime.now().isoformat() + total_amount = round(random.uniform(10, 100) * quantity, 2) # Random total amount + return { + "order_id": order_id, + "customer_id": customer_id, + "prod": product, + "quant_out": quantity, + "ts": timestamp, + "tot_amnt_out": total_amount + } + +if __name__ == "__main__": + + try: + # Produce messages to the Kafka topic + while is_broker_available(): + + if 0 <= datetime.datetime.now().second < 15 or 30 <= datetime.datetime.now().second < 45: + + message = generate_purchase_event() + message_str = json.dumps(message) + # Produce the message to the topic asynchronously + producer.produce(topic, message_str) + time.sleep(1/rate_per_second) + + else: + wait_until() + + finally: + print('Producer closed') + + # Wait for any outstanding messages to be delivered and delivery reports received + producer.flush() diff --git a/04-solution-demos/compare-metrics/readme.md b/04-solution-demos/compare-metrics/readme.md new file mode 100644 index 0000000..520b956 --- /dev/null +++ b/04-solution-demos/compare-metrics/readme.md @@ -0,0 +1,13 @@ +## Compare metrics + +This demo aims to observe the performance metrics of RisingWave and Flink simultaneously. We will be using the same message queues for both demos and the same queries for a fair comparison. + +Navigate to the `producer` and `flink` directories and read their respective readme files to learn how to run both docker compose projects. You should be able to run both docker compose projects simultaneously. + +Be sure to install the Confluent library for Python if it's not already installed on your device. + +```terminal +pip install confluent_kafka +``` + +Note that the metrics between Flink and RisingWave aren't the same but they represent the same concepts. While the usage of Flink here to process streaming data may not be ideal, it is reflective of how a first-time user may approach this. It also allows for a clear side-by-side comparison of RisingWave and Flink. \ No newline at end of file