From 5bec207ee8f04c6388f84b9e1f2d536415c1ace3 Mon Sep 17 00:00:00 2001 From: gligorisaev Date: Fri, 20 Sep 2024 09:06:55 +0200 Subject: [PATCH 1/6] add benchmarks section to the documentation Signed-off-by: gligorisaev --- .github/workflows/download_svgs.yml | 27 + docs/src/benchmarks/index.md | 101 + docs/src/benchmarks/memory-month.svg | 1630 +++++++++++++++++ docs/src/benchmarks/tedgecpuprocent-month.svg | 1111 +++++++++++ plugins/munin/tedgecpuprocent | 73 + plugins/munin/tedgemonitorcpu | 111 ++ scripts/download_svgs.py | 44 + 7 files changed, 3097 insertions(+) create mode 100644 .github/workflows/download_svgs.yml create mode 100644 docs/src/benchmarks/index.md create mode 100644 docs/src/benchmarks/memory-month.svg create mode 100644 docs/src/benchmarks/tedgecpuprocent-month.svg create mode 100644 plugins/munin/tedgecpuprocent create mode 100644 plugins/munin/tedgemonitorcpu create mode 100644 scripts/download_svgs.py diff --git a/.github/workflows/download_svgs.yml b/.github/workflows/download_svgs.yml new file mode 100644 index 00000000000..dbe9cb0efae --- /dev/null +++ b/.github/workflows/download_svgs.yml @@ -0,0 +1,27 @@ +name: Download SVGs + +on: + workflow_dispatch: # You can trigger the workflow manually, or set it on a schedule. + +jobs: + download-svgs: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' # Use Python 3.x + + - name: Install dependencies + run: | + pip install requests + + - name: Run download_svgs.py + env: + MUNIN_USERNAME: ${{ secrets.MUNIN_USERNAME }} + MUNIN_PASSWORD: ${{ secrets.MUNIN_PASSWORD }} + run: | + python3 /ci/build_scripts/download_svgs.py diff --git a/docs/src/benchmarks/index.md b/docs/src/benchmarks/index.md new file mode 100644 index 00000000000..a80b48b02c8 --- /dev/null +++ b/docs/src/benchmarks/index.md @@ -0,0 +1,101 @@ +--- +title: Benchmarks +sidebar_position: 10 +--- + +# Benchmarks + +This section contains performance benchmarks for ThinEdge.io, including detailed CPU and memory usage for key processes, measured on a device running in the OSADL QA Farm. + +## Hardware Information + +The following hardware and software specifications describe the environment in which these benchmarks are measured. The device runs ThinEdge.io and its associated plugins for data collection. + +| Component | Specification | +|----------------|---------------------------------------------------------------| +| **Processor** | Quad-core ARM Cortex-A72 (ARM v8) | +| **Memory** | 1.8 GB RAM | +| **Operating System** | Debian GNU/Linux 12 (bookworm) | +| **MQTT Broker**| Mosquitto (v2.0.18) | +| **Firmware** | ThinEdge.io (v1.1.2) | +| **Plugins Used** | `tedgecpuprocent`, `memory`, `tedge-agent`, `tedge-mapper` | + +> The device is a Raspberry Pi 4 Model B Rev 1.5, a member of the OSADL QA Farm, which continuously collects data on performance and resource consumption. + +## CPU Run-time Consumption of Key ThinEdge.io Tasks + +In this section, we monitor the CPU usage of the most critical ThinEdge.io tasks: `mosquitto`, `tedge-agent`, and `tedge-mapper`. These tasks are essential for device-to-cloud communication, and their performance is crucial for the overall efficiency of the system. + +### Graph of CPU Usage + +Below is a visual representation of the CPU consumption by the main ThinEdge.io processes over the past month. + +![CPU Usage](./tedgecpuprocent-month.svg) + +For each process, the graph shows: +- **Command Name (comm)**: The name of the task running on the device. +- **CPU Usage (cuc)**: The percentage of CPU time utilized by each process. + + +The graph allows you to visually track CPU performance trends and identify potential bottlenecks in the system. + +### Detailed Metrics + +The following table provides a detailed breakdown of the CPU consumption for each task: + +| Task | Description | +|----------------------|----------------------------------------------------------------| +| `mosquitto` | Handles MQTT communications between devices and cloud | +| `tedge-agent` | Coordinates device management, software updates, and telemetry | +| `tedge-mapper_c8y` | Converts telemetry data into the format required by cloud | +| `tedge-mapper-colle` | Converts telemetry data into the format required | + +> Note: The values in this table are updated dynamically once per month. + + +## Memory Usage + +In this section, we monitor the memory consumption of key system components to ensure efficient performance. The memory plugin tracks several metrics to provide a detailed view of how the system memory is being utilized. + +### Graph of Memory Usage + +Below is a visual representation of the memory consumption on the device over the past month. + +![Memory Usage](./memory-month.svg) + +### Monitored Metrics: +- **Total Memory**: The total amount of memory available on the system. +- **Free Memory**: Memory that is available for new processes. +- **Buffers**: Memory used by the system for temporary storage, primarily for block devices (like hard disks). +- **Cached Memory**: Memory that is cached for quicker access to frequently used data. +- **Swap Memory**: Disk space used as virtual memory when physical memory is full. +- **Slab Memory**: Memory used by the kernel for various internal data structures. +- **Shared Memory (shmem)**: Memory used for shared memory segments and tmpfs (temporary file system). +- **Swap Cache**: Memory that keeps track of pages that have been fetched from swap but not yet modified. +- **Page Tables**: Memory used to map virtual memory addresses to physical addresses. + +For each metric, the graph shows the current and historical values, allowing you to monitor how memory is utilized by the system. + +### Detailed Metrics + +The following table provides a breakdown of memory usage: + +| Metric | Description | +|------------------|-----------------------------------------------------------------------------| +| **Total Memory** | The total amount of memory available on the system. | +| **Free Memory** | Memory that is available for new processes. | +| **Buffers** | Memory used for temporary storage for block devices (e.g., hard disks). | +| **Cached Memory** | Memory cached for quicker access to frequently used data. | +| **Swap Memory** | Disk space used as virtual memory. | +| **Slab Memory** | Memory used by the kernel for internal data structures. | +| **Shared Memory** | Memory used for shared memory segments and tmpfs. | +| **Swap Cache** | Memory keeping track of pages fetched from swap but not yet modified. | +| **Page Tables** | Memory used to map virtual memory addresses to physical memory addresses. | + +> The values in this table are updated dynamically once per month. + +By monitoring these metrics, we can ensure that the system is using memory efficiently, and we can detect potential memory leaks or bottlenecks in the system's operation. + +--- + +This section now contains all relevant CPU and memory metrics, helping you track and analyse how system resources are allocated and used over time. These benchmarks are crucial for maintaining overall system performance. diff --git a/docs/src/benchmarks/memory-month.svg b/docs/src/benchmarks/memory-month.svg new file mode 100644 index 00000000000..a6f78216ac7 --- /dev/null +++ b/docs/src/benchmarks/memory-month.svg @@ -0,0 +1,1630 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/benchmarks/tedgecpuprocent-month.svg b/docs/src/benchmarks/tedgecpuprocent-month.svg new file mode 100644 index 00000000000..df3bc5ffcdc --- /dev/null +++ b/docs/src/benchmarks/tedgecpuprocent-month.svg @@ -0,0 +1,1111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/plugins/munin/tedgecpuprocent b/plugins/munin/tedgecpuprocent new file mode 100644 index 00000000000..be22b54e960 --- /dev/null +++ b/plugins/munin/tedgecpuprocent @@ -0,0 +1,73 @@ +#!/bin/sh +# -*- sh -*- + +: << =cut + +=head1 NAME + +tedgecpuprocent - Monitor CPU % consumption of ThinEdge tasks mosquitto, tedge-agent, and tedge-mapper + +=head1 AUTHOR + +Gligor Isaev + +=head1 LICENSE + +GPL-3.0-or-higher + +=head1 MAGIC MARKERS + + #%# family=manual + +=cut + +. "$MUNIN_LIBDIR/plugins/plugin.sh" + +# Get the PIDs of mosquitto, tedge-agent, and tedge-mapper +mosquitto_pid=$(pidof mosquitto) +tedgeagent_pid=$(pidof tedge-agent) +tedgemapper_pids=$(pidof tedge-mapper) + +tasks="" +cpu="" + +# Fetch CPU usage percentage for each process using ps +for i in $mosquitto_pid $tedgeagent_pid; do + commcpu=$(ps -p $i -o comm=,%cpu= | tr -s ' ') + comm=$(echo $commcpu | cut -d' ' -f1) # Get the process name + cpu_usage=$(echo $commcpu | cut -d' ' -f2) # Get the CPU percentage + tasks="$tasks $comm" + cpu="$cpu $cpu_usage" +done + +# Fetch CPU usage percentage for tedge-mapper processes +for i in $tedgemapper_pids; do + commcpu=$(ps -p $i -o args=,%cpu= | tr -s ' ') + comm=$(echo $commcpu | cut -d/ -f4 | cut -d' ' -f1,2 | tr ' ' _) # Extract process name + cpu_usage=$(echo $commcpu | cut -d' ' -f3) # Get CPU percentage + tasks="$tasks $comm" + cpu="$cpu $cpu_usage" +done + +# If config is requested, output graph configuration +if [ "$1" = "config" ]; then + echo 'graph_title Real-time usage CPU percentage of ThinEdge.io tasks' + echo 'graph_args --base 1000 --vertical-label % -l 0' + echo 'graph_category thin-edge' + echo 'graph_vlabel CPU Usage (%)' + echo 'graph_scale no' # Ensures no unit scaling + echo 'graph_info This graph shows CPU usage as a percentage for key ThinEdge.io processes.' + for i in $tasks; do + echo "$i.label $i" # Set the label name for each process + echo "$i.draw LINE2" # Optional: ensure the drawing style is consistent + done + exit 0 +fi + +# Output the actual data for Munin to plot +tasknum=$(echo "$tasks" | tr ' ' "\n" | wc -l) +for i in $(seq 1 "$tasknum"); do + task=$(echo "$tasks" | cut -d' ' -f$i) + cpu_usage=$(echo "$cpu" | cut -d' ' -f$i) + echo "$task.value $cpu_usage" +done diff --git a/plugins/munin/tedgemonitorcpu b/plugins/munin/tedgemonitorcpu new file mode 100644 index 00000000000..fef042a4726 --- /dev/null +++ b/plugins/munin/tedgemonitorcpu @@ -0,0 +1,111 @@ +#PLEASE NOTE THIS PLUGIN IS STILL UNDER CONSTRUCTION + +#!/bin/sh +# -*- sh -*- + +: << =cut + +=head1 NAME + +tedgecpuprocent - Monitor CPU % consumption of ThinEdge tasks mosquitto, tedge-agent, and tedge-mapper and send alarm if deviated + +=head1 AUTHOR + +Gligor Isaev + +=head1 LICENSE + +GPL-3.0-or-higher + +=head1 MAGIC MARKERS + + #%# family=manual + +=cut +. "$MUNIN_LIBDIR/plugins/plugin.sh" + +# Get the PIDs of mosquitto, tedge-agent, and tedge-mapper +mosquitto_pid=$(pidof mosquitto) +tedgeagent_pid=$(pidof tedge-agent) +tedgemapper_pids=$(pidof tedge-mapper) + +tasks="" +cpu="" +mem_usage="" +mem_prev="" + +# Fetch CPU and memory usage percentage for each process using ps +for i in $mosquitto_pid $tedgeagent_pid; do + commcpu=$(ps -p $i -o comm=,%cpu=,%mem= | tr -s ' ') + comm=$(echo $commcpu | cut -d' ' -f1) # Get the process name + cpu_usage=$(echo $commcpu | cut -d' ' -f2) # Get the CPU percentage + mem_now=$(echo $commcpu | cut -d' ' -f3) # Get the Memory percentage + tasks="$tasks $comm" + cpu="$cpu $cpu_usage" + mem_usage="$mem_usage $mem_now" + + # Check for memory increase threshold + mem_prev=$(cat /tmp/${comm}_mem_prev 2>/dev/null || echo "$mem_now") + mem_increase=$(echo "scale=2; $mem_now - $mem_prev" | bc) + threshold=20.0 + + if [ "$(echo "$mem_increase > $threshold" | bc)" -eq 1 ]; then + # Raise MQTT alarm if memory increase is greater than 20% + tedge mqtt pub "te/device/main/a/memory_increase" '{ + "text": "Memory consumption increased by more than 20% for '$comm'", + "severity": "major" + }' --retained + fi + + echo "$mem_now" > /tmp/${comm}_mem_prev # Store the current memory for the next check +done + +# Fetch CPU and memory usage percentage for tedge-mapper processes +for i in $tedgemapper_pids; do + commcpu=$(ps -p $i -o args=,%cpu=,%mem= | tr -s ' ') + comm=$(echo $commcpu | cut -d/ -f4 | cut -d' ' -f1,2 | tr ' ' _) + cpu_usage=$(echo $commcpu | cut -d' ' -f3) + mem_now=$(echo $commcpu | cut -d' ' -f4) + tasks="$tasks $comm" + cpu="$cpu $cpu_usage" + mem_usage="$mem_usage $mem_now" + + mem_prev=$(cat /tmp/${comm}_mem_prev 2>/dev/null || echo "$mem_now") + mem_increase=$(echo "scale=2; $mem_now - $mem_prev" | bc) + + if [ "$(echo "$mem_increase > $threshold" | bc)" -eq 1 ]; then + tedge mqtt pub "te/device/main/a/memory_increase" '{ + "text": "Memory consumption increased by more than 20% for '$comm'", + "severity": "major" + }' --retained + fi + + echo "$mem_now" > /tmp/${comm}_mem_prev +done + +# Output graph configuration if requested +if [ "$1" = "config" ]; then + echo 'graph_title Real-time CPU and Memory usage of ThinEdge.io tasks' + echo 'graph_args --base 1000 --vertical-label % -l 0' + echo 'graph_category thin-edge' + echo 'graph_vlabel CPU & Memory Usage (%)' + echo 'graph_scale no' + echo 'graph_info This graph shows CPU and Memory usage as a percentage for key ThinEdge.io processes.' + for i in $tasks; do + echo "$i_cpu.label $i CPU" + echo "$i_cpu.draw LINE2" + echo "$i_mem.label $i Memory" + echo "$i_mem.draw LINE2" + done + exit 0 +fi + +# Output the actual data for Munin to plot +tasknum=$(echo "$tasks" | tr ' ' "\n" | wc -l) +for i in $(seq 1 "$tasknum"); do + task=$(echo "$tasks" | cut -d' ' -f$i) + cpu_usage=$(echo "$cpu" | cut -d' ' -f$i) + mem_now=$(echo "$mem_usage" | cut -d' ' -f$i) + echo "$task_cpu.value $cpu_usage" + echo "$task_mem.value $mem_now" +done diff --git a/scripts/download_svgs.py b/scripts/download_svgs.py new file mode 100644 index 00000000000..da821c779df --- /dev/null +++ b/scripts/download_svgs.py @@ -0,0 +1,44 @@ +import os +import requests +from requests.auth import HTTPBasicAuth +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +# Get Munin credentials from environment variables +MUNIN_USERNAME = os.getenv('MUNIN_USERNAME') +MUNIN_PASSWORD = os.getenv('MUNIN_PASSWORD') + +# Check if the credentials are available +if not MUNIN_USERNAME or not MUNIN_PASSWORD: + raise EnvironmentError("Munin credentials are not set in the environment variables.") + +# List of SVG filenames to download +svg_files = ['tedgecpuprocent-month.svg', 'memory-month.svg'] + +# URL of the Munin server +base_url = 'https://munin.osadl.org/munin/osadl.org/rackfslot1.osadl.org/' + +# Set the output folder relative to the project folder +project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +output_folder = os.path.join(project_dir, 'docs', 'src', 'benchmarks') + +# Download each SVG file +for svg_file in svg_files: + svg_url = f"{base_url}{svg_file}" + output_path = os.path.join(output_folder, svg_file) + + try: + # Request the SVG file with basic authentication + response = requests.get(svg_url, auth=HTTPBasicAuth(MUNIN_USERNAME, MUNIN_PASSWORD), verify=False) + response.raise_for_status() + + # Write the SVG content to a file + with open(output_path, 'wb') as f: + f.write(response.content) + + print(f"Downloaded {svg_file} to {output_path}") + + except requests.exceptions.RequestException as e: + print(f"Failed to download {svg_file}: {e}") From 4d91c6ac0029433d2f1ed7b17bbc46988c71f181 Mon Sep 17 00:00:00 2001 From: gligorisaev Date: Fri, 20 Sep 2024 09:24:34 +0200 Subject: [PATCH 2/6] correct the Path of the .py Signed-off-by: gligorisaev --- .github/workflows/download_svgs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/download_svgs.yml b/.github/workflows/download_svgs.yml index dbe9cb0efae..1d337f00552 100644 --- a/.github/workflows/download_svgs.yml +++ b/.github/workflows/download_svgs.yml @@ -24,4 +24,4 @@ jobs: MUNIN_USERNAME: ${{ secrets.MUNIN_USERNAME }} MUNIN_PASSWORD: ${{ secrets.MUNIN_PASSWORD }} run: | - python3 /ci/build_scripts/download_svgs.py + python3 /scripts/download_svgs.py From 0cb96bf04f27af87939751100da11961206b395e Mon Sep 17 00:00:00 2001 From: gligorisaev Date: Fri, 20 Sep 2024 09:33:22 +0200 Subject: [PATCH 3/6] change of some text Signed-off-by: gligorisaev --- docs/src/benchmarks/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/benchmarks/index.md b/docs/src/benchmarks/index.md index a80b48b02c8..408d7dec00b 100644 --- a/docs/src/benchmarks/index.md +++ b/docs/src/benchmarks/index.md @@ -41,7 +41,7 @@ The graph allows you to visually track CPU performance trends and identify poten ### Detailed Metrics -The following table provides a detailed breakdown of the CPU consumption for each task: +The following table provides a detailed description of the CPU consumption of measured tasks: | Task | Description | |----------------------|----------------------------------------------------------------| @@ -78,7 +78,7 @@ For each metric, the graph shows the current and historical values, allowing you ### Detailed Metrics -The following table provides a breakdown of memory usage: +The following table provides a description of memory usage metrics: | Metric | Description | |------------------|-----------------------------------------------------------------------------| From 71a8f0a9b6c639c60e77c78005e104cd1d2e0946 Mon Sep 17 00:00:00 2001 From: gligorisaev Date: Fri, 20 Sep 2024 15:04:14 +0200 Subject: [PATCH 4/6] addressed some of the comments in PR Signed-off-by: gligorisaev --- docs/src/benchmarks/index.md | 37 +- docs/src/benchmarks/memory-month.svg | 1630 ------------------------ docs/src/benchmarks/tedgemem-month.svg | 1076 ++++++++++++++++ scripts/download_svgs.py | 2 +- 4 files changed, 1090 insertions(+), 1655 deletions(-) delete mode 100644 docs/src/benchmarks/memory-month.svg create mode 100644 docs/src/benchmarks/tedgemem-month.svg diff --git a/docs/src/benchmarks/index.md b/docs/src/benchmarks/index.md index 408d7dec00b..7b6e443482f 100644 --- a/docs/src/benchmarks/index.md +++ b/docs/src/benchmarks/index.md @@ -55,24 +55,18 @@ The following table provides a detailed description of the CPU consumption of me ## Memory Usage -In this section, we monitor the memory consumption of key system components to ensure efficient performance. The memory plugin tracks several metrics to provide a detailed view of how the system memory is being utilized. +In this section, we monitor the memory consumption of key ThinEdge.io components to ensure efficient performance and to view how the system memory is being utilized. ### Graph of Memory Usage -Below is a visual representation of the memory consumption on the device over the past month. +Below is a visual representation of the memory consumption of ThinEdge.io tasks over the past month. -![Memory Usage](./memory-month.svg) +![Memory Usage](./tedgemem-month.svg) ### Monitored Metrics: -- **Total Memory**: The total amount of memory available on the system. -- **Free Memory**: Memory that is available for new processes. -- **Buffers**: Memory used by the system for temporary storage, primarily for block devices (like hard disks). -- **Cached Memory**: Memory that is cached for quicker access to frequently used data. -- **Swap Memory**: Disk space used as virtual memory when physical memory is full. -- **Slab Memory**: Memory used by the kernel for various internal data structures. -- **Shared Memory (shmem)**: Memory used for shared memory segments and tmpfs (temporary file system). -- **Swap Cache**: Memory that keeps track of pages that have been fetched from swap but not yet modified. -- **Page Tables**: Memory used to map virtual memory addresses to physical addresses. +- **Mosquitto Memory Usage**: Tracks the memory consumed by the mosquitto process, which manages MQTT messages and connections. +- **Tedge-Agent Memory Usage**: Tracks the memory consumed by the tedge-agent process, which handles software updates, configurations, and telemetry operations. +- **Tedge-Mapper Memory Usage**: Tracks the memory consumed by the tedge-mapper process, which converts telemetry data between MQTT and other protocols (e.g., Cumulocity, AWS). For multi-instance tedge-mapper, each instance is tracked individually. For each metric, the graph shows the current and historical values, allowing you to monitor how memory is utilized by the system. @@ -80,17 +74,12 @@ For each metric, the graph shows the current and historical values, allowing you The following table provides a description of memory usage metrics: -| Metric | Description | -|------------------|-----------------------------------------------------------------------------| -| **Total Memory** | The total amount of memory available on the system. | -| **Free Memory** | Memory that is available for new processes. | -| **Buffers** | Memory used for temporary storage for block devices (e.g., hard disks). | -| **Cached Memory** | Memory cached for quicker access to frequently used data. | -| **Swap Memory** | Disk space used as virtual memory. | -| **Slab Memory** | Memory used by the kernel for internal data structures. | -| **Shared Memory** | Memory used for shared memory segments and tmpfs. | -| **Swap Cache** | Memory keeping track of pages fetched from swap but not yet modified. | -| **Page Tables** | Memory used to map virtual memory addresses to physical memory addresses. | +| Metric | Description | +|--------------------------|-----------------------------------------------------------------------------| +| **Mosquitto Memory** | Memory used by the mosquitto process, which manages MQTT messages and connections. | +| **Tedge-Agent Memory** | Memory used by the tedge-agent process, responsible for software updates, configurations, and telemetry operations | +| **Tedge-Mapper Memory** | Memory used by the tedge-mapper process, which converts telemetry data between MQTT and other protocols (e.g., Cumulocity, AWS). Each instance of tedge-mapper is monitored individually if multiple instances exist | + > The values in this table are updated dynamically once per month. @@ -98,4 +87,4 @@ By monitoring these metrics, we can ensure that the system is using memory effic --- -This section now contains all relevant CPU and memory metrics, helping you track and analyse how system resources are allocated and used over time. These benchmarks are crucial for maintaining overall system performance. +This section now contains all relevant memory metrics for ThinEdge tasks, helping you track and analyze how resources are allocated and used by mosquitto, tedge-agent, and tedge-mapper over time. These benchmarks are crucial for maintaining optimal performance of ThinEdge.io components and ensuring efficient memory usage across the system. diff --git a/docs/src/benchmarks/memory-month.svg b/docs/src/benchmarks/memory-month.svg deleted file mode 100644 index a6f78216ac7..00000000000 --- a/docs/src/benchmarks/memory-month.svg +++ /dev/null @@ -1,1630 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/src/benchmarks/tedgemem-month.svg b/docs/src/benchmarks/tedgemem-month.svg new file mode 100644 index 00000000000..d346fc77d03 --- /dev/null +++ b/docs/src/benchmarks/tedgemem-month.svg @@ -0,0 +1,1076 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/download_svgs.py b/scripts/download_svgs.py index da821c779df..9d77e8ad299 100644 --- a/scripts/download_svgs.py +++ b/scripts/download_svgs.py @@ -15,7 +15,7 @@ raise EnvironmentError("Munin credentials are not set in the environment variables.") # List of SVG filenames to download -svg_files = ['tedgecpuprocent-month.svg', 'memory-month.svg'] +svg_files = ['tedgecpuprocent-month.svg', 'tedgemem-month.svg'] # URL of the Munin server base_url = 'https://munin.osadl.org/munin/osadl.org/rackfslot1.osadl.org/' From b916947a1e9cb3985d42cebb916bdadc0b3dc63f Mon Sep 17 00:00:00 2001 From: gligorisaev Date: Mon, 7 Oct 2024 10:05:11 +0200 Subject: [PATCH 5/6] change download script - adressing comment Signed-off-by: gligorisaev --- scripts/download_svgs.sh | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 scripts/download_svgs.sh diff --git a/scripts/download_svgs.sh b/scripts/download_svgs.sh new file mode 100755 index 00000000000..2e74245fbca --- /dev/null +++ b/scripts/download_svgs.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Define your Munin credentials +MUNIN_USERNAME="your_username" +MUNIN_PASSWORD="your_password" + +# List of SVG files to download +SVG_FILES=("tedgecpuprocent-month.svg" "tedgemem-month.svg") + +# Base URL of the Munin server +BASE_URL="https://munin.osadl.org/munin/osadl.org/rackfslot1.osadl.org/" + +# Set the output folder relative to the project folder +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)" +OUTPUT_FOLDER="$PROJECT_DIR/thin-edge.io/docs/src/benchmarks" + +# Print the project directory and output folder for debugging +echo "Project Directory: $PROJECT_DIR" +echo "Output Folder: $OUTPUT_FOLDER" + +# Create the output folder if it doesn't exist +mkdir -p "$OUTPUT_FOLDER" + +# Download each SVG file +for SVG_FILE in "${SVG_FILES[@]}"; do + SVG_URL="${BASE_URL}${SVG_FILE}" + OUTPUT_PATH="${OUTPUT_FOLDER}/${SVG_FILE}" + + # Print the output path for debugging + echo "Downloading $SVG_FILE to $OUTPUT_PATH" + + # Download the file using curl with basic authentication and skipping certificate verification + curl --insecure --user "$MUNIN_USERNAME:$MUNIN_PASSWORD" "$SVG_URL" --output "$OUTPUT_PATH" + + # Check if the download was successful + if [[ $? -eq 0 ]]; then + echo "Downloaded $SVG_FILE to $OUTPUT_PATH" + else + echo "Failed to download $SVG_FILE" + fi +done From 0dec968265aebf30842bf5f44b4e1f8de2320172 Mon Sep 17 00:00:00 2001 From: gligorisaev Date: Mon, 7 Oct 2024 10:08:39 +0200 Subject: [PATCH 6/6] remove uneeded file Signed-off-by: gligorisaev --- .github/workflows/download_svgs.yml | 17 +- docs/src/benchmarks/tedgecpuprocent-month.svg | 1123 +---------------- docs/src/benchmarks/tedgemem-month.svg | 1088 +--------------- scripts/download_svgs.py | 44 - 4 files changed, 32 insertions(+), 2240 deletions(-) delete mode 100644 scripts/download_svgs.py diff --git a/.github/workflows/download_svgs.yml b/.github/workflows/download_svgs.yml index 1d337f00552..f70e50f2d7f 100644 --- a/.github/workflows/download_svgs.yml +++ b/.github/workflows/download_svgs.yml @@ -10,18 +10,17 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' # Use Python 3.x - - - name: Install dependencies + - name: Set up environment variables run: | - pip install requests + echo "MUNIN_USERNAME=${{ secrets.MUNIN_USERNAME }}" >> $GITHUB_ENV + echo "MUNIN_PASSWORD=${{ secrets.MUNIN_PASSWORD }}" >> $GITHUB_ENV + + - name: Make download_svgs.sh executable + run: chmod +x /scripts/download_svgs.sh - - name: Run download_svgs.py + - name: Run download_svgs.sh env: MUNIN_USERNAME: ${{ secrets.MUNIN_USERNAME }} MUNIN_PASSWORD: ${{ secrets.MUNIN_PASSWORD }} run: | - python3 /scripts/download_svgs.py + /scripts/download_svgs.sh diff --git a/docs/src/benchmarks/tedgecpuprocent-month.svg b/docs/src/benchmarks/tedgecpuprocent-month.svg index df3bc5ffcdc..1e6298acc8f 100644 --- a/docs/src/benchmarks/tedgecpuprocent-month.svg +++ b/docs/src/benchmarks/tedgecpuprocent-month.svg @@ -1,1111 +1,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +401 Authorization Required + +

Authorization Required

+

This server could not verify that you +are authorized to access the document +requested. Either you supplied the wrong +credentials (e.g., bad password), or your +browser doesn't understand how to supply +the credentials required.

+ diff --git a/docs/src/benchmarks/tedgemem-month.svg b/docs/src/benchmarks/tedgemem-month.svg index d346fc77d03..1e6298acc8f 100644 --- a/docs/src/benchmarks/tedgemem-month.svg +++ b/docs/src/benchmarks/tedgemem-month.svg @@ -1,1076 +1,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +401 Authorization Required + +

Authorization Required

+

This server could not verify that you +are authorized to access the document +requested. Either you supplied the wrong +credentials (e.g., bad password), or your +browser doesn't understand how to supply +the credentials required.

+ diff --git a/scripts/download_svgs.py b/scripts/download_svgs.py deleted file mode 100644 index 9d77e8ad299..00000000000 --- a/scripts/download_svgs.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import requests -from requests.auth import HTTPBasicAuth -from dotenv import load_dotenv - -# Load environment variables from .env file -load_dotenv() - -# Get Munin credentials from environment variables -MUNIN_USERNAME = os.getenv('MUNIN_USERNAME') -MUNIN_PASSWORD = os.getenv('MUNIN_PASSWORD') - -# Check if the credentials are available -if not MUNIN_USERNAME or not MUNIN_PASSWORD: - raise EnvironmentError("Munin credentials are not set in the environment variables.") - -# List of SVG filenames to download -svg_files = ['tedgecpuprocent-month.svg', 'tedgemem-month.svg'] - -# URL of the Munin server -base_url = 'https://munin.osadl.org/munin/osadl.org/rackfslot1.osadl.org/' - -# Set the output folder relative to the project folder -project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -output_folder = os.path.join(project_dir, 'docs', 'src', 'benchmarks') - -# Download each SVG file -for svg_file in svg_files: - svg_url = f"{base_url}{svg_file}" - output_path = os.path.join(output_folder, svg_file) - - try: - # Request the SVG file with basic authentication - response = requests.get(svg_url, auth=HTTPBasicAuth(MUNIN_USERNAME, MUNIN_PASSWORD), verify=False) - response.raise_for_status() - - # Write the SVG content to a file - with open(output_path, 'wb') as f: - f.write(response.content) - - print(f"Downloaded {svg_file} to {output_path}") - - except requests.exceptions.RequestException as e: - print(f"Failed to download {svg_file}: {e}")