NDEV-2475 collect metrics for requests to Solana

.github/workflows/deploy.py

@@ -1,8 +1,9 @@
 import multiprocessing
 import os
 import re
-import time
+import statistics
 import sys
+from collections import defaultdict
 
 import docker
 import subprocess
@@ -15,6 +16,10 @@
 from python_terraform import Terraform
 from paramiko import SSHClient
 from scp import SCPClient
+try:
+    import pandas as pd
+except ImportError:
+    print("Please install pandas library: 'pip install pandas' and 'pip install tabulate' for requests statistics")
 
 try:
     import click
@@ -293,7 +298,7 @@ def get_all_containers_logs():
 
     ssh_client.connect(hostname=proxy_ip, username='root',
                        key_filename=ssh_key, timeout=120)
-    services = ["postgres", "dbcreation", "indexer", "proxy", "faucet"]
+    services = ["postgres", "dbcreation", "indexer", "proxy", "faucet", "nginx"]
     for service in services:
         upload_remote_logs(ssh_client, service, artifact_logs)
 
@@ -467,5 +472,85 @@ def process_output(output):
                 raise SystemError(message)
 
 
+# Regular expression to match the log format
+log_pattern = re.compile(r"{.*}")
+
+
+def extract_method(request_body):
+    try:
+        request_json = json.loads(request_body)
+        return request_json.get("method", "unknown")
+    except json.JSONDecodeError:
+        return "unknown"
+
+
+def parse_log_file(log_file_path) -> dict:
+    # Read and parse the log file
+    stats = defaultdict(lambda: {"times": list()})
+
+    lines = (line for line in log_file_path.split("\n") if log_pattern.match(line))
+    log_entries = (json.loads(line) for line in lines)
+    formated_requests = (
+        {
+            "request_time": float(log_entry.get("request_time", 0)),
+            "method": extract_method(log_entry.get("jsonrpc_method", "")),
+        }
+        for log_entry in log_entries if extract_method(log_entry.get("jsonrpc_method", "")) != "unknown"
+    )
+    for formated_request in formated_requests:
+        method = formated_request["method"]
+        stats[method]["times"].append(formated_request["request_time"])
+    return stats
+
+
+def calculate_stats(stats):
+    formated_stats = {key: {} for key in stats.keys()}
+    for method, data in stats.items():
+        formated_stats[method]["count"] = len(data["times"])
+        formated_stats[method]["average_time"] = statistics.mean(data["times"])
+        formated_stats[method]["max_time"] = max(data["times"])
+        formated_stats[method]["min_time"] = min(data["times"])
+        formated_stats[method]["median_time"] = statistics.median(data["times"])
+    return formated_stats
+
+
+@cli.command("parse_logs", help="Get logs from nginx")
+@click.option("--solana_ip", default="localhost", help="Solana IP")
+def parse_logs(solana_ip):
+    content = requests.get(f"http://{solana_ip}:8080/logs/access.log").text
+    stats = parse_log_file(content)
+    calculated_stats = calculate_stats(stats)
+    df = pd.DataFrame.from_dict(calculated_stats, orient="index", columns=["count", "min_time", "max_time", "average_time", "median_time"])
+    print(df.to_markdown())
+
+
+class GithubClient:
+
+    def __init__(self, token):
+        self.headers = {"Authorization": f"Bearer {token}",
+                        "Accept": "application/vnd.github+json",
+                        "X-GitHub-Api-Version": "2022-11-28"}
+
+    def add_comment_to_pr(self, msg, pull_number, repo="neon-proxy.py"):
+        data = {"body": f"\n\n{msg}\n\n"}
+        click.echo(f"Sent data: {data}")
+        click.echo(f"Headers: {self.headers}")
+        url = f"https://api.github.com/repos/neonlabsorg/{repo}/issues/{pull_number}/comments"
+        response = requests.post(url, json=data, headers=self.headers)
+        click.echo(f"response: {response.request.url}, response: {response.headers}, response: {response.request.headers}")
+        click.echo(f"Status code: {response.status_code}")
+        if response.status_code != 201:
+            raise RuntimeError(f"Attempt to leave a comment on a PR failed: {response.text}")
+
+
+@cli.command("post_comment", help="Post comment to the PR")
+@click.option("--message", help="Message to post")
+@click.option("--pull_request", help="PR number")
+@click.option("--token", help="Github token")
+def post_comment(message, pull_request, token):
+    gh_client = GithubClient(token)
+    gh_client.add_comment_to_pr(message, pull_request)
+
+
 if __name__ == "__main__":
     cli()

.github/workflows/full_test_suite/main.tf

@@ -77,6 +77,18 @@ resource "hcloud_server" "solana" {
     data.hcloud_ssh_key.ci-ssh-key.id
   ]
 
+  provisioner "file" {
+    source     = "../../../docker-compose/nginx.conf"
+    destination = "/tmp/nginx.conf"
+
+    connection {
+        type        = "ssh"
+        user        = "root"
+        host        = hcloud_server.solana.ipv4_address
+        private_key = file("~/.ssh/ci-stands")
+      }
+  }
+
    provisioner "file" {
     source      = "../../../docker-compose/docker-compose-ci.yml"
     destination = "/tmp/docker-compose-ci.yml"

.github/workflows/full_test_suite/proxy_init.sh

@@ -21,15 +21,14 @@ cd /tmp
 
 # Set required environment variables
 export REVISION=${proxy_image_tag}
-export SOLANA_URL=http:\/\/${solana_ip}:8899
+export SOLANA_URL=http:\/\/${solana_ip}:8080
 export NEON_EVM_COMMIT=${neon_evm_commit}
 export FAUCET_COMMIT=${faucet_model_commit}
 export CI_PP_SOLANA_URL=${ci_pp_solana_url}
 export DOCKERHUB_ORG_NAME=${dockerhub_org_name}
 export PROXY_IMAGE_NAME=${proxy_image_name}
 
 
-
 # Generate docker-compose override file
 cat > docker-compose-ci.override.yml <<EOF
 version: "3"
@@ -118,6 +117,7 @@ SOLANA_DATA='{"jsonrpc":"2.0","id":1,"method":"getHealth"}'
 SOLANA_RESULT='"ok"'
 wait_service "solana" $SOLANA_URL $SOLANA_DATA $SOLANA_RESULT
 
+
 # Up all services
 docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml up -d $SERVICES
 

.github/workflows/full_test_suite/solana_init.sh

@@ -56,9 +56,28 @@ services:
       - "8900:8900"
       - "8001:8001"
       - "8001-8009:8001-8009/udp"
+
+  nginx:
+    image: nginx:latest
+    ports:
+      - "8080:8080"
+    expose:
+      - 8080
+    hostname: nginx
+    container_name: nginx
+    volumes:
+        - /var/log/nginx:/var/log/nginx
+    networks:
+      - net
+    entrypoint: >
+      /bin/sh -c "echo 'Nginx Configuration:' && cat /etc/nginx/nginx.conf && nginx -g 'daemon off;'"
 EOF
 
-
 # wake up Solana
 docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml pull solana
 docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml up -d solana
+
+docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml pull nginx
+docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml up -d nginx
+docker cp nginx.conf $(docker ps -qf "name=nginx"):/etc/nginx/nginx.conf
+docker restart $(docker ps -qf "name=nginx")

.github/workflows/pipeline.yml

@@ -355,6 +355,37 @@ jobs:
       solana_ip: ${{ needs.prepare-infrastructure.outputs.solana_ip }}
       external_call: true
 
+  requests-report:
+    needs:
+      - prepare-infrastructure
+      - basic-tests
+      - build-image
+    if: always()
+    runs-on: test-runner
+    env:
+      SOLANA_IP: ${{ needs.prepare-infrastructure.outputs.solana_ip }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+           fetch-depth: 0
+      - name: Install pandas, tabulate
+        run: |
+          pip install pandas
+          pip install tabulate
+      - name: "Request report"
+        id: request_report
+        run: |
+          stats=$(python3 ./.github/workflows/deploy.py parse_logs --solana_ip=${{ env.SOLANA_IP }})
+          echo "stats<<EOF" >> $GITHUB_OUTPUT
+          echo "$stats" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          echo "stats=$stats"
+      - name: "Post a comment with the report"
+        run: |
+          python3 ./.github/workflows/deploy.py post_comment --message="${{ steps.request_report.outputs.stats }}" \
+          --pull_request="${{ github.event.pull_request.number }}" \
+          --token=${{ secrets.GITHUB_TOKEN }}
+
   #  will be fixed by NDEV-2766
 #  economy-tests:
 #    if: needs.build-image.outputs.full_test_suite=='true'
@@ -391,6 +422,7 @@ jobs:
       - prepare-infrastructure
       - openzeppelin-tests
       - basic-tests
+      - requests-report
       - dapps-tests
       - build-image
     runs-on: test-runner

devel_requirements.txt

@@ -3,3 +3,6 @@ docker==7.1.0
 paramiko==3.4.0
 python-terraform==0.10.1
 scp==0.15.0
+
+pandas==2.2.2
+tabulate==0.9.0

docker-compose/nginx.conf

@@ -0,0 +1,34 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format json_combined escape=json
+      '{ "@timestamp": "$time_iso8601", '
+      '"remote_addr": "$remote_addr", '
+      '"request": "$request", '
+      '"status": "$status", '
+      '"body_bytes_sent": "$body_bytes_sent", '
+      '"http_referer": "$http_referer", '
+      '"http_user_agent": "$http_user_agent", '
+      '"request_time": "$request_time", '
+      '"jsonrpc_method": "$request_body" }';
+
+    access_log /var/log/nginx/access.log json_combined buffer=8k flush=20s;
+
+    server {
+        listen 8080;
+
+        location / {
+            proxy_pass http://solana:8899;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        location /logs {
+            alias /var/log/nginx/;
+        }
+    }
+}