NDEV-2475 collect metrics for requests to Solana (#51)

* NDEV-2475 collect metrics for requests to Solana

.github/workflows/deploy.py

@@ -1,8 +1,9 @@
 import multiprocessing
 import os
 import re
-import time
+import statistics
 import sys
+from collections import defaultdict
 
 import docker
 import subprocess
@@ -15,6 +16,10 @@
 from python_terraform import Terraform
 from paramiko import SSHClient
 from scp import SCPClient
+try:
+    import pandas as pd
+except ImportError:
+    print("Please install pandas library: 'pip install pandas' and 'pip install tabulate' for requests statistics")
 
 try:
     import click
@@ -62,6 +67,8 @@ def cli():
     __file__).parent / "full_test_suite")
 VERSION_BRANCH_TEMPLATE = r"[vt]{1}\d{1,2}\.\d{1,2}\.x.*"
 
+SOLANA_REQUESTS_TITLE = "<summary>Solana Requests Statistics</summary>"
+
 
 def docker_compose(args: str):
     command = f'docker login -u {DOCKER_USERNAME} -p {DOCKER_PASSWORD}'
@@ -467,5 +474,103 @@ def process_output(output):
                 raise SystemError(message)
 
 
+# Regular expression to match the log format
+log_pattern = re.compile(r"{.*}")
+
+
+def extract_method(request_body):
+    try:
+        request_json = json.loads(request_body)
+        return request_json.get("method", "unknown")
+    except json.JSONDecodeError:
+        return "unknown"
+
+
+def parse_log_file(log_file_path) -> dict:
+    # Read and parse the log file
+    stats = defaultdict(lambda: {"times": list()})
+
+    lines = (line for line in log_file_path.split("\n") if log_pattern.match(line))
+    log_entries = (json.loads(line) for line in lines)
+    formated_requests = (
+        {
+            "request_time": float(log_entry.get("request_time", 0)),
+            "method": extract_method(log_entry.get("jsonrpc_method", "")),
+        }
+        for log_entry in log_entries if extract_method(log_entry.get("jsonrpc_method", "")) != "unknown"
+    )
+    for formated_request in formated_requests:
+        method = formated_request["method"]
+        stats[method]["times"].append(formated_request["request_time"])
+    return stats
+
+
+def calculate_stats(stats):
+    formated_stats = {key: {} for key in stats.keys()}
+    for method, data in stats.items():
+        formated_stats[method]["count"] = len(data["times"])
+        formated_stats[method]["average_time"] = statistics.mean(data["times"])
+        formated_stats[method]["max_time"] = max(data["times"])
+        formated_stats[method]["min_time"] = min(data["times"])
+        formated_stats[method]["median_time"] = statistics.median(data["times"])
+    return {k: v for k, v in sorted(formated_stats.items(), key=lambda item: item[1]["count"], reverse=True)}
+
+
+@cli.command("parse_logs", help="Get logs from nginx")
+@click.option("--solana_ip", default="localhost", help="Solana IP")
+def parse_logs(solana_ip):
+    try:
+        content = requests.get(f"http://{solana_ip}:8080/logs/access.log").text
+    except requests.exceptions.InvalidURL as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    stats = parse_log_file(content)
+    calculated_stats = calculate_stats(stats)
+    df = pd.DataFrame.from_dict(calculated_stats, orient="index", columns=["count", "min_time", "max_time", "average_time", "median_time"])
+    print(df.to_markdown())
+
+
+class GithubClient:
+
+    def __init__(self, token):
+        self.headers = {"Authorization": f"Bearer {token}",
+                        "Accept": "application/vnd.github+json",
+                        "X-GitHub-Api-Version": "2022-11-28"}
+
+    def remove_comment_with_title(self, pull_request, title):
+        response = requests.get(pull_request, headers=self.headers)
+        if response.status_code != 200:
+            raise RuntimeError(f"Attempt to get comments on a PR failed: {response.text}")
+        comments = response.json()
+        for comment in comments:
+            if f"<details>{title}" in comment["body"]:
+                response = requests.delete(comment["url"], headers=self.headers)
+                if response.status_code != 204:
+                    raise RuntimeError(f"Attempt to delete a comment on a PR failed: {response.text} {response.request.url}")
+
+    def add_comment_to_pr(self, msg, pull_request, title = SOLANA_REQUESTS_TITLE, remove_previous_comments=True):
+        if remove_previous_comments:
+            self.remove_comment_with_title(pull_request, title)
+        message = f"\n\n{msg}\n\n"
+        if title:
+            message = f"<details>{title}\n\n{message}</details>"
+        data = {"body": message}
+        click.echo(f"Sent data: {data}")
+        click.echo(f"Headers: {self.headers}")
+        response = requests.post(pull_request, json=data, headers=self.headers)
+        click.echo(f"Status code: {response.status_code}")
+        if response.status_code != 201:
+            raise RuntimeError(f"Attempt to leave a comment on a PR failed: {response.text}")
+
+
+@cli.command("post_comment", help="Post comment to the PR")
+@click.option("--message", help="Message to post")
+@click.option("--pull_request", help="Pull Request URL")
+@click.option("--token", help="Github token")
+def post_comment(message, pull_request, token):
+    gh_client = GithubClient(token)
+    gh_client.add_comment_to_pr(message, pull_request)
+
+
 if __name__ == "__main__":
     cli()

.github/workflows/full_test_suite/main.tf

@@ -77,6 +77,18 @@ resource "hcloud_server" "solana" {
     data.hcloud_ssh_key.ci-ssh-key.id
   ]
 
+  provisioner "file" {
+    source     = "../../../docker-compose/nginx.conf"
+    destination = "/tmp/nginx.conf"
+
+    connection {
+        type        = "ssh"
+        user        = "root"
+        host        = hcloud_server.solana.ipv4_address
+        private_key = file("~/.ssh/ci-stands")
+      }
+  }
+
    provisioner "file" {
     source      = "../../../docker-compose/docker-compose-ci.yml"
     destination = "/tmp/docker-compose-ci.yml"

.github/workflows/full_test_suite/proxy_init.sh

@@ -21,15 +21,14 @@ cd /tmp
 
 # Set required environment variables
 export REVISION=${proxy_image_tag}
-export SOLANA_URL=http:\/\/${solana_ip}:8899
+export SOLANA_URL=http:\/\/${solana_ip}:8080
 export NEON_EVM_COMMIT=${neon_evm_commit}
 export FAUCET_COMMIT=${faucet_model_commit}
 export CI_PP_SOLANA_URL=${ci_pp_solana_url}
 export DOCKERHUB_ORG_NAME=${dockerhub_org_name}
 export PROXY_IMAGE_NAME=${proxy_image_name}
 
 
-
 # Generate docker-compose override file
 cat > docker-compose-ci.override.yml <<EOF
 version: "3"
@@ -118,6 +117,7 @@ SOLANA_DATA='{"jsonrpc":"2.0","id":1,"method":"getHealth"}'
 SOLANA_RESULT='"ok"'
 wait_service "solana" $SOLANA_URL $SOLANA_DATA $SOLANA_RESULT
 
+
 # Up all services
 docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml up -d $SERVICES
 

.github/workflows/full_test_suite/solana_init.sh

@@ -56,9 +56,24 @@ services:
       - "8900:8900"
       - "8001:8001"
       - "8001-8009:8001-8009/udp"
+  
+  nginx:
+    image: nginx:latest
+    ports:
+      - "8080:8080"
+    expose:
+      - 8080
+    hostname: nginx
+    container_name: nginx
+    volumes:
+        - /var/log/nginx:/var/log/nginx
+        - /tmp/nginx.conf:/etc/nginx/nginx.conf
+    networks:
+      - net
+    entrypoint: >
+      /bin/sh -c "echo 'Nginx Configuration:' && cat /etc/nginx/nginx.conf && nginx -g 'daemon off;'"
 EOF
 
-
 # wake up Solana
-docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml pull solana
-docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml up -d solana
+docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml pull nginx solana
+docker-compose -f docker-compose-ci.yml -f docker-compose-ci.override.yml up -d nginx solana

.github/workflows/pipeline.yml

@@ -355,6 +355,45 @@ jobs:
       solana_ip: ${{ needs.prepare-infrastructure.outputs.solana_ip }}
       external_call: true
 
+  requests-report:
+    needs:
+      - prepare-infrastructure
+      - basic-tests
+      - build-image
+      - dapps-tests
+      - openzeppelin-tests
+    if: |
+      always() && 
+      contains(fromJSON('["success", "skipped"]'), needs.openzeppelin-tests.result) &&
+      contains(fromJSON('["success", "skipped"]'), needs.dapps-tests.result)
+    runs-on: test-runner
+    env:
+      SOLANA_IP: ${{ needs.prepare-infrastructure.outputs.solana_ip }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+           fetch-depth: 0
+      - name: "Request report"
+        id: request_report
+        run: |
+          stats=$(python3 ./.github/workflows/deploy.py parse_logs --solana_ip=${{ env.SOLANA_IP }})
+          echo "stats<<EOF" >> $GITHUB_OUTPUT
+          echo "$stats" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+          echo "stats=$stats"
+
+          if [[ "${{ github.event.inputs.initial_pr }}" != "" ]]; then
+            echo "pull_request=${{ github.event.inputs.initial_pr }}"  >> $GITHUB_OUTPUT
+          else
+            echo "pull_request=${{ github.event.pull_request.issue_url }}/comments"  >> $GITHUB_OUTPUT
+          fi;
+
+      - name: "Post a comment with the report"
+        run: |
+          python3 ./.github/workflows/deploy.py post_comment --message="${{ steps.request_report.outputs.stats }}" \
+          --pull_request="${{ steps.request_report.outputs.pull_request }}" \
+          --token=${{ secrets.GHTOKEN }}
+
   #  will be fixed by NDEV-2766
 #  economy-tests:
 #    if: needs.build-image.outputs.full_test_suite=='true'
@@ -391,6 +430,7 @@ jobs:
       - prepare-infrastructure
       - openzeppelin-tests
       - basic-tests
+      - requests-report
       - dapps-tests
       - build-image
     runs-on: test-runner

docker-compose/nginx.conf

@@ -0,0 +1,34 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format json_combined escape=json
+      '{ "@timestamp": "$time_iso8601", '
+      '"remote_addr": "$remote_addr", '
+      '"request": "$request", '
+      '"status": "$status", '
+      '"body_bytes_sent": "$body_bytes_sent", '
+      '"http_referer": "$http_referer", '
+      '"http_user_agent": "$http_user_agent", '
+      '"request_time": "$request_time", '
+      '"jsonrpc_method": "$request_body" }';
+
+    access_log /var/log/nginx/access.log json_combined buffer=8k flush=20s;
+
+    server {
+        listen 8080;
+
+        location / {
+            proxy_pass http://solana:8899;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        location /logs {
+            alias /var/log/nginx/;
+        }
+    }
+}