From cd74c878bede083305c1fa36d87a19b5fda7c820 Mon Sep 17 00:00:00 2001 From: Nicolas Bock Date: Tue, 26 Sep 2023 15:44:02 -0600 Subject: [PATCH] Add instructions for development environment And synchronize deployment configs from production with development. Signed-off-by: Nicolas Bock --- README.md | 49 +++++++++++++++++++++++++++----- athena-monitor.yaml | 10 +++++++ athena-processor.yaml | 66 +++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 32 ++++++++------------- 4 files changed, 130 insertions(+), 27 deletions(-) create mode 100644 athena-monitor.yaml create mode 100644 athena-processor.yaml diff --git a/README.md b/README.md index e98f1bf..a0c0943 100644 --- a/README.md +++ b/README.md @@ -31,16 +31,51 @@ The basic flowchart of interaction is as follows ## Hacking -Everything needed to stand up a development environment is contained under a -makefile, docker, docker-compose and golang >= 1.14 are required. +In order to stand up a development environment, you will need -Clone this repository and run the following command to build: +- `make` +- `docker` +- `docker-compose` +- `golang >= 1.16` -```sh -make common-build -``` +For running a docker based installation locally you will need a sandbox account +on Salesforce and a sandbox directory on files.com. Supply + +1. A list of the corresponding credentials in `creds.yaml`, + + ```yaml + db: + dialect: mysql + dsn: "athena:athena@tcp(db:3306)/athena?charset=utf8&parseTime=true" + + filescom: + key : "***" + endpoint: "https://..." + + salesforce: + endpoint: "https://..." + username: "***" + password: "***" + security-token: "***" + ``` + +1. A list of directories to monitor in `athena-monitor-directories.yaml`, + + ```yaml + monitor: + directories: + - "/sandbox/..." + - "/sandbox/..." + ``` + +3. A path for where the report uploads will go in `athena-processor-upload.yaml`, + + ```yaml + processor: + reports-upload-dir: "/sandbox/..." + ``` -For running a docker based installation locally +And finally run ```shell make devel diff --git a/athena-monitor.yaml b/athena-monitor.yaml new file mode 100644 index 0000000..6b13a1a --- /dev/null +++ b/athena-monitor.yaml @@ -0,0 +1,10 @@ +monitor: + files-delta: 1m + poll-every: 10s + directories: + - "/uploads" + - "/uploads/sosreport" + processor-map: + - type: filename + regex: ".*sosreport.*.tar.[xz|gz]+$" + processor: sosreports diff --git a/athena-processor.yaml b/athena-processor.yaml new file mode 100644 index 0000000..63ceb94 --- /dev/null +++ b/athena-processor.yaml @@ -0,0 +1,66 @@ +processor: + batch-comments-every: 5m + base-tmpdir: "/tmp/athena" + subscribers: + sosreports: + sf-comment-enabled: true + sf-comment-public: false + sf-comment: | + Athena processor: {{ processor }} subscriber: {{ subscriber }} has run the following reports. + {% for report in reports -%} + {% for script in report.Scripts -%} + {% if script.Name == "hotsos-short" %} + {% if script.Output != "" -%} + Summary for report: {{ report.Name }} - filepath: {{ report.FilePath }} + ------------------------------------------------------------------------- + {{ script.Output }} + {% endif %} + {% endif %} + {%- endfor -%} + {%- endfor -%} + {%- for report in reports -%} + {%- for script in report.Scripts -%} + {% if script.Name == "hotsos-full" -%} + Full {{ report.Name }} output can be found at: https://files.support.canonical.com/files/{{ script.UploadLocation }} + {% endif %} + {%- endfor -%} + {%- endfor -%} + + reports: + hotsos: + scripts: + hotsos-full: + exit-codes: 0 2 127 126 + run: | + #!/bin/bash + pip3 install hotsos --upgrade &>/dev/null + tar -xf {{filepath}} -C {{basedir}} &>/dev/null + hotsos -s --output-path hotsos-out --all-logs {{basedir}}/$(basename {{filepath}} .tar.xz)/ &>/dev/null + if [ -s hotsos-out/*/summary/full/yaml/hotsos-summary.all.yaml ]; then + cat hotsos-out/*/summary/full/yaml/hotsos-summary.all.yaml + else + echo "No full sosreport generated." + fi + rm -rf hotsos-out + exit 0 + hotsos-short: + exit-codes: 0 2 127 126 + run: | + #!/bin/bash + pip3 install hotsos --upgrade &>/dev/null + tar -xf {{filepath}} -C {{basedir}} &>/dev/null + hotsos -s --output-path hotsos-out --all-logs {{basedir}}/$(basename {{filepath}} .tar.xz)/ &>/dev/null + # check on size of output and use very-short if exceeds SF comment limit + if [ -s hotsos-out/*/summary/short/yaml/hotsos-summary.all.yaml ]; then + # SF comment limit is ~4K but leave some space for headers + if (($(wc -m hotsos-out/*/summary/short/yaml/hotsos-summary.all.yaml| cut -d ' ' -f 1) > 1000)); then + echo "NOTE: using --very-short since output exceeds SF comment char limit - check full summary for issue details." + cat hotsos-out/*/summary/very_short/yaml/hotsos-summary.all.yaml + else + cat hotsos-out/*/summary/short/yaml/hotsos-summary.all.yaml + fi + else + echo "No known bugs or issues found on sosreport." + fi + rm -rf hotsos-out + exit 0 diff --git a/docker-compose.yml b/docker-compose.yml index 3365334..2bf1136 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,5 @@ version: "3.3" services: -# salesforce: -# container_name: salesforceapi -# image: outofcoffee/imposter-sfdc -# ports: -# - 8080:8080 -# volumes: -# - ./mocks:/opt/imposter/config -# command: --configDir /opt/imposter/config --serverUrl http://localhost:8080 -# -# filescom: -# container_name: filescomapi -# image: filescom/files-mock-server:latest -# networks: -# - athena -# restart: always db: container_name: db @@ -29,13 +14,17 @@ services: - athena ports: - "3306:3306" + restart: always athena-monitor: container_name: athena-monitor image: athena/athena-monitor-linux-amd64:main volumes: - - ./development-config.yaml:/etc/athena/main.yaml - command: /athena-monitor --log.level="debug" + - ./creds.yaml:/etc/athena/main.yaml + - ./athena-monitor.yaml:/etc/athena/monitor.yaml + - ./athena-monitor-directories.yaml:/etc/athena/monitor-directories.yaml + - /home/ubuntu/sosreports/tmp:/tmp/athena + command: /athena-monitor --config /etc/athena/main.yaml --config /etc/athena/monitor.yaml --config /etc/athena/monitor-directories.yaml --log.level="debug" depends_on: - nats-streaming - db @@ -47,8 +36,11 @@ services: container_name: athena-processor image: athena/athena-processor-linux-amd64:main volumes: - - ./development-config.yaml:/etc/athena/main.yaml - command: /athena-processor --config /etc/athena/main.yaml --log.level="debug" + - ./creds.yaml:/etc/athena/main.yaml + - ./athena-processor.yaml:/etc/athena/processor.yaml + - ./athena-processor-upload.yaml:/etc/athena/processor-upload.yaml + - /home/ubuntu/sosreports/tmp:/tmp/athena + command: /athena-processor --config /etc/athena/main.yaml --config /etc/athena/processor.yaml --config /etc/athena/processor-upload.yaml --log.level="debug" depends_on: - nats-streaming - db @@ -71,4 +63,4 @@ services: networks: athena: - driver: bridge \ No newline at end of file + driver: bridge