Merge pull request #76 from nicolasbock/devel-deployment

Add instructions for development environment

README.md

@@ -31,16 +31,51 @@ The basic flowchart of interaction is as follows
 
 ## Hacking
 
-Everything needed to stand up a development environment is contained under a
-makefile, docker, docker-compose and golang >= 1.14 are required.
+In order to stand up a development environment, you will need
 
-Clone this repository and run the following command to build:
+- `make`
+- `docker`
+- `docker-compose`
+- `golang >= 1.16`
 
-```sh
-make common-build
-```
+For running a docker based installation locally you will need a sandbox account
+on Salesforce and a sandbox directory on files.com. Supply
+
+1. A list of the corresponding credentials in `creds.yaml`,
+
+      ```yaml
+      db:
+        dialect: mysql
+        dsn: "athena:athena@tcp(db:3306)/athena?charset=utf8&parseTime=true"
+
+      filescom:
+        key : "***"
+        endpoint: "https://..."
+
+      salesforce:
+        endpoint: "https://..."
+        username: "***"
+        password: "***"
+        security-token: "***"
+      ```
+
+1. A list of directories to monitor in `athena-monitor-directories.yaml`,
+
+      ```yaml
+      monitor:
+        directories:
+          - "/sandbox/..."
+          - "/sandbox/..."
+      ```
+
+3. A path for where the report uploads will go in `athena-processor-upload.yaml`,
+
+      ```yaml
+      processor:
+        reports-upload-dir: "/sandbox/..."
+      ```
 
-For running a docker based installation locally
+And finally run
 
 ```shell
 make devel

athena-monitor.yaml

@@ -0,0 +1,10 @@
+monitor:
+  files-delta: 1m
+  poll-every: 10s
+  directories:
+    - "/uploads"
+    - "/uploads/sosreport"
+  processor-map:
+    - type: filename
+      regex: ".*sosreport.*.tar.[xz|gz]+$"
+      processor: sosreports

athena-processor.yaml

@@ -0,0 +1,66 @@
+processor:
+  batch-comments-every: 5m
+  base-tmpdir: "/tmp/athena"
+  subscribers:
+    sosreports:
+      sf-comment-enabled: true
+      sf-comment-public: false
+      sf-comment: |
+        Athena processor: {{ processor }} subscriber: {{ subscriber }} has run the following reports.
+        {% for report in reports -%}
+          {% for script in report.Scripts -%}
+            {% if script.Name == "hotsos-short" %}
+              {% if script.Output != "" -%}
+        Summary for report: {{ report.Name }} - filepath: {{ report.FilePath }}
+        -------------------------------------------------------------------------
+        {{ script.Output }}
+              {% endif %}
+            {% endif %}
+          {%- endfor -%}
+        {%- endfor -%}
+        {%- for report in reports -%}
+          {%- for script in report.Scripts -%}
+            {% if script.Name == "hotsos-full" -%}
+        Full {{ report.Name }} output can be found at: https://files.support.canonical.com/files/{{ script.UploadLocation }}
+            {% endif %}
+          {%- endfor -%}
+        {%- endfor -%}
+
+      reports:
+        hotsos:
+          scripts:
+            hotsos-full:
+              exit-codes: 0 2 127 126
+              run: |
+                #!/bin/bash
+                pip3 install hotsos --upgrade &>/dev/null
+                tar -xf {{filepath}} -C {{basedir}} &>/dev/null
+                hotsos -s --output-path hotsos-out --all-logs {{basedir}}/$(basename {{filepath}} .tar.xz)/ &>/dev/null
+                if [ -s hotsos-out/*/summary/full/yaml/hotsos-summary.all.yaml ]; then
+                  cat hotsos-out/*/summary/full/yaml/hotsos-summary.all.yaml
+                else
+                  echo "No full sosreport generated."
+                fi
+                rm -rf hotsos-out
+                exit 0
+            hotsos-short:
+              exit-codes: 0 2 127 126
+              run: |
+                #!/bin/bash
+                pip3 install hotsos --upgrade &>/dev/null
+                tar -xf {{filepath}} -C {{basedir}} &>/dev/null
+                hotsos -s --output-path hotsos-out --all-logs {{basedir}}/$(basename {{filepath}} .tar.xz)/ &>/dev/null
+                # check on size of output and use very-short if exceeds SF comment limit
+                if [ -s hotsos-out/*/summary/short/yaml/hotsos-summary.all.yaml ]; then
+                  # SF comment limit is ~4K but leave some space for headers
+                  if (($(wc -m hotsos-out/*/summary/short/yaml/hotsos-summary.all.yaml| cut -d ' ' -f 1) > 1000)); then
+                    echo "NOTE: using --very-short since output exceeds SF comment char limit - check full summary for issue details."
+                    cat hotsos-out/*/summary/very_short/yaml/hotsos-summary.all.yaml
+                  else
+                    cat hotsos-out/*/summary/short/yaml/hotsos-summary.all.yaml
+                  fi
+                else
+                  echo "No known bugs or issues found on sosreport."
+                fi
+                rm -rf hotsos-out
+                exit 0

docker-compose.yml

@@ -1,20 +1,5 @@
 version: "3.3"
 services:
-#  salesforce:
-#    container_name: salesforceapi
-#    image: outofcoffee/imposter-sfdc
-#    ports:
-#      - 8080:8080
-#    volumes:
-#      - ./mocks:/opt/imposter/config
-#    command: --configDir /opt/imposter/config --serverUrl http://localhost:8080
-#
-#  filescom:
-#    container_name: filescomapi
-#    image: filescom/files-mock-server:latest
-#    networks:
-#      - athena
-#    restart: always
 
   db:
     container_name: db
@@ -29,13 +14,17 @@ services:
       - athena
     ports:
       - "3306:3306"
+    restart: always
 
   athena-monitor:
     container_name: athena-monitor
     image: athena/athena-monitor-linux-amd64:${BRANCH:-main}
     volumes:
-      - ./development-config.yaml:/etc/athena/main.yaml
-    command: /athena-monitor --log.level="debug"
+      - ./creds.yaml:/etc/athena/main.yaml
+      - ./athena-monitor.yaml:/etc/athena/monitor.yaml
+      - ./athena-monitor-directories.yaml:/etc/athena/monitor-directories.yaml
+      - /home/ubuntu/sosreports/tmp:/tmp/athena
+    command: /athena-monitor --config /etc/athena/main.yaml --config /etc/athena/monitor.yaml --config /etc/athena/monitor-directories.yaml --log.level="debug"
     depends_on:
       - nats-streaming
       - db
@@ -47,8 +36,11 @@ services:
     container_name: athena-processor
     image: athena/athena-processor-linux-amd64:${BRANCH:-main}
     volumes:
-      - ./development-config.yaml:/etc/athena/main.yaml
-    command: /athena-processor --config /etc/athena/main.yaml --log.level="debug"
+      - ./creds.yaml:/etc/athena/main.yaml
+      - ./athena-processor.yaml:/etc/athena/processor.yaml
+      - ./athena-processor-upload.yaml:/etc/athena/processor-upload.yaml
+      - /home/ubuntu/sosreports/tmp:/tmp/athena
+    command: /athena-processor --config /etc/athena/main.yaml --config /etc/athena/processor.yaml --config /etc/athena/processor-upload.yaml --log.level="debug"
     depends_on:
       - nats-streaming
       - db
@@ -71,4 +63,4 @@ services:
 
 networks:
   athena:
-    driver: bridge
+    driver: bridge