added grok rules and created logstack-shipper in dev

.github/workflows/deploy.yml

@@ -105,7 +105,13 @@ jobs:
       - name: drain-staging-space
         uses: cloud-gov/cg-cli-tools@main
         with:
-          command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh
+          command: |
+            apt-get install -y jq && 
+            curl -L -o drain-plugin  https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux && 
+            cf install-plugin -f -r drain-plugin && 
+            rm -f drain-plugin && 
+            mkdir -p /root/.cf/ && touch /root/.cf/config.json && 
+            ./create-space-drain.sh management-staging
           cf_org: gsa-datagov
           cf_space: staging
           cf_username: ${{secrets.CF_SERVICE_USER}}
@@ -123,7 +129,13 @@ jobs:
       - name: drain-prod-space
         uses: cloud-gov/cg-cli-tools@main
         with:
-          command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh
+          command: |
+            apt-get install -y jq && 
+            curl -L -o drain-plugin  https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux && 
+            cf install-plugin -f -r drain-plugin &&
+            rm -f drain-plugin && 
+            mkdir -p /root/.cf/ && touch /root/.cf/config.json && 
+            ./create-space-drain.sh management
           cf_org: gsa-datagov
           cf_space: prod
           cf_username: ${{secrets.CF_SERVICE_USER}}

.github/workflows/dev-deploy.yml

@@ -0,0 +1,75 @@
+---
+name: dev-deploy
+
+on:
+  push:
+    branches:
+      - development
+  workflow_dispatch:
+
+jobs:
+  create-cloudgov-services-development-ssb:
+    name: create services (development-ssb)
+    environment: development-ssb
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: create services
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: ./create-cloudgov-services.sh
+          cf_org: gsa-datagov
+          cf_space: development-ssb
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+
+  deploy-development-ssb:
+    name: deploy (development-ssb)
+    environment: development-ssb
+    runs-on: ubuntu-latest
+    needs:
+      - create-cloudgov-services-development-ssb
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: build dependencies
+        run: |
+          chmod -R 777 logstash
+          PWD=`pwd` make logstash-installation
+      - name: deploy-logstash
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: cf push --vars-file vars.development-ssb.yml --strategy rolling
+          cf_org: gsa-datagov
+          cf_space: development-ssb
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+      - name: smoke test logstash
+        run: |
+          sleep 150  # Logstash is very slow to start up
+          [ "401" = "$(curl -w '%{http_code}' --output /dev/null --silent https://logstash-development-datagov.app.cloud.gov)" ]
+
+  drain-apps-in-development:
+    name: drain everything in development space
+    environment: development-ssb
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-development-ssb
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: drain-development-space
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: |
+            apt-get install -y jq && 
+            curl -L -o drain-plugin https://github.com/cloudfoundry/cf-drain-cli/releases/download/v2.0.0/cf-drain-cli-linux && 
+            cf install-plugin -f -r drain-plugin && 
+            rm -f drain-plugin && 
+            mkdir -p /root/.cf/ && touch /root/.cf/config.json && 
+            ./create-space-drain.sh development-ssb
+          cf_org: gsa-datagov
+          cf_space: development
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}

.gitignore

@@ -2,3 +2,4 @@
 logstash/logstash-oss-*-linux-x86_64.tar.gz
 logstash/plugins.zip
 logstash/awscliv2.zip
+.DS_Store

README.md

@@ -30,6 +30,11 @@ Alternatively, you can auto-drain all apps in a given space by targeting that sp
     cf target -s prod
     ./create-space-drain.sh
 
+Note: at development space, it will use the logstack-shipper in development-ssb space:
+
+    cf target -s development
+    ./create-space-drain.sh development-ssb
+
 After a short delay, logs should begin to flow automatically.
 
 ## Setup

create-cloudgov-services.sh

@@ -13,7 +13,7 @@ space=$(cf target | grep space | cut -d : -f 2 | xargs)
 randpw(){ openssl rand -base64 40 | tr -dc _A-Z-a-z-0-9 | head -c${1:-32};echo; }
 
 # Only create stuff in production and staging spaces
-if [ "$space" = "management" ] || [ "$space" = "management-staging" ]; then
+if [ "$space" = "management" ] || [ "$space" = "management-staging" ] || [ "$space" = "development-ssb" ]; then
     cf service "${app_name}-s3"      > /dev/null 2>&1 || cf create-service s3 basic "${app_name}-s3" --wait&
     cf service "${app_name}-secrets" > /dev/null 2>&1 || 
         cf create-user-provided-service "${app_name}-secrets" -p '{"DRAIN_USER":"'$(randpw)'","DRAIN_PASSWORD":"'$(randpw)'"}' &

create-space-drain.sh

@@ -39,9 +39,9 @@ cat > manifest.yml << EOF
 ---
 applications:
 - name: ${prefix}-space-drain
-  instances: 1
+  instances: 0
   memory: 64M
   no-route: true
 EOF
 
-cf drain-space --drain-name "${prefix}-space-drain" "$drain_url"
+cf drain-space --drain-name "${prefix}-space-drain" "$drain_url"

docker-compose.yml

@@ -15,21 +15,14 @@ services:
       timeout: 10s
       start_period: 10s
     ports:
-      - "127.0.0.1:4566:4566"
-      - "127.0.0.1:4571:4571"
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
     environment:
-      - SERVICES=${SERVICES-}
       - DEBUG=${DEBUG-}
-      - DATA_DIR=${DATA_DIR-}
-      - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-}
-      - HOST_TMP_FOLDER=${TMPDIR:-/tmp/}localstack
       - DOCKER_HOST=unix:///var/run/docker.sock
-      - AWS_DEFAULT_REGION=us-east-1
     volumes:
-      - "${TMPDIR:-/tmp}/localstack:/tmp/localstack"
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"
-      - ./localstack.d:/docker-entrypoint-initaws.d
-
 
   logstash:
     depends_on:

logstash/logstash.conf

@@ -63,11 +63,52 @@ filter{
   # }
   # mutate { add_field => { "[@metadata][target_index]" => "index-%{+YYYY.MM}" } }
 
+  grok {
+      match => { "message" => "%{GREEDYDATA:raw_message_content}" }
+  }
+  grok {
+      match => {
+        "message" => '<%{GREEDYDATA:skip} %{TIMESTAMP_ISO8601:skip} %{DATA:host} %{UUID:proc_id} \[%{DATA:instance_info}\] - \[tags@%{GREEDYDATA:skip} instance_id="%{INT:instance_id}" ip="%{IP:ip}" job="%{GREEDYDATA:skip} space_name="%{DATA:space_name}"\] %{GREEDYDATA:log_data}'
+        tag_on_failure => []
+        overwrite => ["message"]
+        break_on_match => false
+      }
+  }
+  # for message like: catalog-dev.data.gov - [2023-12-15T21:11:49.508628938Z] "GET /0000000 HTTP/1.1" 404 0 21445 "-" "Mozilla/5.0 ..." "127.0.x.x:xxxxx" "10.xx.2.10:xxxxx" x_forwarded_for:"108.xx.xxx.xxx, 64.252.66.xxx, 127.0.x.x" x_forwarded_proto:"https" ... ... b3:xxx
+  grok {
+      match => {
+        "log_data" => '%{HOSTNAME:hostname} - \[%{TIMESTAMP_ISO8601:timestamp}\] "%{WORD:http_method} %{GREEDYDATA:request} %{DATA:http_version}" %{NUMBER:status} %{NUMBER:bytes_sent} %{NUMBER:bytes_received} "%{DATA:http_referer}" "%{DATA:http_user_agent}" %{GREEDYDATA:skip} x_forwarded_for:"%{IP:real_ip}(?:, %{GREEDYDATA:forwarded_ips})*" x_forwarded_proto:%{GREEDYDATA:skip}'
+        tag_on_failure => []
+        overwrite => ["message"]
+        break_on_match => false
+      }
+  }
+  # for message like: 2023-12-14 20:31:53,839 INFO  [ckan.config.middleware.flask_app]  404 /dataset/xxxx render time 0.023 seconds
+  grok {
+      match => {
+        "log_data" => '%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log_level}  \[%{DATA:logger}\]  %{NUMBER:status:int} %{URIPATH:request_path} render time %{NUMBER:render_time:float} seconds'
+        tag_on_failure => []
+        overwrite => ["message"]
+        break_on_match => false
+      }
+  }
+  if (
+      ([log_data] and [log_data] =~ /^NginxLog/)
+      or
+      ([hostname] and [http_status] and [hostname] =~ /^logstash-/ and [http_status] == "200")
+  ) {
+      drop { }
+  }
   mutate {
     add_field => { "[@metadata][NEWRELIC_KEY]" => "${NEWRELIC_LICENSE_KEY:notpresent}" }
     add_field => { "[@metadata][AWS_S3_PROXY]" => "${AWS_S3_PROXY:notpresent}" }  
+    remove_field => ["skip"]
+  }
+  if [space_name] and [space_name] !~ /development/  {
+    mutate {
+      remove_field => ["log_data", "raw_message_content"]
+    }
   }
-
 }
 
 output {

manifest.yml

@@ -5,6 +5,8 @@ applications:
     APP_NAME: ((app_name))
     LS_JAVA_OPTS: ((logstash_java_options))
     https_proxy: ((https_proxy))
+    NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key))
+    NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri))
   buildpacks:
     - binary_buildpack
   health-check-type: process
@@ -17,7 +19,4 @@ applications:
     - route: ((hostname)).((domain))
   services:
     - ((app_name))-s3
-    - ((app_name))-secrets
-env:
-  NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key))
-  NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri))
+    - ((app_name))-secrets

vars.development-ssb.yml

@@ -0,0 +1,20 @@
+---
+# Name prefix for applications in the logstack.
+app_name: logstack
+
+# Options to pass the JVM
+logstash_java_options: -Xms512m -Xmx512m
+# Number of instances of the Logstash to run.
+logstash_instances: 1
+# Amount of memory to provide to the Logstash (Java) application.
+logstash_memory: 1024
+# Assign route for the log shipper. This will be used as the logdrain for other
+# Cloud Foundry applications.
+hostname: logstash-development-datagov
+domain: app.cloud.gov
+https_proxy:
+
+# Optional license key for delivering logs to New Relic
+# See https://github.com/newrelic/logstash-output-plugin
+logstash_newrelic_license_key:
+logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1