Alb s3 integration

docs/API/README.md

@@ -102,3 +102,12 @@ And will result with :
 }
 ```
 
+
+
+####  running the swagger mock server as docker
+
+One-liner docker command (assuming running from repository root):
+
+```text
+docker run -p 9200:4010 -v "$(pwd)/docs/API:/api" -v "$(pwd)/schema:/schema" --name integ-prism -t stoplight/prism mock -h 0.0.0.0 /api/swagger.yaml
+```

docs/Integration-API.md

@@ -244,9 +244,14 @@ the current state of the integration:
   ]
 }
 ```
+The next screen shows the Integration assets status:
+
+![Integration - assets - status](img/Integration-assets-status.png)
+
+
 The next screen shows the maintenance issues:
 
-![todo](...)
+![Maintaining - issue](img/maintaining-issue.png)
 
 Once all the issues are manually resolved by the User, the UX can continue the loading process by the next API
 `PUT _integration/store/$instance_name/activate`
@@ -272,8 +277,6 @@ The loading assets phase will use the existing bulk load api for all the existin
 
 The User can chery pick specific assets to load and use the next UX window for this purpose
 
-![todo](...)
-
 Using the next API
 `PUT _integration/store/$instance_name/load`
 ```json

docs/TableOfContent.md → docs/README.md

@@ -1,4 +1,12 @@
-## Integrations Folder
+## Integrations Documentation Table of Content
+
+The integration repository contains the list of the supported catalogs ([Observability](../schema/observability/README.md) for example)
+
+In addition, it also contains the list of [Integrations](../integrations/README.md) such as [Nginx](../integrations/nginx/info/README.md)
+
+---
+The next documents describe the user workflows, design and architecture of the project. 
+
 - [Integration introduction](Integrations.md)
 - [Simple-Schema](Simple-schema.md)
 - [Integration Structure](Integration-structure.md)

integrations/alb/assets/datasource/datasources.json

@@ -0,0 +1,9 @@
+[
+  {
+    "name" : "myspark",
+    "connector": "jdbc",
+    "properties" : {
+      "url" : "jdbc:hive2://spark-thrift:10000/default"
+    }
+  }
+]

integrations/alb/assets/display/alb-dashboard.ndjson

@@ -0,0 +1,84 @@
+{
+  "attributes": {
+    "title": "AWS ALB Logs Dashboard",
+    "hits": 0,
+    "description": "",
+    "panelsJSON": [
+      {
+        "gridData": {
+          "x": 0,
+          "y": 0,
+          "w": 24,
+          "h": 15,
+          "i": "1"
+        },
+        "version": "7.14.0",
+        "panelIndex": "1",
+        "embeddableConfig": {},
+        "panelRefName": "panel_0"
+      },
+      {
+        "gridData": {
+          "x": 24,
+          "y": 0,
+          "w": 24,
+          "h": 15,
+          "i": "2"
+        },
+        "version": "7.14.0",
+        "panelIndex": "2",
+        "embeddableConfig": {},
+        "panelRefName": "panel_1"
+      },
+      {
+        "gridData": {
+          "x": 0,
+          "y": 15,
+          "w": 48,
+          "h": 15,
+          "i": "3"
+        },
+        "version": "7.14.0",
+        "panelIndex": "3",
+        "embeddableConfig": {},
+        "panelRefName": "panel_2"
+      }
+    ],
+    "optionsJSON": {
+      "hidePanelTitles": false,
+      "useMargins": true
+    },
+    "version": 1,
+    "timeRestore": false,
+    "kibanaSavedObjectMeta": {
+      "searchSourceJSON": {
+        "query": {
+          "language": "kuery",
+          "query": ""
+        },
+        "filter": []
+      }
+    }
+  },
+  "references": [
+    {
+      "name": "panel_0",
+      "type": "visualization",
+      "id": "visualization_id_1"
+    },
+    {
+      "name": "panel_1",
+      "type": "visualization",
+      "id": "visualization_id_2"
+    },
+    {
+      "name": "panel_2",
+      "type": "visualization",
+      "id": "visualization_id_3"
+    }
+  ],
+  "id": "dashboard_id",
+  "type": "dashboard",
+  "updated_at": "2023-03-27T00:00:00.000Z",
+  "version": "WzAsMV0="
+}

integrations/alb/assets/index/alb_logs_metrics.json

@@ -0,0 +1,30 @@
+{
+  "mappings" : {
+    "properties" : {
+      "count2xx" : {
+        "type" : "long"
+      },
+      "count4xx" : {
+        "type" : "long"
+      },
+      "count5xx" : {
+        "type" : "long"
+      },
+      "latencyInSec" : {
+        "type" : "float"
+      },
+      "timestamp" : {
+        "type" : "date"
+      },
+      "totalCount" : {
+        "type" : "long"
+      },
+      "totalReceivedBytes" : {
+        "type" : "long"
+      },
+      "totalSentBytes" : {
+        "type" : "long"
+      }
+    }
+  }
+}

integrations/alb/assets/index/alb_logs_raw.json

@@ -0,0 +1,42 @@
+{
+  "mappings" : {
+    "properties" : {
+      "receivedBytes" : {
+        "type" : "long"
+      },
+      "requestUrl" : {
+        "type" : "text",
+        "fields" : {
+          "keyword" : {
+            "type" : "keyword",
+            "ignore_above" : 256
+          }
+        }
+      },
+      "requestVerb" : {
+        "type" : "text",
+        "fields" : {
+          "keyword" : {
+            "type" : "keyword",
+            "ignore_above" : 256
+          }
+        }
+      },
+      "sentBytes" : {
+        "type" : "long"
+      },
+      "statusCode" : {
+        "type" : "text",
+        "fields" : {
+          "keyword" : {
+            "type" : "keyword",
+            "ignore_above" : 256
+          }
+        }
+      },
+      "timestamp" : {
+        "type" : "date"
+      }
+    }
+  }
+}

integrations/alb/assets/table/alb_logs_temp.sql

@@ -0,0 +1,38 @@
+CREATE TABLE IF NOT EXISTS alb_logs_temp
+(
+    type string,
+    time timestamp,
+    elb string,
+    client_ip string,
+    client_port int,
+    target_ip string,
+    target_port int,
+    request_processing_time double,
+    target_processing_time double,
+    response_processing_time double,
+    elb_status_code int,
+    target_status_code string,
+    received_bytes bigint,
+    sent_bytes bigint,
+    request_verb string,
+    request_url string,
+    request_proto string,
+    user_agent string,
+    ssl_cipher string,
+    ssl_protocol string,
+    target_group_arn string,
+    trace_id string,
+    domain_name string,
+    chosen_cert_arn string,
+    matched_rule_priority string,
+    request_creation_time string,
+    actions_executed string,
+    redirect_url string,
+    lambda_error_reason string,
+    target_port_list string,
+    target_status_code_list string,
+    classification string,
+    classification_reason string
+)
+    USING PARQUET
+    LOCATION 's3a://xxx/'

integrations/alb/assets/view/alb_logs_metrics.sql

@@ -0,0 +1,14 @@
+CREATE MATERIALIZED VIEW alb_logs_metrics
+    AS
+SELECT
+    UNIX_MILLIS(window.start) AS timestamp,
+      COUNT(*) AS totalCount,
+      AVG(target_processing_time) FILTER(WHERE target_processing_time != -1) AS latencyInSec,
+      COUNT(*) FILTER(WHERE target_status_code LIKE '2__') AS count2xx,
+      COUNT(*) FILTER(WHERE target_status_code LIKE '4__') AS count4xx,
+      COUNT(*) FILTER(WHERE target_status_code LIKE '5__') AS count5xx,
+      SUM(received_bytes) AS totalReceivedBytes,
+      SUM(sent_bytes) AS totalSentBytes
+FROM alb_logs_temp
+WHERE client_ip = '10.212.10.101'
+GROUP BY TUMBLE(time, '1 Minute');

integrations/alb/assets/view/alb_logs_raw.sql

@@ -0,0 +1,11 @@
+CREATE MATERIALIZED VIEW alb_logs_raw
+    AS
+SELECT
+    UNIX_MILLIS(time) AS timestamp,
+      request_verb AS requestVerb,
+      request_url AS requestUrl,
+      target_status_code AS statusCode,
+      received_bytes AS receivedBytes,
+      sent_bytes AS sentBytes
+FROM alb_logs_temp
+WHERE client_ip = '10.212.10.101'

integrations/alb/config.json

@@ -0,0 +1,35 @@
+{
+  "name": "alb_logs",
+  "version": {
+    "integ": "0.1.0",
+    "schema": "1.0.0",
+    "resource": "^2.6.0"
+  },
+  "description": "AWS ALB Integration",
+  "catalog": "observability",
+  "components": [
+    "http,communication,cloud,container,aws_elb"
+  ],
+  "collection":[
+    {
+      "logs": [{
+        "info": "ALB logs",
+        "input_type":"logfile",
+        "dataset":"aws.alb",
+        "labels" :["aws","ALB"]
+      }]
+    },
+    {
+      "metrics": [{
+          "info": "ALB metrics signals ",
+          "input_type": "metrics",
+          "dataset": "alb.status",
+          "labels": ["metrics",""]
+        }]
+    }
+  ],
+  "repo": {
+    "github": "https://github.com/opensearch-project/observability/tree/main/integrarions/alb"
+  }
+}
+