enhancement(new sink): new sink for Google chronicle (#13550)

* Add initial sink Signed-off-by: Stephen Wakely <[email protected]> * Started on batching and partitioning Signed-off-by: Stephen Wakely <[email protected]> * It works old style. Signed-off-by: Stephen Wakely <[email protected]> * New style sink working. Signed-off-by: Stephen Wakely <[email protected]> * Encode data for unstructured. Signed-off-by: Stephen Wakely <[email protected]> * WIP Signed-off-by: Stephen Wakely <[email protected]> * Fixed endpoint. Signed-off-by: Stephen Wakely <[email protected]> * Request should be post Signed-off-by: Stephen Wakely <[email protected]> * Tidy code Signed-off-by: Stephen Wakely <[email protected]> * Added content-type header Signed-off-by: Stephen Wakely <[email protected]> * Added integration test. Signed-off-by: Stephen Wakely <[email protected]> * Fixed merge Signed-off-by: Stephen Wakely <[email protected]> * Fixed encoding. Signed-off-by: Stephen Wakely <[email protected]> * Use generated auth keys. Signed-off-by: Stephen Wakely <[email protected]> * Remove old chronicle folder Signed-off-by: Stephen Wakely <[email protected]> * Remove commented module. Signed-off-by: Stephen Wakely <[email protected]> * Tidy module comments. Signed-off-by: Stephen Wakely <[email protected]> * Ackable is no more. Signed-off-by: Stephen Wakely <[email protected]> * Fixed unit tests. Signed-off-by: Stephen Wakely <[email protected]> * Clippy Signed-off-by: Stephen Wakely <[email protected]> * Feedback from Pablo. Signed-off-by: Stephen Wakely <[email protected]> * Added newline Signed-off-by: Stephen Wakely <[email protected]> * Fmt Signed-off-by: Stephen Wakely <[email protected]> * Add docs for google chronicle unstructured sink. Signed-off-by: Stephen Wakely <[email protected]> * Remove Framing from Encoder config. Signed-off-by: Stephen Wakely <[email protected]> * Address feedback. Signed-off-by: Stephen Wakely <[email protected]> * Cue fmt Signed-off-by: Stephen Wakely <[email protected]> * Feedback from Kyle. Signed-off-by: Stephen Wakely <[email protected]> * Compression is not an option. Signed-off-by: Stephen Wakely <[email protected]> * Update goauth. Signed-off-by: Stephen Wakely <[email protected]> * Add integration tests for the unhappy paths. Signed-off-by: Stephen Wakely <[email protected]> * Clippy Signed-off-by: Stephen Wakely <[email protected]> * Add compression:enabled to docs. Signed-off-by: Stephen Wakely <[email protected]> * cue fmt Signed-off-by: Stephen Wakely <[email protected]> * Write using as_tracked_write to save an allocation. Signed-off-by: Stephen Wakely <[email protected]>
vectordotdev · Jul 22, 2022 · 0bac4c4 · 0bac4c4
1 parent 1ad0110
commit 0bac4c4
Show file tree

Hide file tree

Showing 11 changed files with 787 additions and 1 deletion.
diff --git a/Cargo.toml b/Cargo.toml
@@ -582,6 +582,7 @@ sinks-logs = [
   "sinks-azure_blob",
   "sinks-azure_monitor_logs",
   "sinks-blackhole",
+  "sinks-chronicle",
   "sinks-clickhouse",
   "sinks-console",
   "sinks-datadog_archives",
@@ -635,6 +636,7 @@ sinks-axiom = ["sinks-elasticsearch"]
 sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage", "dep:azure_storage_blobs"]
 sinks-azure_monitor_logs = []
 sinks-blackhole = []
+sinks-chronicle = []
 sinks-clickhouse = []
 sinks-console = []
 sinks-datadog_archives = ["sinks-aws_s3", "sinks-azure_blob", "sinks-gcp"]
@@ -688,6 +690,7 @@ all-integration-tests = [
   "aws-integration-tests",
   "axiom-integration-tests",
   "azure-integration-tests",
+  "chronicle-integration-tests",
   "clickhouse-integration-tests",
   "datadog-agent-integration-tests",
   "datadog-logs-integration-tests",
@@ -741,6 +744,7 @@ aws-s3-integration-tests = ["sinks-aws_s3", "sources-aws_s3"]
 aws-sqs-integration-tests = ["sinks-aws_sqs", "sources-aws_sqs"]
 axiom-integration-tests = ["sinks-axiom"]
 azure-blob-integration-tests = ["sinks-azure_blob"]
+chronicle-integration-tests = ["sinks-gcp"]
 clickhouse-integration-tests = ["sinks-clickhouse"]
 datadog-agent-integration-tests = ["sources-datadog_agent"]
 datadog-logs-integration-tests = ["sinks-datadog_logs"]

diff --git a/scripts/integration/chronicleauth.json b/scripts/integration/chronicleauth.json
@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "chronicle-test",
+  "private_key_id": "1",
+  "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIJKQIBAAKCAgEAw/xPtXSFlJgx7qT8C/xNAT/zJRCm854rdw0+0X/m2UnWVLBC\nd92AE8hSIvQCI0wz7y300L0pyUzsHcAw3PA+LDKhjiNKWDlWQsSZ10HpHcmJ9ByM\nxv/gKh5ocgeIzOmycGId9wvGdHd4eZxL86FqN7Ezkixh05lP1Xg5fNNalF+2iWj5\nq9SlI5sBIzbtngtph4Jp9h6g7JaWVcoR2VcAbNK1Hdp1aanbqgroSsRWFJ91NtEK\nq+XzaNpPIXcM9wz1dKAOB34yMlig1UuOZyEqBPZJvXjh/N9TdG6ZTjA7LxvKyGOT\nYHBfIfZgint/UTCN72Qg3xN7f7E4EEWD7jMXdhPm1ERCEOjRlobMj1xw/spFH4O+\nxLAHHp1g4pWM1wTU2BHJH9yqr3GhMghnxQs4H56fx/ML1+onPJqwDzgMiFi5Aqmt\n5Jze5zBfLflUdrcvbeJ5ea5RQLK/f/MuKI2AuFXwXVDqQ86lQ0IJMuTrZBFMjOSq\nInTGkp71zetZ7pXs1r5pP5+aAPtClLn72HRfRuVFALkdXi/lOAQbwuKoLmPSjLAc\nyiPY9fUPlmtDOywTdO81b8tQG8ejSxIqDHd52bhTt5hi5TrzqCa9QIGQOtUJdCSS\nCKIUdTWdcDDyE5PGut/9E+Nd9i3syxfMapOkkM7swbKRjeSKOLNwur6oQusCAwEA\nAQKCAgBUCCcVInT0FZ2zOUJo0TEjTAww5EbxRexVC3TX1wNgP2yKu1iElJCJ6uZs\n3jrcb5wE/atupOz5xuWmi/VnhgriYIMssqz5zvvRfQKJbVVK1FM2O9eQq72t0YPv\nAJQDaGB+F1PJzV62KVOasQ1P1PH6FxyJnEYFuEKmwPnFL8oyvCdEG6KRrodwENIY\nC0Eu9q200JImWVOenaKv/ghRLCCeiGnn9vdI6u+opSycgjp2dTiI3HC668nGN3Se\nYLFKbGACNW5OgCl9tMqhKdhRF8MwhXlwqBgdmJPr5FAPWlbGCCCWczuJqiTgJweV\n2QXqBeuVrVT60Ruu0Im9UFEVHQr9ccvXg2tAy6p3WRq+37phr3RaeVmiZDc5GkC0\nn/1UeSrcIICO1kZKHpuPSjcgty90kXKQdCVNzbHxILND9FFzOBilDSWhdaeGZ+x+\nC+C8pVJeMrVnUqNDsd6hH4uNJQuTR/wUAPYGcW3CPVpy7l+UyoB3hZPVBgZkt9Fa\nCjK6+Wo4aohcdl9/Mk/R8XTeSXT9L38DmFgfCxnV0v9vYUCK6mcxurAxtQ6sNO2i\nuz79gNEhXEAcgarKrF0V/EOfl+uQfB8DEuuatkrw65FEmK//fY1ULI0jv7KFhKC/\njUgIi5p7CTfDXlRsUcFlr/qNHG0O8HvC+nW6zTonA//epfenoQKCAQEA4gq+7g+0\nIjPdrKaXL3n23sgsxzZp5GtedfPH51MaaH93cFhiTSQSiXXGYIbg9KzfNtqX4IiJ\njBk2IVQx2h5T40/d+GGQN03+70gQ5NWycPH9vRFR6mHQR1T1luqEg/7mYQydvTg7\n1KnhUS/lPsEapheztljVKv/cMxgL9qDAk1HoqBAxlgbu6ya6hp06n9vFTUKfg598\nxvZ11R/TUxlCBcnieGPjzsO+qXopepJUarlleN+LhvlOySGvzlTu78/10aKDrMHR\nRG3GTBKLuDQfSI6+diSIpA/FqW5b2uFc4nnBcf+gQt9PPtEReFNjqOXD/Os8Ey1a\nVRI/kyqtBM2gRwKCAQEA3fXMiKSH7qKmkAH3NKSD1wakaJt9Gy5zgSoip6X2kOWi\n/vqalnB9TdynVJaPfJ1K0b/Zm+Fqi658ql5o3jDljJavOdsL4pZMzP+XTP2BNCM9\nvO7u4VHh2dk4yPZhBCTIVxF7dW5BfF31kpGoPA/NNTm2iwe0CSJ7sb3kGwQg9n7t\nK14OcyUvkFrIu9ELY0wXZkPfe9zHeIWB92ZSmhcs/MC6hrWdzYSRrFreFRSGaAKe\nMK7NLvRVVCU+Ln/qvCWReUzQfbKqDB4ayf5DulcvuRzI57znHQur3/EXV6LTmATx\nO6zykbzn9jwzCGKRuytv9DMfmm3Pdzn7/D5NzzlePQKCAQEAk5X8j6cFMAe5Rs8G\ni2MQwujkA7YNcayA4nanIOxro6sGKv4qQL+6srGJMuEd4+MzYrdazqmXVTfo1d4J\nJqcCkrpE7bWV2eY8+7KMBT1Z4tC9oUK7v1LaEJjyOPUqTJyTgu4f17ntpq68hKvG\n/DBM3AFBv19E32xrM5kRxpey5P1n8MOR/KjOOg7xXo8uBc4Shp1nVSM+7xFgYiUD\nb5BgQTNNgNmHlirMRA+VIYiFQaGHFWKv0RLaVAyAozFlSIL5t0RqwNNTdWBPRsAd\nJcoyWseIp862wJfjsPpzUvguE1j36xzCYKc5MISE6ZFY0qAiVPKqKMNF5aGijM4i\nZa50ZQKCAQEAhmXmUjrmAC5Gkt5N1HdlSVdBSAIWj9XhpK1hzIems/gIaUJPRpaY\nmQPp4kNEYN1RR45WcpfcjvW5kOOXtip/ZFB9i5fCK5J8sIh3EexSRsKsCHHY/WA6\nVFG+m5jsB7lKAcaZLfLz/h1arEQ8TRn4VKk8ihTBA+L4aRilPWKTnw4eB+ts7nXu\nxgbiky8yKQ8TAB5K/VIjCiw1FRvFGn68A9OjXhwskBwcpPNkk/kYpXQoWywNG8hg\nvpHKhBd6iDBrVLqZZ/M+GhZEfFjUo8lVIU53rDJq8uB1UyVF0A/M9bjQZSUEwBBV\nmHi9Ie+ldGMJ6CDomvaw7+2RUhxbuwVutQKCAQAOj1TP9RbwkkYI7sZf0BV+ppx2\nhr77O14sIzxZ3/zFREbH4vw48GJOoj8nc3opYQNnUiZwtat3ptnou+1zprX7koTQ\n25rdoCfjrtbw4MUawOV3LvdWVVv5NCmScEyzM+xP8h6aRWQKDxWUYfaGOVBHeuF5\n2g4oSR4s+h1NJvslREr3RiR/m7jbL6VDQhoOUh6o9/bSJW0AqE6Lsx9ws41h22kh\n20Kl6W4adF9XDR9OsT5dznio2gYr2EPt+PTLocPk7s5s7thdWS3oUQ7UA8XlAG6h\nZzT9O+qtMKELxwOgB25fMBcg3kZFHVyqrCQQJSaP82FYdbIHbWiy6VuBWPbd\n-----END RSA PRIVATE KEY-----\n",
+  "client_email": "[email protected]",
+  "client_id": "1",
+  "auth_uri": "http://chronicle-emulator:3000/o/oauth2/auth",
+  "token_uri": "http://chronicle-emulator:3000/token",
+  "auth_provider_x509_cert_url": "https://chronicle-emulator:3000/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.ohno"
+}
diff --git a/scripts/integration/chroniclepub.pem b/scripts/integration/chroniclepub.pem
@@ -0,0 +1,13 @@
+-----BEGIN RSA PUBLIC KEY-----
+MIICCgKCAgEAw/xPtXSFlJgx7qT8C/xNAT/zJRCm854rdw0+0X/m2UnWVLBCd92A
+E8hSIvQCI0wz7y300L0pyUzsHcAw3PA+LDKhjiNKWDlWQsSZ10HpHcmJ9ByMxv/g
+Kh5ocgeIzOmycGId9wvGdHd4eZxL86FqN7Ezkixh05lP1Xg5fNNalF+2iWj5q9Sl
+I5sBIzbtngtph4Jp9h6g7JaWVcoR2VcAbNK1Hdp1aanbqgroSsRWFJ91NtEKq+Xz
+aNpPIXcM9wz1dKAOB34yMlig1UuOZyEqBPZJvXjh/N9TdG6ZTjA7LxvKyGOTYHBf
+IfZgint/UTCN72Qg3xN7f7E4EEWD7jMXdhPm1ERCEOjRlobMj1xw/spFH4O+xLAH
+Hp1g4pWM1wTU2BHJH9yqr3GhMghnxQs4H56fx/ML1+onPJqwDzgMiFi5Aqmt5Jze
+5zBfLflUdrcvbeJ5ea5RQLK/f/MuKI2AuFXwXVDqQ86lQ0IJMuTrZBFMjOSqInTG
+kp71zetZ7pXs1r5pP5+aAPtClLn72HRfRuVFALkdXi/lOAQbwuKoLmPSjLAcyiPY
+9fUPlmtDOywTdO81b8tQG8ejSxIqDHd52bhTt5hi5TrzqCa9QIGQOtUJdCSSCKIU
+dTWdcDDyE5PGut/9E+Nd9i3syxfMapOkkM7swbKRjeSKOLNwur6oQusCAwEAAQ==
+-----END RSA PUBLIC KEY-----
diff --git a/scripts/integration/docker-compose.chronicle.yml b/scripts/integration/docker-compose.chronicle.yml
@@ -0,0 +1,41 @@
+version: "1"
+
+services:
+  chronicle-emulator:
+    image: docker.io/plork/chronicle-emulator:latest
+    ports:
+      - 3000:3000
+    volumes:
+      - ${PWD}/scripts/integration/chroniclepub.pem:/public.pem
+    command: [ "-p", "/public.pem" ]
+  runner:
+    build:
+      context: ${PWD}
+      dockerfile: scripts/integration/Dockerfile
+      args:
+        - RUST_VERSION=${RUST_VERSION}
+    working_dir: /code
+    command:
+      - "cargo"
+      - "nextest"
+      - "run"
+      - "--no-fail-fast"
+      - "--no-default-features"
+      - "--features"
+      - "chronicle-integration-tests"
+      - "--lib"
+      - "${FILTER:-::chronicle_unstructured::}"
+    depends_on:
+      - chronicle-emulator
+    environment:
+      - CHRONICLE_ADDRESS=http://chronicle-emulator:3000
+    volumes:
+      - ${PWD}:/code
+      - cargogit:/usr/local/cargo/git
+      - cargoregistry:/usr/local/cargo/registry
+      - ${PWD}/scripts/integration/chronicleauth.json:/chronicleauth.json
+      - ${PWD}/scripts/integration/invalidchronicleauth.json:/invalidchronicleauth.json
+
+volumes:
+  cargogit: {}
+  cargoregistry: {}
diff --git a/scripts/integration/invalidchronicleauth.json b/scripts/integration/invalidchronicleauth.json
@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "chronicle-test",
+  "private_key_id": "1",
+    "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIICXgIBAAKBgQDouHdVDVz0/M6PGe60Kf/g0nyOxCvbZgiUAZNzFimXDU+RpZ54\n6/oETl6VpRkbp8a4Xb8avll2lsamdHvGcsgnjJXdpp7LfWYLqHEpn0/XFM+womXg\nvglWCDwAsXmrmwpZKEC82mmyFigheyPA/sfuN6z+wa7P5B65xzIdDQX7nQIDAQAB\nAoGBANID/rUDrTrtll8v8Oon6OH0MjIIuOdzKhSfY3h9rKTDf2YaB2xq0KLoMpVr\ne8AoZb5l45t34naR1M3M2xKY7SSDAVJFfg/3Vxeot86DQ23IGLXj7LnNxXnvklXa\nEXaD8LNz/MXxS7/Lu0R+lEtjEkf23+BRb11fL6Q/EDToNHnhAkEA/FnwHhKMc/Bm\nXsS8bENuZP3SV2v7TU6MFTtXJFmsoZBxHnsM8UUi0gq9gBnApmdhy7v2N/Mv9gFI\nviSdr7vm1QJBAOwV3cHAciRHVK71TweOWIJKZBM9ZVut0VDs5GrBYZxGMBiOr3BI\ns7+0ugTKxVimuei6c0KNXw1kg3Vtc5+utakCQQDklAbXBpAomJHxt5zBKBc/7VXx\nEANyk/p5ZOXbLEsdkXuVU3p2tNwEi+v4s9r4H97Kr3goV+SSnbkpWntm6fn9AkBn\nFnE7rlXpA4C12QYGTaDWW7dxM0j0DGUvChH/j6uYuok73+o5hHWAy2DCwOwFduAN\nAIVd1S9hQLeqaf2oB3jpAkEAnRT+bAlMjtUOBO6XPNO4IbYwWJvGMcIEO7zu6AdB\nPJy3/U+bLimxFuYdrs6SnIHIUVdl35AlckHqzT54a5YKqQ==\n-----END RSA PRIVATE KEY-----",
+  "client_email": "[email protected]",
+  "client_id": "1",
+  "auth_uri": "http://chronicle-emulator:3000/o/oauth2/auth",
+  "token_uri": "http://chronicle-emulator:3000/token",
+  "auth_provider_x509_cert_url": "https://chronicle-emulator:3000/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.ohno"
+}