From 83f342047cd305664a2dfa0c5d16fd99f2d042b9 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:38:00 -0500 Subject: [PATCH 01/65] wip --- .github/workflows/ci.yml | 56 ++++++++ action.yml | 29 ++++ action/bindplane.sh | 68 ++++++++++ test/resources/destinations/resource.yaml | 153 ++++++++++++++++++++++ 4 files changed, 306 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 action.yml create mode 100644 action/bindplane.sh create mode 100644 test/resources/destinations/resource.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e5e6614 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: CI +on: + pull_request: + push: + branches: + - main + +permissions: + # Allow action to write raw configs back to the repository. + contents: write + +jobs: + test: + runs-on: ubuntu-latest + matrix: + bindplane_versions: + - 1.40.0 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Start BindPlane + run: | + docker run \ + --name bindplane \ + -e BINDPLANE_USERNAME=admin \ + -e BINDPLANE_PASSWORD=admin \ + -e BINDPLANE_REMOTE_URL=http://localhost:3001 \ + -e BINDPLANE_SESSION_SECRET=2c23c9d3-850f-4062-a5c8-3f9b814ae144 \ + -e BINDPLANE_SECRET_KEY=8a5353f7-bbf4-4eea-846d-a6d54296b781 \ + -e BINDPLANE_LOG_OUTPUT=stdout \ + -e BINDPLANE_ACCEPT_EULA=true \ + -p 3001:3001 \ + ghcr.io/observiq/bindplane-ee:${{ matrix.bindplane_versions }} + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + # These values are hardcode to match the test instance used by + # this workflow. The instance does not persist. Consumers of + # this action should always use secrets when passing in the remote + # url, username, password or api key. + # + # Remote url will never be localhost when running this action. In this + # case, bindplane is literally running within the runner instance. + remote_url: http://localhost:3001 + username: admin + password: admin + destination_path: test/resources/destinations + configuration_path: test/resources/configurations + + - name: Debug Container Logs + if: always() + run: docker logs bindplane diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..1c537ed --- /dev/null +++ b/action.yml @@ -0,0 +1,29 @@ +name: 'BindPlane OP Action' +description: 'GitHub Action for BindPlane OP.' +author: 'observIQ' + +inputs: + remote_url: + description: 'The URL that will be used to connect to BindPlane OP' + required: true + api_key: + description: 'The BindPlane OP API key that will be used to authenticate to BindPlane OP' + username: + description: 'The BindPlane OP username that will be used to authenticate to BindPlane OP' + password: + description: 'The BindPlane OP password that will be used to authenticate to BindPlane OP' + destination_path: + description: 'Path to the file or directory which contains the BindPlane destination resources' + configuration_path: + description: 'Path to the file or directory which contains the BindPlane configuration resources' + +runs: + using: 'shell' + command: | + ./action/bindplane.sh \ + ${{ inputs.remote_url }} \ + ${{ inputs.api_key }} \ + ${{ inputs.username }} \ + ${{ inputs.password }} \ + ${{ inputs.destination_path }} \ + ${{ inputs.configuration_path }} diff --git a/action/bindplane.sh b/action/bindplane.sh new file mode 100644 index 0000000..a0a07fb --- /dev/null +++ b/action/bindplane.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -e + +REMOTE_URL=$1 +API_KEY=$2 +USERNAME=$3 +PASSWORD=$4 +DESTINATION_PATH=$5 +CONFIG_PATH=$6 + +# Validate will ensure that all required variables are set +# and generates the bindplane profile. +validate() { + profile_args="" + + if [ -z "$REMOTE_URL" ]; then + echo "REMOTE_URL is not set." + exit 1 + else + profile_args="$profile_args --remote-url $REMOTE_URL" + fi + + if [ -n "$USERNAME" ] && [ -z "$PASSWORD" ]; then + echo "password is required when username is not set." + exit 1 + else + profile_args="$profile_args --username $USERNAME --password $PASSWORD" + fi + + if [ -z "$USERNAME" ] && [ -z "$API_KEY" ]; then + echo "api key is required when username is not set." + exit 1 + else + profile_args="$profile_args --api-key $API_KEY" + fi + + eval bindplane profile set "action" $profile_args + bindplane profile use "action" +} + +# Apply generic path takes a directory or file path +# and applys it to BindPlane. If the path is a directory +# it will apply all files in the directory using a * glob +# pattern suffix. +apply_generic_path() { + if [ -z "$1" ]; then + return + fi + + if [ -d "$1" ]; then + bindplane apply -f "$1/*" + else + bindplane apply -f "$1" + fi +} + +# Apply will apply resources in the correct order. Re-usable +# resources must exist before they can be referenced by +# a configuration. +apply() { + apply_generic_path "$DESTINATION_PATH" + apply_generic_path "$CONFIG_PATH" +} + +validate +provile +apply diff --git a/test/resources/destinations/resource.yaml b/test/resources/destinations/resource.yaml new file mode 100644 index 0000000..5c7f6f1 --- /dev/null +++ b/test/resources/destinations/resource.yaml @@ -0,0 +1,153 @@ +--- +apiVersion: bindplane.observiq.com/v1 +kind: Configuration +metadata: + id: k8s-cluster + name: k8s-cluster + labels: + platform: kubernetes-deployment +spec: + contentType: "" + measurementInterval: "" + sources: + - id: 01HMS8GVNVFVD5TSWTKSJR1RY5 + type: k8s_cluster + parameters: + - name: cluster_name + value: minikube + - name: node_conditions_to_report + value: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + - name: allocatable_types_to_report + value: + - cpu + - memory + - ephemeral-storage + - storage + - name: collection_interval + value: 60 + - name: distribution + value: kubernetes + - id: 01HMS8GVNVFVD5TSWTKVNZS2JC + displayName: Production events + type: k8s_events + parameters: + - name: cluster_name + value: minikube + - name: namespaces + value: + - kube-system + - production + destinations: + - id: 01HMS8GVNVFVD5TSWTKZZNHB8R + name: bindplane-gateway-agent + selector: + matchLabels: + configuration: k8s-cluster +--- +apiVersion: bindplane.observiq.com/v1 +kind: Configuration +metadata: + id: k8s-gateway + name: k8s-gateway + labels: + platform: kubernetes-gateway +spec: + contentType: "" + measurementInterval: "" + sources: + - id: 01HMS8J2WPEVAX0RSCPW4ZNWT5 + type: otlp + parameters: + - name: telemetry_types + value: + - Metrics + - Logs + - Traces + - name: listen_address + value: 0.0.0.0 + - name: grpc_port + value: 4317 + - name: http_port + value: 4318 + - name: enable_tls + value: false + - name: cert_file + value: "" + - name: key_file + value: "" + - name: mutual_tls + value: false + - name: ca_file + value: "" + - name: enable_grpc_timeout + value: true + - name: grpc_max_connection_idle + value: 60 + - name: grpc_max_connection_age + value: 60 + - name: grpc_max_connection_age_grace + value: 300 + destinations: + - id: 01HMS8J2WPEVAX0RSCQ1NZ5KB6 + name: coralogix + - id: 01HMS8K1AEYPBCBN47ZFYM3X7J + name: elasticsearch-internal + - id: 01HMS8KV7K4N2WKJ3337T9KJ2M + name: prometheus-internal + selector: + matchLabels: + configuration: k8s-gateway +--- +apiVersion: bindplane.observiq.com/v1 +kind: Configuration +metadata: + id: k8s-node + name: k8s-node + labels: + platform: kubernetes-daemonset +spec: + contentType: "" + measurementInterval: "" + sources: + - id: 01HMS8ERRSFYE4KGQVQ2ZXXDZK + type: k8s_container + parameters: + - name: cluster_name + value: minikube + - name: log_source + value: file + - name: file_path + value: + - /var/log/containers/*.log + - name: exclude_file_path + value: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + - name: journald_path + value: "" + - name: start_at + value: end + - id: 01HMS8ERRSFYE4KGQVQ8DQC5X0 + type: k8s_kubelet + parameters: + - name: cluster_name + value: minikube + - name: metric_groups + value: + - node + - pod + - container + - volume + - name: collection_interval + value: 60 + destinations: + - id: 01HMS8HBK216AD6V30GKX968XK + name: bindplane-gateway-agent + selector: + matchLabels: + configuration: k8s-node From 2364c0f8b05e4a3ec5b61801dd79163f56c75ecd Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:39:54 -0500 Subject: [PATCH 02/65] fix matrix --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5e6614..329b671 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,10 @@ permissions: jobs: test: runs-on: ubuntu-latest - matrix: - bindplane_versions: - - 1.40.0 + strategy: + matrix: + bindplane_versions: + - 1.40.0 steps: - name: Checkout uses: actions/checkout@v4 From 5438ff10ea2f2516d816935cdbe17fc6b67bca69 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:42:35 -0500 Subject: [PATCH 03/65] install cli --- action/bindplane.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/action/bindplane.sh b/action/bindplane.sh index a0a07fb..f936baa 100644 --- a/action/bindplane.sh +++ b/action/bindplane.sh @@ -9,6 +9,16 @@ PASSWORD=$4 DESTINATION_PATH=$5 CONFIG_PATH=$6 +install_bindplane_cli() { + curl -Ls \ + -o bindplane.zip \ + https://storage.googleapis.com/bindplane-op-releases/bindplane/latest/bindplane-ee-linux-amd64.zip + + unzip bindplane.zip + + ./bindplane --help > /dev/null +} + # Validate will ensure that all required variables are set # and generates the bindplane profile. validate() { @@ -63,6 +73,7 @@ apply() { apply_generic_path "$CONFIG_PATH" } +install_bindplane_cli validate provile apply From c438e12d071051edd7535c0e0ff1ec8e4c117518 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:43:16 -0500 Subject: [PATCH 04/65] pull image --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 329b671..ade2a22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: with: fetch-depth: 0 + - name: Pull BindPlane + run: docker pull ghcr.io/observiq/bindplane-ee:${{ matrix.bindplane_versions }} + - name: Start BindPlane run: | docker run \ From d6a80d3ef88b2a90569de95c04df596dc0575e8b Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:43:32 -0500 Subject: [PATCH 05/65] run bindplane in background --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ade2a22..80c639a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: - name: Start BindPlane run: | docker run \ + -d --name bindplane \ -e BINDPLANE_USERNAME=admin \ -e BINDPLANE_PASSWORD=admin \ From 4d9490597fd091310c61322cf61d62da92289d26 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:44:36 -0500 Subject: [PATCH 06/65] arg syntax --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80c639a..d9f3697 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - name: Start BindPlane run: | docker run \ - -d + -d \ --name bindplane \ -e BINDPLANE_USERNAME=admin \ -e BINDPLANE_PASSWORD=admin \ From f3c5971c83707c0dcbc6fad197df728a5941e34d Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:47:12 -0500 Subject: [PATCH 07/65] fix action name and uses field --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9f3697..6a8bf14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,8 +40,10 @@ jobs: -p 3001:3001 \ ghcr.io/observiq/bindplane-ee:${{ matrix.bindplane_versions }} - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 + - name: Run BindPlane Action + # This should be replaced with a release action. + # /@ + uses: ./ with: # These values are hardcode to match the test instance used by # this workflow. The instance does not persist. Consumers of From 5263a6e391fae0ded3b659bc8079ec282cc9ccfa Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:53:52 -0500 Subject: [PATCH 08/65] switch to docker --- Dockerfile | 8 ++++++++ action.yml | 18 +++++++++--------- action/bindplane.sh => entrypoint.sh | 0 3 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 Dockerfile rename action/bindplane.sh => entrypoint.sh (100%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c0140c6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +# Container image that runs your code +FROM alpine:3.10 + +RUN apk add --no-cache bash + +COPY --chmod=0755 entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/action.yml b/action.yml index 1c537ed..a292962 100644 --- a/action.yml +++ b/action.yml @@ -18,12 +18,12 @@ inputs: description: 'Path to the file or directory which contains the BindPlane configuration resources' runs: - using: 'shell' - command: | - ./action/bindplane.sh \ - ${{ inputs.remote_url }} \ - ${{ inputs.api_key }} \ - ${{ inputs.username }} \ - ${{ inputs.password }} \ - ${{ inputs.destination_path }} \ - ${{ inputs.configuration_path }} + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.remote_url }} + - ${{ inputs.api_key }} + - ${{ inputs.username }} + - ${{ inputs.password }} + - ${{ inputs.destination_path }} + - ${{ inputs.configuration_path }} diff --git a/action/bindplane.sh b/entrypoint.sh similarity index 100% rename from action/bindplane.sh rename to entrypoint.sh From 5009f841440cf404d5f027abbc8e0441e6c44e3e Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:54:43 -0500 Subject: [PATCH 09/65] cURL --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c0140c6..a38e123 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Container image that runs your code FROM alpine:3.10 -RUN apk add --no-cache bash +RUN apk add --no-cache bash curl COPY --chmod=0755 entrypoint.sh /entrypoint.sh From a1b7617fd6ae070aec66b9d70ad985e229f4d79a Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:56:28 -0500 Subject: [PATCH 10/65] put bindplane binary in path --- entrypoint.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index f936baa..05bb603 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,9 +14,12 @@ install_bindplane_cli() { -o bindplane.zip \ https://storage.googleapis.com/bindplane-op-releases/bindplane/latest/bindplane-ee-linux-amd64.zip - unzip bindplane.zip + mkdir -p ~/bin + export PATH=$PATH:~/bin - ./bindplane --help > /dev/null + unzip bindplane.zip -d ~/bin + + bindplane --help > /dev/null } # Validate will ensure that all required variables are set From 74bfe20bb58cfc134f166b1c03a2e64d9b5101c1 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 14:59:59 -0500 Subject: [PATCH 11/65] fix args and add shellcheck to ci --- .github/workflows/ci.yml | 14 ++++++++++++++ entrypoint.sh | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a8bf14..1cdf19f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,20 @@ permissions: contents: write jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install ShellCheck + run: sudo apt-get install shellcheck + + - name: Run ShellCheck + run: shellcheck -x -s bash entrypoint.sh + test: runs-on: ubuntu-latest strategy: diff --git a/entrypoint.sh b/entrypoint.sh index 05bb603..253882d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -44,11 +44,11 @@ validate() { if [ -z "$USERNAME" ] && [ -z "$API_KEY" ]; then echo "api key is required when username is not set." exit 1 - else + elif [ -n "$API_KEY" ]; then profile_args="$profile_args --api-key $API_KEY" - fi + fi - eval bindplane profile set "action" $profile_args + eval bindplane profile set "action" "$profile_args" bindplane profile use "action" } From 79f74e12e6060829e2c9a12b2b7d8d1b6be13678 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:01:38 -0500 Subject: [PATCH 12/65] remove missing function call --- entrypoint.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 253882d..541d634 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -68,15 +68,10 @@ apply_generic_path() { fi } +install_bindplane_cli +validate # Apply will apply resources in the correct order. Re-usable # resources must exist before they can be referenced by # a configuration. -apply() { - apply_generic_path "$DESTINATION_PATH" - apply_generic_path "$CONFIG_PATH" -} - -install_bindplane_cli -validate -provile -apply +apply_generic_path "$DESTINATION_PATH" +apply_generic_path "$CONFIG_PATH" From bc582cdb088dffe6d18402d2ffbabd5684554e0e Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:02:34 -0500 Subject: [PATCH 13/65] Add debug resource get --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cdf19f..e5e2243 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,12 @@ jobs: destination_path: test/resources/destinations configuration_path: test/resources/configurations + - name: Get Resources + if: always() + run: | + bindplane get destinations + bindplane get configurations + - name: Debug Container Logs if: always() run: docker logs bindplane From 7ce99fdf0f494a246ec12f1fbdc0c76ea8a58eb2 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:03:29 -0500 Subject: [PATCH 14/65] fix paths --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5e2243..0cb2372 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,8 @@ jobs: remote_url: http://localhost:3001 username: admin password: admin - destination_path: test/resources/destinations + # Testing tailing and non trailing slash + destination_path: test/resources/destinations/ configuration_path: test/resources/configurations - name: Get Resources From 5499a9048a6e40f216bd7c7197686b0e4b7c580b Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:06:09 -0500 Subject: [PATCH 15/65] remove dir support --- .github/workflows/ci.yml | 4 ++-- README.md | 39 ++++++++++++++++++++++++++++++++++++++- action.yml | 4 ++-- entrypoint.sh | 20 ++------------------ 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cb2372..11ae2c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,8 +70,8 @@ jobs: username: admin password: admin # Testing tailing and non trailing slash - destination_path: test/resources/destinations/ - configuration_path: test/resources/configurations + destination_path: test/resources/destinations/resource.yaml + configuration_path: test/resources/configurations/resource.yaml - name: Get Resources if: always() diff --git a/README.md b/README.md index 209d125..932daf2 100644 --- a/README.md +++ b/README.md @@ -1 +1,38 @@ -# bindplane-op-action \ No newline at end of file +# bindplane-op-action + +## Usage + +### Export Resources + +```bash +bindplane get destination -o yaml --export > destination.yaml +bindplane get configuration -o yaml --export > configuration.yaml +``` + +### Workflow + +```yaml +name: goreleaser + +on: + pull_request: + push: + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} + api_key: ${{ secrets.BINDPLANE_API_KEY }} +``` diff --git a/action.yml b/action.yml index a292962..69a3fcb 100644 --- a/action.yml +++ b/action.yml @@ -13,9 +13,9 @@ inputs: password: description: 'The BindPlane OP password that will be used to authenticate to BindPlane OP' destination_path: - description: 'Path to the file or directory which contains the BindPlane destination resources' + description: 'Path to the file which contains the BindPlane destination resources' configuration_path: - description: 'Path to the file or directory which contains the BindPlane configuration resources' + description: 'Path to the file which contains the BindPlane configuration resources' runs: using: 'docker' diff --git a/entrypoint.sh b/entrypoint.sh index 541d634..a5f1f3d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -52,26 +52,10 @@ validate() { bindplane profile use "action" } -# Apply generic path takes a directory or file path -# and applys it to BindPlane. If the path is a directory -# it will apply all files in the directory using a * glob -# pattern suffix. -apply_generic_path() { - if [ -z "$1" ]; then - return - fi - - if [ -d "$1" ]; then - bindplane apply -f "$1/*" - else - bindplane apply -f "$1" - fi -} - install_bindplane_cli validate # Apply will apply resources in the correct order. Re-usable # resources must exist before they can be referenced by # a configuration. -apply_generic_path "$DESTINATION_PATH" -apply_generic_path "$CONFIG_PATH" +bindplane apply "$DESTINATION_PATH" +bindplane apply "$CONFIG_PATH" From baa2db742347e9361847b24500eaac0836a05802 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:10:31 -0500 Subject: [PATCH 16/65] fix resources for tests --- test/resources/configurations/resource.yaml | 153 ++++++++ test/resources/destinations/resource.yaml | 364 ++++++++++++-------- 2 files changed, 376 insertions(+), 141 deletions(-) create mode 100644 test/resources/configurations/resource.yaml diff --git a/test/resources/configurations/resource.yaml b/test/resources/configurations/resource.yaml new file mode 100644 index 0000000..5c7f6f1 --- /dev/null +++ b/test/resources/configurations/resource.yaml @@ -0,0 +1,153 @@ +--- +apiVersion: bindplane.observiq.com/v1 +kind: Configuration +metadata: + id: k8s-cluster + name: k8s-cluster + labels: + platform: kubernetes-deployment +spec: + contentType: "" + measurementInterval: "" + sources: + - id: 01HMS8GVNVFVD5TSWTKSJR1RY5 + type: k8s_cluster + parameters: + - name: cluster_name + value: minikube + - name: node_conditions_to_report + value: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + - name: allocatable_types_to_report + value: + - cpu + - memory + - ephemeral-storage + - storage + - name: collection_interval + value: 60 + - name: distribution + value: kubernetes + - id: 01HMS8GVNVFVD5TSWTKVNZS2JC + displayName: Production events + type: k8s_events + parameters: + - name: cluster_name + value: minikube + - name: namespaces + value: + - kube-system + - production + destinations: + - id: 01HMS8GVNVFVD5TSWTKZZNHB8R + name: bindplane-gateway-agent + selector: + matchLabels: + configuration: k8s-cluster +--- +apiVersion: bindplane.observiq.com/v1 +kind: Configuration +metadata: + id: k8s-gateway + name: k8s-gateway + labels: + platform: kubernetes-gateway +spec: + contentType: "" + measurementInterval: "" + sources: + - id: 01HMS8J2WPEVAX0RSCPW4ZNWT5 + type: otlp + parameters: + - name: telemetry_types + value: + - Metrics + - Logs + - Traces + - name: listen_address + value: 0.0.0.0 + - name: grpc_port + value: 4317 + - name: http_port + value: 4318 + - name: enable_tls + value: false + - name: cert_file + value: "" + - name: key_file + value: "" + - name: mutual_tls + value: false + - name: ca_file + value: "" + - name: enable_grpc_timeout + value: true + - name: grpc_max_connection_idle + value: 60 + - name: grpc_max_connection_age + value: 60 + - name: grpc_max_connection_age_grace + value: 300 + destinations: + - id: 01HMS8J2WPEVAX0RSCQ1NZ5KB6 + name: coralogix + - id: 01HMS8K1AEYPBCBN47ZFYM3X7J + name: elasticsearch-internal + - id: 01HMS8KV7K4N2WKJ3337T9KJ2M + name: prometheus-internal + selector: + matchLabels: + configuration: k8s-gateway +--- +apiVersion: bindplane.observiq.com/v1 +kind: Configuration +metadata: + id: k8s-node + name: k8s-node + labels: + platform: kubernetes-daemonset +spec: + contentType: "" + measurementInterval: "" + sources: + - id: 01HMS8ERRSFYE4KGQVQ2ZXXDZK + type: k8s_container + parameters: + - name: cluster_name + value: minikube + - name: log_source + value: file + - name: file_path + value: + - /var/log/containers/*.log + - name: exclude_file_path + value: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + - name: journald_path + value: "" + - name: start_at + value: end + - id: 01HMS8ERRSFYE4KGQVQ8DQC5X0 + type: k8s_kubelet + parameters: + - name: cluster_name + value: minikube + - name: metric_groups + value: + - node + - pod + - container + - volume + - name: collection_interval + value: 60 + destinations: + - id: 01HMS8HBK216AD6V30GKX968XK + name: bindplane-gateway-agent + selector: + matchLabels: + configuration: k8s-node diff --git a/test/resources/destinations/resource.yaml b/test/resources/destinations/resource.yaml index 5c7f6f1..99b091d 100644 --- a/test/resources/destinations/resource.yaml +++ b/test/resources/destinations/resource.yaml @@ -1,153 +1,235 @@ --- apiVersion: bindplane.observiq.com/v1 -kind: Configuration +kind: Destination metadata: - id: k8s-cluster - name: k8s-cluster - labels: - platform: kubernetes-deployment + id: bindplane-gateway-agent + name: bindplane-gateway-agent spec: - contentType: "" - measurementInterval: "" - sources: - - id: 01HMS8GVNVFVD5TSWTKSJR1RY5 - type: k8s_cluster - parameters: - - name: cluster_name - value: minikube - - name: node_conditions_to_report - value: - - Ready - - DiskPressure - - MemoryPressure - - PIDPressure - - NetworkUnavailable - - name: allocatable_types_to_report - value: - - cpu - - memory - - ephemeral-storage - - storage - - name: collection_interval - value: 60 - - name: distribution - value: kubernetes - - id: 01HMS8GVNVFVD5TSWTKVNZS2JC - displayName: Production events - type: k8s_events - parameters: - - name: cluster_name - value: minikube - - name: namespaces - value: - - kube-system - - production - destinations: - - id: 01HMS8GVNVFVD5TSWTKZZNHB8R - name: bindplane-gateway-agent - selector: - matchLabels: - configuration: k8s-cluster + type: otlp_grpc + parameters: + - name: hostname + value: bindplane-gateway-agent.bindplane-agent.svc.cluster.local + - name: grpc_port + value: 4317 + - name: http_port + value: 4318 + - name: protocol + value: grpc + - name: headers + value: {} + - name: http_compression + value: gzip + - name: grpc_compression + value: gzip + - name: enable_tls + value: false + - name: insecure_skip_verify + value: false + - name: ca_file + value: "" + - name: mutual_tls + value: false + - name: cert_file + value: "" + - name: key_file + value: "" + - name: retry_on_failure_enabled + value: true + - name: retry_on_failure_initial_interval + value: 5 + - name: retry_on_failure_max_interval + value: 30 + - name: retry_on_failure_max_elapsed_time + value: 300 + - name: sending_queue_enabled + value: true + - name: sending_queue_num_consumers + value: 10 + - name: sending_queue_queue_size + value: 5000 + - name: persistent_queue_enabled + value: true + - name: persistent_queue_directory + value: $OIQ_OTEL_COLLECTOR_HOME/storage +status: + latest: true --- apiVersion: bindplane.observiq.com/v1 -kind: Configuration +kind: Destination metadata: - id: k8s-gateway - name: k8s-gateway - labels: - platform: kubernetes-gateway + id: coralogix + name: coralogix spec: - contentType: "" - measurementInterval: "" - sources: - - id: 01HMS8J2WPEVAX0RSCPW4ZNWT5 - type: otlp - parameters: - - name: telemetry_types - value: - - Metrics - - Logs - - Traces - - name: listen_address - value: 0.0.0.0 - - name: grpc_port - value: 4317 - - name: http_port - value: 4318 - - name: enable_tls - value: false - - name: cert_file - value: "" - - name: key_file - value: "" - - name: mutual_tls - value: false - - name: ca_file - value: "" - - name: enable_grpc_timeout - value: true - - name: grpc_max_connection_idle - value: 60 - - name: grpc_max_connection_age - value: 60 - - name: grpc_max_connection_age_grace - value: 300 - destinations: - - id: 01HMS8J2WPEVAX0RSCQ1NZ5KB6 - name: coralogix - - id: 01HMS8K1AEYPBCBN47ZFYM3X7J - name: elasticsearch-internal - - id: 01HMS8KV7K4N2WKJ3337T9KJ2M - name: prometheus-internal - selector: - matchLabels: - configuration: k8s-gateway + type: coralogix + parameters: + - name: private_key + value: (sensitive) + sensitive: true + - name: application_name + value: bindplane-test + - name: subsystem_name + value: test + - name: region + value: USA1 + - name: domain + value: "" + - name: resource_attributes + value: true + - name: application_name_attributes + value: [] + - name: subsystem_name_attributes + value: [] + - name: enable_metrics + value: true + - name: enable_logs + value: true + - name: enable_traces + value: true + - name: timeout + value: 5 + - name: compression + value: gzip + - name: retry_on_failure_enabled + value: true + - name: retry_on_failure_initial_interval + value: 5 + - name: retry_on_failure_max_interval + value: 30 + - name: retry_on_failure_max_elapsed_time + value: 300 + - name: sending_queue_enabled + value: true + - name: sending_queue_num_consumers + value: 10 + - name: sending_queue_queue_size + value: 5000 + - name: persistent_queue_enabled + value: true + - name: persistent_queue_directory + value: $OIQ_OTEL_COLLECTOR_HOME/storage +status: + latest: true --- apiVersion: bindplane.observiq.com/v1 -kind: Configuration +kind: Destination metadata: - id: k8s-node - name: k8s-node - labels: - platform: kubernetes-daemonset + id: elasticsearch-internal + name: elasticsearch-internal spec: - contentType: "" - measurementInterval: "" - sources: - - id: 01HMS8ERRSFYE4KGQVQ2ZXXDZK - type: k8s_container - parameters: - - name: cluster_name - value: minikube - - name: log_source - value: file - - name: file_path - value: - - /var/log/containers/*.log - - name: exclude_file_path - value: - - /var/log/containers/observiq-*-collector-* - - /var/log/containers/bindplane-*-agent-* - - name: journald_path - value: "" - - name: start_at - value: end - - id: 01HMS8ERRSFYE4KGQVQ8DQC5X0 - type: k8s_kubelet - parameters: - - name: cluster_name - value: minikube - - name: metric_groups - value: - - node - - pod - - container - - volume - - name: collection_interval - value: 60 - destinations: - - id: 01HMS8HBK216AD6V30GKX968XK - name: bindplane-gateway-agent - selector: - matchLabels: - configuration: k8s-node + type: elasticsearch + parameters: + - name: enable_elastic_cloud + value: false + - name: endpoints + value: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + - name: cloudid + value: "" + - name: enable_logs + value: true + - name: logs_index + value: logs-generic-default + - name: enable_traces + value: true + - name: traces_index + value: traces-generic-default + - name: pipeline + value: "" + - name: enable_auth + value: false + - name: auth_type + value: basic + - name: user + value: "" + - name: password + value: (sensitive) + sensitive: true + - name: api_key + value: (sensitive) + sensitive: true + - name: configure_tls + value: false + - name: insecure_skip_verify + value: false + - name: ca_file + value: "" + - name: mutual_tls + value: false + - name: cert_file + value: "" + - name: key_file + value: "" + - name: retry_on_failure_enabled + value: true + - name: num_workers + value: 0 + - name: sending_queue_enabled + value: true + - name: sending_queue_num_consumers + value: 10 + - name: sending_queue_queue_size + value: 5000 + - name: persistent_queue_enabled + value: true + - name: persistent_queue_directory + value: $OIQ_OTEL_COLLECTOR_HOME/storage +status: + latest: true +--- +apiVersion: bindplane.observiq.com/v1 +kind: Destination +metadata: + id: prometheus-internal + name: prometheus-internal +spec: + type: prometheus_remote_write + parameters: + - name: hostname + value: prom-prod.corp.net + - name: port + value: 9009 + - name: path + value: /api/v1/write + - name: add_unit_and_type + value: true + - name: namespace + value: "" + - name: compression + value: gzip + - name: enable_resource_to_telemetry_conversion + value: true + - name: headers + value: {} + - name: external_labels + value: {} + - name: enable_tls + value: false + - name: strict_tls_verify + value: false + - name: ca_file + value: "" + - name: mutual_tls + value: false + - name: cert_file + value: "" + - name: key_file + value: "" + - name: enable_write_ahead_log + value: false + - name: wal_storage_path + value: prometheus_rw + - name: wal_buffer_size + value: 300 + - name: wal_truncate_frequency + value: 60 + - name: retry_on_failure_enabled + value: true + - name: retry_on_failure_initial_interval + value: 5 + - name: retry_on_failure_max_interval + value: 30 + - name: retry_on_failure_max_elapsed_time + value: 300 +status: + latest: true From 72071ccd886257042dea86c1b6b6caeda8269b25 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:12:53 -0500 Subject: [PATCH 17/65] wait for bindplane to become ready --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11ae2c8..1adbd36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,15 @@ jobs: -p 3001:3001 \ ghcr.io/observiq/bindplane-ee:${{ matrix.bindplane_versions }} + - name: Wait for BindPlane + uses: nick-fields/retry@v2 + with: + timeout_minutes: 1 + polling_interval_seconds: 5 + max_attempts: 10 + shell: bash + command: docker exec bindplane /bindplane get agent + - name: Run BindPlane Action # This should be replaced with a release action. # /@ From 60f48ed53cda6017830b271331486dbb2dd42e7f Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:17:33 -0500 Subject: [PATCH 18/65] try setting bindplane host --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1adbd36..50c45b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: docker run \ -d \ --name bindplane \ + -e BINDPLANE_HOST=0.0.0.0 \ + -e BINDPLANE_PORT=3001 \ -e BINDPLANE_USERNAME=admin \ -e BINDPLANE_PASSWORD=admin \ -e BINDPLANE_REMOTE_URL=http://localhost:3001 \ From 9135e209b5344a6de2aa75b1f256947781597199 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:21:06 -0500 Subject: [PATCH 19/65] detect runner ip --- .github/workflows/ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50c45b2..e80c178 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,10 @@ jobs: with: fetch-depth: 0 + - name: Detect Runner IP + id: ip + uses: haythem/public-ip@v1.2 + - name: Pull BindPlane run: docker pull ghcr.io/observiq/bindplane-ee:${{ matrix.bindplane_versions }} @@ -44,11 +48,9 @@ jobs: docker run \ -d \ --name bindplane \ - -e BINDPLANE_HOST=0.0.0.0 \ - -e BINDPLANE_PORT=3001 \ -e BINDPLANE_USERNAME=admin \ -e BINDPLANE_PASSWORD=admin \ - -e BINDPLANE_REMOTE_URL=http://localhost:3001 \ + -e BINDPLANE_REMOTE_URL=http://${{ steps.ip.outputs.ipv4 }}:3001 \ -e BINDPLANE_SESSION_SECRET=2c23c9d3-850f-4062-a5c8-3f9b814ae144 \ -e BINDPLANE_SECRET_KEY=8a5353f7-bbf4-4eea-846d-a6d54296b781 \ -e BINDPLANE_LOG_OUTPUT=stdout \ @@ -75,9 +77,11 @@ jobs: # this action should always use secrets when passing in the remote # url, username, password or api key. # - # Remote url will never be localhost when running this action. In this - # case, bindplane is literally running within the runner instance. - remote_url: http://localhost:3001 + # Remote url will never be localhost when running this action. The action + # executes in a container and localhost will always be the container's network + # and not the network of the bindplane instance, even if that instance + # is running within this runner. + remote_url: http://${{ steps.ip.outputs.ipv4 }}:3001 username: admin password: admin # Testing tailing and non trailing slash From e0db5acd3c5098b12e93d6a153d1336bb8905e1d Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:27:57 -0500 Subject: [PATCH 20/65] detect internal ip --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e80c178..687d0ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,10 @@ jobs: fetch-depth: 0 - name: Detect Runner IP - id: ip - uses: haythem/public-ip@v1.2 + run: echo "MAIN_IP=$(ip addr show | grep 'inet ' | grep -v '127.0.0.1' | awk '{print $2}' | cut -f1 -d'/')" >> $GITHUB_ENV + + - name: Print Runner IP + run: echo $MAIN_IP - name: Pull BindPlane run: docker pull ghcr.io/observiq/bindplane-ee:${{ matrix.bindplane_versions }} @@ -50,7 +52,7 @@ jobs: --name bindplane \ -e BINDPLANE_USERNAME=admin \ -e BINDPLANE_PASSWORD=admin \ - -e BINDPLANE_REMOTE_URL=http://${{ steps.ip.outputs.ipv4 }}:3001 \ + -e BINDPLANE_REMOTE_URL=http://${MAIN_IP}:3001 \ -e BINDPLANE_SESSION_SECRET=2c23c9d3-850f-4062-a5c8-3f9b814ae144 \ -e BINDPLANE_SECRET_KEY=8a5353f7-bbf4-4eea-846d-a6d54296b781 \ -e BINDPLANE_LOG_OUTPUT=stdout \ @@ -81,7 +83,7 @@ jobs: # executes in a container and localhost will always be the container's network # and not the network of the bindplane instance, even if that instance # is running within this runner. - remote_url: http://${{ steps.ip.outputs.ipv4 }}:3001 + remote_url: http://${{ env.MAIN_IP }}:3001 username: admin password: admin # Testing tailing and non trailing slash From d9f9f814a9c2f122e10d936ac375667909cdbcc4 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:35:35 -0500 Subject: [PATCH 21/65] debug --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 687d0ff..24bddca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: fetch-depth: 0 - name: Detect Runner IP - run: echo "MAIN_IP=$(ip addr show | grep 'inet ' | grep -v '127.0.0.1' | awk '{print $2}' | cut -f1 -d'/')" >> $GITHUB_ENV + run: ip addr show | grep 'inet ' | grep -v '127.0.0.1' | awk '{print $2}' | cut -f1 -d'/' - name: Print Runner IP run: echo $MAIN_IP From 9d59f6b6d8d2a124c64e9431df93b1558494a76b Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:51:57 -0500 Subject: [PATCH 22/65] use first addr --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24bddca..5345406 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: fetch-depth: 0 - name: Detect Runner IP - run: ip addr show | grep 'inet ' | grep -v '127.0.0.1' | awk '{print $2}' | cut -f1 -d'/' + run: echo "MAIN_IP=$(ip addr show | grep 'inet ' | grep -v '127.0.0.1' | awk '{print $2}' | cut -f1 -d'/' | head -n 1)" >> $GITHUB_ENV - name: Print Runner IP run: echo $MAIN_IP From c3a45f39f00336f684b6e64a1a9285a65ff5a3b4 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 15:54:07 -0500 Subject: [PATCH 23/65] exec into container --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5345406..937c308 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,8 +93,8 @@ jobs: - name: Get Resources if: always() run: | - bindplane get destinations - bindplane get configurations + docker exec bindplane /bindplane get destinations + docker exec bindplane /bindplane get configurations - name: Debug Container Logs if: always() From 34311b8be1323c79a8192e2dcc0bc1ba0688c8ee Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:00:21 -0500 Subject: [PATCH 24/65] test additional version --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 937c308..d532b27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: matrix: bindplane_versions: - 1.40.0 + - 1.37.0 steps: - name: Checkout uses: actions/checkout@v4 From 00e4c3f54c7436ab95c907d3141c3fa6618e6942 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:11:23 -0500 Subject: [PATCH 25/65] try write back without commit --- .github/workflows/ci.yml | 5 +++++ action.yml | 6 ++++++ entrypoint.sh | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d532b27..13444c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: push: branches: - main + - dev + paths: + - 'test/resources/**' permissions: # Allow action to write raw configs back to the repository. @@ -90,6 +93,8 @@ jobs: # Testing tailing and non trailing slash destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml + configuration_output_path: test/otel/output-${{ matrix.bindplane_versions }}.yaml + configuration_output_branch: dev - name: Get Resources if: always() diff --git a/action.yml b/action.yml index 69a3fcb..4fbf88d 100644 --- a/action.yml +++ b/action.yml @@ -16,6 +16,10 @@ inputs: description: 'Path to the file which contains the BindPlane destination resources' configuration_path: description: 'Path to the file which contains the BindPlane configuration resources' + configuration_output_path: + description: 'Path to the file which will contain the rendered OTEL format of the configuration resources' + configuration_output_branch: + description: 'The branch to commit the rendered OTEL format of the configuration resources to. This branch must also be the branch the action is running on' runs: using: 'docker' @@ -27,3 +31,5 @@ runs: - ${{ inputs.password }} - ${{ inputs.destination_path }} - ${{ inputs.configuration_path }} + - ${{ inputs.configuration_output_path }} + - ${{ inputs.configuration_output_branch }} diff --git a/entrypoint.sh b/entrypoint.sh index a5f1f3d..f97da72 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,6 +8,8 @@ USERNAME=$3 PASSWORD=$4 DESTINATION_PATH=$5 CONFIG_PATH=$6 +OUTPUT_PATH=$7 +OUTPUT_BRANCH=$8 install_bindplane_cli() { curl -Ls \ @@ -52,6 +54,14 @@ validate() { bindplane profile use "action" } +write_back() { + mkdir tmp + for config in $(bindplane get config | awk 'NR>1 {print $1}'); do + bindplane get config "$config" -o raw > "tmp/$config.yaml" + done + ls tmp +} + install_bindplane_cli validate # Apply will apply resources in the correct order. Re-usable @@ -59,3 +69,4 @@ validate # a configuration. bindplane apply "$DESTINATION_PATH" bindplane apply "$CONFIG_PATH" +write_back From 8366507a39c872b44e76821ba058686976df2b8e Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:15:49 -0500 Subject: [PATCH 26/65] test write back --- entrypoint.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index f97da72..d1043e8 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,6 +11,9 @@ CONFIG_PATH=$6 OUTPUT_PATH=$7 OUTPUT_BRANCH=$8 +BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/}) +echo "Current branch is $BRANCH_NAME" + install_bindplane_cli() { curl -Ls \ -o bindplane.zip \ @@ -55,6 +58,16 @@ validate() { } write_back() { + # If output branch matches current branch, write back to repo + if [ "$BRANCH_NAME" == "$OUTPUT_BRANCH" ]; then + echo "Writing back to repo" + # TODO, move logic here + else + echo "Skipping repo write. Current branch ${BRANCH_NAME} does not match output branch ${OUTPUT_BRANCH}." + fi + + # TODO handle output path dir checks + mkdir tmp for config in $(bindplane get config | awk 'NR>1 {print $1}'); do bindplane get config "$config" -o raw > "tmp/$config.yaml" From 7bede5cbc38fc52c68856425ad6feff6ab89286f Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:22:05 -0500 Subject: [PATCH 27/65] ci From fbbcc9dfc42f0e7ac2edc402de6b8b02e46e7be1 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:22:36 -0500 Subject: [PATCH 28/65] test write back --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13444c5..0d2f081 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,8 @@ on: branches: - main - dev - paths: - - 'test/resources/**' + # paths: + # - 'test/resources/**' permissions: # Allow action to write raw configs back to the repository. From 497c7de91d5984a57aae5b90e13bcad3d4c308b7 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:24:09 -0500 Subject: [PATCH 29/65] test write back --- entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index d1043e8..9c4f49c 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -83,3 +83,5 @@ validate bindplane apply "$DESTINATION_PATH" bindplane apply "$CONFIG_PATH" write_back + +git status From 6907d650eab11eb8dfab1eca41e1829bec8bd314 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:29:35 -0500 Subject: [PATCH 30/65] test write back --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a38e123..b0c2834 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Container image that runs your code FROM alpine:3.10 -RUN apk add --no-cache bash curl +RUN apk add --no-cache bash curl git COPY --chmod=0755 entrypoint.sh /entrypoint.sh From 4044089a115f8fc89a28b904099f5c2e24c5e7de Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:35:30 -0500 Subject: [PATCH 31/65] try write back --- .github/workflows/ci.yml | 2 +- action.yml | 6 +++--- entrypoint.sh | 28 ++++++++++++++++------------ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d2f081..b67f254 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Testing tailing and non trailing slash destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml - configuration_output_path: test/otel/output-${{ matrix.bindplane_versions }}.yaml + configuration_output_dir: test/otel/{{ matrix.bindplane_versions }} configuration_output_branch: dev - name: Get Resources diff --git a/action.yml b/action.yml index 4fbf88d..c690996 100644 --- a/action.yml +++ b/action.yml @@ -16,8 +16,8 @@ inputs: description: 'Path to the file which contains the BindPlane destination resources' configuration_path: description: 'Path to the file which contains the BindPlane configuration resources' - configuration_output_path: - description: 'Path to the file which will contain the rendered OTEL format of the configuration resources' + configuration_output_dir: + description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' configuration_output_branch: description: 'The branch to commit the rendered OTEL format of the configuration resources to. This branch must also be the branch the action is running on' @@ -31,5 +31,5 @@ runs: - ${{ inputs.password }} - ${{ inputs.destination_path }} - ${{ inputs.configuration_path }} - - ${{ inputs.configuration_output_path }} + - ${{ inputs.configuration_output_dir }} - ${{ inputs.configuration_output_branch }} diff --git a/entrypoint.sh b/entrypoint.sh index 9c4f49c..4882840 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,7 +8,7 @@ USERNAME=$3 PASSWORD=$4 DESTINATION_PATH=$5 CONFIG_PATH=$6 -OUTPUT_PATH=$7 +OUTPUT_DIR=$7 OUTPUT_BRANCH=$8 BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/}) @@ -51,28 +51,33 @@ validate() { exit 1 elif [ -n "$API_KEY" ]; then profile_args="$profile_args --api-key $API_KEY" - fi + fi + + if [ -n "$OUTPUT_BRANCH" ] && [ -z "$OUTPUT_DIR" ]; then + echo "OUTPUT_DIR is required when OUTPUT_BRANCH is set." + exit 1 + fi eval bindplane profile set "action" "$profile_args" bindplane profile use "action" } write_back() { - # If output branch matches current branch, write back to repo - if [ "$BRANCH_NAME" == "$OUTPUT_BRANCH" ]; then - echo "Writing back to repo" - # TODO, move logic here - else + if [ "$BRANCH_NAME" != "$OUTPUT_BRANCH" ]; then echo "Skipping repo write. Current branch ${BRANCH_NAME} does not match output branch ${OUTPUT_BRANCH}." + return fi - # TODO handle output path dir checks + mkdir -p "$OUTPUT_DIR" - mkdir tmp for config in $(bindplane get config | awk 'NR>1 {print $1}'); do - bindplane get config "$config" -o raw > "tmp/$config.yaml" + out_file="$OUTPUT_DIR/$config.yaml" + bindplane get config "$config" -o raw > "$out_file" + git add "$out_file" done - ls tmp + + git commit -m "BindPlane OP Action: Update OTEL Configs" + git push } install_bindplane_cli @@ -84,4 +89,3 @@ bindplane apply "$DESTINATION_PATH" bindplane apply "$CONFIG_PATH" write_back -git status From 0e13d4b1d88b5eec8aecb1c62bfbc9421053c4a1 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:43:29 -0500 Subject: [PATCH 32/65] add github auth --- .github/workflows/ci.yml | 2 ++ action.yml | 3 +++ entrypoint.sh | 33 ++++++++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b67f254..6ec15b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,8 @@ jobs: configuration_path: test/resources/configurations/resource.yaml configuration_output_dir: test/otel/{{ matrix.bindplane_versions }} configuration_output_branch: dev + # Token should have contents: write permissions + token: ${{ secrets.GITHUB_TOKEN }} - name: Get Resources if: always() diff --git a/action.yml b/action.yml index c690996..efeee17 100644 --- a/action.yml +++ b/action.yml @@ -20,6 +20,8 @@ inputs: description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' configuration_output_branch: description: 'The branch to commit the rendered OTEL format of the configuration resources to. This branch must also be the branch the action is running on' + token: + description: 'The GitHub token used to authenticate to GitHub when writing OTEL configs back to the repo' runs: using: 'docker' @@ -33,3 +35,4 @@ runs: - ${{ inputs.configuration_path }} - ${{ inputs.configuration_output_dir }} - ${{ inputs.configuration_output_branch }} + - ${{ inputs.token }} diff --git a/entrypoint.sh b/entrypoint.sh index 4882840..d23806f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -10,8 +10,9 @@ DESTINATION_PATH=$5 CONFIG_PATH=$6 OUTPUT_DIR=$7 OUTPUT_BRANCH=$8 +GITHUB_TOKEN=$9 -BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/}) +BRANCH_NAME=${GITHUB_REF#refs/heads/} echo "Current branch is $BRANCH_NAME" install_bindplane_cli() { @@ -53,9 +54,29 @@ validate() { profile_args="$profile_args --api-key $API_KEY" fi - if [ -n "$OUTPUT_BRANCH" ] && [ -z "$OUTPUT_DIR" ]; then - echo "OUTPUT_DIR is required when OUTPUT_BRANCH is set." - exit 1 + if [ -n "$OUTPUT_BRANCH" ]; then + if [ -z "$OUTPUT_DIR" ]; then + echo "OUTPUT_DIR is required when OUTPUT_BRANCH is set." + exit 1 + fi + + if [ -z "$GITHUB_TOKEN" ]; then + echo "GITHUB_TOKEN is required when OUTPUT_BRANCH is set." + exit 1 + fi + + # GITHUB_ACTOR and GITHUB_REPOSITORY are set by the github actions runtime + # but we do expect them to be set. + + if [ -z "$GITHUB_ACTOR" ]; then + echo "GITHUB_ACTOR is required when OUTPUT_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." + exit 1 + fi + + if [ -z "$GITHUB_REPOSITORY" ]; then + echo "GITHUB_REPOSITORY is required when OUTPUT_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." + exit 1 + fi fi eval bindplane profile set "action" "$profile_args" @@ -76,8 +97,10 @@ write_back() { git add "$out_file" done + git config --global user.email "bindplane-op-action" + git config --global user.name "bindplane-op-action" git commit -m "BindPlane OP Action: Update OTEL Configs" - git push + git push "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" "HEAD:$OUTPUT_BRANCH" } install_bindplane_cli From de9b4a2f27c96c1d349188a3347f8927577154ad Mon Sep 17 00:00:00 2001 From: bindplane-op-action Date: Mon, 22 Jan 2024 21:43:56 +0000 Subject: [PATCH 33/65] BindPlane OP Action: Update OTEL Configs --- .../k8s-cluster.yaml | 78 ++++++++ .../k8s-gateway.yaml | 176 ++++++++++++++++++ .../k8s-node.yaml | 85 +++++++++ 3 files changed, 339 insertions(+) create mode 100644 test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml create mode 100644 test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml create mode 100644 test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml diff --git a/test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml b/test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml new file mode 100644 index 0000000..8ae61a5 --- /dev/null +++ b/test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml @@ -0,0 +1,78 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-cluster:1 +receivers: + k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + allocatable_types_to_report: + - cpu + - memory + - ephemeral-storage + - storage + auth_type: serviceAccount + collection_interval: 60s + distribution: kubernetes + node_conditions_to_report: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + auth_type: serviceAccount + namespaces: + - kube-system + - production +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: + receivers: + - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + processors: + - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: + receivers: + - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + processors: + - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + diff --git a/test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml b/test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml new file mode 100644 index 0000000..55193b1 --- /dev/null +++ b/test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml @@ -0,0 +1,176 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-gateway:1 +receivers: + otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + keepalive: + server_parameters: + max_connection_age: 60s + max_connection_age_grace: 300s + max_connection_idle: 60s + time: 2h + timeout: 20s + http: + endpoint: 0.0.0.0:4318 +processors: + resourcedetection/coralogix: + detectors: + - system + override: false + system: + hostname_sources: + - os +exporters: + coralogix/coralogix__logs: + application_name: bindplane-test + logs: + compression: gzip + endpoint: otel-logs.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__metrics: + application_name: bindplane-test + metrics: + compression: gzip + endpoint: otel-metrics.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__traces: + application_name: bindplane-test + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + traces: + compression: gzip + endpoint: otel-traces.coralogix.us:443 + elasticsearch/elasticsearch-internal__logs: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + logs_index: logs-generic-default + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + elasticsearch/elasticsearch-internal__traces: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + traces_index: traces-generic-default + prometheusremotewrite/prometheus-internal: + add_metric_suffixes: true + compression: gzip + endpoint: http://prom-prod.corp.net:9009//api/v1/write + resource_to_telemetry_conversion: + enabled: true + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s +extensions: + file_storage/coralogix: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + file_storage/elasticsearch-internal: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/coralogix + - file_storage/elasticsearch-internal + pipelines: + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__logs + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__logs + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__metrics + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - prometheusremotewrite/prometheus-internal + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__traces + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__traces + diff --git a/test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml b/test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml new file mode 100644 index 0000000..52883fa --- /dev/null +++ b/test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml @@ -0,0 +1,85 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-node:1 +receivers: + kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + auth_type: serviceAccount + collection_interval: 60s + endpoint: ${KUBE_NODE_NAME}:10250 + insecure_skip_verify: true + metric_groups: + - node + - pod + - container + - volume + plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: + parameters: + body_json_parsing: true + exclude_file_log_path: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + log_paths: + - /var/log/containers/*.log + log_source: file + start_at: end + path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: + receivers: + - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container + processors: + - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: + receivers: + - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + processors: + - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + From 237fe4406ebc128607d9d3150390c05a0f888378 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:48:48 -0500 Subject: [PATCH 34/65] clone repo --- .github/workflows/ci.yml | 2 +- entrypoint.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ec15b4..3398166 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Testing tailing and non trailing slash destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml - configuration_output_dir: test/otel/{{ matrix.bindplane_versions }} + configuration_output_dir: test/otel/${{ matrix.bindplane_versions }} configuration_output_branch: dev # Token should have contents: write permissions token: ${{ secrets.GITHUB_TOKEN }} diff --git a/entrypoint.sh b/entrypoint.sh index d23806f..4eee4a1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -89,10 +89,12 @@ write_back() { return fi - mkdir -p "$OUTPUT_DIR" + git clone https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git + + mkdir -p "$GITHUB_REPOSITORY/$OUTPUT_DIR" for config in $(bindplane get config | awk 'NR>1 {print $1}'); do - out_file="$OUTPUT_DIR/$config.yaml" + out_file="$GITHUB_REPOSITORY/$OUTPUT_DIR/$config.yaml" bindplane get config "$config" -o raw > "$out_file" git add "$out_file" done From d23dbd0681c8e13195d5d89dc5c640ada19eac88 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:50:49 -0500 Subject: [PATCH 35/65] try again --- entrypoint.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 4eee4a1..3923215 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -89,7 +89,12 @@ write_back() { return fi - git clone https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git + # Clone the repo on the current branch + # and use depth 1 to avoid cloning the entire history. + git clone \ + --depth 1 \ + --branch "$BRANCH_NAME" \ + https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git mkdir -p "$GITHUB_REPOSITORY/$OUTPUT_DIR" @@ -102,7 +107,7 @@ write_back() { git config --global user.email "bindplane-op-action" git config --global user.name "bindplane-op-action" git commit -m "BindPlane OP Action: Update OTEL Configs" - git push "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" "HEAD:$OUTPUT_BRANCH" + git push "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" } install_bindplane_cli From 45a79d467c0a70db86aca29385f92e893816ca7d Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:51:21 -0500 Subject: [PATCH 36/65] latest --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 932daf2..87e3787 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,18 @@ bindplane get configuration -o yaml --export > configuration.yaml ### Workflow ```yaml -name: goreleaser +name: CI +# When raw config write back is configured, it is important +# to run this workflow only when changes to the resources +# are detected. This can prevent a CI infinite loop. on: - pull_request: push: + branches: + - main + paths: + - 'test/resources/**' + permissions: contents: write From a470a35cce3afc71c3de11c0cef65bd96904fa6c Mon Sep 17 00:00:00 2001 From: bindplane-op-action Date: Mon, 22 Jan 2024 21:52:05 +0000 Subject: [PATCH 37/65] BindPlane OP Action: Update OTEL Configs --- .../test/otel/1.37.0/k8s-cluster.yaml | 78 ++++++++ .../test/otel/1.37.0/k8s-gateway.yaml | 176 ++++++++++++++++++ .../test/otel/1.37.0/k8s-node.yaml | 85 +++++++++ 3 files changed, 339 insertions(+) create mode 100644 jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml create mode 100644 jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml create mode 100644 jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml diff --git a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml new file mode 100644 index 0000000..8ae61a5 --- /dev/null +++ b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml @@ -0,0 +1,78 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-cluster:1 +receivers: + k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + allocatable_types_to_report: + - cpu + - memory + - ephemeral-storage + - storage + auth_type: serviceAccount + collection_interval: 60s + distribution: kubernetes + node_conditions_to_report: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + auth_type: serviceAccount + namespaces: + - kube-system + - production +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: + receivers: + - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + processors: + - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: + receivers: + - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + processors: + - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + diff --git a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml new file mode 100644 index 0000000..55193b1 --- /dev/null +++ b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml @@ -0,0 +1,176 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-gateway:1 +receivers: + otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + keepalive: + server_parameters: + max_connection_age: 60s + max_connection_age_grace: 300s + max_connection_idle: 60s + time: 2h + timeout: 20s + http: + endpoint: 0.0.0.0:4318 +processors: + resourcedetection/coralogix: + detectors: + - system + override: false + system: + hostname_sources: + - os +exporters: + coralogix/coralogix__logs: + application_name: bindplane-test + logs: + compression: gzip + endpoint: otel-logs.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__metrics: + application_name: bindplane-test + metrics: + compression: gzip + endpoint: otel-metrics.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__traces: + application_name: bindplane-test + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + traces: + compression: gzip + endpoint: otel-traces.coralogix.us:443 + elasticsearch/elasticsearch-internal__logs: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + logs_index: logs-generic-default + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + elasticsearch/elasticsearch-internal__traces: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + traces_index: traces-generic-default + prometheusremotewrite/prometheus-internal: + add_metric_suffixes: true + compression: gzip + endpoint: http://prom-prod.corp.net:9009//api/v1/write + resource_to_telemetry_conversion: + enabled: true + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s +extensions: + file_storage/coralogix: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + file_storage/elasticsearch-internal: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/coralogix + - file_storage/elasticsearch-internal + pipelines: + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__logs + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__logs + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__metrics + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - prometheusremotewrite/prometheus-internal + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__traces + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__traces + diff --git a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml new file mode 100644 index 0000000..52883fa --- /dev/null +++ b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml @@ -0,0 +1,85 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-node:1 +receivers: + kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + auth_type: serviceAccount + collection_interval: 60s + endpoint: ${KUBE_NODE_NAME}:10250 + insecure_skip_verify: true + metric_groups: + - node + - pod + - container + - volume + plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: + parameters: + body_json_parsing: true + exclude_file_log_path: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + log_paths: + - /var/log/containers/*.log + log_source: file + start_at: end + path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: + receivers: + - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container + processors: + - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: + receivers: + - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + processors: + - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + From 3170b22857d6080629d12aedca6d30e8be006a91 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:51:56 -0500 Subject: [PATCH 38/65] shellcheck --- entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 3923215..b604dcd 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -94,7 +94,7 @@ write_back() { git clone \ --depth 1 \ --branch "$BRANCH_NAME" \ - https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git + https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git mkdir -p "$GITHUB_REPOSITORY/$OUTPUT_DIR" @@ -107,7 +107,7 @@ write_back() { git config --global user.email "bindplane-op-action" git config --global user.name "bindplane-op-action" git commit -m "BindPlane OP Action: Update OTEL Configs" - git push "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" + git push } install_bindplane_cli From 51b1fce65b1859b4ca5f0d115fd870f0bec023bd Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:52:49 -0500 Subject: [PATCH 39/65] prevent out of order commits --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3398166..35709a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,11 @@ permissions: # Allow action to write raw configs back to the repository. contents: write +# Run commits in order to prevent out of order write back commits. +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: false + jobs: shellcheck: runs-on: ubuntu-latest From 0b30f5c1084dad9be5f59703f6b4ecf911dc053a Mon Sep 17 00:00:00 2001 From: bindplane-op-action Date: Mon, 22 Jan 2024 21:53:20 +0000 Subject: [PATCH 40/65] BindPlane OP Action: Update OTEL Configs --- .../test/otel/1.40.0/k8s-cluster.yaml | 78 ++++++++ .../test/otel/1.40.0/k8s-gateway.yaml | 176 ++++++++++++++++++ .../test/otel/1.40.0/k8s-node.yaml | 85 +++++++++ 3 files changed, 339 insertions(+) create mode 100644 jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml create mode 100644 jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml create mode 100644 jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml diff --git a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml new file mode 100644 index 0000000..c400d08 --- /dev/null +++ b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml @@ -0,0 +1,78 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-cluster:1 +receivers: + k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + allocatable_types_to_report: + - cpu + - memory + - ephemeral-storage + - storage + auth_type: serviceAccount + collection_interval: 1m0s + distribution: kubernetes + node_conditions_to_report: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + auth_type: serviceAccount + namespaces: + - kube-system + - production +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: + receivers: + - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + processors: + - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: + receivers: + - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + processors: + - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + diff --git a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml new file mode 100644 index 0000000..ab35830 --- /dev/null +++ b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml @@ -0,0 +1,176 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-gateway:1 +receivers: + otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + keepalive: + server_parameters: + max_connection_age: 60s + max_connection_age_grace: 300s + max_connection_idle: 60s + time: 2h + timeout: 20s + http: + endpoint: 0.0.0.0:4318 +processors: + resourcedetection/coralogix: + detectors: + - system + override: false + system: + hostname_sources: + - os +exporters: + coralogix/coralogix__logs: + application_name: bindplane-test + logs: + compression: gzip + endpoint: otel-logs.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__metrics: + application_name: bindplane-test + metrics: + compression: gzip + endpoint: otel-metrics.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__traces: + application_name: bindplane-test + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + traces: + compression: gzip + endpoint: otel-traces.coralogix.us:443 + elasticsearch/elasticsearch-internal__logs: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + logs_index: logs-generic-default + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + elasticsearch/elasticsearch-internal__traces: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + traces_index: traces-generic-default + prometheusremotewrite/prometheus-internal: + add_metric_suffixes: true + compression: gzip + endpoint: http://prom-prod.corp.net:9009/api/v1/write + resource_to_telemetry_conversion: + enabled: true + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s +extensions: + file_storage/coralogix: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + file_storage/elasticsearch-internal: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/coralogix + - file_storage/elasticsearch-internal + pipelines: + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__logs + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__logs + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__metrics + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - prometheusremotewrite/prometheus-internal + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__traces + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__traces + diff --git a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml new file mode 100644 index 0000000..536d39c --- /dev/null +++ b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml @@ -0,0 +1,85 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-node:1 +receivers: + kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + auth_type: serviceAccount + collection_interval: 1m0s + endpoint: ${KUBE_NODE_NAME}:10250 + insecure_skip_verify: true + metric_groups: + - node + - pod + - container + - volume + plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: + parameters: + body_json_parsing: true + exclude_file_log_path: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + log_paths: + - /var/log/containers/*.log + log_source: file + start_at: end + path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: + receivers: + - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container + processors: + - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: + receivers: + - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + processors: + - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + From a6fb59a85c4d564e1a8b30814f100b19b44a5395 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:53:55 -0500 Subject: [PATCH 41/65] shellcheck --- entrypoint.sh | 2 +- .../k8s-cluster.yaml | 78 -------- .../k8s-gateway.yaml | 176 ------------------ .../k8s-node.yaml | 85 --------- 4 files changed, 1 insertion(+), 340 deletions(-) delete mode 100644 test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml delete mode 100644 test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml delete mode 100644 test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml diff --git a/entrypoint.sh b/entrypoint.sh index b604dcd..d7986eb 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -94,7 +94,7 @@ write_back() { git clone \ --depth 1 \ --branch "$BRANCH_NAME" \ - https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git + "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" mkdir -p "$GITHUB_REPOSITORY/$OUTPUT_DIR" diff --git a/test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml b/test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml deleted file mode 100644 index 8ae61a5..0000000 --- a/test/otel/{{ matrix.bindplane_versions }}/k8s-cluster.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-cluster:1 -receivers: - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: - allocatable_types_to_report: - - cpu - - memory - - ephemeral-storage - - storage - auth_type: serviceAccount - collection_interval: 60s - distribution: kubernetes - node_conditions_to_report: - - Ready - - DiskPressure - - MemoryPressure - - PIDPressure - - NetworkUnavailable - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: - auth_type: serviceAccount - namespaces: - - kube-system - - production -processors: - batch/bindplane-gateway-agent: null - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube -exporters: - otlp/bindplane-gateway-agent: - compression: gzip - endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/bindplane-gateway-agent - tls: - insecure: true -extensions: - file_storage/bindplane-gateway-agent: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/bindplane-gateway-agent - pipelines: - logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: - receivers: - - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC - processors: - - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: - receivers: - - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 - processors: - - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - diff --git a/test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml b/test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml deleted file mode 100644 index 55193b1..0000000 --- a/test/otel/{{ matrix.bindplane_versions }}/k8s-gateway.yaml +++ /dev/null @@ -1,176 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-gateway:1 -receivers: - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - keepalive: - server_parameters: - max_connection_age: 60s - max_connection_age_grace: 300s - max_connection_idle: 60s - time: 2h - timeout: 20s - http: - endpoint: 0.0.0.0:4318 -processors: - resourcedetection/coralogix: - detectors: - - system - override: false - system: - hostname_sources: - - os -exporters: - coralogix/coralogix__logs: - application_name: bindplane-test - logs: - compression: gzip - endpoint: otel-logs.coralogix.us:443 - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - coralogix/coralogix__metrics: - application_name: bindplane-test - metrics: - compression: gzip - endpoint: otel-metrics.coralogix.us:443 - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - coralogix/coralogix__traces: - application_name: bindplane-test - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - traces: - compression: gzip - endpoint: otel-traces.coralogix.us:443 - elasticsearch/elasticsearch-internal__logs: - endpoints: - - https://es-0.corp.net:9200 - - https://es-1.corp.net:9200 - logs_index: logs-generic-default - num_workers: 0 - pipeline: null - retry: - enabled: true - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/elasticsearch-internal - tls: null - elasticsearch/elasticsearch-internal__traces: - endpoints: - - https://es-0.corp.net:9200 - - https://es-1.corp.net:9200 - num_workers: 0 - pipeline: null - retry: - enabled: true - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/elasticsearch-internal - tls: null - traces_index: traces-generic-default - prometheusremotewrite/prometheus-internal: - add_metric_suffixes: true - compression: gzip - endpoint: http://prom-prod.corp.net:9009//api/v1/write - resource_to_telemetry_conversion: - enabled: true - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s -extensions: - file_storage/coralogix: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - file_storage/elasticsearch-internal: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/coralogix - - file_storage/elasticsearch-internal - pipelines: - logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__logs - logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - elasticsearch/elasticsearch-internal__logs - metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__metrics - metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - prometheusremotewrite/prometheus-internal - traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__traces - traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - elasticsearch/elasticsearch-internal__traces - diff --git a/test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml b/test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml deleted file mode 100644 index 52883fa..0000000 --- a/test/otel/{{ matrix.bindplane_versions }}/k8s-node.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-node:1 -receivers: - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: - auth_type: serviceAccount - collection_interval: 60s - endpoint: ${KUBE_NODE_NAME}:10250 - insecure_skip_verify: true - metric_groups: - - node - - pod - - container - - volume - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: - parameters: - body_json_parsing: true - exclude_file_log_path: - - /var/log/containers/observiq-*-collector-* - - /var/log/containers/bindplane-*-agent-* - log_paths: - - /var/log/containers/*.log - log_source: file - start_at: end - path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml -processors: - batch/bindplane-gateway-agent: null - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - - action: insert - key: k8s.node.name - value: ${KUBE_NODE_NAME} - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - - action: insert - key: k8s.node.name - value: ${KUBE_NODE_NAME} -exporters: - otlp/bindplane-gateway-agent: - compression: gzip - endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/bindplane-gateway-agent - tls: - insecure: true -extensions: - file_storage/bindplane-gateway-agent: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/bindplane-gateway-agent - pipelines: - logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: - receivers: - - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container - processors: - - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: - receivers: - - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 - processors: - - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - From 0c39ec27a0e108c5d81f181538bd78c5c060436d Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:55:54 -0500 Subject: [PATCH 42/65] return early if nothing to commit --- entrypoint.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index d7986eb..03d1d97 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -96,7 +96,9 @@ write_back() { --branch "$BRANCH_NAME" \ "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - mkdir -p "$GITHUB_REPOSITORY/$OUTPUT_DIR" + cd "$GITHUB_REPOSITORY" + + mkdir -p "$OUTPUT_DIR" for config in $(bindplane get config | awk 'NR>1 {print $1}'); do out_file="$GITHUB_REPOSITORY/$OUTPUT_DIR/$config.yaml" @@ -104,6 +106,12 @@ write_back() { git add "$out_file" done + # check if git status is clean, return early + if [[ -z $(git status --porcelain) ]]; then + echo "No changes detected. Skipping commit." + return + fi + git config --global user.email "bindplane-op-action" git config --global user.name "bindplane-op-action" git commit -m "BindPlane OP Action: Update OTEL Configs" From a62cc353d64753df95b6388f40d19e972f999733 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:57:57 -0500 Subject: [PATCH 43/65] fix out path --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 03d1d97..525c5c1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -101,7 +101,7 @@ write_back() { mkdir -p "$OUTPUT_DIR" for config in $(bindplane get config | awk 'NR>1 {print $1}'); do - out_file="$GITHUB_REPOSITORY/$OUTPUT_DIR/$config.yaml" + out_file="$OUTPUT_DIR/$config.yaml" bindplane get config "$config" -o raw > "$out_file" git add "$out_file" done From b2d756614ebc16b1585c94faf2a2bff5fbd279ea Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 16:59:38 -0500 Subject: [PATCH 44/65] clone repo outside of orig context --- entrypoint.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 525c5c1..c0129bd 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -94,9 +94,10 @@ write_back() { git clone \ --depth 1 \ --branch "$BRANCH_NAME" \ - "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + ../out_repo - cd "$GITHUB_REPOSITORY" + cd "../out_repo" mkdir -p "$OUTPUT_DIR" From 1faa264c47992452c42365a02c111666cf19a665 Mon Sep 17 00:00:00 2001 From: bindplane-op-action Date: Mon, 22 Jan 2024 22:00:05 +0000 Subject: [PATCH 45/65] BindPlane OP Action: Update OTEL Configs --- test/otel/1.40.0/k8s-cluster.yaml | 78 +++++++++++++ test/otel/1.40.0/k8s-gateway.yaml | 176 ++++++++++++++++++++++++++++++ test/otel/1.40.0/k8s-node.yaml | 85 +++++++++++++++ 3 files changed, 339 insertions(+) create mode 100644 test/otel/1.40.0/k8s-cluster.yaml create mode 100644 test/otel/1.40.0/k8s-gateway.yaml create mode 100644 test/otel/1.40.0/k8s-node.yaml diff --git a/test/otel/1.40.0/k8s-cluster.yaml b/test/otel/1.40.0/k8s-cluster.yaml new file mode 100644 index 0000000..c400d08 --- /dev/null +++ b/test/otel/1.40.0/k8s-cluster.yaml @@ -0,0 +1,78 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-cluster:1 +receivers: + k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + allocatable_types_to_report: + - cpu + - memory + - ephemeral-storage + - storage + auth_type: serviceAccount + collection_interval: 1m0s + distribution: kubernetes + node_conditions_to_report: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + auth_type: serviceAccount + namespaces: + - kube-system + - production +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: + receivers: + - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + processors: + - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: + receivers: + - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + processors: + - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + diff --git a/test/otel/1.40.0/k8s-gateway.yaml b/test/otel/1.40.0/k8s-gateway.yaml new file mode 100644 index 0000000..ab35830 --- /dev/null +++ b/test/otel/1.40.0/k8s-gateway.yaml @@ -0,0 +1,176 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-gateway:1 +receivers: + otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + keepalive: + server_parameters: + max_connection_age: 60s + max_connection_age_grace: 300s + max_connection_idle: 60s + time: 2h + timeout: 20s + http: + endpoint: 0.0.0.0:4318 +processors: + resourcedetection/coralogix: + detectors: + - system + override: false + system: + hostname_sources: + - os +exporters: + coralogix/coralogix__logs: + application_name: bindplane-test + logs: + compression: gzip + endpoint: otel-logs.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__metrics: + application_name: bindplane-test + metrics: + compression: gzip + endpoint: otel-metrics.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__traces: + application_name: bindplane-test + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + traces: + compression: gzip + endpoint: otel-traces.coralogix.us:443 + elasticsearch/elasticsearch-internal__logs: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + logs_index: logs-generic-default + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + elasticsearch/elasticsearch-internal__traces: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + traces_index: traces-generic-default + prometheusremotewrite/prometheus-internal: + add_metric_suffixes: true + compression: gzip + endpoint: http://prom-prod.corp.net:9009/api/v1/write + resource_to_telemetry_conversion: + enabled: true + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s +extensions: + file_storage/coralogix: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + file_storage/elasticsearch-internal: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/coralogix + - file_storage/elasticsearch-internal + pipelines: + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__logs + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__logs + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__metrics + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - prometheusremotewrite/prometheus-internal + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__traces + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__traces + diff --git a/test/otel/1.40.0/k8s-node.yaml b/test/otel/1.40.0/k8s-node.yaml new file mode 100644 index 0000000..536d39c --- /dev/null +++ b/test/otel/1.40.0/k8s-node.yaml @@ -0,0 +1,85 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-node:1 +receivers: + kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + auth_type: serviceAccount + collection_interval: 1m0s + endpoint: ${KUBE_NODE_NAME}:10250 + insecure_skip_verify: true + metric_groups: + - node + - pod + - container + - volume + plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: + parameters: + body_json_parsing: true + exclude_file_log_path: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + log_paths: + - /var/log/containers/*.log + log_source: file + start_at: end + path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: + receivers: + - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container + processors: + - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: + receivers: + - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + processors: + - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + From f4b0905bd9d0fb91019d2be10c9e8e869ad8b9db Mon Sep 17 00:00:00 2001 From: bindplane-op-action Date: Mon, 22 Jan 2024 22:00:16 +0000 Subject: [PATCH 46/65] BindPlane OP Action: Update OTEL Configs --- test/otel/1.37.0/k8s-cluster.yaml | 78 +++++++++++++ test/otel/1.37.0/k8s-gateway.yaml | 176 ++++++++++++++++++++++++++++++ test/otel/1.37.0/k8s-node.yaml | 85 +++++++++++++++ 3 files changed, 339 insertions(+) create mode 100644 test/otel/1.37.0/k8s-cluster.yaml create mode 100644 test/otel/1.37.0/k8s-gateway.yaml create mode 100644 test/otel/1.37.0/k8s-node.yaml diff --git a/test/otel/1.37.0/k8s-cluster.yaml b/test/otel/1.37.0/k8s-cluster.yaml new file mode 100644 index 0000000..8ae61a5 --- /dev/null +++ b/test/otel/1.37.0/k8s-cluster.yaml @@ -0,0 +1,78 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-cluster:1 +receivers: + k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + allocatable_types_to_report: + - cpu + - memory + - ephemeral-storage + - storage + auth_type: serviceAccount + collection_interval: 60s + distribution: kubernetes + node_conditions_to_report: + - Ready + - DiskPressure + - MemoryPressure + - PIDPressure + - NetworkUnavailable + k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + auth_type: serviceAccount + namespaces: + - kube-system + - production +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: + receivers: + - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + processors: + - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: + receivers: + - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + processors: + - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + diff --git a/test/otel/1.37.0/k8s-gateway.yaml b/test/otel/1.37.0/k8s-gateway.yaml new file mode 100644 index 0000000..55193b1 --- /dev/null +++ b/test/otel/1.37.0/k8s-gateway.yaml @@ -0,0 +1,176 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-gateway:1 +receivers: + otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + keepalive: + server_parameters: + max_connection_age: 60s + max_connection_age_grace: 300s + max_connection_idle: 60s + time: 2h + timeout: 20s + http: + endpoint: 0.0.0.0:4318 +processors: + resourcedetection/coralogix: + detectors: + - system + override: false + system: + hostname_sources: + - os +exporters: + coralogix/coralogix__logs: + application_name: bindplane-test + logs: + compression: gzip + endpoint: otel-logs.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__metrics: + application_name: bindplane-test + metrics: + compression: gzip + endpoint: otel-metrics.coralogix.us:443 + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + coralogix/coralogix__traces: + application_name: bindplane-test + private_key: (sensitive) + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/coralogix + subsystem_name: test + timeout: 5s + traces: + compression: gzip + endpoint: otel-traces.coralogix.us:443 + elasticsearch/elasticsearch-internal__logs: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + logs_index: logs-generic-default + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + elasticsearch/elasticsearch-internal__traces: + endpoints: + - https://es-0.corp.net:9200 + - https://es-1.corp.net:9200 + num_workers: 0 + pipeline: null + retry: + enabled: true + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/elasticsearch-internal + tls: null + traces_index: traces-generic-default + prometheusremotewrite/prometheus-internal: + add_metric_suffixes: true + compression: gzip + endpoint: http://prom-prod.corp.net:9009//api/v1/write + resource_to_telemetry_conversion: + enabled: true + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s +extensions: + file_storage/coralogix: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + file_storage/elasticsearch-internal: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/coralogix + - file_storage/elasticsearch-internal + pipelines: + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__logs + logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__logs + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__metrics + metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - prometheusremotewrite/prometheus-internal + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: + - resourcedetection/coralogix + exporters: + - coralogix/coralogix__traces + traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: + receivers: + - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 + processors: [] + exporters: + - elasticsearch/elasticsearch-internal__traces + diff --git a/test/otel/1.37.0/k8s-node.yaml b/test/otel/1.37.0/k8s-node.yaml new file mode 100644 index 0000000..52883fa --- /dev/null +++ b/test/otel/1.37.0/k8s-node.yaml @@ -0,0 +1,85 @@ +# This configuration is managed by BindPlane OP. +# Configuration: k8s-node:1 +receivers: + kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + auth_type: serviceAccount + collection_interval: 60s + endpoint: ${KUBE_NODE_NAME}:10250 + insecure_skip_verify: true + metric_groups: + - node + - pod + - container + - volume + plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: + parameters: + body_json_parsing: true + exclude_file_log_path: + - /var/log/containers/observiq-*-collector-* + - /var/log/containers/bindplane-*-agent-* + log_paths: + - /var/log/containers/*.log + log_source: file + start_at: end + path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml +processors: + batch/bindplane-gateway-agent: null + resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: + attributes: + - action: insert + key: k8s.cluster.name + value: minikube + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} +exporters: + otlp/bindplane-gateway-agent: + compression: gzip + endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 300s + max_interval: 30s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 5000 + storage: file_storage/bindplane-gateway-agent + tls: + insecure: true +extensions: + file_storage/bindplane-gateway-agent: + compaction: + directory: $OIQ_OTEL_COLLECTOR_HOME/storage + on_rebound: true + directory: $OIQ_OTEL_COLLECTOR_HOME/storage +service: + extensions: + - file_storage/bindplane-gateway-agent + pipelines: + logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: + receivers: + - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container + processors: + - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: + receivers: + - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + processors: + - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 + - batch/bindplane-gateway-agent + exporters: + - otlp/bindplane-gateway-agent + From 3a1b309b0fa7c90f4ec00d7da0958d060b909e9c Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 17:00:48 -0500 Subject: [PATCH 47/65] comment --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35709a4..ecc7425 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + # This matrix allows us to test multiple bindplane versions. + # When writing back to the repo, we write to directories based + # on the bindplane version. bindplane_versions: - 1.40.0 - 1.37.0 From 05712c9eebf4d0adea14672070af3c4c3478fa8e Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 17:01:14 -0500 Subject: [PATCH 48/65] always run --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecc7425..c28b291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,6 @@ on: branches: - main - dev - # paths: - # - 'test/resources/**' permissions: # Allow action to write raw configs back to the repository. From ff1690590acfa9be1b3748cc892d7fafe2248d85 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 17:06:04 -0500 Subject: [PATCH 49/65] fix workflow example --- README.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 87e3787..be5181b 100644 --- a/README.md +++ b/README.md @@ -11,20 +11,17 @@ bindplane get configuration -o yaml --export > configuration.yaml ### Workflow +The following workflow can be used as an example. + ```yaml -name: CI +name: bindplane -# When raw config write back is configured, it is important -# to run this workflow only when changes to the resources -# are detected. This can prevent a CI infinite loop. on: push: branches: - main - paths: - - 'test/resources/**' - +# Write back requires access to the repo permissions: contents: write @@ -38,8 +35,16 @@ jobs: fetch-depth: 0 - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 + uses: jsirianni/bindplane-op-action@main with: remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} - api_key: ${{ secrets.BINDPLANE_API_KEY }} + username: ${{ secrets.BINDPLANE_USERNAME }} + password: ${{ secrets.BINDPLANE_PASSWORD }} + api_key: "" # Optional replacement for username and password + destination_path: test/resources/destinations/resource.yaml + configuration_path: test/resources/configurations/resource.yaml + # Write back requires these three options + configuration_output_dir: test/otel/${{ matrix.bindplane_versions }} + configuration_output_branch: dev + token: ${{ secrets.GITHUB_TOKEN }} ``` From 0f84df2d835b03af5f001427c647a053650b9ccd Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 17:12:33 -0500 Subject: [PATCH 50/65] fix rm --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index be5181b..34854f8 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ on: permissions: contents: write +# Run commits in order to prevent out of order write back commits. +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: false + jobs: goreleaser: runs-on: ubuntu-latest From 7c53c334816a882cd214369b26f910130bc059a4 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 17:17:26 -0500 Subject: [PATCH 51/65] skip apply if not on target branch --- .github/workflows/ci.yml | 3 +-- action.yml | 6 +++--- entrypoint.sh | 32 +++++++++++++++----------------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c28b291..6126682 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,11 +96,10 @@ jobs: remote_url: http://${{ env.MAIN_IP }}:3001 username: admin password: admin - # Testing tailing and non trailing slash destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml configuration_output_dir: test/otel/${{ matrix.bindplane_versions }} - configuration_output_branch: dev + target_branch: dev # Token should have contents: write permissions token: ${{ secrets.GITHUB_TOKEN }} diff --git a/action.yml b/action.yml index efeee17..34a538a 100644 --- a/action.yml +++ b/action.yml @@ -18,8 +18,8 @@ inputs: description: 'Path to the file which contains the BindPlane configuration resources' configuration_output_dir: description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' - configuration_output_branch: - description: 'The branch to commit the rendered OTEL format of the configuration resources to. This branch must also be the branch the action is running on' + target_branch: + description: 'Resource apply and OTEL config write back will only happen when this branch is the current branch of the action' token: description: 'The GitHub token used to authenticate to GitHub when writing OTEL configs back to the repo' @@ -34,5 +34,5 @@ runs: - ${{ inputs.destination_path }} - ${{ inputs.configuration_path }} - ${{ inputs.configuration_output_dir }} - - ${{ inputs.configuration_output_branch }} + - ${{ inputs.target_branch }} - ${{ inputs.token }} diff --git a/entrypoint.sh b/entrypoint.sh index c0129bd..38fd68f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,7 +9,7 @@ PASSWORD=$4 DESTINATION_PATH=$5 CONFIG_PATH=$6 OUTPUT_DIR=$7 -OUTPUT_BRANCH=$8 +TARGET_BRANCH=$8 GITHUB_TOKEN=$9 BRANCH_NAME=${GITHUB_REF#refs/heads/} @@ -54,14 +54,14 @@ validate() { profile_args="$profile_args --api-key $API_KEY" fi - if [ -n "$OUTPUT_BRANCH" ]; then + if [ -n "$TARGET_BRANCH" ]; then if [ -z "$OUTPUT_DIR" ]; then - echo "OUTPUT_DIR is required when OUTPUT_BRANCH is set." + echo "OUTPUT_DIR is required when TARGET_BRANCH is set." exit 1 fi if [ -z "$GITHUB_TOKEN" ]; then - echo "GITHUB_TOKEN is required when OUTPUT_BRANCH is set." + echo "GITHUB_TOKEN is required when TARGET_BRANCH is set." exit 1 fi @@ -69,12 +69,12 @@ validate() { # but we do expect them to be set. if [ -z "$GITHUB_ACTOR" ]; then - echo "GITHUB_ACTOR is required when OUTPUT_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." + echo "GITHUB_ACTOR is required when TARGET_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." exit 1 fi if [ -z "$GITHUB_REPOSITORY" ]; then - echo "GITHUB_REPOSITORY is required when OUTPUT_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." + echo "GITHUB_REPOSITORY is required when TARGET_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." exit 1 fi fi @@ -84,11 +84,6 @@ validate() { } write_back() { - if [ "$BRANCH_NAME" != "$OUTPUT_BRANCH" ]; then - echo "Skipping repo write. Current branch ${BRANCH_NAME} does not match output branch ${OUTPUT_BRANCH}." - return - fi - # Clone the repo on the current branch # and use depth 1 to avoid cloning the entire history. git clone \ @@ -121,10 +116,13 @@ write_back() { install_bindplane_cli validate -# Apply will apply resources in the correct order. Re-usable -# resources must exist before they can be referenced by -# a configuration. -bindplane apply "$DESTINATION_PATH" -bindplane apply "$CONFIG_PATH" -write_back + +if [ "$BRANCH_NAME" != "$TARGET_BRANCH" ]; then + echo "Skipping apply and repo write. Current branch ${BRANCH_NAME} does not match target branch ${TARGET_BRANCH}." +else + bindplane apply "$DESTINATION_PATH" + bindplane apply "$CONFIG_PATH" + write_back +fi + From 3a8a47e572f403980e9bcabf8f418402fafa11d0 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Mon, 22 Jan 2024 17:28:00 -0500 Subject: [PATCH 52/65] update org --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 34854f8..0358192 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,7 @@ jobs: with: fetch-depth: 0 - - name: Run GoReleaser - uses: jsirianni/bindplane-op-action@main + - uses: observIQ/bindplane-op-action@main with: remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} username: ${{ secrets.BINDPLANE_USERNAME }} From 0be28d31b87fb34fa19569e3478362b23db2dbd3 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 13:39:13 -0500 Subject: [PATCH 53/65] rename params --- .github/workflows/ci.yml | 8 ++--- README.md | 8 ++--- action.yml | 39 ++++++++++++++++++------ entrypoint.sh | 66 ++++++++++++++++++++-------------------- 4 files changed, 70 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6126682..fe70dec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,15 +87,15 @@ jobs: # These values are hardcode to match the test instance used by # this workflow. The instance does not persist. Consumers of # this action should always use secrets when passing in the remote - # url, username, password or api key. + # url, bindplane_username, bindplane_password or api key. # # Remote url will never be localhost when running this action. The action # executes in a container and localhost will always be the container's network # and not the network of the bindplane instance, even if that instance # is running within this runner. - remote_url: http://${{ env.MAIN_IP }}:3001 - username: admin - password: admin + bindplane_remote_url: http://${{ env.MAIN_IP }}:3001 + bindplane_username: admin + bindplane_password: admin destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml configuration_output_dir: test/otel/${{ matrix.bindplane_versions }} diff --git a/README.md b/README.md index 0358192..7eac15d 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,10 @@ jobs: - uses: observIQ/bindplane-op-action@main with: - remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} - username: ${{ secrets.BINDPLANE_USERNAME }} - password: ${{ secrets.BINDPLANE_PASSWORD }} - api_key: "" # Optional replacement for username and password + bindplane_remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} + bindplane_username: ${{ secrets.BINDPLANE_USERNAME }} + bindplane_password: ${{ secrets.BINDPLANE_PASSWORD }} + bindplane_api_key: "" # Optional replacement for bindplane_username and bindplane_password destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml # Write back requires these three options diff --git a/action.yml b/action.yml index 34a538a..7450358 100644 --- a/action.yml +++ b/action.yml @@ -3,23 +3,42 @@ description: 'GitHub Action for BindPlane OP.' author: 'observIQ' inputs: - remote_url: + # BindPlane configuration will be used to interact with the BindPlane OP's API + # using the BindPlane CLI bundled with the action. + bindplane_remote_url: description: 'The URL that will be used to connect to BindPlane OP' required: true - api_key: + bindplane_api_key: description: 'The BindPlane OP API key that will be used to authenticate to BindPlane OP' - username: - description: 'The BindPlane OP username that will be used to authenticate to BindPlane OP' - password: - description: 'The BindPlane OP password that will be used to authenticate to BindPlane OP' + bindplane_username: + description: 'The BindPlane OP bindplane_username that will be used to authenticate to BindPlane OP' + bindplane_password: + description: 'The BindPlane OP bindplane_password that will be used to authenticate to BindPlane OP' + + # This path should be a file that contains a BindPlane destination resource(s). destination_path: description: 'Path to the file which contains the BindPlane destination resources' + + # This path should be a file that contains a BindPlane configuration resource(s). configuration_path: description: 'Path to the file which contains the BindPlane configuration resources' + + # Raw OTEL config write back to the repo is + # configured with the following inputs. + + # This directory is the output directory that will be used when writing the raw OTEL configurations + # back to the repo. configuration_output_dir: description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' + + # This is the branch the action should be running against. When running against this branch, repo write + # back will be triggered. Configurations are commited directly to this branch. target_branch: description: 'Resource apply and OTEL config write back will only happen when this branch is the current branch of the action' + + # This is the Github token that will be used to authenticate to Github when writing OTEL configs back to the repo. + # This is generally passed in with the workflows secrets.GITHUB_TOKEN. It requires that the workflow have the + # contents.write permission configured. token: description: 'The GitHub token used to authenticate to GitHub when writing OTEL configs back to the repo' @@ -27,10 +46,10 @@ runs: using: 'docker' image: 'Dockerfile' args: - - ${{ inputs.remote_url }} - - ${{ inputs.api_key }} - - ${{ inputs.username }} - - ${{ inputs.password }} + - ${{ inputs.bindplane_remote_url }} + - ${{ inputs.bindplane_api_key }} + - ${{ inputs.bindplane_username }} + - ${{ inputs.bindplane_password }} - ${{ inputs.destination_path }} - ${{ inputs.configuration_path }} - ${{ inputs.configuration_output_dir }} diff --git a/entrypoint.sh b/entrypoint.sh index 38fd68f..45f488b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,15 +2,15 @@ set -e -REMOTE_URL=$1 -API_KEY=$2 -USERNAME=$3 -PASSWORD=$4 -DESTINATION_PATH=$5 -CONFIG_PATH=$6 -OUTPUT_DIR=$7 -TARGET_BRANCH=$8 -GITHUB_TOKEN=$9 +bindplane_remote_url=$1 +bindplane_api_key=$2 +bindplane_username=$3 +bindplane_password=$4 +destination_path=$5 +configuration_path=$6 +configuration_output_dir=$7 +target_branch=$8 +token=$9 BRANCH_NAME=${GITHUB_REF#refs/heads/} echo "Current branch is $BRANCH_NAME" @@ -33,35 +33,35 @@ install_bindplane_cli() { validate() { profile_args="" - if [ -z "$REMOTE_URL" ]; then - echo "REMOTE_URL is not set." + if [ -z "$bindplane_remote_url" ]; then + echo "bindplane_remote_url is not set." exit 1 else - profile_args="$profile_args --remote-url $REMOTE_URL" + profile_args="$profile_args --remote-url $bindplane_remote_url" fi - if [ -n "$USERNAME" ] && [ -z "$PASSWORD" ]; then - echo "password is required when username is not set." + if [ -n "$bindplane_username" ] && [ -z "$bindplane_password" ]; then + echo "bindplane_password is required when bindplane_username is not set." exit 1 else - profile_args="$profile_args --username $USERNAME --password $PASSWORD" + profile_args="$profile_args --bindplane_username $bindplane_username --bindplane_password $bindplane_password" fi - if [ -z "$USERNAME" ] && [ -z "$API_KEY" ]; then - echo "api key is required when username is not set." + if [ -z "$bindplane_username" ] && [ -z "$bindplane_api_key" ]; then + echo "api key is required when bindplane_username is not set." exit 1 - elif [ -n "$API_KEY" ]; then - profile_args="$profile_args --api-key $API_KEY" + elif [ -n "$bindplane_api_key" ]; then + profile_args="$profile_args --api-key $bindplane_api_key" fi - if [ -n "$TARGET_BRANCH" ]; then - if [ -z "$OUTPUT_DIR" ]; then - echo "OUTPUT_DIR is required when TARGET_BRANCH is set." + if [ -n "$target_branch" ]; then + if [ -z "$configuration_output_dir" ]; then + echo "configuration_output_dir is required when target_branch is set." exit 1 fi - if [ -z "$GITHUB_TOKEN" ]; then - echo "GITHUB_TOKEN is required when TARGET_BRANCH is set." + if [ -z "$token" ]; then + echo "token is required when target_branch is set." exit 1 fi @@ -69,12 +69,12 @@ validate() { # but we do expect them to be set. if [ -z "$GITHUB_ACTOR" ]; then - echo "GITHUB_ACTOR is required when TARGET_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." + echo "GITHUB_ACTOR is required when target_branch is set. This is likely a bug in the action. Please reach out to obserIQ support." exit 1 fi if [ -z "$GITHUB_REPOSITORY" ]; then - echo "GITHUB_REPOSITORY is required when TARGET_BRANCH is set. This is likely a bug in the action. Please reach out to obserIQ support." + echo "GITHUB_REPOSITORY is required when target_branch is set. This is likely a bug in the action. Please reach out to obserIQ support." exit 1 fi fi @@ -89,15 +89,15 @@ write_back() { git clone \ --depth 1 \ --branch "$BRANCH_NAME" \ - "https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + "https://${GITHUB_ACTOR}:${token}@github.com/${GITHUB_REPOSITORY}.git" \ ../out_repo cd "../out_repo" - mkdir -p "$OUTPUT_DIR" + mkdir -p "$configuration_output_dir" for config in $(bindplane get config | awk 'NR>1 {print $1}'); do - out_file="$OUTPUT_DIR/$config.yaml" + out_file="$configuration_output_dir/$config.yaml" bindplane get config "$config" -o raw > "$out_file" git add "$out_file" done @@ -117,11 +117,11 @@ write_back() { install_bindplane_cli validate -if [ "$BRANCH_NAME" != "$TARGET_BRANCH" ]; then - echo "Skipping apply and repo write. Current branch ${BRANCH_NAME} does not match target branch ${TARGET_BRANCH}." +if [ "$BRANCH_NAME" != "$target_branch" ]; then + echo "Skipping apply and repo write. Current branch ${BRANCH_NAME} does not match target branch ${target_branch}." else - bindplane apply "$DESTINATION_PATH" - bindplane apply "$CONFIG_PATH" + bindplane apply "$destination_path" + bindplane apply "$configuration_path" write_back fi From 88f9324a1b37b17f527d5649a8596a29621bd03e Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 13:40:46 -0500 Subject: [PATCH 54/65] fix bad rename --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 45f488b..fa748e2 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -44,7 +44,7 @@ validate() { echo "bindplane_password is required when bindplane_username is not set." exit 1 else - profile_args="$profile_args --bindplane_username $bindplane_username --bindplane_password $bindplane_password" + profile_args="$profile_args --username $bindplane_username --password $bindplane_password" fi if [ -z "$bindplane_username" ] && [ -z "$bindplane_api_key" ]; then From 1c7ab46c7a6a6cf1b32977803f37cd67768035b3 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 13:52:31 -0500 Subject: [PATCH 55/65] enable bool for write back. Move branch check outside of write back because it always matters --- .github/workflows/ci.yml | 1 + README.md | 7 +++++-- action.yml | 20 ++++++++++++-------- entrypoint.sh | 38 +++++++++++++++++++++++++++----------- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe70dec..7fa3927 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,7 @@ jobs: target_branch: dev # Token should have contents: write permissions token: ${{ secrets.GITHUB_TOKEN }} + enable_otel_config_write_back: true - name: Get Resources if: always() diff --git a/README.md b/README.md index 7eac15d..f9f3de0 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,13 @@ jobs: bindplane_remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} bindplane_username: ${{ secrets.BINDPLANE_USERNAME }} bindplane_password: ${{ secrets.BINDPLANE_PASSWORD }} - bindplane_api_key: "" # Optional replacement for bindplane_username and bindplane_password + bindplane_api_key: "" # Optional replacement for username and password + destination_path: test/resources/destinations/resource.yaml configuration_path: test/resources/configurations/resource.yaml - # Write back requires these three options + + # Write raw OTEL configs back to the repo. + enable_otel_config_write_back: true configuration_output_dir: test/otel/${{ matrix.bindplane_versions }} configuration_output_branch: dev token: ${{ secrets.GITHUB_TOKEN }} diff --git a/action.yml b/action.yml index 7450358..a1438fc 100644 --- a/action.yml +++ b/action.yml @@ -15,6 +15,11 @@ inputs: bindplane_password: description: 'The BindPlane OP bindplane_password that will be used to authenticate to BindPlane OP' + # This is the branch the action should be running against. When running against this branch, repo write + # back will be triggered. Configurations are commited directly to this branch. + target_branch: + description: 'Resource apply and OTEL config write back will only happen when this branch is the current branch of the action' + # This path should be a file that contains a BindPlane destination resource(s). destination_path: description: 'Path to the file which contains the BindPlane destination resources' @@ -23,19 +28,17 @@ inputs: configuration_path: description: 'Path to the file which contains the BindPlane configuration resources' - # Raw OTEL config write back to the repo is - # configured with the following inputs. + # Enable OTEL raw config write back will allow the action to write the raw OTEL configuration + # back to the repo to the configured directory and branch. + enable_otel_config_write_back: + description: 'Enable OTEL raw config write back' + default: false # This directory is the output directory that will be used when writing the raw OTEL configurations # back to the repo. configuration_output_dir: description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' - # This is the branch the action should be running against. When running against this branch, repo write - # back will be triggered. Configurations are commited directly to this branch. - target_branch: - description: 'Resource apply and OTEL config write back will only happen when this branch is the current branch of the action' - # This is the Github token that will be used to authenticate to Github when writing OTEL configs back to the repo. # This is generally passed in with the workflows secrets.GITHUB_TOKEN. It requires that the workflow have the # contents.write permission configured. @@ -50,8 +53,9 @@ runs: - ${{ inputs.bindplane_api_key }} - ${{ inputs.bindplane_username }} - ${{ inputs.bindplane_password }} + - ${{ inputs.target_branch }} - ${{ inputs.destination_path }} - ${{ inputs.configuration_path }} + - ${{ inputs.enable_otel_config_write_back }} - ${{ inputs.configuration_output_dir }} - - ${{ inputs.target_branch }} - ${{ inputs.token }} diff --git a/entrypoint.sh b/entrypoint.sh index fa748e2..af4f1ca 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,15 +2,16 @@ set -e -bindplane_remote_url=$1 -bindplane_api_key=$2 -bindplane_username=$3 -bindplane_password=$4 -destination_path=$5 -configuration_path=$6 -configuration_output_dir=$7 -target_branch=$8 -token=$9 +bindplane_remote_url=${1} +bindplane_api_key=${2} +bindplane_username=${3} +bindplane_password=${4} +target_branch=${5} +destination_path=${6} +configuration_path=${7} +enable_otel_config_write_back=${8} +configuration_output_dir=${9} +token=${10} BRANCH_NAME=${GITHUB_REF#refs/heads/} echo "Current branch is $BRANCH_NAME" @@ -33,6 +34,15 @@ install_bindplane_cli() { validate() { profile_args="" + # Target branch is always required. When not set, the script will not + # know which branch it should apply configurations from or write back + # raw otel configs. + if [ -z "$target_branch" ]; then + echo "target_branch is required when enable_otel_config_write_back is true." + exit 1 + fi + + if [ -z "$bindplane_remote_url" ]; then echo "bindplane_remote_url is not set." exit 1 @@ -54,7 +64,9 @@ validate() { profile_args="$profile_args --api-key $bindplane_api_key" fi - if [ -n "$target_branch" ]; then + # configuration_output_dir, target_branch, and token are only required + # when enable_otel_config_write_back is true. + if [ "$enable_otel_config_write_back" = true ]; then if [ -z "$configuration_output_dir" ]; then echo "configuration_output_dir is required when target_branch is set." exit 1 @@ -122,7 +134,11 @@ if [ "$BRANCH_NAME" != "$target_branch" ]; then else bindplane apply "$destination_path" bindplane apply "$configuration_path" - write_back + + # call write_back when enable_otel_config_write_back is true + if [ "$enable_otel_config_write_back" = true ]; then + write_back + fi fi From 82f2f2ed6e07fdcbee19b6e639a953d0982ab813 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 14:02:05 -0500 Subject: [PATCH 56/65] some cleanup --- action.yml | 21 ++------------------- entrypoint.sh | 32 +++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/action.yml b/action.yml index a1438fc..513ee8d 100644 --- a/action.yml +++ b/action.yml @@ -3,8 +3,6 @@ description: 'GitHub Action for BindPlane OP.' author: 'observIQ' inputs: - # BindPlane configuration will be used to interact with the BindPlane OP's API - # using the BindPlane CLI bundled with the action. bindplane_remote_url: description: 'The URL that will be used to connect to BindPlane OP' required: true @@ -14,40 +12,25 @@ inputs: description: 'The BindPlane OP bindplane_username that will be used to authenticate to BindPlane OP' bindplane_password: description: 'The BindPlane OP bindplane_password that will be used to authenticate to BindPlane OP' - - # This is the branch the action should be running against. When running against this branch, repo write - # back will be triggered. Configurations are commited directly to this branch. target_branch: description: 'Resource apply and OTEL config write back will only happen when this branch is the current branch of the action' - - # This path should be a file that contains a BindPlane destination resource(s). destination_path: description: 'Path to the file which contains the BindPlane destination resources' - - # This path should be a file that contains a BindPlane configuration resource(s). configuration_path: description: 'Path to the file which contains the BindPlane configuration resources' - - # Enable OTEL raw config write back will allow the action to write the raw OTEL configuration - # back to the repo to the configured directory and branch. enable_otel_config_write_back: description: 'Enable OTEL raw config write back' default: false - - # This directory is the output directory that will be used when writing the raw OTEL configurations - # back to the repo. configuration_output_dir: description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' - - # This is the Github token that will be used to authenticate to Github when writing OTEL configs back to the repo. - # This is generally passed in with the workflows secrets.GITHUB_TOKEN. It requires that the workflow have the - # contents.write permission configured. token: description: 'The GitHub token used to authenticate to GitHub when writing OTEL configs back to the repo' runs: using: 'docker' image: 'Dockerfile' + # Arg order must match the order of the arg parsing + # in entrypoint.sh. args: - ${{ inputs.bindplane_remote_url }} - ${{ inputs.bindplane_api_key }} diff --git a/entrypoint.sh b/entrypoint.sh index af4f1ca..2e2ec6d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,6 +2,7 @@ set -e +# This order must match the order in acitons.yml bindplane_remote_url=${1} bindplane_api_key=${2} bindplane_username=${3} @@ -13,6 +14,8 @@ enable_otel_config_write_back=${8} configuration_output_dir=${9} token=${10} +# This branch name will be compared to target_branch to determine if the action +# should apply or write back configurations. BRANCH_NAME=${GITHUB_REF#refs/heads/} echo "Current branch is $BRANCH_NAME" @@ -42,7 +45,6 @@ validate() { exit 1 fi - if [ -z "$bindplane_remote_url" ]; then echo "bindplane_remote_url is not set." exit 1 @@ -126,19 +128,31 @@ write_back() { git push } -install_bindplane_cli -validate +main() { + # Short circuit if the current branch does not match the target branch, + # there is nothing to do. + if [ "$BRANCH_NAME" != "$target_branch" ]; then + echo "Skipping apply and repo write. Current branch ${BRANCH_NAME} does not match target branch ${target_branch}." + exit 0 + fi + + # Install the CLI right away in order to construct + # a client profile. + install_bindplane_cli -if [ "$BRANCH_NAME" != "$target_branch" ]; then - echo "Skipping apply and repo write. Current branch ${BRANCH_NAME} does not match target branch ${target_branch}." -else + # Ensure required options are set and configure + # the client profile. + validate + + # Apply resources in the correct order. bindplane apply "$destination_path" bindplane apply "$configuration_path" - # call write_back when enable_otel_config_write_back is true + # When write back is enabled, write the raw otel configs + # back to the repository. if [ "$enable_otel_config_write_back" = true ]; then write_back fi -fi - +} +main From ceb959797e1945f5b408c8172b52a01b6c32fb2d Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 14:24:38 -0500 Subject: [PATCH 57/65] readme --- README.md | 52 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f9f3de0..f612a10 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,45 @@ # bindplane-op-action +The BindPlane OP action can be used to deploy configurations to your BindPlane OP +server. It also supports exporting the OpenTelemetry configurations back to the repository. + +## Configuration + +| Parameter | Default | Description | +| :---------------------------- | :--------- | :------------------------------ | +| bindplane_remote_url | required | The endpoint that will be used to connect to BindPalne OP. | +| bindplane_api_key | | API key used to authenticate to BindPlane. Required when BindPlane multi account is enabled or when running on BindPlane Cloud | +| bindplane_username | | Username used to authenticate to BindPlane. Not required if API key is set. | +| bindplane_password | | Password used to authenticate to BindPlane. +| target_branch | required | The branch that the action will use when applying resources to bindplane or when writing otel configs back to the repo. | +| destination_path | | Path to the file which contains the BindPlane destination resources | +| configuration_path | | Path to the file which contains the BindPlane configuration resources | +| enable_otel_config_write_back | `false` | Whether or not the action should write the raw OpenTelemetry configurations back to the repository. | +| configuration_output_dir | | When write back is enabled, this is the path that will be written to. | +| token | | The Github token that will be used to write to the repo. Usually secrets.GITHUB_TOKEN is sufficient. Requires the `contents.write` permission. | + ## Usage ### Export Resources +To get started, you must handle exporting your existing resources to the repository. Use +the `bindplane get` commands with the `--export` flag. + ```bash bindplane get destination -o yaml --export > destination.yaml bindplane get configuration -o yaml --export > configuration.yaml ``` +With the resources exported to the repository, you can move on to configuring the action +using a new workflow. + ### Workflow -The following workflow can be used as an example. +The following workflow can be used as an example. It uses the same file paths +created in the [Export Resources](#export-resources) section. + +This example will write the raw OTEL configurations back to the repository at the +path `otel/` ```yaml name: bindplane @@ -44,14 +72,20 @@ jobs: bindplane_remote_url: ${{ secrets.BINDPLANE_REMOTE_URL }} bindplane_username: ${{ secrets.BINDPLANE_USERNAME }} bindplane_password: ${{ secrets.BINDPLANE_PASSWORD }} - bindplane_api_key: "" # Optional replacement for username and password - - destination_path: test/resources/destinations/resource.yaml - configuration_path: test/resources/configurations/resource.yaml - - # Write raw OTEL configs back to the repo. + target_branch: main + destination_path: destination.yaml + configuration_path: configuration.yaml enable_otel_config_write_back: true - configuration_output_dir: test/otel/${{ matrix.bindplane_versions }} - configuration_output_branch: dev + configuration_output_dir: otel/ token: ${{ secrets.GITHUB_TOKEN }} ``` + +After the action is executed, you can expect to see OTEL configurations +in the `otel/` directory. + +``` +otel +├── k8s-cluster.yaml +├── k8s-gateway.yaml +└── k8s-node.yaml +``` From a25a5eb2d53846b6e8db626b84e5b414b0852621 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 14:25:22 -0500 Subject: [PATCH 58/65] badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f612a10..87fa9a0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![CI](https://github.com/observIQ/bindplane-op-action/actions/workflows/ci.yml/badge.svg)](https://github.com/observIQ/bindplane-op-action/actions/workflows/ci.yml) + # bindplane-op-action The BindPlane OP action can be used to deploy configurations to your BindPlane OP From a2c93e8826b694bc54622620fedd688fc8780eec Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 14:31:11 -0500 Subject: [PATCH 59/65] readme job name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87fa9a0..ca59726 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ concurrency: cancel-in-progress: false jobs: - goreleaser: + bindplane: runs-on: ubuntu-latest steps: - name: Checkout From 5a051b9ee2fa3d44e351d4dede5f107991e1b195 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Tue, 23 Jan 2024 14:42:48 -0500 Subject: [PATCH 60/65] grab pending version --- entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 2e2ec6d..7400261 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -112,7 +112,9 @@ write_back() { for config in $(bindplane get config | awk 'NR>1 {print $1}'); do out_file="$configuration_output_dir/$config.yaml" - bindplane get config "$config" -o raw > "$out_file" + # It is safe to always ask for "pending". BindPlane will return + # the current version if there is no pending version. + bindplane get config "${config}:pending" -o raw > "$out_file" git add "$out_file" done From 2c3e0c509ad0464ad44a625ecc4db6f30ae23881 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Wed, 24 Jan 2024 09:16:25 -0500 Subject: [PATCH 61/65] add logging useful for debugging running action --- entrypoint.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 7400261..1adfe25 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -138,23 +138,35 @@ main() { exit 0 fi + echo "Target branch ${target_branch} matches current branch ${BRANCH_NAME}." + # Install the CLI right away in order to construct # a client profile. + echo "Installing bindplane CLI." install_bindplane_cli # Ensure required options are set and configure # the client profile. + echo "Validating options and configuring client profile." validate # Apply resources in the correct order. + echo "Applying resources." + + echo "Applying destination path: $destination_path" bindplane apply "$destination_path" + + echo "Applying configuration path: $configuration_path" bindplane apply "$configuration_path" # When write back is enabled, write the raw otel configs # back to the repository. if [ "$enable_otel_config_write_back" = true ]; then + echo "Writing back raw otel configs." write_back fi + + echo "Done." } main From b91574a1311e263eea8230e164f0e36b73cef5c3 Mon Sep 17 00:00:00 2001 From: Joe Sirianni Date: Wed, 24 Jan 2024 17:25:10 -0500 Subject: [PATCH 62/65] remove unused files --- .../test/otel/1.37.0/k8s-cluster.yaml | 78 -------- .../test/otel/1.37.0/k8s-gateway.yaml | 176 ------------------ .../test/otel/1.37.0/k8s-node.yaml | 85 --------- .../test/otel/1.40.0/k8s-cluster.yaml | 78 -------- .../test/otel/1.40.0/k8s-gateway.yaml | 176 ------------------ .../test/otel/1.40.0/k8s-node.yaml | 85 --------- 6 files changed, 678 deletions(-) delete mode 100644 jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml delete mode 100644 jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml delete mode 100644 jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml delete mode 100644 jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml delete mode 100644 jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml delete mode 100644 jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml diff --git a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml deleted file mode 100644 index 8ae61a5..0000000 --- a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-cluster.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-cluster:1 -receivers: - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: - allocatable_types_to_report: - - cpu - - memory - - ephemeral-storage - - storage - auth_type: serviceAccount - collection_interval: 60s - distribution: kubernetes - node_conditions_to_report: - - Ready - - DiskPressure - - MemoryPressure - - PIDPressure - - NetworkUnavailable - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: - auth_type: serviceAccount - namespaces: - - kube-system - - production -processors: - batch/bindplane-gateway-agent: null - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube -exporters: - otlp/bindplane-gateway-agent: - compression: gzip - endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/bindplane-gateway-agent - tls: - insecure: true -extensions: - file_storage/bindplane-gateway-agent: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/bindplane-gateway-agent - pipelines: - logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: - receivers: - - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC - processors: - - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: - receivers: - - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 - processors: - - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - diff --git a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml deleted file mode 100644 index 55193b1..0000000 --- a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-gateway.yaml +++ /dev/null @@ -1,176 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-gateway:1 -receivers: - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - keepalive: - server_parameters: - max_connection_age: 60s - max_connection_age_grace: 300s - max_connection_idle: 60s - time: 2h - timeout: 20s - http: - endpoint: 0.0.0.0:4318 -processors: - resourcedetection/coralogix: - detectors: - - system - override: false - system: - hostname_sources: - - os -exporters: - coralogix/coralogix__logs: - application_name: bindplane-test - logs: - compression: gzip - endpoint: otel-logs.coralogix.us:443 - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - coralogix/coralogix__metrics: - application_name: bindplane-test - metrics: - compression: gzip - endpoint: otel-metrics.coralogix.us:443 - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - coralogix/coralogix__traces: - application_name: bindplane-test - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - traces: - compression: gzip - endpoint: otel-traces.coralogix.us:443 - elasticsearch/elasticsearch-internal__logs: - endpoints: - - https://es-0.corp.net:9200 - - https://es-1.corp.net:9200 - logs_index: logs-generic-default - num_workers: 0 - pipeline: null - retry: - enabled: true - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/elasticsearch-internal - tls: null - elasticsearch/elasticsearch-internal__traces: - endpoints: - - https://es-0.corp.net:9200 - - https://es-1.corp.net:9200 - num_workers: 0 - pipeline: null - retry: - enabled: true - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/elasticsearch-internal - tls: null - traces_index: traces-generic-default - prometheusremotewrite/prometheus-internal: - add_metric_suffixes: true - compression: gzip - endpoint: http://prom-prod.corp.net:9009//api/v1/write - resource_to_telemetry_conversion: - enabled: true - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s -extensions: - file_storage/coralogix: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - file_storage/elasticsearch-internal: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/coralogix - - file_storage/elasticsearch-internal - pipelines: - logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__logs - logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - elasticsearch/elasticsearch-internal__logs - metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__metrics - metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - prometheusremotewrite/prometheus-internal - traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__traces - traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - elasticsearch/elasticsearch-internal__traces - diff --git a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml b/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml deleted file mode 100644 index 52883fa..0000000 --- a/jsirianni/bindplane-op-action/test/otel/1.37.0/k8s-node.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-node:1 -receivers: - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: - auth_type: serviceAccount - collection_interval: 60s - endpoint: ${KUBE_NODE_NAME}:10250 - insecure_skip_verify: true - metric_groups: - - node - - pod - - container - - volume - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: - parameters: - body_json_parsing: true - exclude_file_log_path: - - /var/log/containers/observiq-*-collector-* - - /var/log/containers/bindplane-*-agent-* - log_paths: - - /var/log/containers/*.log - log_source: file - start_at: end - path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml -processors: - batch/bindplane-gateway-agent: null - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - - action: insert - key: k8s.node.name - value: ${KUBE_NODE_NAME} - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - - action: insert - key: k8s.node.name - value: ${KUBE_NODE_NAME} -exporters: - otlp/bindplane-gateway-agent: - compression: gzip - endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/bindplane-gateway-agent - tls: - insecure: true -extensions: - file_storage/bindplane-gateway-agent: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/bindplane-gateway-agent - pipelines: - logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: - receivers: - - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container - processors: - - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: - receivers: - - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 - processors: - - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - diff --git a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml deleted file mode 100644 index c400d08..0000000 --- a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-cluster.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-cluster:1 -receivers: - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: - allocatable_types_to_report: - - cpu - - memory - - ephemeral-storage - - storage - auth_type: serviceAccount - collection_interval: 1m0s - distribution: kubernetes - node_conditions_to_report: - - Ready - - DiskPressure - - MemoryPressure - - PIDPressure - - NetworkUnavailable - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: - auth_type: serviceAccount - namespaces: - - kube-system - - production -processors: - batch/bindplane-gateway-agent: null - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube -exporters: - otlp/bindplane-gateway-agent: - compression: gzip - endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/bindplane-gateway-agent - tls: - insecure: true -extensions: - file_storage/bindplane-gateway-agent: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/bindplane-gateway-agent - pipelines: - logs/source1_01HMS8GVNVFVD5TSWTKVNZS2JC__bindplane-gateway-agent-0: - receivers: - - k8s_events/source1_01HMS8GVNVFVD5TSWTKVNZS2JC - processors: - - resource/source1_01HMS8GVNVFVD5TSWTKVNZS2JC - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - metrics/source0_01HMS8GVNVFVD5TSWTKSJR1RY5__bindplane-gateway-agent-0: - receivers: - - k8s_cluster/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 - processors: - - resource/source0_01HMS8GVNVFVD5TSWTKSJR1RY5 - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - diff --git a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml deleted file mode 100644 index ab35830..0000000 --- a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-gateway.yaml +++ /dev/null @@ -1,176 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-gateway:1 -receivers: - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - keepalive: - server_parameters: - max_connection_age: 60s - max_connection_age_grace: 300s - max_connection_idle: 60s - time: 2h - timeout: 20s - http: - endpoint: 0.0.0.0:4318 -processors: - resourcedetection/coralogix: - detectors: - - system - override: false - system: - hostname_sources: - - os -exporters: - coralogix/coralogix__logs: - application_name: bindplane-test - logs: - compression: gzip - endpoint: otel-logs.coralogix.us:443 - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - coralogix/coralogix__metrics: - application_name: bindplane-test - metrics: - compression: gzip - endpoint: otel-metrics.coralogix.us:443 - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - coralogix/coralogix__traces: - application_name: bindplane-test - private_key: (sensitive) - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/coralogix - subsystem_name: test - timeout: 5s - traces: - compression: gzip - endpoint: otel-traces.coralogix.us:443 - elasticsearch/elasticsearch-internal__logs: - endpoints: - - https://es-0.corp.net:9200 - - https://es-1.corp.net:9200 - logs_index: logs-generic-default - num_workers: 0 - pipeline: null - retry: - enabled: true - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/elasticsearch-internal - tls: null - elasticsearch/elasticsearch-internal__traces: - endpoints: - - https://es-0.corp.net:9200 - - https://es-1.corp.net:9200 - num_workers: 0 - pipeline: null - retry: - enabled: true - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/elasticsearch-internal - tls: null - traces_index: traces-generic-default - prometheusremotewrite/prometheus-internal: - add_metric_suffixes: true - compression: gzip - endpoint: http://prom-prod.corp.net:9009/api/v1/write - resource_to_telemetry_conversion: - enabled: true - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s -extensions: - file_storage/coralogix: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - file_storage/elasticsearch-internal: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/coralogix - - file_storage/elasticsearch-internal - pipelines: - logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__logs - logs/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - elasticsearch/elasticsearch-internal__logs - metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__metrics - metrics/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__prometheus-internal-2: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - prometheusremotewrite/prometheus-internal - traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__coralogix-0: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: - - resourcedetection/coralogix - exporters: - - coralogix/coralogix__traces - traces/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5__elasticsearch-internal-1: - receivers: - - otlp/source0_01HMS8J2WPEVAX0RSCPW4ZNWT5 - processors: [] - exporters: - - elasticsearch/elasticsearch-internal__traces - diff --git a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml b/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml deleted file mode 100644 index 536d39c..0000000 --- a/jsirianni/bindplane-op-action/test/otel/1.40.0/k8s-node.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# This configuration is managed by BindPlane OP. -# Configuration: k8s-node:1 -receivers: - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: - auth_type: serviceAccount - collection_interval: 1m0s - endpoint: ${KUBE_NODE_NAME}:10250 - insecure_skip_verify: true - metric_groups: - - node - - pod - - container - - volume - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container: - parameters: - body_json_parsing: true - exclude_file_log_path: - - /var/log/containers/observiq-*-collector-* - - /var/log/containers/bindplane-*-agent-* - log_paths: - - /var/log/containers/*.log - log_source: file - start_at: end - path: $OIQ_OTEL_COLLECTOR_HOME/plugins/container_logs.yaml -processors: - batch/bindplane-gateway-agent: null - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - - action: insert - key: k8s.node.name - value: ${KUBE_NODE_NAME} - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0: - attributes: - - action: insert - key: k8s.cluster.name - value: minikube - - action: insert - key: k8s.node.name - value: ${KUBE_NODE_NAME} -exporters: - otlp/bindplane-gateway-agent: - compression: gzip - endpoint: bindplane-gateway-agent.bindplane-agent.svc.cluster.local:4317 - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 300s - max_interval: 30s - sending_queue: - enabled: true - num_consumers: 10 - queue_size: 5000 - storage: file_storage/bindplane-gateway-agent - tls: - insecure: true -extensions: - file_storage/bindplane-gateway-agent: - compaction: - directory: $OIQ_OTEL_COLLECTOR_HOME/storage - on_rebound: true - directory: $OIQ_OTEL_COLLECTOR_HOME/storage -service: - extensions: - - file_storage/bindplane-gateway-agent - pipelines: - logs/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__bindplane-gateway-agent-0: - receivers: - - plugin/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK__container - processors: - - resource/source0_01HMS8ERRSFYE4KGQVQ2ZXXDZK - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - metrics/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0__bindplane-gateway-agent-0: - receivers: - - kubeletstats/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 - processors: - - resource/source1_01HMS8ERRSFYE4KGQVQ8DQC5X0 - - batch/bindplane-gateway-agent - exporters: - - otlp/bindplane-gateway-agent - From c6e551524098f5a48a790bdee80d942dc54be486 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Fri, 26 Jan 2024 11:28:44 -0500 Subject: [PATCH 63/65] use latest instead of pending --- entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 1adfe25..191a24b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -112,9 +112,9 @@ write_back() { for config in $(bindplane get config | awk 'NR>1 {print $1}'); do out_file="$configuration_output_dir/$config.yaml" - # It is safe to always ask for "pending". BindPlane will return - # the current version if there is no pending version. - bindplane get config "${config}:pending" -o raw > "$out_file" + # It is safe to always ask for "latest". BindPlane will return + # the current version if there is no latest version. + bindplane get config "${config}:latest" -o raw > "$out_file" git add "$out_file" done From beb2524d7d9695f58bfd6edb2c4831bedbf89886 Mon Sep 17 00:00:00 2001 From: jsirianni Date: Fri, 26 Jan 2024 11:36:14 -0500 Subject: [PATCH 64/65] cleanup zip output --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 191a24b..7399f51 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -27,7 +27,7 @@ install_bindplane_cli() { mkdir -p ~/bin export PATH=$PATH:~/bin - unzip bindplane.zip -d ~/bin + unzip bindplane.zip -d ~/bin > /dev/null bindplane --help > /dev/null } From 39340e7df6958d503e478ae94ecb39070794f5dc Mon Sep 17 00:00:00 2001 From: Joseph Sirianni Date: Mon, 29 Jan 2024 09:09:18 -0500 Subject: [PATCH 65/65] feat: Auto rollout (#4) * wip: auto rollout * test against this branch * add auto rollout to workflow * print config apply output * check rollout status * check for pending rollout * shellcheck: Fix loop logic * install jq * add enable_auto_rollout to readme * revert this change, use external repo for testing this pr * check for stable configs as well * remove branch from ci, we will test using an external repo * use case --- .github/workflows/ci.yml | 1 + Dockerfile | 2 +- README.md | 2 ++ action.yml | 4 ++++ entrypoint.sh | 25 ++++++++++++++++++++++++- 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fa3927..f0db9eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,7 @@ jobs: # Token should have contents: write permissions token: ${{ secrets.GITHUB_TOKEN }} enable_otel_config_write_back: true + enable_auto_rollout: true - name: Get Resources if: always() diff --git a/Dockerfile b/Dockerfile index b0c2834..085168b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Container image that runs your code FROM alpine:3.10 -RUN apk add --no-cache bash curl git +RUN apk add --no-cache bash curl git jq COPY --chmod=0755 entrypoint.sh /entrypoint.sh diff --git a/README.md b/README.md index ca59726..8eb3e7f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ server. It also supports exporting the OpenTelemetry configurations back to the | enable_otel_config_write_back | `false` | Whether or not the action should write the raw OpenTelemetry configurations back to the repository. | | configuration_output_dir | | When write back is enabled, this is the path that will be written to. | | token | | The Github token that will be used to write to the repo. Usually secrets.GITHUB_TOKEN is sufficient. Requires the `contents.write` permission. | +| enable_auto_rollout | `false` | When enabled, the action will trigger a rollout for any configuration that has been updated. | ## Usage @@ -80,6 +81,7 @@ jobs: enable_otel_config_write_back: true configuration_output_dir: otel/ token: ${{ secrets.GITHUB_TOKEN }} + enable_auto_rollout: true ``` After the action is executed, you can expect to see OTEL configurations diff --git a/action.yml b/action.yml index 513ee8d..dcffaf9 100644 --- a/action.yml +++ b/action.yml @@ -25,6 +25,9 @@ inputs: description: 'Path to the directory which will contain the rendered OTEL format of the configuration resources' token: description: 'The GitHub token used to authenticate to GitHub when writing OTEL configs back to the repo' + enable_auto_rollout: + description: 'When enabled, the action will trigger a rollout for all configurations that have been updated' + default: false runs: using: 'docker' @@ -42,3 +45,4 @@ runs: - ${{ inputs.enable_otel_config_write_back }} - ${{ inputs.configuration_output_dir }} - ${{ inputs.token }} + - ${{ inputs.enable_auto_rollout }} diff --git a/entrypoint.sh b/entrypoint.sh index 7399f51..6bb66e7 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,6 +13,7 @@ configuration_path=${7} enable_otel_config_write_back=${8} configuration_output_dir=${9} token=${10} +enable_auto_rollout=${11} # This branch name will be compared to target_branch to determine if the action # should apply or write back configurations. @@ -157,7 +158,29 @@ main() { bindplane apply "$destination_path" echo "Applying configuration path: $configuration_path" - bindplane apply "$configuration_path" + bindplane apply "$configuration_path" > configuration.out + cat configuration.out + + # When auto rollout is enabled + if [ "$enable_auto_rollout" = true ]; then + echo "Auto rollout enabled." + awk '{print $2}' < configuration.out | while IFS= read -r config + do + status=$(bindplane rollout status "${config}" -o json | jq .status) + case "$status" in + 0) + echo "Configuration ${config} has a pending rollout, triggering rollout." + bindplane rollout start "$config" + ;; + 4) + echo "Configuration ${config} is stable, skipping rollout." + ;; + *) + echo "Configuration ${config} has an unknown status, skipping rollout." + ;; + esac + done + fi # When write back is enabled, write the raw otel configs # back to the repository.