From 8597e58d4eb487e04b95f3eb7850e78e7130600f Mon Sep 17 00:00:00 2001 From: Marcus Aspin Date: Fri, 21 Jun 2024 09:32:29 +0100 Subject: [PATCH] PI-2200 Add Sentry alert for high throughput (#3934) * PI-2200 Add Sentry alert for high throughput * Adjust threshold to 200% over a period of 24 hours, and sum across all environments --- .github/workflows/bootstrap.yml | 93 +++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 18e86d14bb..3d9685bdea 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -142,8 +142,7 @@ jobs: run: | response=$(curl https://sentry.io/api/0/teams/ministryofjustice/probation-integration/projects/ --fail \ -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \ - -H 'Content-Type: application/json' \ - -d '{"name":"${{ inputs.project_name }}"}') + -H 'Content-Type: application/json' --data '{"name":"${{ inputs.project_name }}"}') echo "slug=$(echo "$response" | jq -r '.slug')" | tee -a "$GITHUB_OUTPUT" env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} @@ -152,8 +151,7 @@ jobs: run: | curl -X PUT "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/" --fail \ -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \ - -H 'Content-Type: application/json' \ - -d '{"platform":"kotlin"}' + -H 'Content-Type: application/json' --data '{"platform":"kotlin"}' env: SLUG: ${{ steps.project.outputs.slug }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} @@ -172,30 +170,103 @@ jobs: run: | curl -X PUT "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/keys/$KEY_ID/" --fail \ -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \ - -H 'Content-Type: application/json' \ - -d '{"rateLimit":{"count":10,"window":60}}' + -H 'Content-Type: application/json' --data '{"rateLimit":{"count":10,"window":60}}' env: SLUG: ${{ steps.project.outputs.slug }} KEY_ID: ${{ steps.client_key.outputs.id }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - - name: Create alert + - name: Create alert for new issues run: | - curl -X POST "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/rules/?duplicateRule=true&wizardV3=true" --fail \ + curl "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/rules/?duplicateRule=true&wizardV3=true" --fail \ -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \ - -H 'Content-Type: application/json' \ - -d '{"conditions":[{"id":"sentry.rules.conditions.first_seen_event.FirstSeenEventCondition","name":"A new issue is created"},{"id":"sentry.rules.conditions.regression_event.RegressionEventCondition","name":"The issue changes state from resolved to unresolved"},{"id":"sentry.rules.conditions.reappeared_event.ReappearedEventCondition","name":"The issue changes state from ignored to unresolved"}],"filters":[],"actions":[{"workspace":"50134","id":"sentry.integrations.slack.notify_action.SlackNotifyServiceAction","channel":"probation-integration-notifications","channel_id":"C033HPR0W91","tags":"environment","name":"Send a notification to the MOJ Digital & Technology Slack workspace to probation-integration-notifications (optionally, an ID: C033HPR0W91) and show tags [] in notification"}],"actionMatch":"any","filterMatch":"all","frequency":5,"name":"New issues","dateCreated":"2023-01-23T12:03:42.969890Z","owner":"team:1611212","environment":null,"projects":["${{ steps.project.outputs.slug }}"]}' + -H 'Content-Type: application/json' --data "$(jq --arg env "$env" --arg project_slug "$SLUG" -n '{ + "actionMatch": "any", + "actions": [ + { + "channel": "probation-integration-notifications", + "channel_id": "C033HPR0W91", + "id": "sentry.integrations.slack.notify_action.SlackNotifyServiceAction", + "name": "Send a notification to the MOJ Digital & Technology Slack workspace to probation-integration-notifications (optionally, an ID: C033HPR0W91) and show tags [] in notification", + "tags": "environment", + "workspace": "50134" + } + ], + "conditions": [ + { "id": "sentry.rules.conditions.first_seen_event.FirstSeenEventCondition", "name": "A new issue is created" }, + { "id": "sentry.rules.conditions.regression_event.RegressionEventCondition", "name": "The issue changes state from resolved to unresolved" }, + { "id": "sentry.rules.conditions.reappeared_event.ReappearedEventCondition", "name": "The issue changes state from ignored to unresolved" } + ], + "environment": null, + "filterMatch": "all", + "filters": [], + "frequency": 5, + "name": "New issues", + "owner": "team:1611212", + "projects": [$project_slug] + }')" env: SLUG: ${{ steps.project.outputs.slug }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + - name: Create alerts for high throughput + run: | + curl "https://sentry.io/api/0/organizations/ministryofjustice/alert-rules/?duplicateRule=true&wizardV3=true" --fail \ + -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \ + -H 'Content-Type: application/json' --data "$(jq --arg project_slug "$SLUG" --arg project_name "$PROJECT_NAME" -n '{ + "aggregate": "count()", + "alertType": "throughput", + "comparisonDelta": 10080, + "dataset": "generic_metrics", + "environment": null, + "eventTypes": [ + "transaction" + ], + "name": ($project_name + " throughput"), + "owner": "team:1611212", + "projects": [ + $project_slug + ], + "query": "", + "queryType": 1, + "resolveThreshold": null, + "thresholdPeriod": 1, + "thresholdType": 0, + "timeWindow": 1440, + "triggers": [ + { + "actions": [], + "alertThreshold": 500, + "label": "critical" + }, + { + "actions": [ + { + "inputChannelId": "C033HPR0W91", + "integrationId": 50134, + "options": null, + "targetIdentifier": "#probation-integration-notifications", + "targetType": "specific", + "type": "slack" + } + ], + "alertThreshold": 200, + "label": "warning" + } + ] + }')" + env: + SLUG: ${{ steps.project.outputs.slug }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + PROJECT_NAME: ${{ inputs.project_name }} + sentry-secrets: runs-on: ubuntu-latest if: ${{ inputs.create_sentry_project }} needs: sentry-setup strategy: matrix: - environment: ["test", "preprod", "prod"] + environment: [ "test", "preprod", "prod" ] environment: ${{ matrix.environment }} steps: - uses: actions/checkout@v4