Skip to content

Commit

Permalink
PI-2200 Add Sentry alert for high throughput (#3934)
Browse files Browse the repository at this point in the history 
* PI-2200 Add Sentry alert for high throughput

* Adjust threshold to 200% over a period of 24 hours, and sum across all environments
  • Loading branch information
@marcus-bcl
marcus-bcl authored Jun 21, 2024
1 parent b062409 commit 8597e58
Showing 1 changed file with 82 additions and 11 deletions.
93 changes: 82 additions & 11 deletions .github/workflows/bootstrap.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,7 @@ jobs:
run: |
response=$(curl https://sentry.io/api/0/teams/ministryofjustice/probation-integration/projects/ --fail \
-H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \
-H 'Content-Type: application/json' \
-d '{"name":"${{ inputs.project_name }}"}')
-H 'Content-Type: application/json' --data '{"name":"${{ inputs.project_name }}"}')
echo "slug=$(echo "$response" | jq -r '.slug')" | tee -a "$GITHUB_OUTPUT"
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
Expand All @@ -152,8 +151,7 @@ jobs:
run: |
curl -X PUT "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/" --fail \
-H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \
-H 'Content-Type: application/json' \
-d '{"platform":"kotlin"}'
-H 'Content-Type: application/json' --data '{"platform":"kotlin"}'
env:
SLUG: ${{ steps.project.outputs.slug }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
Expand All @@ -172,30 +170,103 @@ jobs:
run: |
curl -X PUT "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/keys/$KEY_ID/" --fail \
-H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \
-H 'Content-Type: application/json' \
-d '{"rateLimit":{"count":10,"window":60}}'
-H 'Content-Type: application/json' --data '{"rateLimit":{"count":10,"window":60}}'
env:
SLUG: ${{ steps.project.outputs.slug }}
KEY_ID: ${{ steps.client_key.outputs.id }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

- name: Create alert
- name: Create alert for new issues
run: |
curl -X POST "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/rules/?duplicateRule=true&wizardV3=true" --fail \
curl "https://sentry.io/api/0/projects/ministryofjustice/$SLUG/rules/?duplicateRule=true&wizardV3=true" --fail \
-H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \
-H 'Content-Type: application/json' \
-d '{"conditions":[{"id":"sentry.rules.conditions.first_seen_event.FirstSeenEventCondition","name":"A new issue is created"},{"id":"sentry.rules.conditions.regression_event.RegressionEventCondition","name":"The issue changes state from resolved to unresolved"},{"id":"sentry.rules.conditions.reappeared_event.ReappearedEventCondition","name":"The issue changes state from ignored to unresolved"}],"filters":[],"actions":[{"workspace":"50134","id":"sentry.integrations.slack.notify_action.SlackNotifyServiceAction","channel":"probation-integration-notifications","channel_id":"C033HPR0W91","tags":"environment","name":"Send a notification to the MOJ Digital & Technology Slack workspace to probation-integration-notifications (optionally, an ID: C033HPR0W91) and show tags [] in notification"}],"actionMatch":"any","filterMatch":"all","frequency":5,"name":"New issues","dateCreated":"2023-01-23T12:03:42.969890Z","owner":"team:1611212","environment":null,"projects":["${{ steps.project.outputs.slug }}"]}'
-H 'Content-Type: application/json' --data "$(jq --arg env "$env" --arg project_slug "$SLUG" -n '{
"actionMatch": "any",
"actions": [
{
"channel": "probation-integration-notifications",
"channel_id": "C033HPR0W91",
"id": "sentry.integrations.slack.notify_action.SlackNotifyServiceAction",
"name": "Send a notification to the MOJ Digital & Technology Slack workspace to probation-integration-notifications (optionally, an ID: C033HPR0W91) and show tags [] in notification",
"tags": "environment",
"workspace": "50134"
}
],
"conditions": [
{ "id": "sentry.rules.conditions.first_seen_event.FirstSeenEventCondition", "name": "A new issue is created" },
{ "id": "sentry.rules.conditions.regression_event.RegressionEventCondition", "name": "The issue changes state from resolved to unresolved" },
{ "id": "sentry.rules.conditions.reappeared_event.ReappearedEventCondition", "name": "The issue changes state from ignored to unresolved" }
],
"environment": null,
"filterMatch": "all",
"filters": [],
"frequency": 5,
"name": "New issues",
"owner": "team:1611212",
"projects": [$project_slug]
}')"
env:
SLUG: ${{ steps.project.outputs.slug }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

- name: Create alerts for high throughput
run: |
curl "https://sentry.io/api/0/organizations/ministryofjustice/alert-rules/?duplicateRule=true&wizardV3=true" --fail \
-H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \
-H 'Content-Type: application/json' --data "$(jq --arg project_slug "$SLUG" --arg project_name "$PROJECT_NAME" -n '{
"aggregate": "count()",
"alertType": "throughput",
"comparisonDelta": 10080,
"dataset": "generic_metrics",
"environment": null,
"eventTypes": [
"transaction"
],
"name": ($project_name + " throughput"),
"owner": "team:1611212",
"projects": [
$project_slug
],
"query": "",
"queryType": 1,
"resolveThreshold": null,
"thresholdPeriod": 1,
"thresholdType": 0,
"timeWindow": 1440,
"triggers": [
{
"actions": [],
"alertThreshold": 500,
"label": "critical"
},
{
"actions": [
{
"inputChannelId": "C033HPR0W91",
"integrationId": 50134,
"options": null,
"targetIdentifier": "#probation-integration-notifications",
"targetType": "specific",
"type": "slack"
}
],
"alertThreshold": 200,
"label": "warning"
}
]
}')"
env:
SLUG: ${{ steps.project.outputs.slug }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
PROJECT_NAME: ${{ inputs.project_name }}

sentry-secrets:
runs-on: ubuntu-latest
if: ${{ inputs.create_sentry_project }}
needs: sentry-setup
strategy:
matrix:
environment: ["test", "preprod", "prod"]
environment: [ "test", "preprod", "prod" ]
environment: ${{ matrix.environment }}
steps:
- uses: actions/checkout@v4
Expand Down

0 comments on commit 8597e58

Please sign in to comment.