From 3244100fd010f44c56b28d70521da52b40e2fe33 Mon Sep 17 00:00:00 2001 From: "Alexis (AJ) Jasso" Date: Thu, 11 Apr 2024 11:36:18 -0700 Subject: [PATCH 1/5] Add update_monitors.sh script and readme for datadog migration --- dd_migration/Readme.md | 13 ++++++ dd_migration/update_monitors.sh | 74 +++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 dd_migration/Readme.md create mode 100755 dd_migration/update_monitors.sh diff --git a/dd_migration/Readme.md b/dd_migration/Readme.md new file mode 100644 index 0000000..e1ac7d8 --- /dev/null +++ b/dd_migration/Readme.md @@ -0,0 +1,13 @@ +# Datadog Migration Tools + +Scripts and terraform files to aid in migration for customers using datadog for alerting + +## Tools + +## `update_monitors.sh` + +This script will search a datadog instance for all monitors with a given notification method (such as `@slack-platform-alerts`) and create a new monitor with the same name and all the same settings, but with a new notification method (such as `@firehydrant-platform-alerts`). Optionally, the `UPDATE_IN_PLACE` variable in the script can be set to `true`, in which case the existing monitor will be modified to use the new notification method instead. + +This script requires that a datadog API key be exported as `DD_API_KEY`, as well as an application key exported as `DD_APP_KEY`. Note that the application key must have all scopes needed for modifying all expected alerts and so should be created by someone with admin access. In the script, the datadog site parameter `DD_SITE_PARAMETER` needs to be set to connect with any site other than the default US1. See this page for the appropriate value: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site + +Known issue: All monitors in question need to have some text (apart from the notification method) in the `message` field for the monitor. Datadog allows this field to contain the notification method only, but then returns an empty string for the message as part of the API and thus it can't be modified in this way. \ No newline at end of file diff --git a/dd_migration/update_monitors.sh b/dd_migration/update_monitors.sh new file mode 100755 index 0000000..052c753 --- /dev/null +++ b/dd_migration/update_monitors.sh @@ -0,0 +1,74 @@ +#! /bin/bash + +# By default, we will make a copy of any alert with notification OLD_NOTIFICATION that is identical in all other ways but replaces OLD_NOTIFICATION +# with NEW_NOTIFICATION. Setting UPDATE_IN_PLACE here changes that behavior to modify the existing alert instead of making a new one. +UPDATE_IN_PLACE="false" +# The existing notification as it appears in the Datadog message. For example @slack-platform-alerts +OLD_NOTIFICATION="" +# The new notification as it would appear in the Datadog message. For example @firehydrant-platform-alerts +NEW_NOTIFICATION="" +# The appropriate Datadog site parameter from the table here: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site +DD_SITE_PARAMETER="datadoghq.com" + +if [ -z "$DD_API_KEY" ] +then + echo "Please export your datadog API key as DD_API_KEY" + exit 1 +fi + +if [ -z "$DD_APP_KEY" ] +then + echo "Please export your datadog application key as DD_APP_KEY" + exit 1 +fi + +URL_ENCODED_OLD_NOTIFICATION=$(echo ${OLD_NOTIFICATION:1}| jq -Rr @uri) + +RAW_SEARCH=$(curl -s --fail-with-body 'https://api.'"$DD_SITE_PARAMETER"'/api/v1/monitor/search?query=notification%3A"'"$URL_ENCODED_OLD_NOTIFICATION"'"' -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY") + +PAGES=$(echo $RAW_SEARCH | jq .metadata.page_count) +MONITOR_COUNT=$(echo $RAW_SEARCH | jq .metadata.total_count) + +MONITORS=() +for ((i=0; i<$PAGES; i++)); do + MONITORS+=$(curl -s --fail-with-body 'https://api.'"$DD_SITE_PARAMETER"'/api/v1/monitor/search?query=notification%3A"'"$URL_ENCODED_OLD_NOTIFICATION"'"&page='"$i" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY" | jq '.monitors | .[] | .id') + MONITORS+=" " +done + +echo "updating $MONITOR_COUNT monitors..." + +for monitorid in $MONITORS; do + rawmonitor=$(curl -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY") + new_message=$(echo $rawmonitor | jq .message | sed "s/$OLD_NOTIFICATION/$NEW_NOTIFICATION/g") + if [ $UPDATE_IN_PLACE == "true" ] + then + curl -X PUT -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: $DD_API_KEY" \ + -H "DD-APPLICATION-KEY: $DD_APP_KEY" \ + -d { "message": $new_message } + + echo "updated $(echo $rawmonitor | jq .name) in place" + else + payload='{ + "name": '"$(echo $rawmonitor | jq .name)"', + "message": '"$new_message"', + "options": '"$(echo $rawmonitor | jq .options)"', + "priority": '"$(echo $rawmonitor | jq .priority)"', + "query": '"$(echo $rawmonitor | jq .query)"', + "restricted_roles": '"$(echo $rawmonitor | jq .restricted_roles)"', + "tags": '"$(echo $rawmonitor | jq .tags)"', + "type": '"$(echo $rawmonitor | jq .type)"' +}' + + curl -X POST -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor" \ + -H "Accept: application/json" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: $DD_API_KEY" \ + -H "DD-APPLICATION-KEY: $DD_APP_KEY" \ + -d "$payload" + + echo "created new copy of $(echo $rawmonitor | jq .name) with notifications targeting $NEW_NOTIFICATION" + fi +done From 3247494bf127a3b75cf105bf1c06ce526b06d8f2 Mon Sep 17 00:00:00 2001 From: "Alexis (AJ) Jasso" Date: Fri, 12 Apr 2024 09:53:32 -0700 Subject: [PATCH 2/5] Add and document datadog-webhook.tf --- dd_migration/Readme.md | 8 +++++++- dd_migration/datadog-webhook.tf | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 dd_migration/datadog-webhook.tf diff --git a/dd_migration/Readme.md b/dd_migration/Readme.md index e1ac7d8..957f6ad 100644 --- a/dd_migration/Readme.md +++ b/dd_migration/Readme.md @@ -10,4 +10,10 @@ This script will search a datadog instance for all monitors with a given notific This script requires that a datadog API key be exported as `DD_API_KEY`, as well as an application key exported as `DD_APP_KEY`. Note that the application key must have all scopes needed for modifying all expected alerts and so should be created by someone with admin access. In the script, the datadog site parameter `DD_SITE_PARAMETER` needs to be set to connect with any site other than the default US1. See this page for the appropriate value: https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site -Known issue: All monitors in question need to have some text (apart from the notification method) in the `message` field for the monitor. Datadog allows this field to contain the notification method only, but then returns an empty string for the message as part of the API and thus it can't be modified in this way. \ No newline at end of file +Known issue: All monitors in question need to have some text (apart from the notification method) in the `message` field for the monitor. Datadog allows this field to contain the notification method only, but then returns an empty string for the message as part of the API and thus it can't be modified in this way. + +## `datadog-webhook.tf` + +This is an example terraform file for creating a signals compatible webhook at datadog. It requires the API key and Application key as described above, as well as the base datadog API url as described in the provider setup here: https://registry.terraform.io/providers/DataDog/datadog/latest/docs + +The team-specific URL would be added where indicated. This would create a notification method addressable in datadog as `@webhook-firehydrant-myteam` \ No newline at end of file diff --git a/dd_migration/datadog-webhook.tf b/dd_migration/datadog-webhook.tf new file mode 100644 index 0000000..f4ce7f2 --- /dev/null +++ b/dd_migration/datadog-webhook.tf @@ -0,0 +1,33 @@ +terraform { + required_version = "~> 1.5.5" + + required_providers { + datadog = { + source = "DataDog/datadog" + version = ">= 3.38.0" + } + } +} + +provider "datadog" { + api_key = "" + app_key = "" + api_url = "" +} + +resource "datadog_webhook" "firehydrant-myteam" { + name = "firehydrant-myteam" + url = "" + encode_as = "json" + payload = < Date: Fri, 12 Apr 2024 09:58:52 -0700 Subject: [PATCH 3/5] Add nice set commands --- dd_migration/update_monitors.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dd_migration/update_monitors.sh b/dd_migration/update_monitors.sh index 052c753..9fddbf7 100755 --- a/dd_migration/update_monitors.sh +++ b/dd_migration/update_monitors.sh @@ -1,4 +1,5 @@ -#! /bin/bash +#! /usr/bin/env bash +set -eo pipefail # By default, we will make a copy of any alert with notification OLD_NOTIFICATION that is identical in all other ways but replaces OLD_NOTIFICATION # with NEW_NOTIFICATION. Setting UPDATE_IN_PLACE here changes that behavior to modify the existing alert instead of making a new one. @@ -22,6 +23,8 @@ then exit 1 fi +set -u + URL_ENCODED_OLD_NOTIFICATION=$(echo ${OLD_NOTIFICATION:1}| jq -Rr @uri) RAW_SEARCH=$(curl -s --fail-with-body 'https://api.'"$DD_SITE_PARAMETER"'/api/v1/monitor/search?query=notification%3A"'"$URL_ENCODED_OLD_NOTIFICATION"'"' -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY") From 60d8d251057509c6cc74e2cbbf87430868c66d41 Mon Sep 17 00:00:00 2001 From: "Alexis (AJ) Jasso" Date: Fri, 12 Apr 2024 14:51:48 -0700 Subject: [PATCH 4/5] Add continue prompt with list of monitor names --- dd_migration/update_monitors.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/dd_migration/update_monitors.sh b/dd_migration/update_monitors.sh index 9fddbf7..e1711bc 100755 --- a/dd_migration/update_monitors.sh +++ b/dd_migration/update_monitors.sh @@ -38,7 +38,31 @@ for ((i=0; i<$PAGES; i++)); do MONITORS+=" " done -echo "updating $MONITOR_COUNT monitors..." +echo "Found $MONITOR_COUNT monitors..." + +monitornames=() +for monitorid in $MONITORS; do + rawmonitor=$(curl -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY") + monitornames+=$(echo $rawmonitor | jq .name) + monitornames+='\n' +done + +if [ $UPDATE_IN_PLACE == "true" ] +then + echo -e "The following monitors will be updated to point to $NEW_NOTIFICATION:\n" +else + echo -e "Copies of the following monitors will be created pointing to $NEW_NOTIFICATION:\n" +fi + +echo -e $monitornames + +echo "Continue?" +select yn in "Yes" "No"; do + case $yn in + Yes ) break;; + No ) exit;; + esac +done for monitorid in $MONITORS; do rawmonitor=$(curl -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY") From 2ab73f4e6646359170f7a8def26434b2ca5599cc Mon Sep 17 00:00:00 2001 From: "Alexis (AJ) Jasso" Date: Fri, 12 Apr 2024 15:17:34 -0700 Subject: [PATCH 5/5] Fix prompt to default No --- dd_migration/update_monitors.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dd_migration/update_monitors.sh b/dd_migration/update_monitors.sh index e1711bc..bb3e734 100755 --- a/dd_migration/update_monitors.sh +++ b/dd_migration/update_monitors.sh @@ -56,13 +56,11 @@ fi echo -e $monitornames -echo "Continue?" -select yn in "Yes" "No"; do - case $yn in - Yes ) break;; - No ) exit;; - esac -done +read -p "Type Yes to continue: " yn +case $yn in + Yes ) ;; + * ) exit 1;; +esac for monitorid in $MONITORS; do rawmonitor=$(curl -s --fail-with-body "https://api.$DD_SITE_PARAMETER/api/v1/monitor/$monitorid" -H "Accept: application/json" -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY")