From a93ef6b99a4a6d7c3b7ed40d9b86a687deb56a90 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 5 Jul 2024 11:10:35 +0300 Subject: [PATCH 1/4] add script to set blobstore credentials --- elasticsearch/blobstore_credentials.sh | 135 +++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 elasticsearch/blobstore_credentials.sh diff --git a/elasticsearch/blobstore_credentials.sh b/elasticsearch/blobstore_credentials.sh new file mode 100644 index 00000000..2aca8852 --- /dev/null +++ b/elasticsearch/blobstore_credentials.sh @@ -0,0 +1,135 @@ +#!/bin/sh + +# Configure Elasticsearch S3 credentials to elasticsearch-keystore +# set s3.access_key and s3.secret_key, also for Backblaze +ES_PATH="/usr/share/elasticsearch" +KEY_NAME_PREFIX="s3.client.default" +ENV_FILE="prod" + +LOGIN_USER=$(who am i | awk '{ print $1 }') +if [ "x$LOGIN_USER" = x ]; then + # XXX fall back to whoami (look by uid) + echo could not find login user 1>&2 + exit 1 +fi + +run_as_login_user() { + su $LOGIN_USER -c "$*" +} + +help() { + echo "Usage: $0 -b [-c ]" + echo "" + echo "Options:" + echo " -b Specify the blobstore type (s3 or b2)" + echo " -c Specify the Elasticsearch container name (if running in Docker)" + echo " -h Show this help message" + echo "" +} + +zzz() { + echo $1 | tr 'A-Za-z' 'N-ZA-Mn-za-m' +} + +while getopts "b:h" opt; do + case $opt in + b) + BLOBSTORE="$OPTARG" + ;; + h) + help + exit 0 + ;; + *) + help + exit 1 + ;; + esac +done + +if [ $(whoami) != "root" ]; then + echo "This script must be run as root." + exit 1 +fi + +if [ -z "$BLOBSTORE" ]; then + show_help + exit 1 +fi + +if [ "$BLOBSTORE" != "s3" ] && [ "$BLOBSTORE" != "b2" ]; then + echo "Invalid blobstore type. Use 's3' or 'b2'." + exit 1 +fi + +PRIVATE_CONF_DIR="es-credentials-setup" +mkdir -p $PRIVATE_CONF_DIR +chmod go-rwx $PRIVATE_CONF_DIR + +cd $PRIVATE_CONF_DIR +CONFIG_REPO_PREFIX=$(zzz tvg@tvguho.pbz:zrqvnpybhq) +CONFIG_REPO_NAME=$(zzz fgbel-vaqrkre-pbasvt) +PRIVATE_CONF_REPO=$(pwd)/$CONFIG_REPO_NAME + +echo "Cloning $CONFIG_REPO_NAME repo" 1>&2 +if ! git clone "$CONFIG_REPO_PREFIX/$CONFIG_REPO_NAME.git" >/dev/null 2>&1; then + echo "FATAL: could not clone config repo" 1>&2 + exit 1 +fi + +PRIVATE_CONF_FILE="$PRIVATE_CONF_REPO/$ENV_FILE.sh" +cd .. + +if [ ! -f "$PRIVATE_CONF_FILE" ]; then + echo "FATAL: could not access $PRIVATE_CONF_FILE" 1>&2 + exit 1 +fi + +. "$PRIVATE_CONF_FILE" + +rm -rf $PRIVATE_CONF_DIR + +if [ "$BLOBSTORE" = "s3" ]; then + ACCESS_KEY=$S3_ACCESS_KEY + SECRET_KEY=$S3_SECRET_KEY +elif [ "$BLOBSTORE" = "b2" ]; then + ACCESS_KEY=$B2_ACCESS_KEY + SECRET_KEY=$B2_SECRET_KEY +fi + +check_elasticsearch() { + if curl -s "http://localhost:9200" >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +add_credentials() { + echo "$ACCESS_KEY" | $ES_PATH/bin/elasticsearch-keystore add --stdin --force "$KEY_NAME_PREFIX.access_key" + echo "$SECRET_KEY" | $ES_PATH/bin/elasticsearch-keystore add --stdin --force "$KEY_NAME_PREFIX.secret_key" +} + +# Function to reload Elasticsearch secure settings +reload_secure_settings() { + curl -X POST "http://localhost:9200/_nodes/reload_secure_settings" -H "Content-Type: application/json" -d '{}' +} + +if check_elasticsearch; then + add_credentialsgi + if [ $? -eq 0 ]; then + reload_secure_settings + if [ $? -eq 0 ]; then + echo "Credentials for $BLOBSTORE added and secure settings reloaded successfully." + else + echo "Failed to reload secure settings." + exit 1 + fi + else + echo "Failed to add credentials to the keystore." + exit 1 + fi +else + echo "Elasticsearch is not running or reachable. Exiting." + exit 1 +fi From a2d27d868b98621b693a85e10ef7a5d2e74163fc Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 5 Jul 2024 13:00:25 +0300 Subject: [PATCH 2/4] run as login_user --- elasticsearch/blobstore_credentials.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/elasticsearch/blobstore_credentials.sh b/elasticsearch/blobstore_credentials.sh index 2aca8852..50fd5e82 100644 --- a/elasticsearch/blobstore_credentials.sh +++ b/elasticsearch/blobstore_credentials.sh @@ -63,7 +63,7 @@ if [ "$BLOBSTORE" != "s3" ] && [ "$BLOBSTORE" != "b2" ]; then fi PRIVATE_CONF_DIR="es-credentials-setup" -mkdir -p $PRIVATE_CONF_DIR +run_as_login_user mkdir -p $PRIVATE_CONF_DIR chmod go-rwx $PRIVATE_CONF_DIR cd $PRIVATE_CONF_DIR @@ -72,7 +72,7 @@ CONFIG_REPO_NAME=$(zzz fgbel-vaqrkre-pbasvt) PRIVATE_CONF_REPO=$(pwd)/$CONFIG_REPO_NAME echo "Cloning $CONFIG_REPO_NAME repo" 1>&2 -if ! git clone "$CONFIG_REPO_PREFIX/$CONFIG_REPO_NAME.git" >/dev/null 2>&1; then +if ! run_as_login_user git clone "$CONFIG_REPO_PREFIX/$CONFIG_REPO_NAME.git" >/dev/null 2>&1; then echo "FATAL: could not clone config repo" 1>&2 exit 1 fi @@ -90,11 +90,11 @@ fi rm -rf $PRIVATE_CONF_DIR if [ "$BLOBSTORE" = "s3" ]; then - ACCESS_KEY=$S3_ACCESS_KEY - SECRET_KEY=$S3_SECRET_KEY + ACCESS_KEY=$ELASTICSEARCH_SNAPSHOT_S3_ACCESS_KEY + SECRET_KEY=$ELASTICSEARCH_SNAPSHOT_S3_SECRET_KEY elif [ "$BLOBSTORE" = "b2" ]; then - ACCESS_KEY=$B2_ACCESS_KEY - SECRET_KEY=$B2_SECRET_KEY + ACCESS_KEY=$ELASTICSEARCH_SNAPSHOT_B2_ACCESS_KEY + SECRET_KEY=$ELASTICSEARCH_SNAPSHOT_B2_SECRET_KEY fi check_elasticsearch() { From d144b1da712d36a2f4a3d9867fb3801d6f1cc0ac Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 5 Jul 2024 13:55:29 +0300 Subject: [PATCH 3/4] updates: login and doc --- elasticsearch/blobstore_credentials.sh | 49 ++++++++++++++++++-------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/elasticsearch/blobstore_credentials.sh b/elasticsearch/blobstore_credentials.sh index 50fd5e82..cc82d24f 100644 --- a/elasticsearch/blobstore_credentials.sh +++ b/elasticsearch/blobstore_credentials.sh @@ -1,7 +1,22 @@ #!/bin/sh -# Configure Elasticsearch S3 credentials to elasticsearch-keystore -# set s3.access_key and s3.secret_key, also for Backblaze +# This script configures Elasticsearch to use cloud storage credentials for snapshot and restore operations. +# It supports both Amazon S3 and Backblaze B2 as the cloud storage options. +# +# Usage: ./script.sh -b +# +# Options: +# -b Specify the cloud storage type ('s3' for Amazon S3, 'b2' for Backblaze B2). +# -h Display this help message and exit. +# +# Notes: +# - This script must be run as the root user. +# - Ensure Elasticsearch is running and accessible at http://localhost:9200 before executing this script. + +# Example: +# ./elasticsearch/blobstore_credentials.sh -b s3 +# + ES_PATH="/usr/share/elasticsearch" KEY_NAME_PREFIX="s3.client.default" ENV_FILE="prod" @@ -22,11 +37,14 @@ help() { echo "" echo "Options:" echo " -b Specify the blobstore type (s3 or b2)" - echo " -c Specify the Elasticsearch container name (if running in Docker)" echo " -h Show this help message" echo "" } +log() { + echo "$1" + } + zzz() { echo $1 | tr 'A-Za-z' 'N-ZA-Mn-za-m' } @@ -48,32 +66,33 @@ while getopts "b:h" opt; do done if [ $(whoami) != "root" ]; then - echo "This script must be run as root." + log "ERROR: This script must be run as root." exit 1 fi if [ -z "$BLOBSTORE" ]; then - show_help + help exit 1 fi if [ "$BLOBSTORE" != "s3" ] && [ "$BLOBSTORE" != "b2" ]; then - echo "Invalid blobstore type. Use 's3' or 'b2'." + log "ERROR: Invalid blobstore type. Use 's3' or 'b2'." exit 1 fi PRIVATE_CONF_DIR="es-credentials-setup" run_as_login_user mkdir -p $PRIVATE_CONF_DIR chmod go-rwx $PRIVATE_CONF_DIR +log "INFO: Created private configuration directory $PRIVATE_CONF_DIR" cd $PRIVATE_CONF_DIR CONFIG_REPO_PREFIX=$(zzz tvg@tvguho.pbz:zrqvnpybhq) CONFIG_REPO_NAME=$(zzz fgbel-vaqrkre-pbasvt) PRIVATE_CONF_REPO=$(pwd)/$CONFIG_REPO_NAME -echo "Cloning $CONFIG_REPO_NAME repo" 1>&2 +log "INFO: Cloning $CONFIG_REPO_NAME repo" 1>&2 if ! run_as_login_user git clone "$CONFIG_REPO_PREFIX/$CONFIG_REPO_NAME.git" >/dev/null 2>&1; then - echo "FATAL: could not clone config repo" 1>&2 + log "FATAL: could not clone config repo" 1>&2 exit 1 fi @@ -81,7 +100,7 @@ PRIVATE_CONF_FILE="$PRIVATE_CONF_REPO/$ENV_FILE.sh" cd .. if [ ! -f "$PRIVATE_CONF_FILE" ]; then - echo "FATAL: could not access $PRIVATE_CONF_FILE" 1>&2 + log "FATAL: could not access $PRIVATE_CONF_FILE" 1>&2 exit 1 fi @@ -106,30 +125,32 @@ check_elasticsearch() { } add_credentials() { + log "INFO: Adding credentials to the Elasticsearch keystore" echo "$ACCESS_KEY" | $ES_PATH/bin/elasticsearch-keystore add --stdin --force "$KEY_NAME_PREFIX.access_key" echo "$SECRET_KEY" | $ES_PATH/bin/elasticsearch-keystore add --stdin --force "$KEY_NAME_PREFIX.secret_key" } # Function to reload Elasticsearch secure settings reload_secure_settings() { + log "INFO: Reloading Elasticsearch secure settings" curl -X POST "http://localhost:9200/_nodes/reload_secure_settings" -H "Content-Type: application/json" -d '{}' } if check_elasticsearch; then - add_credentialsgi + add_credentials if [ $? -eq 0 ]; then reload_secure_settings if [ $? -eq 0 ]; then - echo "Credentials for $BLOBSTORE added and secure settings reloaded successfully." + log "INFO: Credentials for $BLOBSTORE added and secure settings reloaded successfully." else - echo "Failed to reload secure settings." + log "ERROR: Failed to reload secure settings." exit 1 fi else - echo "Failed to add credentials to the keystore." + log "ERROR: Failed to add credentials to the keystore." exit 1 fi else - echo "Elasticsearch is not running or reachable. Exiting." + log "ERROR: Elasticsearch is not running or reachable. Exiting." exit 1 fi From 87bab3798d9271ef090ba43581c3ce3b5fe8cd56 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Date: Fri, 5 Jul 2024 13:58:04 +0300 Subject: [PATCH 4/4] updates: login and doc --- doc/elasticsearch/elasticsearch_configuration.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/elasticsearch/elasticsearch_configuration.md b/doc/elasticsearch/elasticsearch_configuration.md index bf1f10bc..fccdf5c3 100644 --- a/doc/elasticsearch/elasticsearch_configuration.md +++ b/doc/elasticsearch/elasticsearch_configuration.md @@ -93,14 +93,13 @@ The RPM and Debian packages will configure this setting automatically. No furthe ### Configuring S3 Repository for Snapshot/Restore -We are using S3 for Elasticsearch Snapshot & Restore +We are using Elasticsearch's S3 Repository plugin for Elasticsearch Snapshot & Restore The bucket `mediacloud-elasticsearch-snapshots` has been created with the relevant policy to allow ES snapshot access to the S3 bucket. The AWS secure credntials are added to Elasticsearch's keystore as follows: - `bin/elasticsearch-keystore add s3.client.default.access_key` - `bin/elasticsearch-keystore add s3.client.default.secret_key` + `./elasticsearch/blobstore_credentials.sh` #### Register S3 repository