Skip to content

Commit

Permalink
Merge pull request #13 from tnelson-doghouse/passing-database-name-1
Browse files Browse the repository at this point in the history 
Passing database name 1
  • Loading branch information
@shreddedbacon
shreddedbacon authored Nov 22, 2024
2 parents 1582301 + 19a3d52 commit 2ace668
Show file tree
Hide file tree
Showing 4 changed files with 256 additions and 25 deletions.
17 changes: 16 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,19 @@ The mtk config file should be provided as a base64 encoded string, this allows f
```
$ cat example.mtk.yml | base64 | tr -d '\n\r'
cmV3cml0ZToKICAjIERydXBhbCA4CiAgdXNlcnNfZmllbGRfZGF0YToKICAgIG1haWw6IGNvbmNhdCh1aWQsICJAU0FOSVRJU0VEIikKICAgIHBhc3M6ICciU0FOSVRJWkVEX1BBU1NXT1JEIicKICAgICMgV2UgZG9uJ3QgYWxsb3cgbm8gZGlydHkgbmFtZXMgYXJvdW5kIGhlcmUhCiAgICBuYW1lOiBjb25jYXQodWlkLCAiLVNBTklUSVNFRCIpCiAgICBpbml0OiAnIlNBTklUSVNFRF9JTklUIicKICAjIERydXBhbCA3CiAgdXNlcnM6CiAgICBtYWlsOiBjb25jYXQodWlkLCAiQFNBTklUSVNFRCIpCiAgICBwYXNzOiAnIlNBTklUSVpFRF9QQVNTV09SRCInCgp3aGVyZToKICAjIE9ubHkgaW5jbHVkZSBib2R5IGZpZWxkIGRhdGEgZm9yIGN1cnJlbnQgcmV2aXNpb25zLgogIG5vZGVfcmV2aXNpb25fX2JvZHk6IHwtCiAgICAgIHJldmlzaW9uX2lkIElOIChTRUxFQ1QgdmlkIEZST00gbm9kZSkKCm5vZGF0YToKICAtIGNhY2hlKgogIC0gY2FwdGNoYV9zZXNzaW9ucwogIC0gaGlzdG9yeQogIC0gZmxvb2QKICAtIGJhdGNoCiAgLSBxdWV1ZQogIC0gc2Vzc2lvbnMKICAtIHNlbWFwaG9yZQogIC0gc2VhcmNoX2FwaV90YXNrCiAgLSBzZWFyY2hfZGF0YXNldAogIC0gc2VhcmNoX2luZGV4CiAgLSBzZWFyY2hfdG90YWwKICAtIHdhdGNoZG9nCiAgLSB3ZWJmb3JtX3N1Ym1pc3Npb25fZGF0YQoKaWdub3JlOgogIC0gX19BQ1FVSUFfTU9OSVRPUklOR19fCgo=
```
```

## Variables

An example of the GraphQL needed to create an advanced task is available in
createDumpSanitisedDB.gql. The variables which can be passed in are also
documented in this file (in the "displayName" field of each entry).

## Process

The process goes through X basic stages:

1. Set up all the initial variables
2. MTK creates a database dump that's basically a sanitised .sql file
3. Make docker-style container with sanitised DB (using podman)
4. Save new container to registry
2 changes: 1 addition & 1 deletion createDumpSanitisedDB.gql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,4 @@ mutation createDumpSanitisedDB {
}
}
}
}
}
134 changes: 134 additions & 0 deletions createDumpSanitisedDB_setDBVariables.gql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
mutation createDumpSanitisedDB {
addAdvancedTaskDefinition(
input:{
name: "Sanitised Database Image Build"
description: "Test making a sanitised database dump image"
confirmationText: "Are you sure you wish to create a sanitised database dump now?"
type: IMAGE
permission: MAINTAINER
image: "uselagoon/database-image-task:v0.0.1"
project: 123
advancedTaskDefinitionArguments: [
{
name: "BUILDER_DOCKER_COMPOSE_SERVICE_NAME",
displayName: "OPTIONAL: The name of the docker-compose service to backup (defaults to mariadb)",
type: STRING
optional: true
},
{
name: "BUILDER_BACKUP_IMAGE_NAME",
displayName: "The name of the resulting image to build without tag (eg, for dockerhub myproject/image, or for custom registry quay.io/myproject/image, can also be a pattern like '${registry}/${organization}/${project}/${environment}/${service}-data')",
type: STRING
# defaultValue: "${registry}/${organization}/${project}/${environment}/${service}-data"
},
{
name: "BUILDER_IMAGE_NAME",
displayName: "OPTIONAL: The name of the builder source image to do the initial db import (defaults to mariadb:10.6)",
type: STRING
optional: true
},
{
name: "BUILDER_CLEAN_IMAGE_NAME",
displayName: "OPTIONAL: The name of the clean source image that will be built into (defaults to uselagoon/mariadb-10.6-drupal:latest)",
type: STRING
optional: true
},
{
name: "BUILDER_MTK_YAML_BASE64",
displayName: "OPTIONAL: The base64 encoded value of the mtk dump file to use (defaults to no config)",
type: STRING
optional: true
},
{
name: "BUILDER_REGISTRY_USERNAME",
displayName: "The username to log in to registry with",
type: STRING
# defaultValue: "username"
},
{
name: "BUILDER_REGISTRY_PASSWORD",
displayName: "The password to log in to registry with",
type: STRING
# defaultValue: "password"
},
{
name: "BUILDER_REGISTRY_HOST",
displayName: "OPTIONAL: If not using dockerhub, define the registry to use (eg quay.io)",
type: STRING
optional: true
},
{
name: "BUILDER_REGISTRY_ORGANIZATION",
displayName: "OPTIONAL: if you want to provide an organization base for the backup image name to parse",
type: STRING
optional: true
},
# Database hostname
{
name: "BUILDER_MTK_DUMP_HOSTNAME",
displayName: "OPTIONAL: The database hostname you want to use",
type: STRING
optional: true
},
{
name: "BUILDER_MTK_DUMP_HOSTNAME_NAME",
displayName: "OPTIONAL: The name of the variable that contains the database hostname you want to use",
type: STRING
optional: true
},
# Database name
{
name: "BUILDER_MTK_DUMP_DATABASE",
displayName: "OPTIONAL: The database name you want to use",
type: STRING
optional: true
},
{
name: "BUILDER_MTK_DUMP_DATABASE_NAME",
displayName: "OPTIONAL: The name of the variable that contains the database name you want to use",
type: STRING
optional: true
},
# Database username
{
name: "BUILDER_MTK_DUMP_USERNAME",
displayName: "OPTIONAL: The database username you want to use",
type: STRING
optional: true
},
{
name: "BUILDER_MTK_DUMP_USERNAME_NAME",
displayName: "OPTIONAL: The name of the variable that contains the database username you want to use",
type: STRING
optional: true
},
# Database password
{
name: "BUILDER_MTK_DUMP_PASSWORD",
displayName: "OPTIONAL: The database password you want to use",
type: STRING
optional: true
},
{
name: "BUILDER_MTK_DUMP_PASSWORD_NAME",
displayName: "OPTIONAL: The name of the variable that contains the database password you want to use",
type: STRING
optional: true
}
]
}
){
... on AdvancedTaskDefinitionImage {
id
name
description
image
confirmationText
advancedTaskDefinitionArguments {
type
name
displayName
}
}
}
}
128 changes: 105 additions & 23 deletions mariadb-image-builder
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,98 @@ function projectEnvironmentVariableCheck() {
echo "$existingValue"
}

# function to return (echo) one of the MTK environment variables. Order of things to check:
# - Existing value
# - Passed variable name
# - Created from LAGOON_SERVICE_NAME
#
# Parameters are:
# - <name> Name of MTK variable to set
# - <search key> Search key to use when searching JSON
#
# Also reads:
# Variable | Check
# ---------------------------------------------------+--------------------------------
# BUILDER_MTK_DUMP_<name> | projectEnvironmentVariableCheck
# BUILDER_MTK_DUMP_<name>_NAME | projectEnvironmentVariableCheck
# The variable named in BUILDER_MTK_DUMP_<name>_NAME | No
# LAGOON_SERVICE_NAME | Should already be checked
function calc_MTK_variable() {
local mtkVariableName="$1"
local jsonSearchKey="$2"

# check for argument
[ "$mtkVariableName" ] || return

##### If MTK_DUMP_* has been set
# Example values
# If MTK_DUMP_DATABASE = foodb
# $final_variable_name = MTK_DUMP_DATABASE
# ${!final_variable_name} = foodb
local final_variable_name="MTK_DUMP_${mtkVariableName}"
local source_variable_name="BUILDER_${final_variable_name}"
declare "$source_variable_name"=$(projectEnvironmentVariableCheck "$source_variable_name" "${!source_variable_name}" "$jsonSearchKey")

if [ -n "${!source_variable_name}" ]; then
echo "${!source_variable_name}"
return
fi

##### If MTK_DUMP_*_NAME has been set
# Example values
# If DB_NAME_CENTRAL=foodb and MTK_DUMP_DATABASE_NAME = DB_NAME_CENTRAL
# $nameholder_variable_name = MTK_DUMP_DATABASE_NAME
# $use_var = DB_NAME_CENTRAL
# ${!use_var} = foodb
local nameholder_variable_name="${final_variable_name}_NAME"
declare "$nameholder_variable_name"=$(projectEnvironmentVariableCheck "$nameholder_variable_name" "${!nameholder_variable_name}")

if [ -n "${!nameholder_variable_name}" ]; then
local use_var=${!nameholder_variable_name}
if [ "${!use_var}" ]; then
echo "${!use_var}"
return
else
echo "Warning: Could not find value for $nameholder_variable_name > $use_var" >&2
fi
fi

##### Special settings for special types
case $mtkVariableName in
DATABASE | USERNAME | PASSWORD)
# Do nothing
;;
HOSTNAME)
# Set the DB_X host variables to what would be the lagoon service variable values.
# These are generated automatically from the LAGOON_SERVICE_NAME
# these will use bash references later; eg: ${!DB_HOST}
local DB_HOST="${LAGOON_SERVICE_NAME}_HOST"
local DB_READREPLICA_HOSTS="${LAGOON_SERVICE_NAME}_READREPLICA_HOSTS"

# Picks one of the read replicas to use when dumping the database
local DB_HOSTNAME
DB_HOSTNAME=$(echo "${!DB_READREPLICA_HOSTS}" | perl -F, -ane '$index=int(rand(scalar(@F)));print $F[$index]')
# if `DB_HOSTNAME` is empty, then need to fall back to mariadb host
if [ -z "$DB_HOSTNAME" ]; then
echo "Warning: No read replica found" >&2
DB_HOSTNAME="${!DB_HOST}"
fi
echo "$DB_HOSTNAME"
return
;;
CONFIG)
echo "Error: Can't directly handle '$mtkVariableName' variable -- please do separately" >&2
exit 1
;;
*)
echo "Error: Unrecognised MTK variable type: $mtkVariableName" >&2
exit 1
esac

##### Fall back to LAGOON_SERVICE_NAME
echo "${LAGOON_SERVICE_NAME}_USERNAME"
}

echo "======================="
echo "Starting image-builder"
echo "======================="
Expand Down Expand Up @@ -165,6 +257,13 @@ BUILDER_MTK_YAML_BASE64=$(projectEnvironmentVariableCheck BUILDER_MTK_YAML_BASE6
# optional
BUILDER_PUSH_TAGS=$(projectEnvironmentVariableCheck BUILDER_PUSH_TAGS "both")

# Set up the MTK variables
MTK_DUMP_HOSTNAME=$(calc_MTK_variable HOSTNAME "$MTK_DUMP_HOSTNAME")
MTK_DUMP_DATABASE=$(calc_MTK_variable DATABASE "$MTK_DUMP_DATABASE")
MTK_DUMP_USERNAME=$(calc_MTK_variable USERNAME "$MTK_DUMP_USERNAME")
MTK_DUMP_PASSWORD=$(calc_MTK_variable PASSWORD "$MTK_DUMP_PASSWORD")


# check the pattern
if [ -z $BUILDER_BACKUP_IMAGE_NAME ]; then
# default pattern to parse if no image provided
Expand All @@ -175,7 +274,7 @@ else
fi
echo $BUILDER_BACKUP_IMAGE_NAME

# error out if username and password aren't provided
# error out if registry username and password aren't provided
if [ -z $BUILDER_REGISTRY_USERNAME ]; then
echo "BUILDER_REGISTRY_USERNAME not defined"
exit 1
Expand All @@ -189,7 +288,6 @@ fi
# Generic variables
date=`date -I`
san_db_dump_filename="sanitised-dump.sql"
san_db_tarfile="container-${date}.tar"
backup_image_tag=${BUILDER_BACKUP_IMAGE_TAG:-"backup-${date}"}
backup_image_full="${BUILDER_BACKUP_IMAGE_NAME}:${backup_image_tag}"

Expand All @@ -199,30 +297,14 @@ backup_image_full="${BUILDER_BACKUP_IMAGE_NAME}:${backup_image_tag}"
echo
echo "=== Phase 2: MTK dump ==="

# set the DB_X variables to what would be the lagoon service variable values, these are handled automatically from the LAGOON_SERVICE_NAME
# these will use bash references later; eg: ${!DB_HOST}
DB_HOST="${LAGOON_SERVICE_NAME}_HOST"
DB_READREPLICA_HOSTS="${LAGOON_SERVICE_NAME}_READREPLICA_HOSTS"
DB_USERNAME="${LAGOON_SERVICE_NAME}_USERNAME"
DB_PASSWORD="${LAGOON_SERVICE_NAME}_PASSWORD"
DB_DATABASE="${LAGOON_SERVICE_NAME}_DATABASE"

# Picks one of the read replicas to use when dumping the database
DB_HOSTNAME=`echo "${!DB_READREPLICA_HOSTS}" | perl -F, -ane '$index=int(rand(scalar(@F)));print $F[$index]'`
# if `DB_HOSTNAME` is empty, then need to fall back to mariadb host
if [ -z $DB_HOSTNAME ]; then
DB_HOSTNAME="${!DB_HOST}"
fi
export MTK_DUMP_HOSTNAME MTK_DUMP_DATABASE MTK_DUMP_USERNAME MTK_DUMP_PASSWORD

# dump the MTK YAML to the mtk file if it has been provided, otherwise mtk will just dump the entire database as is
if [ ! -z $BUILDER_MTK_YAML_BASE64 ]; then
echo $BUILDER_MTK_YAML_BASE64 | base64 -d > mtk.yml
export MTK_DUMP_CONFIG="mtk.yml"
if [ -n "$BUILDER_MTK_YAML_BASE64" ]; then
mtk_filename=mtk.yml
echo "$BUILDER_MTK_YAML_BASE64" | base64 -d > $mtk_filename
export MTK_DUMP_CONFIG="$mtk_filename"
fi
export MTK_DUMP_HOSTNAME="$DB_HOSTNAME"
export MTK_DUMP_USERNAME="${!DB_USERNAME}"
export MTK_DUMP_PASSWORD="${!DB_PASSWORD}"
export MTK_DUMP_DATABASE="${!DB_DATABASE}"

# Run MTK here
mtk-dump > "$san_db_dump_filename"
Expand Down

0 comments on commit 2ace668

Please sign in to comment.