Updated the Backup script to work with single node

Removed hardcoded values for DNS zone in userdata scripts
Updated the NSGs ports based on graphdb_node_count
Updated user_data.tf
Changed the name of the LB target group to avoid conflicts when scaling from 1 to 3 AZs
Updated the monitoring, to not deploy cluster alarms when a single node is deployed
Updated the availability_tests
Updated how the VPC azs are calculated based on the graphdb_node_count
Added calculations for the subnets based on the graphdb_node_count
Removed route53_availability_content_match from modules\monitoring
Moved route53_availability_http_string_type to root level.
Changed the availability test to support single node deployment.
Added dynamic change of the availability tests http string type based on tls_enabled
Updated the README.md
Updated CHANGELOG.md
Made duplicate code into functions and moved it to 00_functions.sh
Updated README.md example for existing subnets
Renamed 10_start_graphdb_services.sh.tpl to 10_start_single_graphdb_services.sh.tpl

CHANGELOG.md

@@ -1,5 +1,12 @@
 # GraphDB AWS Terraform Module Changelog
 
+# 1.2.0
+* Added support for single node deployment
+  * Added new userdata script `10_start_graphdb_services.sh.tpl` for single node setup.
+  * Made cluster-related userdata scripts executable only when graphdb_node_count is greater than 1.
+* Removed hardcoded values from the userdata scripts.
+* Changed the availability tests http_string_type to be calculated based on TLS being enabled.
+
 ## 1.0.1
 
 * Updated GraphDB version to [10.6.4](https://graphdb.ontotext.com/documentation/10.6/release-notes.html#graphdb-10-6-4)

README.md

@@ -329,10 +329,19 @@ If you have existing VPC created in your account, you can use that VPC to deploy
 What you need to do is to specify values for the following variables
 ```hcl
 vpc_id = "vpc-12345678"
-vpc_public_subnet_ids = ["public-subnet-1","public-subnet2","public-subnet-3"]
-vpc_private_subnet_ids = ["private-subnet-1","private-subnet-2","private-subnet-3"]
+vpc_public_subnet_ids = ["subnet-123456","subnet-234567","subnet-345678"]
+vpc_private_subnet_ids = ["subnet-456789","subnet-567891","subnet-678912"]
 ```
 
+## Single Node Deployment
+
+This Terraform module has the ability to deploy a single instance of GraphDB.
+To deploy a single instance you just need to set `graphdb_node_count` to 1, everything else happens automatically.
+
+**Important:** While scaling from a single node deployment to a cluster (e.g., from 1 node to 3 nodes) is possible,
+it is not recommended. Synchronizing the repository across all nodes can be time-consuming,
+potentially causing scripts to time out.
+
 ## Updating configurations on an active deployment
 
 ### Updating Configurations

graphdb.properties

@@ -0,0 +1 @@
+laina=laina

main.tf

@@ -2,6 +2,27 @@ data "aws_region" "current" {}
 
 data "aws_caller_identity" "current" {}
 
+locals {
+  # Reduce to one subnet if node_count is 1
+  effective_private_subnet_cidrs = var.graphdb_node_count == 1 ? [var.vpc_private_subnet_cidrs[0]] : var.vpc_private_subnet_cidrs
+  effective_public_subnet_cidrs  = var.graphdb_node_count == 1 ? [var.vpc_public_subnet_cidrs[0]] : var.vpc_public_subnet_cidrs
+  # Determine the appropriate subnets based on node_count
+  lb_subnets = (var.graphdb_node_count == 1 ?
+    (var.vpc_id == "" ?
+      (var.lb_internal ? [module.vpc[0].private_subnet_ids[0]] : [module.vpc[0].public_subnet_ids[0]]) :
+      (var.lb_internal ? [var.vpc_private_subnet_ids[0]] : [var.vpc_public_subnet_ids[0]])
+    ) :
+    (var.vpc_id == "" ?
+      (var.lb_internal ? module.vpc[0].private_subnet_ids : module.vpc[0].public_subnet_ids) :
+      (var.lb_internal ? var.vpc_private_subnet_ids : var.vpc_public_subnet_ids)
+  ))
+  # Check if node_count is 1 and select only one subnet if true
+  graphdb_subnets = (var.graphdb_node_count == 1 ?
+    [(var.vpc_id != "" ? var.vpc_private_subnet_ids : module.vpc[0].private_subnet_ids)[0]] :
+    (var.vpc_id != "" ? var.vpc_private_subnet_ids : module.vpc[0].private_subnet_ids)
+  )
+}
+
 module "vpc" {
   source = "./modules/vpc"
 
@@ -10,8 +31,8 @@ module "vpc" {
   resource_name_prefix                            = var.resource_name_prefix
   vpc_dns_hostnames                               = var.vpc_dns_hostnames
   vpc_dns_support                                 = var.vpc_dns_support
-  vpc_private_subnet_cidrs                        = var.vpc_private_subnet_cidrs
-  vpc_public_subnet_cidrs                         = var.vpc_public_subnet_cidrs
+  vpc_private_subnet_cidrs                        = local.effective_private_subnet_cidrs
+  vpc_public_subnet_cidrs                         = local.effective_public_subnet_cidrs
   vpc_cidr_block                                  = var.vpc_cidr_block
   single_nat_gateway                              = var.single_nat_gateway
   enable_nat_gateway                              = var.enable_nat_gateway
@@ -21,6 +42,7 @@ module "vpc" {
   vpc_endpoint_service_accept_connection_requests = var.vpc_endpoint_service_accept_connection_requests
   vpc_enable_flow_logs                            = var.vpc_enable_flow_logs
   vpc_flow_log_bucket_arn                         = var.vpc_enable_flow_logs && var.deploy_logging_module ? module.logging[0].graphdb_logging_bucket_arn : null
+  graphdb_node_count                              = var.graphdb_node_count
 }
 
 module "backup" {
@@ -93,7 +115,7 @@ module "load_balancer" {
 
   resource_name_prefix          = var.resource_name_prefix
   vpc_id                        = var.vpc_id != "" ? var.vpc_id : module.vpc[0].vpc_id
-  lb_subnets                    = var.vpc_id == "" ? (var.lb_internal ? module.vpc[0].private_subnet_ids : module.vpc[0].public_subnet_ids) : (var.lb_internal ? var.vpc_private_subnet_ids : var.vpc_public_subnet_ids)
+  lb_subnets                    = local.lb_subnets
   lb_internal                   = var.lb_internal
   lb_deregistration_delay       = var.lb_deregistration_delay
   lb_health_check_path          = var.lb_health_check_path
@@ -103,6 +125,7 @@ module "load_balancer" {
   lb_tls_policy                 = var.lb_tls_policy
   lb_access_logs_bucket_name    = var.lb_enable_access_logs && var.deploy_logging_module ? module.logging[0].graphdb_logging_bucket_name : null
   lb_enable_access_logs         = var.lb_enable_access_logs
+  graphdb_node_count            = var.graphdb_node_count
 }
 
 locals {
@@ -111,6 +134,11 @@ locals {
   )
 }
 
+locals {
+  lb_tls_enabled              = var.lb_tls_certificate_arn != null ? true : false
+  calculated_http_string_type = local.lb_tls_enabled == true ? "HTTPS" : "HTTP"
+}
+
 module "monitoring" {
   source = "./modules/monitoring"
   providers = {
@@ -129,6 +157,8 @@ module "monitoring" {
   cloudwatch_log_group_retention_in_days = var.monitoring_log_group_retention_in_days
   route53_availability_request_url       = module.load_balancer.lb_dns_name
   route53_availability_measure_latency   = var.monitoring_route53_measure_latency
+  graphdb_node_count                     = var.graphdb_node_count
+  route53_availability_http_string_type  = local.calculated_http_string_type
 }
 
 module "graphdb" {
@@ -142,13 +172,13 @@ module "graphdb" {
 
   allowed_inbound_cidrs     = var.allowed_inbound_cidrs_lb
   allowed_inbound_cidrs_ssh = var.allowed_inbound_cidrs_ssh
-  graphdb_subnets           = var.vpc_id != "" ? var.vpc_private_subnet_ids : module.vpc[0].private_subnet_ids
+  graphdb_subnets           = local.graphdb_subnets
   graphdb_target_group_arns = local.graphdb_target_group_arns
   vpc_id                    = var.vpc_id != "" ? var.vpc_id : module.vpc[0].vpc_id
 
   # Network Load Balancer
   lb_enable_private_access = var.lb_internal ? var.lb_enable_private_access : false
-  lb_subnets               = var.vpc_id == "" ? (var.lb_internal ? module.vpc[0].private_subnet_ids : module.vpc[0].public_subnet_ids) : (var.lb_internal ? var.vpc_private_subnet_ids : var.vpc_public_subnet_ids)
+  lb_subnets               = local.lb_subnets
   graphdb_lb_dns_name      = module.load_balancer.lb_dns_name
 
   # GraphDB Configurations

modules/graphdb/nsg.tf

@@ -1,3 +1,7 @@
+locals {
+  calculated_graphdb_port = var.graphdb_node_count == 1 ? 7201 : 7200
+}
+
 resource "aws_security_group" "graphdb_security_group" {
   name        = var.resource_name_prefix
   description = "Security group for GraphDB components"
@@ -8,13 +12,15 @@ resource "aws_security_group_rule" "graphdb_internal_http" {
   description       = "Allow GraphDB proxies and nodes to communicate (HTTP)."
   security_group_id = aws_security_group.graphdb_security_group.id
   type              = "ingress"
-  from_port         = 7200
+  from_port         = local.calculated_graphdb_port
   to_port           = 7201
   protocol          = "tcp"
   cidr_blocks       = local.subnet_cidr_blocks
 }
 
 resource "aws_security_group_rule" "graphdb_internal_raft" {
+  count = var.graphdb_node_count != 1 ? 1 : 0
+
   description       = "Allow GraphDB proxies and nodes to communicate (Raft)."
   security_group_id = aws_security_group.graphdb_security_group.id
   type              = "ingress"
@@ -25,7 +31,8 @@ resource "aws_security_group_rule" "graphdb_internal_raft" {
 }
 
 resource "aws_security_group_rule" "graphdb_ssh_inbound" {
-  count             = var.allowed_inbound_cidrs_ssh != null ? 1 : 0
+  count = var.allowed_inbound_cidrs_ssh != null ? 1 : 0
+
   description       = "Allow specified CIDRs SSH access to the GraphDB instances."
   security_group_id = aws_security_group.graphdb_security_group.id
   type              = "ingress"
@@ -51,8 +58,8 @@ resource "aws_security_group_rule" "graphdb_network_lb_ingress" {
   description       = "CIRDs allowed to access GraphDB."
   security_group_id = aws_security_group.graphdb_security_group.id
   type              = "ingress"
-  from_port         = 7200
-  to_port           = 7200
+  from_port         = local.calculated_graphdb_port
+  to_port           = local.calculated_graphdb_port
   protocol          = "tcp"
   cidr_blocks       = var.allowed_inbound_cidrs
 }
@@ -64,7 +71,7 @@ resource "aws_security_group_rule" "graphdb_lb_healthchecks" {
   description       = "Allow the load balancer to healthcheck the GraphDB nodes and access the proxies."
   security_group_id = aws_security_group.graphdb_security_group.id
   type              = "ingress"
-  from_port         = 7200
+  from_port         = local.calculated_graphdb_port
   to_port           = 7201
   protocol          = "tcp"
   cidr_blocks       = local.lb_subnet_cidr_blocks

modules/graphdb/templates/00_functions.sh

@@ -6,3 +6,124 @@
 log_with_timestamp() {
   echo "$(date '+%Y-%m-%d %H:%M:%S'): $1"
 }
+
+# Function which waits for all DNS records to be created
+wait_dns_records() {
+  local ZONE_ID="$1"
+  local ROUTE53_ZONE_DNS_NAME="$2"
+  local ASG_NAME="$3"
+  local NODE_COUNT=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names "$ASG_NAME" --query "AutoScalingGroups[0].DesiredCapacity" --output text)
+  local all_dns_records=($(aws route53 list-resource-record-sets --hosted-zone-id "$${ZONE_ID}" --query "ResourceRecordSets[?contains(Name, '.$${ROUTE53_ZONE_DNS_NAME}') == \`true\`].Name" --output text))
+  local all_dns_records_count="$${#all_dns_records[@]}"
+
+  if [ "$${all_dns_records_count}" -ne $${NODE_COUNT} ]; then
+    sleep 5
+    log_with_timestamp "Private DNS zone record count is $${all_dns_records_count}, expecting $${NODE_COUNT}"
+    wait_dns_records "$ZONE_ID" "$ROUTE53_ZONE_DNS_NAME" "$ASG_NAME"
+  else
+    log_with_timestamp "Private DNS zone record count is $${all_dns_records_count}, expecting $${NODE_COUNT}"
+  fi
+}
+
+# Function which checks if GraphDB is started, we assume it is when the infrastructure endpoint is reached
+check_gdb() {
+  if [ -z "$1" ]; then
+    log_with_timestamp "Error: IP address or hostname is not provided."
+    return 1
+  fi
+
+  local gdb_address="http://$1:7201/rest/monitor/infrastructure"
+  if curl -s --head -u "admin:$${GRAPHDB_ADMIN_PASSWORD}" --fail "$${gdb_address}" >/dev/null; then
+    log_with_timestamp "Success, GraphDB node $${gdb_address} is available"
+    return 0
+  else
+    log_with_timestamp "GraphDB node $${gdb_address} is not available yet"
+    return 1
+  fi
+}
+
+check_all_dns_records() {
+  local ZONE_ID="$1"
+  local ROUTE53_ZONE_DNS_NAME="$2"
+  local RETRY_DELAY="$${3:-5}"  # Default retry delay to 5 seconds if not provided
+
+  # Retrieve existing DNS records
+  local EXISTING_DNS_RECORDS
+  EXISTING_DNS_RECORDS=$(aws route53 list-resource-record-sets --hosted-zone-id "$${ZONE_ID}" --query "ResourceRecordSets[?contains(Name, '.$${ROUTE53_ZONE_DNS_NAME}') == \`true\`].Name")
+
+  # Convert the output into an array
+  local EXISTING_DNS_RECORDS_ARRAY
+  readarray -t EXISTING_DNS_RECORDS_ARRAY <<<"$(echo "$EXISTING_DNS_RECORDS" | jq -r '.[] | rtrimstr(".")')"
+
+  # Wait for all instances to be running
+  for record in "$${EXISTING_DNS_RECORDS_ARRAY[@]}"; do
+    log_with_timestamp "Pinging $record"
+    if [ -n "$record" ]; then
+      while ! check_gdb "$record"; do
+        log_with_timestamp "Waiting for GDB $record to start"
+        sleep "$RETRY_DELAY"
+      done
+    else
+      log_with_timestamp "Error: address is empty."
+    fi
+  done
+
+  log_with_timestamp "All GDB instances are available."
+}
+
+configure_graphdb_security() {
+  local GRAPHDB_PASSWORD=$1
+  local GRAPHDB_URL=$${2:-"http://localhost:7201"}
+
+  IS_SECURITY_ENABLED=$(curl -s -X GET \
+    --header 'Accept: application/json' \
+    -u "admin:$GRAPHDB_PASSWORD" \
+    "$${GRAPHDB_URL}/rest/security")
+
+  # Check if GDB security is enabled
+  if [[ $IS_SECURITY_ENABLED == "true" ]]; then
+    log_with_timestamp "Security is enabled"
+  else
+    # Set the admin password
+    SET_PASSWORD=$(
+      curl --location -s -w "%%{http_code}" \
+        --request PATCH "$${GRAPHDB_URL}/rest/security/users/admin" \
+        --header 'Content-Type: application/json' \
+        --data "{ \"password\": \"$GRAPHDB_PASSWORD\" }"
+    )
+    if [[ "$SET_PASSWORD" == 200 ]]; then
+      log_with_timestamp "Set GraphDB password successfully"
+    else
+      log_with_timestamp "Failed setting GraphDB password. Please check the logs!"
+    fi
+
+    # Enable the security
+    ENABLED_SECURITY=$(curl -X POST -s -w "%%{http_code}" \
+      --header 'Content-Type: application/json' \
+      --header 'Accept: */*' \
+      -d 'true' "$${GRAPHDB_URL}/rest/security")
+
+    if [[ "$ENABLED_SECURITY" == 200 ]]; then
+      log_with_timestamp "Enabled GraphDB security successfully"
+    else
+      log_with_timestamp "Failed enabling GraphDB security. Please check the logs!"
+    fi
+  fi
+}
+
+# Function to check if the GraphDB license has been applied
+check_license() {
+  # Define the URL to check
+  local URL="http://localhost:7201/rest/graphdb-settings/license"
+
+  # Send an HTTP GET request and store the response in a variable
+  local response=$(curl -s "$URL")
+
+  # Check if the response contains the word "free"
+  if [[ "$response" == *"free"* ]]; then
+    log_with_timestamp "Free license detected, EE license required, exiting!"
+    exit 1
+  else
+    echo "License is mounted"
+  fi
+}

modules/graphdb/templates/05_gdb_backup_conf.sh.tpl

@@ -20,18 +20,16 @@ if [ ${deploy_backup} == "true" ]; then
   cat <<-EOF >/usr/bin/graphdb_backup
 #!/bin/bash
 
-set -euxo pipefail
+set -euo pipefail
 
 GRAPHDB_ADMIN_PASSWORD="\$(aws --cli-connect-timeout 300 ssm get-parameter --region ${region} --name "/${name}/graphdb/admin_password" --with-decryption | jq -r .Parameter.Value | base64 -d)"
-NODE_STATE="\$(curl --silent --fail --user "admin:\$GRAPHDB_ADMIN_PASSWORD" localhost:7201/rest/cluster/node/status | jq -r .nodeState)"
-
-if [ "\$NODE_STATE" != "LEADER" ]; then
-  echo "current node is not a leader, but \$NODE_STATE"
-  exit 0
-fi
+NODE_STATE="\$(curl --silent -u "admin:\$GRAPHDB_ADMIN_PASSWORD" http://localhost:7201/rest/cluster/node/status | jq -r .nodeState)"
 
 function trigger_backup {
   local backup_name="\$(date +'%Y-%m-%d_%H-%M-%S').tar"
+  current_time=$(date +"%T %Y-%m-%d")
+  start_time=$(date +%s)
+  echo "Creating backup $backup_name at $start_time"
 
   curl \
     -vvv --fail \
@@ -59,9 +57,23 @@ function rotate_backups {
   done
 }
 
-if ! trigger_backup; then
-  echo "failed to create backup"
-  exit 1
+# Checks if GraphDB is running in cluster
+IS_CLUSTER=\$(
+  curl -s -o /dev/null \
+    -u "admin:\$GRAPHDB_ADMIN_PASSWORD" \
+    -w "%%{http_code}" \
+    http://localhost:7200/rest/monitor/cluster
+)
+
+if [ "\$IS_CLUSTER" == 200 ]; then
+  # Checks if the current GraphDB instance is Leader, otherwise exits.
+  if [ "\$NODE_STATE" != "LEADER" ]; then
+    echo "current node is not a leader, but \$NODE_STATE"
+    exit 0
+  fi
+  (trigger_backup && echo "") | tee -a /var/opt/graphdb/node/graphdb_backup.log
+elif [ "\$IS_CLUSTER" == 503 ]; then
+  (trigger_backup && echo "") | tee -a /var/opt/graphdb/node/graphdb_backup.log
 fi
 
 rotate_backups