feat(events): add postgres docker file

lib/workload/stateless/filemanager/database/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:15
+
+COPY migrations/ /docker-entrypoint-initdb.d/

lib/workload/stateless/filemanager/database/database.sh

@@ -0,0 +1,74 @@
+function usage() {
+  usage="$(basename "$0") [-h] [-u <USER>] [-c init|migrate|reset] -- administer the filemanager database.
+
+  commands:
+    init  initialize the database
+    migrate  migrate the database using files in the migrations directory
+    reset  drop and recreate the database
+
+  options:
+      -h  show this help text
+      -u  set the user for the database [default: POSTGRES_USER environment variable or not set]
+      -c  the command for this script to run [default: not set, required]"
+
+  echo "$usage"
+}
+
+function run_command() {
+  if [[ "$command" == "init" ]]; then
+    /bin/bash init_database.sh "${args[@]}"
+  elif [[ "$command" == "migrate" ]]; then
+    /bin/bash migrate.sh "${args[@]}"
+  elif [[ "$command" == "reset" ]]; then
+    /bin/bash reset_database.sh "${args[@]}"
+  else
+    printf "unknown command: -%s\n" "$command" >&2
+    usage >&2
+    exit 1
+  fi
+}
+
+function set_args() {
+    if [[ -n "${POSTGRES_USER}" ]]; then
+      echo "Using ${POSTGRES_USER} as the postgres user"
+      args+=( "--username" )
+      args+=( "POSTGRES_USER" )
+    else
+      echo "No user supplied, using default"
+    fi
+}
+
+args=()
+command=""
+while getopts ':hu:c:' option; do
+  case "$option" in
+    h)
+      usage
+      exit
+      ;;
+    u)
+      args+=( "--username" )
+      args+=( "$OPTARG" )
+      ;;
+    c)
+      command="$OPTARG"
+      ;;
+    :)
+      printf "missing argument for -%s\n" "$OPTARG" >&2
+      usage >&2
+      exit 1
+      ;;
+    \?)
+      printf "invalid option: -%s\n" "$OPTARG" >&2
+      usage >&2
+      exit 1
+      ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+if [[ ${#args[@]} -eq 0 ]]; then
+  set_args
+fi
+
+run_command

lib/workload/stateless/filemanager/database/init_database.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-createdb filemanager
+createdb "$@" filemanager

lib/workload/stateless/filemanager/database/migrate.sh

@@ -1,16 +1,6 @@
 #!/bin/bash
 
-args=()
-
-if [ -z "$1" ]
-  then
-    echo "No user supplied, using default"
-  else
-    args+=( "-u" )
-    args+=( "$1" )
-fi
-
 find ./migrations -type f -print0 | while IFS= read -r -d $'\0' file; do
     echo "Running migration for $file"
-    psql "${args[@]}" -d filemanager -a -f "$file"
+    psql "$@" -d filemanager -a -f "$file"
 done

lib/workload/stateless/filemanager/database/reset_database.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-dropdb filemanager
-createdb filemanager
+dropdb "$@" filemanager
+createdb "$@" filemanager

lib/workload/stateless/filemanager/docker-compose.yml

@@ -0,0 +1,16 @@
+# NOTE:
+# If you update image version, make sure to update in docker-compose.ci.yml as well
+
+version: '3.1'
+
+services:
+  db:
+    build: database
+    container_name: filemanager_db
+    restart: always
+    environment:
+      - POSTGRES_DATABASE=filemanager
+      - POSTGRES_USER=filemanager
+      - POSTGRES_PASSWORD=filemanager
+    ports:
+      - "5432:5432"

lib/workload/stateless/filemanager/filemanager/src/database/mod.rs

@@ -1,4 +1,4 @@
-pub mod aws;
+pub mod s3;
 
 use crate::error::Error::DbClientError;
 use crate::error::Result;

lib/workload/stateless/filemanager/filemanager/src/database/aws/ingester.rs → lib/workload/stateless/filemanager/filemanager/src/database/s3/ingester.rs

@@ -4,8 +4,8 @@ use futures::StreamExt;
 use sqlx::{Executor, Postgres, query_file, QueryBuilder};
 use uuid::Uuid;
 use crate::database::DbClient;
-use crate::events::aws::{Events, FlatS3EventMessage, FlatS3EventMessages, TransposedS3EventMessages};
-use crate::events::aws::s3::S3;
+use crate::events::s3::{Events, FlatS3EventMessage, FlatS3EventMessages, TransposedS3EventMessages};
+use crate::events::s3::s3::S3;
 use crate::error::Result;
 
 /// An ingester for S3 events.
@@ -48,11 +48,11 @@ impl Ingester {
             last_modified_dates
         } = object_created;
 
-        query_file!("queries/ingester/insert_objects.sql", object_ids, buckets, keys, sizes, e_tags, event_times, last_modified_dates, vec![None; object_ids.len()], portal_run_ids)
+        query_file!("../database/queries/ingester/insert_objects.sql", object_ids, buckets, keys, sizes, e_tags, event_times, last_modified_dates, vec![None; object_ids.len()], portal_run_ids)
             .execute(&mut self.db.pool)
             .await?;
 
-        query_file!("queries/ingester/aws/insert_s3_objects.sql", object_ids, storage_classes)
+        query_file!("../database/queries/ingester/aws/insert_s3_objects.sql", object_ids, storage_classes)
             .execute(&mut self.db.pool)
             .await?;
 
@@ -70,7 +70,7 @@ impl Ingester {
             last_modified_dates
         } = object_removed;
 
-        query_file!("queries/update_deleted.sql", keys, buckets, event_times)
+        query_file!("../database/queries/update_deleted.sql", keys, buckets, event_times)
             .execute(&mut self.db.pool)
             .await?;
 

lib/workload/stateless/filemanager/filemanager/src/events/mod.rs

@@ -1,9 +1,9 @@
 //! This module handles converting storage events into database objects
 //!
 
-pub mod aws;
+pub mod s3;
 
-use crate::events::aws::Events;
+use crate::events::s3::Events;
 use crate::error::Result;
 
 /// This trait processes raw events into a common type that can easily be consumed by the database.