Draft: s3invsync setup -- inject AWS credentials for script for LINC

.gitignore

@@ -0,0 +1 @@
+.idea/

s3invsync/README.md

@@ -0,0 +1,120 @@
+## Setup Steps on AWS Side
+
+### Enable Versioning and Inventory Configuration Management
+
+The following is example output from `aws s3api list-bucket-inventory-configurations --bucket <bucket-name-here>`
+
+In the `OptionalFields` section, you can see the fields needed to properly handle a sync of the S3 bucket via `s3invsync`
+
+```shell
+{
+    "InventoryConfigurationList": [
+        {
+            "Destination": {
+                "S3BucketDestination": {
+                    "AccountId": "151312473579",
+                    "Bucket": "arn:aws:s3:::<bucket-name-here>",
+                    "Format": "CSV"
+                }
+            },
+            "IsEnabled": true,
+            "Id": "versioned-s3invsync",
+            "IncludedObjectVersions": "All",
+            "OptionalFields": [
+                "Size",
+                "LastModifiedDate",
+                "ETag",
+                "IsMultipartUploaded",
+                "ChecksumAlgorithm"
+            ],
+            "Schedule": {
+                "Frequency": "Daily"
+            }
+        }
+    ],
+    "IsTruncated": false
+}
+```
+
+### Issued a new user:
+
+```
+{
+    "UserId": "AIDASGOXNDXVU6GU3SSNH",
+    "Account": "151312473579",
+    "Arn": "arn:aws:iam::151312473579:user/engaging-user"
+}
+```
+
+with IAM policy attached:
+
+```shell
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        ...other stuff
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:GetObject",
+                "s3:GetObjectVersion",
+                "s3:GetObjectAttributes",
+                "s3:GetBucketLocation",
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
+            ],
+            "Resource": [
+                "arn:aws:s3:::your-bucket-name",
+                "arn:aws:s3:::your-bucket-name/*"
+            ]
+        }
+        ...
+    ]
+}
+```
+
+**Note:** Ensure that user is read-only!
+
+### Setup credentials in Engaging
+
+```shell
+mkdir -p ~/.aws
+cat > ~/.aws/credentials <<EOF
+[default]
+aws_access_key_id=<access_key>
+aws_secret_access_key=<aws_secret_access_key>
+EOF
+
+cat > ~/.aws/config <<EOF
+[default]
+region=us-east-2
+EOF
+```
+
+### Corresponding S3 bucket policy (for example)
+
+```shell
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        ...other stuff -- added the clause below
+        {
+            "Sid": "AllowUserAccess",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam::151312473579:user/engaging-user"
+            },
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetObject",
+                "s3:GetObjectVersion"
+            ],
+            "Resource": [
+                "arn:aws:s3:::mgh-neuroglancer",
+                "arn:aws:s3:::mgh-neuroglancer/*"
+            ]
+        },
+        ...
+    ]
+}
+```

s3invsync/linc_s3invsync.sbatch

@@ -0,0 +1,58 @@
+#!/bin/bash
+#SBATCH --job-name=smart_job
+#SBATCH --partition=ou_bcs_normal
+#SBATCH --time=11:30:00   # Between 11 and 12 hours
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mem=16G
+#SBATCH [email protected]
+#SBATCH --mail-type=BEGIN,END,FAIL
+
+TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
+
+OUTPUT_DIR="$HOME/slurm_logs/$TIMESTAMP"
+mkdir -p "$OUTPUT_DIR"
+
+# Redirect standard output and error manually since SBATCH can't use runtime variables
+exec > >(tee -a "$OUTPUT_DIR/job_output_$TIMESTAMP.txt") 2> >(tee -a "$OUTPUT_DIR/job_error_$TIMESTAMP.txt" >&2)
+
+# Find an available node within `ou_bcs_normal` that meets time requirements
+# ...probably unnecessary, but included to make sure that the node is consistent since MIT Engaging is changing often
+SELECTED_NODE=$(sinfo --format="%P %N %T %l" | awk '
+function time_to_minutes(t) {
+    split(t, a, ":");
+    return (length(a) == 3) ? (a[1] * 60 + a[2]) : a[1];
+}
+$1 == "ou_bcs_normal" && time_to_minutes($4) >= 660 && time_to_minutes($4) <= 720 && $3 ~ /idle|mix/ {print $2; exit}')
+
+if [ -z "$SELECTED_NODE" ]; then
+    echo "No available idle or mixed nodes in ou_bcs_normal with an 11-12 hour time limit. Exiting."
+    exit 1
+fi
+
+echo "Selected node: $SELECTED_NODE"
+
+echo "Setting up AWS CLI..."
+AWS_CLI_DIR="$HOME/aws-cli-temp"
+mkdir -p "$AWS_CLI_DIR"
+cd "$AWS_CLI_DIR"
+
+python3 -m venv aws-cli-venv
+source aws-cli-venv/bin/activate
+pip install --upgrade pip
+pip install awscli jq
+aws configure list
+aws --version
+aws sts get-caller-identity
+
+echo "AWS credentials successfully retrieved."
+
+## Cleanup AWS CLI
+deactivate 
+rm -rf "$AWS_CLI_DIR"
+
+echo "Running linc_s3invsync job at $TIMESTAMP..."
+
+cargo install s3invsync --version 0.2.0
+
+s3invsync --allow-new-nonempty --ok-errors all s3://linc-brain-mit-staging-us-east-2/linc-brain-mit-staging-us-east-2/staging-configuration /orcd/data/linc/001/s3lincbrain/