88 add clowder connection containers back to the docker compose (#89)

* update traefik * fix inconsistent environment variable * remove commented out dependencies * depend * correct dependency * clean up hostip, minio url * add missing environment variables, match docker-compose with nginx * remove inline comment * change everything to latest * split to two yaml file and add example to deploy with clowder or not * remove volume * changelog entry * fix typo
ncsa · Jan 18, 2024 · 72f7d96 · 72f7d96
1 parent 2bfadc7
commit 72f7d96
Show file tree

Hide file tree

Showing 6 changed files with 160 additions and 131 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Rabbitmq username and password [#61](https://github.com/ncsa/standalone-smm-analytics/issues/90)
 
+### Added
+- Docker compose file for clowder connection [#88](https://github.com/ncsa/standalone-smm-analytics/issues/88)
+
 ## [Beta] - 10-26-2023
 
 ### Added

diff --git a/containerized_analytics/smile/docker-command-smile-traefik.sh b/containerized_analytics/smile/docker-command-smile-traefik.sh
@@ -82,6 +82,9 @@ export CLOWDER_ON=false
 # start
 docker-compose -f docker-compose-smile-traefik.yml -d up
 
+# start with clowder add-on
+# docker-compose -f docker-compose-smile-traefik.yml -f docker-compose-smile-clowder.yml -d up
+
 # stop
 # docker-compose -f docker-compose-smile.yml down -v
 

diff --git a/containerized_analytics/smile/docker-command-smile.sh b/containerized_analytics/smile/docker-command-smile.sh
@@ -69,11 +69,14 @@ export GOOGLE_CLIENT_SECRET=<<google client secret>>
 
 export CLOWDER_BASE_URL=https://clowder.server.com/
 export CLOWDER_GLOBAL_KEY=<<clowder global key>>
-export CLOWSER_ON=false
+export CLOWDER_ON=false
 
 # start
 docker-compose -f docker-compose-smile.yml -d up
 
+# start with clowder add-on
+# docker-compose -f docker-compose-smile.yml -f docker-compose-smile-clowder.yml -d up
+
 # stop
 # docker-compose -f docker-compose-smile.yml down -v
 

diff --git a/containerized_analytics/smile/docker-compose-smile-clowder.yml b/containerized_analytics/smile/docker-compose-smile-clowder.yml
@@ -0,0 +1,68 @@
+version: "3.3"
+
+services:
+  clowder-upload-file:
+    image: socialmediamacroscope/clowder_upload_file:latest
+    container_name: clowder_upload_file
+    depends_on:
+      - rabbitmq
+    command: python ./rabbitmq_handler.py
+    environment:
+      - AWS_ACCESSKEY=${AWS_ACCESSKEY}
+      - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET}
+      - BUCKET_NAME=${BUCKET_NAME}
+      - CLOWDER_BASE_URL=${CLOWDER_BASE_URL}
+      - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
+      - MINIO_URL=${MINIO_URL}
+      - QUEUE_NAME=lambda_upload_clowder
+      - RABBITMQ_HOST=${RABBITMQ_HOST}
+    restart: unless-stopped
+
+  clowder-list:
+    image: socialmediamacroscope/clowder_list:latest
+    container_name: clowder_list
+    depends_on:
+      - rabbitmq
+    command: python ./rabbitmq_handler.py
+    environment:
+      - CLOWDER_BASE_URL=${CLOWDER_BASE_URL}
+      - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY}
+      - QUEUE_NAME=lambda_list_clowder
+      - RABBITMQ_HOST=${RABBITMQ_HOST}
+    restart: unless-stopped
+
+  clowder-create-space:
+    image: socialmediamacroscope/clowder_create_space:latest
+    container_name: clowder_create_space
+    depends_on:
+      - rabbitmq
+    command: python ./rabbitmq_handler.py
+    environment:
+      - CLOWDER_BASE_URL=${CLOWDER_BASE_URL}
+      - QUEUE_NAME=clowder_create_space
+      - RABBITMQ_HOST=${RABBITMQ_HOST}
+    restart: unless-stopped
+
+  clowder-create-dataset:
+    image: socialmediamacroscope/clowder_create_dataset:latest
+    container_name: clowder_create_dataset
+    depends_on:
+      - rabbitmq
+    command: python ./rabbitmq_handler.py
+    environment:
+      - CLOWDER_BASE_URL=${CLOWDER_BASE_URL}
+      - QUEUE_NAME=lambda_invoke_clowder
+      - RABBITMQ_HOST=${RABBITMQ_HOST}
+    restart: unless-stopped
+
+  clowder-create-collection:
+    image: socialmediamacroscope/clowder_create_collection:latest
+    container_name: clowder_create_collection
+    depends_on:
+      - rabbitmq
+    command: python ./rabbitmq_handler.py
+    environment:
+      - CLOWDER_BASE_URL=${CLOWDER_BASE_URL}
+      - QUEUE_NAME=clowder_create_collection
+      - RABBITMQ_HOST=${RABBITMQ_HOST}
+    restart: unless-stopped
diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml
@@ -82,13 +82,12 @@ services:
       - traefik.tcp.http.routers.redis.tls.certresolver=myresolver
 
   algorithm-classification-predict:
-    image: socialmediamacroscope/classification_predict:0.1.2
+    image: socialmediamacroscope/classification_predict:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -98,13 +97,12 @@ services:
     restart: unless-stopped
 
   algorithm-classification-split:
-    image: socialmediamacroscope/classification_split:0.1.2
+    image: socialmediamacroscope/classification_split:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -114,13 +112,12 @@ services:
     restart: unless-stopped
 
   algorithm-classification-train:
-    image: socialmediamacroscope/classification_train:0.1.2
+    image: socialmediamacroscope/classification_train:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -130,13 +127,12 @@ services:
     restart: unless-stopped
 
   algorithm-histogram:
-    image: socialmediamacroscope/histogram:0.1.1
+    image: socialmediamacroscope/histogram:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      - MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -146,13 +142,12 @@ services:
     restart: unless-stopped
 
   algorithm-network-analysis:
-    image: socialmediamacroscope/network_analysis:0.1.2
+    image: socialmediamacroscope/network_analysis:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -166,13 +161,12 @@ services:
     restart: unless-stopped
 
   algorithm-preprocessing:
-    image: socialmediamacroscope/preprocessing:0.1.2
+    image: socialmediamacroscope/preprocessing:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -186,13 +180,12 @@ services:
     restart: unless-stopped
 
   algorithm-sentiment-analysis:
-    image: socialmediamacroscope/sentiment_analysis:0.1.4
+    image: socialmediamacroscope/sentiment_analysis:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      - MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -206,13 +199,12 @@ services:
     restart: unless-stopped
 
   algorithm-screen-name-prompt:
-    image: socialmediamacroscope/screen_name_prompt:0.1.2
+    image: socialmediamacroscope/screen_name_prompt:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./screen_name_prompt.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -222,13 +214,12 @@ services:
     restart: unless-stopped
 
   algorithm-topic-modeling:
-    image: socialmediamacroscope/topic_modeling:0.1.2
+    image: socialmediamacroscope/topic_modeling:latest
     depends_on:
       - rabbitmq
       - minio
     command: python ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -242,13 +233,12 @@ services:
     restart: unless-stopped
 
   algorithm-name-entity-recognition:
-    image: socialmediamacroscope/name_entity_recognition:0.1.1
+    image: socialmediamacroscope/name_entity_recognition:latest
     depends_on:
       - rabbitmq
       - minio
     command: python3 ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -262,13 +252,12 @@ services:
     restart: unless-stopped
 
   algorithm-autophrase:
-    image: socialmediamacroscope/autophrase:0.1.3
+    image: socialmediamacroscope/autophrase:latest
     depends_on:
       - rabbitmq
       - minio
     command: python3 ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -282,13 +271,12 @@ services:
     restart: unless-stopped
 
   image-crawler:
-    image: socialmediamacroscope/image_crawler:0.1.2
+    image: socialmediamacroscope/image_crawler:latest
     depends_on:
       - rabbitmq
       - minio
     command: python3 ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - AWS_ACCESSKEY=${AWS_ACCESSKEY}
@@ -302,13 +290,12 @@ services:
     restart: unless-stopped
 
   collect-reddit-comment:
-    image: socialmediamacroscope/collect_reddit_comment:0.1.2
+    image: socialmediamacroscope/collect_reddit_comment:latest
     depends_on:
       - rabbitmq
       - minio
     command: python3 ./rabbitmq_handler.py
     environment:
-      #- MINIO_URL=${MINIO_URL}
       - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - RABBITMQ_HOST=${RABBITMQ_HOST}
       - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID}
@@ -324,50 +311,39 @@ services:
     restart: unless-stopped
 
   smile-server:
-    image: socialmediamacroscope/smile_server:twitter-v2
+    image: socialmediamacroscope/smile_server:latest
     container_name: smile_server
     depends_on:
       - rabbitmq
       - minio
       - redis
-      - algorithm-classification-predict
-      - algorithm-classification-split
-      - algorithm-classification-train
-      - algorithm-histogram
-      - algorithm-network-analysis
-      - algorithm-preprocessing
-      - algorithm-sentiment-analysis
-      - algorithm-autophrase
-      - algorithm-name-entity-recognition
-      - algorithm-topic-modeling
-      - algorithm-screen-name-prompt
-#      - clowder-create-collection
-#      - clowder-create-dataset
-#      - clowder-create-space
-#      - clowder-list
-#      - clowder-upload-file
+      - smile-graphql
     command: npm run docker-start
     environment:
       - HOME=${HOME}
       - DOCKERIZED=${DOCKERIZED}
+      - CLOWDER_ON=${CLOWDER_ON}
       - LOCAL_ALGORITHM=${LOCAL_ALGORITHM}
       - GA_KEY=${GA_KEY}
       - MINIO_URL=${MINIO_URL}
-      - MINIO_PUBLIC_ACCESS_URL=${MINIO_URL}
+      - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL}
       - REDIS_URL=${REDIS_URL}
       - SMILE_GRAPHQL_URL=${SMILE_GRAPHQL_URL}
       - RABBITMQ_URL=${RABBITMQ_URL}
       - BUCKET_NAME=${BUCKET_NAME}
-      - AWS_ACCESSKEY=${MINIO_ROOT_USER}
-      - AWS_ACCESSKEYSECRET=${MINIO_ROOT_PASSWORD}
+      - AWS_ACCESSKEY=${AWS_ACCESSKEY}
+      - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET}
       - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY}
       - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET}
+      - TWITTER_ON=${TWITTER_ON}
       - TWITTER_V2_CLIENT_ID=${TWITTER_V2_CLIENT_ID}
       - TWITTER_V2_CLIENT_SECRET=${TWITTER_V2_CLIENT_SECRET}
       - TWITTER_V2_CALLBACK_URL=${TWITTER_V2_CALLBACK_URL}
+      - REDDIT_ON=${REDDIT_ON}
       - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID}
       - REDDIT_CLIENT_SECRET=${REDDIT_CLIENT_SECRET}
       - REDDIT_CALLBACK_URL=${REDDIT_CALLBACK_URL}
+      - SHARE_EXPIRE_IN=${SHARE_EXPIRE_IN}
       - BOX_CLIENT_ID=${BOX_CLIENT_ID}
       - BOX_CLIENT_SECRET=${BOX_CLIENT_SECRET}
       - DROPBOX_CLIENT_ID=${DROPBOX_CLIENT_ID}
@@ -395,10 +371,8 @@ services:
       - traefik.http.routers.smile-server.tls.certresolver=myresolver
 
   smile-graphql:
-    image: socialmediamacroscope/smile_graphql:twitter-v2
+    image: socialmediamacroscope/smile_graphql:latest
     container_name: graphql
-    depends_on:
-      - smile-server
     command: npm run docker-start
     environment:
       - DOCKERIZED=${DOCKERIZED}