From 7258ae0caa05c6d68d79823e8792215401977d22 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 10:17:24 -0600 Subject: [PATCH 01/13] update traefik --- .../smile/docker-compose-smile-traefik.yml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index a94617f9..ecbed2d8 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -413,6 +413,77 @@ services: - traefik.http.services.graphql.loadbalancer.server.port=5050 - traefik.http.routers.graphql.tls.certresolver=myresolver + clowder-upload-file : + image: socialmediamacroscope/clowder_upload_file:pr-77 + container_name: clowder_upload_file + depends_on: + - smile-server + command: python ./rabbitmq_handler.py + environment: + - AWS_ACCESSKEY=${AWS_ACCESSKEY} + - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} + - BUCKET_NAME=${BUCKET_NAME} + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - HOST_IP=${HOST_IP} + - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} + - MINIO_URL=${MINIO_URL} + - QUEUE_NAME=lambda_upload_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-list: + image: socialmediamacroscope/clowder_list:0.1.0 + container_name: clowder_list + depends_on: + - smile-server + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY} + - HOST_IP=${HOST_IP} + - QUEUE_NAME=lambda_list_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-space: + image: socialmediamacroscope/clowder_create_space:0.1.0 + container_name: clowder_create_space + depends_on: + - smile-server + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - HOST_IP=${HOST_IP} + - QUEUE_NAME=clowder_create_space + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-dataset: + image: socialmediamacroscope/clowder_create_dataset:0.1.0 + container_name: clowder_create_dataset + depends_on: + - smile-server + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - HOST_IP=${HOST_IP} + - QUEUE_NAME=lambda_invoke_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-collection: + image: socialmediamacroscope/clowder_create_collection:0.1.0 + container_name: clowder_create_collection + depends_on: + - smile-server + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - HOST_IP=${HOST_IP} + - QUEUE_NAME=clowder_create_collection + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + volumes: smile_content_data: driver: local From 0894762a836c8dcf233962e6f9c80cddc683748e Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 10:30:32 -0600 Subject: [PATCH 02/13] fix inconsistent environment variable --- .../smile/docker-compose-smile-traefik.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index ecbed2d8..57ee453e 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -358,8 +358,8 @@ services: - SMILE_GRAPHQL_URL=${SMILE_GRAPHQL_URL} - RABBITMQ_URL=${RABBITMQ_URL} - BUCKET_NAME=${BUCKET_NAME} - - AWS_ACCESSKEY=${MINIO_ROOT_USER} - - AWS_ACCESSKEYSECRET=${MINIO_ROOT_PASSWORD} + - AWS_ACCESSKEY=${AWS_ACCESSKEY} + - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY} - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} - TWITTER_V2_CLIENT_ID=${TWITTER_V2_CLIENT_ID} From fc787eba5e2147bd476a1f026ca048842050d318 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 11:03:18 -0600 Subject: [PATCH 03/13] remove commented out dependencies --- .../smile/docker-compose-smile-traefik.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index 57ee453e..3d9cd75b 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -341,11 +341,6 @@ services: - algorithm-name-entity-recognition - algorithm-topic-modeling - algorithm-screen-name-prompt -# - clowder-create-collection -# - clowder-create-dataset -# - clowder-create-space -# - clowder-list -# - clowder-upload-file command: npm run docker-start environment: - HOME=${HOME} From 8989bf8d628cb9ee37b77250f06f684fec725fff Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 11:13:59 -0600 Subject: [PATCH 04/13] depend --- .../smile/docker-compose-smile-traefik.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index 3d9cd75b..7ac9dc18 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -412,7 +412,7 @@ services: image: socialmediamacroscope/clowder_upload_file:pr-77 container_name: clowder_upload_file depends_on: - - smile-server + - rabbitmq command: python ./rabbitmq_handler.py environment: - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -430,7 +430,7 @@ services: image: socialmediamacroscope/clowder_list:0.1.0 container_name: clowder_list depends_on: - - smile-server + - rabbitmq command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} @@ -444,7 +444,7 @@ services: image: socialmediamacroscope/clowder_create_space:0.1.0 container_name: clowder_create_space depends_on: - - smile-server + - rabbitmq command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} @@ -457,7 +457,7 @@ services: image: socialmediamacroscope/clowder_create_dataset:0.1.0 container_name: clowder_create_dataset depends_on: - - smile-server + - rabbitmq command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} @@ -470,7 +470,7 @@ services: image: socialmediamacroscope/clowder_create_collection:0.1.0 container_name: clowder_create_collection depends_on: - - smile-server + - rabbitmq command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} From 0a94f9cb79503024570f960e1a4c9f094402f80c Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 11:18:05 -0600 Subject: [PATCH 05/13] correct dependency --- .../smile/docker-compose-smile-traefik.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index 7ac9dc18..f8751e0d 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -330,17 +330,7 @@ services: - rabbitmq - minio - redis - - algorithm-classification-predict - - algorithm-classification-split - - algorithm-classification-train - - algorithm-histogram - - algorithm-network-analysis - - algorithm-preprocessing - - algorithm-sentiment-analysis - - algorithm-autophrase - - algorithm-name-entity-recognition - - algorithm-topic-modeling - - algorithm-screen-name-prompt + - smile-graphql command: npm run docker-start environment: - HOME=${HOME} @@ -392,8 +382,6 @@ services: smile-graphql: image: socialmediamacroscope/smile_graphql:twitter-v2 container_name: graphql - depends_on: - - smile-server command: npm run docker-start environment: - DOCKERIZED=${DOCKERIZED} From 5a04e56c10c242473bb8c23bba24b17ecf9bbd8d Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 11:35:59 -0600 Subject: [PATCH 06/13] clean up hostip, minio url --- .../smile/docker-compose-smile-traefik.yml | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index f8751e0d..88b783a3 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -88,7 +88,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -104,7 +103,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -120,7 +118,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -136,7 +133,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - - MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -152,7 +148,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -172,7 +167,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -192,7 +186,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - - MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -212,7 +205,6 @@ services: - minio command: python ./screen_name_prompt.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -228,7 +220,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -248,7 +239,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -268,7 +258,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -288,7 +277,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -308,7 +296,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID} @@ -338,7 +325,7 @@ services: - LOCAL_ALGORITHM=${LOCAL_ALGORITHM} - GA_KEY=${GA_KEY} - MINIO_URL=${MINIO_URL} - - MINIO_PUBLIC_ACCESS_URL=${MINIO_URL} + - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - REDIS_URL=${REDIS_URL} - SMILE_GRAPHQL_URL=${SMILE_GRAPHQL_URL} - RABBITMQ_URL=${RABBITMQ_URL} @@ -407,7 +394,6 @@ services: - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} - BUCKET_NAME=${BUCKET_NAME} - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - HOST_IP=${HOST_IP} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - MINIO_URL=${MINIO_URL} - QUEUE_NAME=lambda_upload_clowder @@ -423,7 +409,6 @@ services: environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY} - - HOST_IP=${HOST_IP} - QUEUE_NAME=lambda_list_clowder - RABBITMQ_HOST=${RABBITMQ_HOST} restart: unless-stopped @@ -436,7 +421,6 @@ services: command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - HOST_IP=${HOST_IP} - QUEUE_NAME=clowder_create_space - RABBITMQ_HOST=${RABBITMQ_HOST} restart: unless-stopped @@ -449,7 +433,6 @@ services: command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - HOST_IP=${HOST_IP} - QUEUE_NAME=lambda_invoke_clowder - RABBITMQ_HOST=${RABBITMQ_HOST} restart: unless-stopped @@ -462,7 +445,6 @@ services: command: python ./rabbitmq_handler.py environment: - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - HOST_IP=${HOST_IP} - QUEUE_NAME=clowder_create_collection - RABBITMQ_HOST=${RABBITMQ_HOST} restart: unless-stopped From e6a76d501cf4b63b9ac4ff514565cdce4d63b3f1 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 15:06:43 -0600 Subject: [PATCH 07/13] add missing environment variables, match docker-compose with nginx --- .../smile/docker-compose-smile-traefik.yml | 44 ++--- .../smile/docker-compose-smile.yml | 163 ++++++++++++------ 2 files changed, 134 insertions(+), 73 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index 88b783a3..a5ebb774 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -82,7 +82,7 @@ services: - traefik.tcp.http.routers.redis.tls.certresolver=myresolver algorithm-classification-predict: - image: socialmediamacroscope/classification_predict:0.1.2 + image: socialmediamacroscope/classification_predict:latest depends_on: - rabbitmq - minio @@ -97,7 +97,7 @@ services: restart: unless-stopped algorithm-classification-split: - image: socialmediamacroscope/classification_split:0.1.2 + image: socialmediamacroscope/classification_split:latest depends_on: - rabbitmq - minio @@ -112,7 +112,7 @@ services: restart: unless-stopped algorithm-classification-train: - image: socialmediamacroscope/classification_train:0.1.2 + image: socialmediamacroscope/classification_train:latest depends_on: - rabbitmq - minio @@ -127,7 +127,7 @@ services: restart: unless-stopped algorithm-histogram: - image: socialmediamacroscope/histogram:0.1.1 + image: socialmediamacroscope/histogram:latest depends_on: - rabbitmq - minio @@ -142,7 +142,7 @@ services: restart: unless-stopped algorithm-network-analysis: - image: socialmediamacroscope/network_analysis:0.1.2 + image: socialmediamacroscope/network_analysis:latest depends_on: - rabbitmq - minio @@ -161,7 +161,7 @@ services: restart: unless-stopped algorithm-preprocessing: - image: socialmediamacroscope/preprocessing:0.1.2 + image: socialmediamacroscope/preprocessing:latest depends_on: - rabbitmq - minio @@ -180,7 +180,7 @@ services: restart: unless-stopped algorithm-sentiment-analysis: - image: socialmediamacroscope/sentiment_analysis:0.1.4 + image: socialmediamacroscope/sentiment_analysis:latest depends_on: - rabbitmq - minio @@ -199,7 +199,7 @@ services: restart: unless-stopped algorithm-screen-name-prompt: - image: socialmediamacroscope/screen_name_prompt:0.1.2 + image: socialmediamacroscope/screen_name_prompt:latest depends_on: - rabbitmq - minio @@ -214,7 +214,7 @@ services: restart: unless-stopped algorithm-topic-modeling: - image: socialmediamacroscope/topic_modeling:0.1.2 + image: socialmediamacroscope/topic_modeling:latest depends_on: - rabbitmq - minio @@ -233,7 +233,7 @@ services: restart: unless-stopped algorithm-name-entity-recognition: - image: socialmediamacroscope/name_entity_recognition:0.1.1 + image: socialmediamacroscope/name_entity_recognition:latest depends_on: - rabbitmq - minio @@ -252,7 +252,7 @@ services: restart: unless-stopped algorithm-autophrase: - image: socialmediamacroscope/autophrase:0.1.3 + image: socialmediamacroscope/autophrase:latest depends_on: - rabbitmq - minio @@ -271,7 +271,7 @@ services: restart: unless-stopped image-crawler: - image: socialmediamacroscope/image_crawler:0.1.2 + image: socialmediamacroscope/image_crawler:latest depends_on: - rabbitmq - minio @@ -290,7 +290,7 @@ services: restart: unless-stopped collect-reddit-comment: - image: socialmediamacroscope/collect_reddit_comment:0.1.2 + image: socialmediamacroscope/collect_reddit_comment:latest depends_on: - rabbitmq - minio @@ -311,7 +311,7 @@ services: restart: unless-stopped smile-server: - image: socialmediamacroscope/smile_server:twitter-v2 + image: socialmediamacroscope/smile_server:latest container_name: smile_server depends_on: - rabbitmq @@ -322,6 +322,7 @@ services: environment: - HOME=${HOME} - DOCKERIZED=${DOCKERIZED} + - CLOWDER_ON=${CLOWDER_ON} - LOCAL_ALGORITHM=${LOCAL_ALGORITHM} - GA_KEY=${GA_KEY} - MINIO_URL=${MINIO_URL} @@ -334,12 +335,15 @@ services: - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY} - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} + - TWITTER_ON=${TWITTER_ON} - TWITTER_V2_CLIENT_ID=${TWITTER_V2_CLIENT_ID} - TWITTER_V2_CLIENT_SECRET=${TWITTER_V2_CLIENT_SECRET} - TWITTER_V2_CALLBACK_URL=${TWITTER_V2_CALLBACK_URL} + - REDDIT_ON=${REDDIT_ON} - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID} - REDDIT_CLIENT_SECRET=${REDDIT_CLIENT_SECRET} - REDDIT_CALLBACK_URL=${REDDIT_CALLBACK_URL} + - SHARE_EXPIRE_IN=${SHARE_EXPIRE_IN} - BOX_CLIENT_ID=${BOX_CLIENT_ID} - BOX_CLIENT_SECRET=${BOX_CLIENT_SECRET} - DROPBOX_CLIENT_ID=${DROPBOX_CLIENT_ID} @@ -367,7 +371,7 @@ services: - traefik.http.routers.smile-server.tls.certresolver=myresolver smile-graphql: - image: socialmediamacroscope/smile_graphql:twitter-v2 + image: socialmediamacroscope/smile_graphql:latest container_name: graphql command: npm run docker-start environment: @@ -384,7 +388,7 @@ services: - traefik.http.routers.graphql.tls.certresolver=myresolver clowder-upload-file : - image: socialmediamacroscope/clowder_upload_file:pr-77 + image: socialmediamacroscope/clowder_upload_file:latest container_name: clowder_upload_file depends_on: - rabbitmq @@ -401,7 +405,7 @@ services: restart: unless-stopped clowder-list: - image: socialmediamacroscope/clowder_list:0.1.0 + image: socialmediamacroscope/clowder_list:latest container_name: clowder_list depends_on: - rabbitmq @@ -414,7 +418,7 @@ services: restart: unless-stopped clowder-create-space: - image: socialmediamacroscope/clowder_create_space:0.1.0 + image: socialmediamacroscope/clowder_create_space:latest container_name: clowder_create_space depends_on: - rabbitmq @@ -426,7 +430,7 @@ services: restart: unless-stopped clowder-create-dataset: - image: socialmediamacroscope/clowder_create_dataset:0.1.0 + image: socialmediamacroscope/clowder_create_dataset:latest container_name: clowder_create_dataset depends_on: - rabbitmq @@ -438,7 +442,7 @@ services: restart: unless-stopped clowder-create-collection: - image: socialmediamacroscope/clowder_create_collection:0.1.0 + image: socialmediamacroscope/clowder_create_collection:latest container_name: clowder_create_collection depends_on: - rabbitmq diff --git a/containerized_analytics/smile/docker-compose-smile.yml b/containerized_analytics/smile/docker-compose-smile.yml index c9a6d93f..8a6c04b4 100644 --- a/containerized_analytics/smile/docker-compose-smile.yml +++ b/containerized_analytics/smile/docker-compose-smile.yml @@ -9,16 +9,17 @@ services: minio: image: minio/minio + container_name: minio hostname: minio ports: - 9000:9000 - 9001:9001 volumes: - - "smile_content_data:/tmp" - command: server /tmp + - "smile_content_data:/data" environment: - - MINIO_ACCESS_KEY=${AWS_ACCESSKEY} - - MINIO_SECRET_KEY=${AWS_ACCESSKEYSECRET} + - MINIO_ROOT_USER=${AWS_ACCESSKEY} + - MINIO_ROOT_PASSWORD=${AWS_ACCESSKEYSECRET} + command: server /data --console-address ":9001" restart: unless-stopped redis: @@ -297,57 +298,49 @@ services: - rabbitmq - minio - redis - - algorithm-classification-predict - - algorithm-classification-split - - algorithm-classification-train - - algorithm-histogram - - algorithm-network-analysis - - algorithm-preprocessing - - algorithm-sentiment-analysis - - algorithm-autophrase - - algorithm-name-entity-recognition - - algorithm-topic-modeling - - algorithm-screen-name-prompt - #- clowder-create-collection - #- clowder-create-dataset - #- clowder-create-space - #- clowder-list - #- clowder-upload-file + - smile-graphql ports: - 8001:8001 command: npm run docker-start environment: - - HOME=${HOME} - - DOCKERIZED=${DOCKERIZED} - - LOCAL_ALGORITHM=${LOCAL_ALGORITHM} - - GA_KEY=${GA_KEY} - - MINIO_URL=${MINIO_URL} - - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - - REDIS_URL=${REDIS_URL} - - SMILE_GRAPHQL_URL=${SMILE_GRAPHQL_URL} - - RABBITMQ_URL=${RABBITMQ_URL} - - BUCKET_NAME=${BUCKET_NAME} - - AWS_ACCESSKEY=${AWS_ACCESSKEY} - - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} - - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY} - - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} - - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID} - - REDDIT_CLIENT_SECRET=${REDDIT_CLIENT_SECRET} - - REDDIT_CALLBACK_URL=${REDDIT_CALLBACK_URL} - - BOX_CLIENT_ID=${BOX_CLIENT_ID} - - BOX_CLIENT_SECRET=${BOX_CLIENT_SECRET} - - DROPBOX_CLIENT_ID=${DROPBOX_CLIENT_ID} - - DROPBOX_CLIENT_SECRET=${DROPBOX_CLIENT_SECRET} - - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} - - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} - - EMAIL_HOST=${EMAIL_HOST} - - EMAIL_PORT=${EMAIL_PORT} - - EMAIL_FROM_ADDRESS=${EMAIL_FROM_ADDRESS} - - EMAIL_PASSWORD=${EMAIL_PASSWORD} - - SINGLE_USER=${SINGLE_USER} - - CILOGON_CLIENT_ID=${CILOGON_CLIENT_ID} - - CILOGON_CLIENT_SECRET=${CILOGON_CLIENT_SECRET} - - CILOGON_CALLBACK_URL=${CILOGON_CALLBACK_URL} + - HOME=${HOME} + - DOCKERIZED=${DOCKERIZED} + - CLOWDER_ON=${CLOWDER_ON} + - LOCAL_ALGORITHM=${LOCAL_ALGORITHM} + - GA_KEY=${GA_KEY} + - MINIO_URL=${MINIO_URL} + - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} + - REDIS_URL=${REDIS_URL} + - SMILE_GRAPHQL_URL=${SMILE_GRAPHQL_URL} + - RABBITMQ_URL=${RABBITMQ_URL} + - BUCKET_NAME=${BUCKET_NAME} + - AWS_ACCESSKEY=${AWS_ACCESSKEY} + - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} + - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY} + - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} + - TWITTER_ON=${TWITTER_ON} + - TWITTER_V2_CLIENT_ID=${TWITTER_V2_CLIENT_ID} + - TWITTER_V2_CLIENT_SECRET=${TWITTER_V2_CLIENT_SECRET} + - TWITTER_V2_CALLBACK_URL=${TWITTER_V2_CALLBACK_URL} + - REDDIT_ON=${REDDIT_ON} + - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID} + - REDDIT_CLIENT_SECRET=${REDDIT_CLIENT_SECRET} + - REDDIT_CALLBACK_URL=${REDDIT_CALLBACK_URL} + - SHARE_EXPIRE_IN=${SHARE_EXPIRE_IN} + - BOX_CLIENT_ID=${BOX_CLIENT_ID} + - BOX_CLIENT_SECRET=${BOX_CLIENT_SECRET} + - DROPBOX_CLIENT_ID=${DROPBOX_CLIENT_ID} + - DROPBOX_CLIENT_SECRET=${DROPBOX_CLIENT_SECRET} + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - EMAIL_HOST=${EMAIL_HOST} + - EMAIL_PORT=${EMAIL_PORT} + - EMAIL_FROM_ADDRESS=${EMAIL_FROM_ADDRESS} + - EMAIL_PASSWORD=${EMAIL_PASSWORD} + - SINGLE_USER=${SINGLE_USER} + - CILOGON_CLIENT_ID=${CILOGON_CLIENT_ID} + - CILOGON_CLIENT_SECRET=${CILOGON_CLIENT_SECRET} + - CILOGON_CALLBACK_URL=${CILOGON_CALLBACK_URL} volumes: - "smile_content_data:/tmp" - "smile_tag:${HOME}/smile" @@ -355,8 +348,6 @@ services: smile-graphql: image: socialmediamacroscope/smile_graphql:0.2.7 - depends_on: - - smile-server ports: - 5050:5050 command: npm run docker-start @@ -366,6 +357,72 @@ services: - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} restart: unless-stopped + clowder-upload-file: + image: socialmediamacroscope/clowder_upload_file:latest + container_name: clowder_upload_file + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - AWS_ACCESSKEY=${AWS_ACCESSKEY} + - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} + - BUCKET_NAME=${BUCKET_NAME} + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} + - MINIO_URL=${MINIO_URL} + - QUEUE_NAME=lambda_upload_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-list: + image: socialmediamacroscope/clowder_list:latest + container_name: clowder_list + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY} + - QUEUE_NAME=lambda_list_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-space: + image: socialmediamacroscope/clowder_create_space:latest + container_name: clowder_create_space + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - QUEUE_NAME=clowder_create_space + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-dataset: + image: socialmediamacroscope/clowder_create_dataset:latest + container_name: clowder_create_dataset + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - QUEUE_NAME=lambda_invoke_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-collection: + image: socialmediamacroscope/clowder_create_collection:latest + container_name: clowder_create_collection + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - QUEUE_NAME=clowder_create_collection + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + volumes: smile_content_data: driver: local From d70e9cbae90e5618a63f12f48fd61db3995abcdc Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 15:08:08 -0600 Subject: [PATCH 08/13] remove inline comment --- .../smile/docker-compose-smile.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile.yml b/containerized_analytics/smile/docker-compose-smile.yml index 8a6c04b4..8a85cbee 100644 --- a/containerized_analytics/smile/docker-compose-smile.yml +++ b/containerized_analytics/smile/docker-compose-smile.yml @@ -57,7 +57,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -73,7 +72,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -89,7 +87,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -105,7 +102,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - - MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -121,7 +117,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -141,7 +136,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -161,7 +155,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - - MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -181,7 +174,6 @@ services: - minio command: python ./screen_name_prompt.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -197,7 +189,6 @@ services: - minio command: python ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -217,7 +208,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -237,7 +227,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -257,7 +246,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - AWS_ACCESSKEY=${AWS_ACCESSKEY} @@ -277,7 +265,6 @@ services: - minio command: python3 ./rabbitmq_handler.py environment: - #- MINIO_URL=${MINIO_URL} - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - RABBITMQ_HOST=${RABBITMQ_HOST} - REDDIT_CLIENT_ID=${REDDIT_CLIENT_ID} From 8352d4609df881e69170726e93473cdb4fdf341d Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 15:13:06 -0600 Subject: [PATCH 09/13] change everything to latest --- .../smile/docker-compose-smile.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile.yml b/containerized_analytics/smile/docker-compose-smile.yml index 8a85cbee..c2e60032 100644 --- a/containerized_analytics/smile/docker-compose-smile.yml +++ b/containerized_analytics/smile/docker-compose-smile.yml @@ -51,7 +51,7 @@ services: restart: unless-stopped algorithm-classification-predict: - image: socialmediamacroscope/classification_predict:0.1.2 + image: socialmediamacroscope/classification_predict:latest depends_on: - rabbitmq - minio @@ -66,7 +66,7 @@ services: restart: unless-stopped algorithm-classification-split: - image: socialmediamacroscope/classification_split:0.1.2 + image: socialmediamacroscope/classification_split:latest depends_on: - rabbitmq - minio @@ -81,7 +81,7 @@ services: restart: unless-stopped algorithm-classification-train: - image: socialmediamacroscope/classification_train:0.1.2 + image: socialmediamacroscope/classification_train:latest depends_on: - rabbitmq - minio @@ -96,7 +96,7 @@ services: restart: unless-stopped algorithm-histogram: - image: socialmediamacroscope/histogram:0.1.1 + image: socialmediamacroscope/histogram:latest depends_on: - rabbitmq - minio @@ -111,7 +111,7 @@ services: restart: unless-stopped algorithm-network-analysis: - image: socialmediamacroscope/network_analysis:0.1.2 + image: socialmediamacroscope/network_analysis:latest depends_on: - rabbitmq - minio @@ -130,7 +130,7 @@ services: restart: unless-stopped algorithm-preprocessing: - image: socialmediamacroscope/preprocessing:0.1.2 + image: socialmediamacroscope/preprocessing:latest depends_on: - rabbitmq - minio @@ -149,7 +149,7 @@ services: restart: unless-stopped algorithm-sentiment-analysis: - image: socialmediamacroscope/sentiment_analysis:0.1.4 + image: socialmediamacroscope/sentiment_analysis:latest depends_on: - rabbitmq - minio @@ -168,7 +168,7 @@ services: restart: unless-stopped algorithm-screen-name-prompt: - image: socialmediamacroscope/screen_name_prompt:0.1.2 + image: socialmediamacroscope/screen_name_prompt:latest depends_on: - rabbitmq - minio @@ -183,7 +183,7 @@ services: restart: unless-stopped algorithm-topic-modeling: - image: socialmediamacroscope/topic_modeling:0.1.2 + image: socialmediamacroscope/topic_modeling:latest depends_on: - rabbitmq - minio @@ -202,7 +202,7 @@ services: restart: unless-stopped algorithm-name-entity-recognition: - image: socialmediamacroscope/name_entity_recognition:0.1.1 + image: socialmediamacroscope/name_entity_recognition:latest depends_on: - rabbitmq - minio @@ -221,7 +221,7 @@ services: restart: unless-stopped algorithm-autophrase: - image: socialmediamacroscope/autophrase:0.1.3 + image: socialmediamacroscope/autophrase:latest depends_on: - rabbitmq - minio @@ -240,7 +240,7 @@ services: restart: unless-stopped image-crawler: - image: socialmediamacroscope/image_crawler:0.1.2 + image: socialmediamacroscope/image_crawler:latest depends_on: - rabbitmq - minio @@ -259,7 +259,7 @@ services: restart: unless-stopped collect-reddit-comment: - image: socialmediamacroscope/collect_reddit_comment:0.1.2 + image: socialmediamacroscope/collect_reddit_comment:latest depends_on: - rabbitmq - minio @@ -280,7 +280,7 @@ services: restart: unless-stopped smile-server: - image: socialmediamacroscope/smile_server:0.2.7 + image: socialmediamacroscope/smile_server:latest depends_on: - rabbitmq - minio @@ -334,7 +334,7 @@ services: restart: unless-stopped smile-graphql: - image: socialmediamacroscope/smile_graphql:0.2.7 + image: socialmediamacroscope/smile_graphql:latest ports: - 5050:5050 command: npm run docker-start From b7bc6213cf79e3b2e623bfd315de46e63ac1c59c Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 15:26:34 -0600 Subject: [PATCH 10/13] split to two yaml file and add example to deploy with clowder or not --- .../smile/docker-command-smile-traefik.sh | 3 + .../smile/docker-command-smile.sh | 3 + .../smile/docker-compose-smile-clowder.yml | 91 +++++++++++++++++++ .../smile/docker-compose-smile-traefik.yml | 66 -------------- .../smile/docker-compose-smile.yml | 66 -------------- 5 files changed, 97 insertions(+), 132 deletions(-) create mode 100644 containerized_analytics/smile/docker-compose-smile-clowder.yml diff --git a/containerized_analytics/smile/docker-command-smile-traefik.sh b/containerized_analytics/smile/docker-command-smile-traefik.sh index af23c8a1..e4fdf507 100644 --- a/containerized_analytics/smile/docker-command-smile-traefik.sh +++ b/containerized_analytics/smile/docker-command-smile-traefik.sh @@ -82,6 +82,9 @@ export CLOWDER_ON=false # start docker-compose -f docker-compose-smile-traefik.yml -d up +# start with clowder add-on +# docker-compose -f docker-compose-smile-traefik.yml -f docker-compose-smile-clowder.yml -d up + # stop # docker-compose -f docker-compose-smile.yml down -v diff --git a/containerized_analytics/smile/docker-command-smile.sh b/containerized_analytics/smile/docker-command-smile.sh index 420b45ce..2932048a 100644 --- a/containerized_analytics/smile/docker-command-smile.sh +++ b/containerized_analytics/smile/docker-command-smile.sh @@ -74,6 +74,9 @@ export CLOWSER_ON=false # start docker-compose -f docker-compose-smile.yml -d up +# start with clowder add-on +# docker-compose -f docker-compose-smile.yml -f docker-compose-smile-clowder.yml -d up + # stop # docker-compose -f docker-compose-smile.yml down -v diff --git a/containerized_analytics/smile/docker-compose-smile-clowder.yml b/containerized_analytics/smile/docker-compose-smile-clowder.yml new file mode 100644 index 00000000..dd5565cd --- /dev/null +++ b/containerized_analytics/smile/docker-compose-smile-clowder.yml @@ -0,0 +1,91 @@ +version: "3.3" + +services: + clowder-upload-file: + image: socialmediamacroscope/clowder_upload_file:latest + container_name: clowder_upload_file + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - AWS_ACCESSKEY=${AWS_ACCESSKEY} + - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} + - BUCKET_NAME=${BUCKET_NAME} + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} + - MINIO_URL=${MINIO_URL} + - QUEUE_NAME=lambda_upload_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-list: + image: socialmediamacroscope/clowder_list:latest + container_name: clowder_list + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY} + - QUEUE_NAME=lambda_list_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-space: + image: socialmediamacroscope/clowder_create_space:latest + container_name: clowder_create_space + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - QUEUE_NAME=clowder_create_space + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-dataset: + image: socialmediamacroscope/clowder_create_dataset:latest + container_name: clowder_create_dataset + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - QUEUE_NAME=lambda_invoke_clowder + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + + clowder-create-collection: + image: socialmediamacroscope/clowder_create_collection:latest + container_name: clowder_create_collection + depends_on: + - rabbitmq + command: python ./rabbitmq_handler.py + environment: + - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} + - QUEUE_NAME=clowder_create_collection + - RABBITMQ_HOST=${RABBITMQ_HOST} + restart: unless-stopped + +volumes: + smile_content_data: + driver: local + driver_opts: + o: bind + type: none + # put your own local data path here + device: "${HOME}/smile_data" + + smile_user: + driver: local + driver_opts: + o: bind + type: none + device: "${HOME}/smile_user" + + smile_tag: + driver: local + driver_opts: + o: bind + type: none + device: "${HOME}/smile" diff --git a/containerized_analytics/smile/docker-compose-smile-traefik.yml b/containerized_analytics/smile/docker-compose-smile-traefik.yml index a5ebb774..5177d0d1 100644 --- a/containerized_analytics/smile/docker-compose-smile-traefik.yml +++ b/containerized_analytics/smile/docker-compose-smile-traefik.yml @@ -387,72 +387,6 @@ services: - traefik.http.services.graphql.loadbalancer.server.port=5050 - traefik.http.routers.graphql.tls.certresolver=myresolver - clowder-upload-file : - image: socialmediamacroscope/clowder_upload_file:latest - container_name: clowder_upload_file - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - AWS_ACCESSKEY=${AWS_ACCESSKEY} - - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} - - BUCKET_NAME=${BUCKET_NAME} - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - - MINIO_URL=${MINIO_URL} - - QUEUE_NAME=lambda_upload_clowder - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-list: - image: socialmediamacroscope/clowder_list:latest - container_name: clowder_list - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY} - - QUEUE_NAME=lambda_list_clowder - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-create-space: - image: socialmediamacroscope/clowder_create_space:latest - container_name: clowder_create_space - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - QUEUE_NAME=clowder_create_space - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-create-dataset: - image: socialmediamacroscope/clowder_create_dataset:latest - container_name: clowder_create_dataset - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - QUEUE_NAME=lambda_invoke_clowder - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-create-collection: - image: socialmediamacroscope/clowder_create_collection:latest - container_name: clowder_create_collection - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - QUEUE_NAME=clowder_create_collection - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - volumes: smile_content_data: driver: local diff --git a/containerized_analytics/smile/docker-compose-smile.yml b/containerized_analytics/smile/docker-compose-smile.yml index c2e60032..c8595fb8 100644 --- a/containerized_analytics/smile/docker-compose-smile.yml +++ b/containerized_analytics/smile/docker-compose-smile.yml @@ -344,72 +344,6 @@ services: - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET} restart: unless-stopped - clowder-upload-file: - image: socialmediamacroscope/clowder_upload_file:latest - container_name: clowder_upload_file - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - AWS_ACCESSKEY=${AWS_ACCESSKEY} - - AWS_ACCESSKEYSECRET=${AWS_ACCESSKEYSECRET} - - BUCKET_NAME=${BUCKET_NAME} - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - MINIO_PUBLIC_ACCESS_URL=${MINIO_PUBLIC_ACCESS_URL} - - MINIO_URL=${MINIO_URL} - - QUEUE_NAME=lambda_upload_clowder - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-list: - image: socialmediamacroscope/clowder_list:latest - container_name: clowder_list - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - CLOWDER_GLOBAL_KEY=${CLOWDER_GLOBAL_KEY} - - QUEUE_NAME=lambda_list_clowder - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-create-space: - image: socialmediamacroscope/clowder_create_space:latest - container_name: clowder_create_space - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - QUEUE_NAME=clowder_create_space - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-create-dataset: - image: socialmediamacroscope/clowder_create_dataset:latest - container_name: clowder_create_dataset - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - QUEUE_NAME=lambda_invoke_clowder - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - - clowder-create-collection: - image: socialmediamacroscope/clowder_create_collection:latest - container_name: clowder_create_collection - depends_on: - - rabbitmq - command: python ./rabbitmq_handler.py - environment: - - CLOWDER_BASE_URL=${CLOWDER_BASE_URL} - - QUEUE_NAME=clowder_create_collection - - RABBITMQ_HOST=${RABBITMQ_HOST} - restart: unless-stopped - volumes: smile_content_data: driver: local From 46cbf6a9b4c6247b3e40fe7afce9383c6deae12b Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 15:29:23 -0600 Subject: [PATCH 11/13] remove volume --- .../smile/docker-compose-smile-clowder.yml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/containerized_analytics/smile/docker-compose-smile-clowder.yml b/containerized_analytics/smile/docker-compose-smile-clowder.yml index dd5565cd..56ffd334 100644 --- a/containerized_analytics/smile/docker-compose-smile-clowder.yml +++ b/containerized_analytics/smile/docker-compose-smile-clowder.yml @@ -66,26 +66,3 @@ services: - QUEUE_NAME=clowder_create_collection - RABBITMQ_HOST=${RABBITMQ_HOST} restart: unless-stopped - -volumes: - smile_content_data: - driver: local - driver_opts: - o: bind - type: none - # put your own local data path here - device: "${HOME}/smile_data" - - smile_user: - driver: local - driver_opts: - o: bind - type: none - device: "${HOME}/smile_user" - - smile_tag: - driver: local - driver_opts: - o: bind - type: none - device: "${HOME}/smile" From f2616ecc7a35d861e9f5cf04d1f57a686779554c Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 27 Nov 2023 15:31:34 -0600 Subject: [PATCH 12/13] changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f309edda..c46dfea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Rabbitmq username and password [#61](https://github.com/ncsa/standalone-smm-analytics/issues/90) +### Added +- Docker compose file for clowder connection [#88](https://github.com/ncsa/standalone-smm-analytics/issues/88) + ## [Beta] - 10-26-2023 ### Added From 38ca019cc5b77b06618101a5ac37689d05c88396 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Thu, 11 Jan 2024 12:13:49 -0600 Subject: [PATCH 13/13] fix typo --- containerized_analytics/smile/docker-command-smile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containerized_analytics/smile/docker-command-smile.sh b/containerized_analytics/smile/docker-command-smile.sh index 2932048a..97de5b24 100644 --- a/containerized_analytics/smile/docker-command-smile.sh +++ b/containerized_analytics/smile/docker-command-smile.sh @@ -69,7 +69,7 @@ export GOOGLE_CLIENT_SECRET=<> export CLOWDER_BASE_URL=https://clowder.server.com/ export CLOWDER_GLOBAL_KEY=<> -export CLOWSER_ON=false +export CLOWDER_ON=false # start docker-compose -f docker-compose-smile.yml -d up