add initial docker compose file for testing

autotm/preprocessing/dictionaries_preparation.py

@@ -218,10 +218,20 @@ def prepearing_cooc_dict(
     docs_count = data.shape[0]
 
     logger.debug("Performing calculate_cooc_dicts")
+<<<<<<< Updated upstream
     cooc_dicts = calculate_cooc(batches_path=BATCHES_DIR, vocab=vocab_words, window_size=cooc_window)
     logger.debug("Performed calculate_cooc_dicts")
     cooc_df_dict, cooc_df_term_dict = cooc_dicts.cooc_df, cooc_dicts.cooc_df_term
     cooc_tf_dict, cooc_tf_term_dict = cooc_dicts.cooc_tf, cooc_dicts.cooc_tf_term
+=======
+    df_dicts, tf_dicts = calculate_cooc_dicts(
+        vocab_words, data, n_cores=n_cores, window=cooc_window
+    )
+    logger.debug("Performed calculate_cooc_dicts")
+
+    cooc_df_dict, cooc_df_term_dict = df_dicts[0], df_dicts[1]
+    cooc_tf_dict, cooc_tf_term_dict = tf_dicts[0], tf_dicts[1]
+>>>>>>> Stashed changes
 
     pairs_count = cooc_tf_dict[RESERVED_TUPLE]
 

tests/deploy/compose.yaml

@@ -0,0 +1,133 @@
+services:
+  redis:
+    image: redis:6.2
+    ports:
+      - "6379:6379"
+    networks:
+      - test_autotm_net
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+
+  rabbitmq:
+    image: rabbitmq:3.8-management-alpin
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    networks:
+      - test_autotm_net
+    configs:
+      - source: rabbitmq_conf
+        target: /etc/rabbitmq/rabbitmq.conf
+      - source: rabbitmq_enabled_plugins
+        target: /etc/rabbitmq/enabled_plugins
+      - source: rabbitmq_consumer_settings
+        target: /etc/rabbitmq/conf.d/consumer-settings.conf
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 30s
+      timeout: 30s
+      retries: 3
+
+  flower:
+    image: mher/flower
+    depends_on:
+      - rabbitmq
+    ports:
+      - "5555:5555"
+    networks:
+      - test_autotm_net
+    environment:
+      CELERY_BROKER_URL: "amqp://guest:guest@rabbitmq:5672"
+      CELERY_RESULT_BACKEND: "redis://redis:6379/1"
+
+# todo: "/var/lib/mysql"
+  mlflow-db:
+    image: mysql/mysql-server:5.7.28
+    ports:
+      - "3306:3306"
+    networks:
+      - test_autotm_net
+    environment:
+      MYSQL_DATABASE: "mlflow"
+      MYSQL_USER: "mlflow"
+      MYSQL_PASSWORD: "mlflow"
+      MYSQL_ROOT_PASSWORD: "mlflow"
+    configs:
+      - source: mysql_conf
+        target: /etc/mysql/my.cnf
+    healthcheck:
+      test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ]
+      timeout: 20s
+      retries: 10
+
+# todo: set correct image
+# todo: "/var/lib/mlruns"
+# todo: shared artifact folder
+  mlflow:
+    image: "mlflow-webserver:latest"
+    command: [
+      "--backend-store-uri", "mysql+pymysql://mlflow:mlflow@mlflow-db:3306/mlflow",
+      "--default-artifact-root", "/var/lib/mlruns",
+      "--host", "0.0.0.0"
+    ]
+    depends_on:
+      - mlflow-db
+    ports:
+      - "5000:5000"
+    networks:
+      - test_autotm_net
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:5000 || exit 1" ]
+      timeout: 10s
+      retries: 10
+
+# todo: /var/lib/mongodb
+  mongo:
+    image: "mongo:4.4.6-bionic"
+    command: [
+      "--dbpath", "/var/lib/mongodb",
+      "--bind_ip_all"
+    ]
+    ports:
+      - "27017:27017"
+    networks:
+      - test_autotm_net
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: "mongoadmin"
+      MONGO_INITDB_ROOT_PASSWORD: "secret"
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongo mongo:27017/test --quiet 1
+      timeout: 10s
+      retries: 10
+
+  mongo-express:
+    image: "mongo-express:latest"
+    ports:
+      - "8081:8081"
+    networks:
+      - test_autotm_net
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: "mongoadmin"
+      ME_CONFIG_MONGODB_ADMINPASSWORD: "secret"
+      ME_CONFIG_MONGODB_SERVER: "mongo"
+      ME_CONFIG_MONGODB_PORT: "27017"
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f http://localhost:8081 || exit 1" ]
+      timeout: 10s
+      retries: 10
+
+configs:
+  rabbitmq_consumer_settings:
+    file: rabbitmq/consumer-settings.conf
+
+  rabbitmq_conf:
+    file: rabbitmq/rabbitmq.conf
+
+  rabbitmq_enabled_plugins:
+    file: rabbitmq/enabled_plugins
+
+  mysql_conf:
+    file: mysql/my.cnf
+
+networks:
+  test_autotm_net:

tests/deploy/mysql/my.cnf

@@ -0,0 +1,4 @@
+[mysqld]
+max_connections=500
+wait_timeout=15
+interactive_timeout=15

tests/deploy/rabbitmq/consumer-settings.conf

@@ -0,0 +1,6 @@
+## Consumer timeout
+## If a message delivered to a consumer has not been acknowledge before this timer
+## triggers the channel will be force closed by the broker. This ensure that
+## faultly consumers that never ack will not hold on to messages indefinitely.
+##
+consumer_timeout = 1800000

tests/deploy/rabbitmq/enabled_plugins

@@ -0,0 +1 @@
+[rabbitmq_management,rabbitmq_prometheus].

tests/deploy/rabbitmq/rabbitmq.conf

@@ -0,0 +1,2 @@
+management.tcp.port = 15672
+management.tcp.ip   = 0.0.0.0

tests/distributed/conftest.py

@@ -0,0 +1,23 @@
+import pytest
+import os
+
+
+# Override the following 3 fixtures to make pytest-docker running with externally started docker compose
+@pytest.fixture(scope="session")
+def docker_compose_project_name():
+    return ""
+
+
+@pytest.fixture(scope="session")
+def docker_setup():
+    return None
+
+
+@pytest.fixture(scope="session")
+def docker_cleanup():
+    return None
+
+
+@pytest.fixture(scope="session")
+def docker_compose_file(pytestconfig):
+    return os.path.join(str(pytestconfig.rootpath), "tests", "deploy", "compose.yaml")

tests/distributed/test_distributed_fit_predict.py

@@ -0,0 +1,2 @@
+def test_distributed_fit_predict():
+    pass