From c551f4a4dca3437167ac802ab0df6032e071aed4 Mon Sep 17 00:00:00 2001 From: fonhorst Date: Sat, 6 Jan 2024 15:01:04 +0300 Subject: [PATCH] add initial docker compose file for testing --- .../preprocessing/dictionaries_preparation.py | 10 ++ tests/__init__.py | 0 tests/deploy/compose.yaml | 133 ++++++++++++++++++ tests/deploy/mysql/my.cnf | 4 + tests/deploy/rabbitmq/consumer-settings.conf | 6 + tests/deploy/rabbitmq/enabled_plugins | 1 + tests/deploy/rabbitmq/rabbitmq.conf | 2 + tests/distributed/__init__.py | 0 tests/distributed/conftest.py | 23 +++ .../test_distributed_fit_predict.py | 2 + 10 files changed, 181 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/deploy/compose.yaml create mode 100644 tests/deploy/mysql/my.cnf create mode 100644 tests/deploy/rabbitmq/consumer-settings.conf create mode 100644 tests/deploy/rabbitmq/enabled_plugins create mode 100644 tests/deploy/rabbitmq/rabbitmq.conf create mode 100644 tests/distributed/__init__.py create mode 100644 tests/distributed/conftest.py create mode 100644 tests/distributed/test_distributed_fit_predict.py diff --git a/autotm/preprocessing/dictionaries_preparation.py b/autotm/preprocessing/dictionaries_preparation.py index 29efe64..1f7f651 100644 --- a/autotm/preprocessing/dictionaries_preparation.py +++ b/autotm/preprocessing/dictionaries_preparation.py @@ -218,10 +218,20 @@ def prepearing_cooc_dict( docs_count = data.shape[0] logger.debug("Performing calculate_cooc_dicts") +<<<<<<< Updated upstream cooc_dicts = calculate_cooc(batches_path=BATCHES_DIR, vocab=vocab_words, window_size=cooc_window) logger.debug("Performed calculate_cooc_dicts") cooc_df_dict, cooc_df_term_dict = cooc_dicts.cooc_df, cooc_dicts.cooc_df_term cooc_tf_dict, cooc_tf_term_dict = cooc_dicts.cooc_tf, cooc_dicts.cooc_tf_term +======= + df_dicts, tf_dicts = calculate_cooc_dicts( + vocab_words, data, n_cores=n_cores, window=cooc_window + ) + logger.debug("Performed calculate_cooc_dicts") + + cooc_df_dict, cooc_df_term_dict = df_dicts[0], df_dicts[1] + cooc_tf_dict, cooc_tf_term_dict = tf_dicts[0], tf_dicts[1] +>>>>>>> Stashed changes pairs_count = cooc_tf_dict[RESERVED_TUPLE] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/deploy/compose.yaml b/tests/deploy/compose.yaml new file mode 100644 index 0000000..9e07e41 --- /dev/null +++ b/tests/deploy/compose.yaml @@ -0,0 +1,133 @@ +services: + redis: + image: redis:6.2 + ports: + - "6379:6379" + networks: + - test_autotm_net + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + + rabbitmq: + image: rabbitmq:3.8-management-alpin + ports: + - "5672:5672" + - "15672:15672" + networks: + - test_autotm_net + configs: + - source: rabbitmq_conf + target: /etc/rabbitmq/rabbitmq.conf + - source: rabbitmq_enabled_plugins + target: /etc/rabbitmq/enabled_plugins + - source: rabbitmq_consumer_settings + target: /etc/rabbitmq/conf.d/consumer-settings.conf + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 30s + timeout: 30s + retries: 3 + + flower: + image: mher/flower + depends_on: + - rabbitmq + ports: + - "5555:5555" + networks: + - test_autotm_net + environment: + CELERY_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + CELERY_RESULT_BACKEND: "redis://redis:6379/1" + +# todo: "/var/lib/mysql" + mlflow-db: + image: mysql/mysql-server:5.7.28 + ports: + - "3306:3306" + networks: + - test_autotm_net + environment: + MYSQL_DATABASE: "mlflow" + MYSQL_USER: "mlflow" + MYSQL_PASSWORD: "mlflow" + MYSQL_ROOT_PASSWORD: "mlflow" + configs: + - source: mysql_conf + target: /etc/mysql/my.cnf + healthcheck: + test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] + timeout: 20s + retries: 10 + +# todo: set correct image +# todo: "/var/lib/mlruns" +# todo: shared artifact folder + mlflow: + image: "mlflow-webserver:latest" + command: [ + "--backend-store-uri", "mysql+pymysql://mlflow:mlflow@mlflow-db:3306/mlflow", + "--default-artifact-root", "/var/lib/mlruns", + "--host", "0.0.0.0" + ] + depends_on: + - mlflow-db + ports: + - "5000:5000" + networks: + - test_autotm_net + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:5000 || exit 1" ] + timeout: 10s + retries: 10 + +# todo: /var/lib/mongodb + mongo: + image: "mongo:4.4.6-bionic" + command: [ + "--dbpath", "/var/lib/mongodb", + "--bind_ip_all" + ] + ports: + - "27017:27017" + networks: + - test_autotm_net + environment: + MONGO_INITDB_ROOT_USERNAME: "mongoadmin" + MONGO_INITDB_ROOT_PASSWORD: "secret" + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongo mongo:27017/test --quiet 1 + timeout: 10s + retries: 10 + + mongo-express: + image: "mongo-express:latest" + ports: + - "8081:8081" + networks: + - test_autotm_net + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: "mongoadmin" + ME_CONFIG_MONGODB_ADMINPASSWORD: "secret" + ME_CONFIG_MONGODB_SERVER: "mongo" + ME_CONFIG_MONGODB_PORT: "27017" + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8081 || exit 1" ] + timeout: 10s + retries: 10 + +configs: + rabbitmq_consumer_settings: + file: rabbitmq/consumer-settings.conf + + rabbitmq_conf: + file: rabbitmq/rabbitmq.conf + + rabbitmq_enabled_plugins: + file: rabbitmq/enabled_plugins + + mysql_conf: + file: mysql/my.cnf + +networks: + test_autotm_net: diff --git a/tests/deploy/mysql/my.cnf b/tests/deploy/mysql/my.cnf new file mode 100644 index 0000000..6482a26 --- /dev/null +++ b/tests/deploy/mysql/my.cnf @@ -0,0 +1,4 @@ +[mysqld] +max_connections=500 +wait_timeout=15 +interactive_timeout=15 diff --git a/tests/deploy/rabbitmq/consumer-settings.conf b/tests/deploy/rabbitmq/consumer-settings.conf new file mode 100644 index 0000000..acd4dfc --- /dev/null +++ b/tests/deploy/rabbitmq/consumer-settings.conf @@ -0,0 +1,6 @@ +## Consumer timeout +## If a message delivered to a consumer has not been acknowledge before this timer +## triggers the channel will be force closed by the broker. This ensure that +## faultly consumers that never ack will not hold on to messages indefinitely. +## +consumer_timeout = 1800000 diff --git a/tests/deploy/rabbitmq/enabled_plugins b/tests/deploy/rabbitmq/enabled_plugins new file mode 100644 index 0000000..318ea04 --- /dev/null +++ b/tests/deploy/rabbitmq/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_management,rabbitmq_prometheus]. \ No newline at end of file diff --git a/tests/deploy/rabbitmq/rabbitmq.conf b/tests/deploy/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..36e9f8e --- /dev/null +++ b/tests/deploy/rabbitmq/rabbitmq.conf @@ -0,0 +1,2 @@ +management.tcp.port = 15672 +management.tcp.ip = 0.0.0.0 diff --git a/tests/distributed/__init__.py b/tests/distributed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/distributed/conftest.py b/tests/distributed/conftest.py new file mode 100644 index 0000000..4b67629 --- /dev/null +++ b/tests/distributed/conftest.py @@ -0,0 +1,23 @@ +import pytest +import os + + +# Override the following 3 fixtures to make pytest-docker running with externally started docker compose +@pytest.fixture(scope="session") +def docker_compose_project_name(): + return "" + + +@pytest.fixture(scope="session") +def docker_setup(): + return None + + +@pytest.fixture(scope="session") +def docker_cleanup(): + return None + + +@pytest.fixture(scope="session") +def docker_compose_file(pytestconfig): + return os.path.join(str(pytestconfig.rootpath), "tests", "deploy", "compose.yaml") diff --git a/tests/distributed/test_distributed_fit_predict.py b/tests/distributed/test_distributed_fit_predict.py new file mode 100644 index 0000000..086b15c --- /dev/null +++ b/tests/distributed/test_distributed_fit_predict.py @@ -0,0 +1,2 @@ +def test_distributed_fit_predict(): + pass