From 50ee5086fe7726e904e4a4751e3916faf0e2ccd5 Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Wed, 24 Apr 2024 20:02:35 -0400 Subject: [PATCH] improve ci --- .github/workflows/main.yaml | 12 +++++------- README.md | 4 ++-- scripts/start_celery_and_flower.sh | 6 ++++-- src/poc_celery/poc1/__init__.py | 2 +- src/poc_celery/poc2/__init__.py | 2 +- src/poc_celery/poc2/celery_app.py | 2 +- .../poc2/{tasks_collectors.py => tasks.py} | 0 tests/poc1/test_tasks_async.py | 4 ++-- tests/poc1/test_tasks_collectors.py | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) rename src/poc_celery/poc2/{tasks_collectors.py => tasks.py} (100%) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 34e8e88..6025094 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -12,8 +12,9 @@ jobs: strategy: matrix: python_version: ["3.10"] + poc_id: ["poc1", "poc2"] concurrency: - group: ci-tests-${{ github.ref }} + group: ci-tests-${{ matrix.poc_id }}-${{ github.ref }} cancel-in-progress: true defaults: @@ -45,13 +46,10 @@ jobs: run: docker run --name redis -d redis redis-server --save 60 1 --loglevel warning - name: Start Celery worker - run: celery -A poc_celery.celery_app worker --loglevel=DEBUG & + run: celery -A poc_celery.${{ matrix.poc_id }}.celery_app worker --loglevel=DEBUG & - - name: Run pytest for Collectors - run: pytest -vvv tests/test_tasks_collectors.py - - - name: Run pytest for Async tasks - run: pytest -vvv tests/test_tasks_async.py + - name: Run pytest + run: pytest -vvv tests/${{ matrix.poc_id }} - name: Run linter run: | diff --git a/README.md b/README.md index b949aee..b098d69 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,8 @@ bash scripts/setup.sh This command executes the script that: 1. **Starts a Celery Worker**: Launches a Celery worker instance using - `poc_celery.celery_app` as the application module. This worker listens for - tasks dispatched to the queues and executes them as they arrive. + `poc_celery.poc1.celery_app` as the application module. This worker listens + for tasks dispatched to the queues and executes them as they arrive. 2. **Launches Flower**: Initiates Flower on the default port (5555), allowing you to access a web-based user interface to monitor and manage the Celery diff --git a/scripts/start_celery_and_flower.sh b/scripts/start_celery_and_flower.sh index 816d7ba..52252fd 100755 --- a/scripts/start_celery_and_flower.sh +++ b/scripts/start_celery_and_flower.sh @@ -1,5 +1,7 @@ #!/bin/bash +POC_ID=${1:-poc1} + # Fetch the Rabbitmq IP address by directly invoking the get_amqp_ip function AMQP_IP=$(python -c 'from poc_celery.get_container_ip import get_amqp_ip; print(get_amqp_ip())') @@ -13,10 +15,10 @@ echo "Rabbitmq IP: $AMQP_IP" # Start the Celery worker echo "Starting Celery worker..." -celery -A poc_celery.celery_app worker --loglevel=INFO & +celery -A poc_celery.${POC_ID}.celery_app worker --loglevel=INFO & # Start Flower echo "Starting Flower with Rabbitmq at $AMQP_IP..." -celery -A poc_celery.celery_app flower --broker=amqp://guest:guest@{AMQP_IP}:5672 & +celery -A poc_celery.${POC_ID}.celery_app flower --broker=amqp://guest:guest@{AMQP_IP}:5672 & echo "Celery and Flower have been started." diff --git a/src/poc_celery/poc1/__init__.py b/src/poc_celery/poc1/__init__.py index f2d66a9..6bc74b2 100644 --- a/src/poc_celery/poc1/__init__.py +++ b/src/poc_celery/poc1/__init__.py @@ -1,3 +1,3 @@ -from poc_celery.celery_app import app as celery_app +from poc_celery.poc1.celery_app import app as celery_app __all__ = ("celery_app",) diff --git a/src/poc_celery/poc2/__init__.py b/src/poc_celery/poc2/__init__.py index f2d66a9..1203398 100644 --- a/src/poc_celery/poc2/__init__.py +++ b/src/poc_celery/poc2/__init__.py @@ -1,3 +1,3 @@ -from poc_celery.celery_app import app as celery_app +from poc_celery.poc2.celery_app import app as celery_app __all__ = ("celery_app",) diff --git a/src/poc_celery/poc2/celery_app.py b/src/poc_celery/poc2/celery_app.py index 5f072e1..13938e3 100644 --- a/src/poc_celery/poc2/celery_app.py +++ b/src/poc_celery/poc2/celery_app.py @@ -12,7 +12,7 @@ broker=f"amqp://guest:guest@{AMQP_IP}:5672", backend=f"redis://{REDIS_IP}:6379/0", include=[ - "poc_celery.poc2.tasks_collectors", + "poc_celery.poc2.tasks", ], ) diff --git a/src/poc_celery/poc2/tasks_collectors.py b/src/poc_celery/poc2/tasks.py similarity index 100% rename from src/poc_celery/poc2/tasks_collectors.py rename to src/poc_celery/poc2/tasks.py diff --git a/tests/poc1/test_tasks_async.py b/tests/poc1/test_tasks_async.py index 91144f4..7972967 100644 --- a/tests/poc1/test_tasks_async.py +++ b/tests/poc1/test_tasks_async.py @@ -4,7 +4,7 @@ import pytest -from poc_celery.tasks_async import DATA_DIR, clean_data, create_project +from poc_celery.poc1.tasks_async import DATA_DIR, clean_data, create_project @pytest.fixture @@ -56,7 +56,7 @@ def test_create_project(mock_file_io): async def test_create_project_stress(mock_file_io): file_path = str(DATA_DIR / "collectors.txt") - num_calls = 100000 + num_calls = 10 calls = [ [1, 1, 3], diff --git a/tests/poc1/test_tasks_collectors.py b/tests/poc1/test_tasks_collectors.py index ada8b4e..313ceda 100644 --- a/tests/poc1/test_tasks_collectors.py +++ b/tests/poc1/test_tasks_collectors.py @@ -4,7 +4,7 @@ import pytest -from poc_celery.tasks_collectors import ( +from poc_celery.poc1.tasks_collectors import ( collector_request, generate_collector_request, ) @@ -29,8 +29,8 @@ def test_generate_collector_request(): assert isinstance(request_id, str), "The request_id should be a string." -@patch("poc_celery.tasks_collectors.collector_gathering.s") -@patch("poc_celery.tasks_collectors.group") +@patch("poc_celery.poc1.tasks_collectors.collector_gathering.s") +@patch("poc_celery.poc1.tasks_collectors.group") def test_collector_request_triggers_sub_collectors( mock_group, mock_collector_gathering_s ):