improve ci

.github/workflows/main.yaml

@@ -12,8 +12,9 @@ jobs:
     strategy:
       matrix:
         python_version: ["3.10"]
+        poc_id: ["poc1", "poc2"]
     concurrency:
-      group: ci-tests-${{ github.ref }}
+      group: ci-tests-${{ matrix.poc_id }}-${{ github.ref }}
       cancel-in-progress: true
 
     defaults:
@@ -45,13 +46,10 @@ jobs:
         run: docker run --name redis -d redis redis-server --save 60 1 --loglevel warning
 
       - name: Start Celery worker
-        run: celery -A poc_celery.celery_app worker --loglevel=DEBUG &
+        run: celery -A poc_celery.${{ matrix.poc_id }}.celery_app worker --loglevel=DEBUG &
 
-      - name: Run pytest for Collectors
-        run: pytest -vvv tests/test_tasks_collectors.py
-
-      - name: Run pytest for Async tasks
-        run: pytest -vvv tests/test_tasks_async.py
+      - name: Run pytest
+        run: pytest -vvv tests/${{ matrix.poc_id }}
 
       - name: Run linter
         run: |

README.md

@@ -165,8 +165,8 @@ bash scripts/setup.sh
 This command executes the script that:
 
 1. **Starts a Celery Worker**: Launches a Celery worker instance using
-   `poc_celery.celery_app` as the application module. This worker listens for
-   tasks dispatched to the queues and executes them as they arrive.
+   `poc_celery.poc1.celery_app` as the application module. This worker listens
+   for tasks dispatched to the queues and executes them as they arrive.
 
 2. **Launches Flower**: Initiates Flower on the default port (5555), allowing
    you to access a web-based user interface to monitor and manage the Celery

scripts/start_celery_and_flower.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+POC_ID=${1:-poc1}
+
 # Fetch the Rabbitmq IP address by directly invoking the get_amqp_ip function
 AMQP_IP=$(python -c 'from poc_celery.get_container_ip import get_amqp_ip; print(get_amqp_ip())')
 
@@ -13,10 +15,10 @@ echo "Rabbitmq IP: $AMQP_IP"
 
 # Start the Celery worker
 echo "Starting Celery worker..."
-celery -A poc_celery.celery_app worker --loglevel=INFO &
+celery -A poc_celery.${POC_ID}.celery_app worker --loglevel=INFO &
 
 # Start Flower
 echo "Starting Flower with Rabbitmq at $AMQP_IP..."
-celery -A poc_celery.celery_app flower --broker=amqp://guest:guest@{AMQP_IP}:5672 &
+celery -A poc_celery.${POC_ID}.celery_app flower --broker=amqp://guest:guest@{AMQP_IP}:5672 &
 
 echo "Celery and Flower have been started."

src/poc_celery/poc1/__init__.py

@@ -1,3 +1,3 @@
-from poc_celery.celery_app import app as celery_app
+from poc_celery.poc1.celery_app import app as celery_app
 
 __all__ = ("celery_app",)

src/poc_celery/poc2/__init__.py

@@ -1,3 +1,3 @@
-from poc_celery.celery_app import app as celery_app
+from poc_celery.poc2.celery_app import app as celery_app
 
 __all__ = ("celery_app",)

src/poc_celery/poc2/celery_app.py

@@ -12,7 +12,7 @@
     broker=f"amqp://guest:guest@{AMQP_IP}:5672",
     backend=f"redis://{REDIS_IP}:6379/0",
     include=[
-        "poc_celery.poc2.tasks_collectors",
+        "poc_celery.poc2.tasks",
     ],
 )
 

tests/poc1/test_tasks_async.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from poc_celery.tasks_async import DATA_DIR, clean_data, create_project
+from poc_celery.poc1.tasks_async import DATA_DIR, clean_data, create_project
 
 
 @pytest.fixture
@@ -56,7 +56,7 @@ def test_create_project(mock_file_io):
 async def test_create_project_stress(mock_file_io):
     file_path = str(DATA_DIR / "collectors.txt")
 
-    num_calls = 100000
+    num_calls = 10
 
     calls = [
         [1, 1, 3],

tests/poc1/test_tasks_collectors.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from poc_celery.tasks_collectors import (
+from poc_celery.poc1.tasks_collectors import (
     collector_request,
     generate_collector_request,
 )
@@ -29,8 +29,8 @@ def test_generate_collector_request():
     assert isinstance(request_id, str), "The request_id should be a string."
 
 
-@patch("poc_celery.tasks_collectors.collector_gathering.s")
-@patch("poc_celery.tasks_collectors.group")
+@patch("poc_celery.poc1.tasks_collectors.collector_gathering.s")
+@patch("poc_celery.poc1.tasks_collectors.group")
 def test_collector_request_triggers_sub_collectors(
     mock_group, mock_collector_gathering_s
 ):