refactor: Improve the intial structure

.github/workflows/main.yaml

@@ -8,33 +8,32 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 10
     strategy:
       matrix:
         python_version: ["3.10"]
     concurrency:
-      group: ci-${{ github.ref }}
+      group: ci-deploy-${{ github.ref }}
       cancel-in-progress: true
 
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
-        mamba-version: "*"
         environment-file: conda/base.yaml
         channels: conda-forge,nodefaults
-        activate-environment: celery-collectors
-        use-mamba: true
-        miniforge-variant: Mambaforge
+        activate-environment: poc-celery
+        auto-update-conda: true
+        conda-solver: libmamba
 
     - name: Prepare path
-      run: mkdir "data/"
+      run: mkdir -p "data/"
 
     - name: Run Rabbitmq container
       run: docker run --name rabbitmq  -d --rm -p 5672:5672 rabbitmq
@@ -46,8 +45,7 @@ jobs:
       run: celery -A main.celery_app worker --loglevel=DEBUG &
 
     - name: Run pytest for Collectors
-      run: pytest -vvv main/tests/tests_tasks_collectors.py
-
+      run: pytest -vvv test_tasks_collectors.py
 
     - name: Run pytest for Async tasks
-      run: pytest -vvv main/tests/tests_tasks_async.py
+      run: pytest -vvv test_tasks_async.py

.gitignore

@@ -158,3 +158,5 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+/data

README.md

@@ -86,7 +86,7 @@ This workflow demonstrates the power of Celery for handling complex asynchronous
 3. Activate the new environment:
 
    ```bash
-   mamba activate celery-collectors
+   mamba activate poc-celery
    ```
 
 ### RabbitMQ and Redis Setup Using Docker
@@ -98,6 +98,12 @@ docker run --name rabbitmq -d -p 5672:5672 rabbitmq
 docker run --name redis -d -p 6379:6379 redis
 ```
 
+or
+
+```bash
+bash scripts/setup.sh
+```
+
 These commands start RabbitMQ and Redis servers in Docker containers named `rabbitmq` and `redis`, respectively.
 
 ## Monitoring Celery Tasks with Flower
@@ -109,13 +115,13 @@ To facilitate an efficient development and monitoring environment, we've prepare
 To start both the Celery worker and Flower, navigate to your project's root directory and run:
 
 ```bash
-bash main/scripts/start_celery_and_flower.sh
+bash scripts/setup.sh
 ```
 
 This command executes the script that:
 
 1. **Starts a Celery Worker**: Launches a Celery worker instance using `main.celery_app` as the application module. This worker listens for tasks dispatched to the queues and executes them as they arrive.
-   
+
 2. **Launches Flower**: Initiates Flower on the default port (5555), allowing you to access a web-based user interface to monitor and manage the Celery worker and tasks. Flower provides insights into task progress, worker status, task history, and much more, making it an invaluable tool for debugging and optimizing your task workflows.
 
 

conda/base.yaml

@@ -1,4 +1,4 @@
-name: celery-collectors
+name: poc-celery
 channels:
   - nodefaults
   - conda-forge
@@ -10,6 +10,6 @@ dependencies:
     - rabbitmq
     - redis
     - pytest
-    - pytest-celery 
+    - pytest-celery
     - factory_boy
     - pytest-asyncio

main/celery_app.py

@@ -1,15 +1,15 @@
 
 from celery import Celery
 
-from main.scripts.get_container_ip import get_amqp_ip, get_redis_ip
+from main.get_container_ip import get_amqp_ip, get_redis_ip
 
 # Get the Rabbitmq container IP address
 AMQP_IP = get_amqp_ip()
 REDIS_IP = get_redis_ip()
 
 # Create a Celery instance with Rabbitmq as the broker and result backend
 app = Celery(
-    'celery-collectors',
+    'poc-celery',
     broker=f'amqp://guest:guest@{AMQP_IP}:5672',
     backend=f'redis://{REDIS_IP}:6379/0',
     include=[

main/scripts/get_container_ip.py → main/get_container_ip.py

@@ -1,6 +1,7 @@
 import subprocess
 import json
 
+
 def get_amqp_ip():
     """
     Get the IP address of the Rabbitmq container.
@@ -19,16 +20,14 @@ def get_amqp_ip():
     """
     # Run the docker inspect command to get information about the rabbitmq container
     result = subprocess.run(["docker", "inspect", "rabbitmq"], capture_output=True, text=True)
-    
+
     # Parse the output as JSON
     container_info = json.loads(result.stdout)
-    
+
     # Extract the IP address from the container information
     ip_address = container_info[0]['NetworkSettings']['IPAddress']
     return ip_address
 
-import subprocess
-import json
 
 def get_redis_ip():
     """
@@ -48,10 +47,10 @@ def get_redis_ip():
     """
     # Run the docker inspect command to get information about the redis container
     result = subprocess.run(["docker", "inspect", "redis"], capture_output=True, text=True)
-    
+
     # Parse the output as JSON
     container_info = json.loads(result.stdout)
-    
+
     # Extract the IP address from the container information
     ip_address = container_info[0]['NetworkSettings']['IPAddress']
     return ip_address

main/tasks_async.py

@@ -4,7 +4,7 @@
 from main.celery_app import app
 
 # app = Celery('tasks', broker='your_broker_url', backend='your_backend_url')
-DATA_DIR = Path("data").absolute()
+DATA_DIR = Path(__file__).parent.parent / "data"
 
 @app.task
 def clean_data(file_path):

scripts/setup.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+docker run --rm --name rabbitmq -d -p 5672:5672 rabbitmq || true
+docker run --rm --name redis -d -p 6379:6379 redis || true
+bash scripts/start_celery_and_flower.sh &

main/scripts/start_celery_and_flower.sh → scripts/start_celery_and_flower.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Fetch the Rabbitmq IP address by directly invoking the get_amqp_ip function
-AMQP_IP=$(python -c 'from main.scripts.get_container_ip import get_amqp_ip; print(get_amqp_ip())')
+AMQP_IP=$(python -c 'from main.get_container_ip import get_amqp_ip; print(get_amqp_ip())')
 
 # Validate the fetched IP
 if [ -z "$AMQP_IP" ]; then