Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat:setup celery for local development #100

Closed
wants to merge 5 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: Docker setup to streamline build and deployment
Moved Dockerfile and entry script out of local_deployment for a cleaner structure. Updated docker-compose and Makefile to reflect these changes, using a unified image to leverage caching. Removed Poetry setup in favor of using pip with requirements.txt for dependency management.
hareshkainthdbt committed Dec 10, 2024
commit 9c27d1793fd8517bb2addb4d5e14812002159cb7
17 changes: 12 additions & 5 deletions local_deployment/Dockerfile → Dockerfile
Original file line number Diff line number Diff line change
@@ -33,10 +33,17 @@ RUN apt install -y curl && \
apt install -y nodejs

WORKDIR /app
COPY . /app

# Install poetry and project dependencies
RUN pip install poetry==1.8.3 && \
poetry install --without dev
RUN pip install poetry

# Copy only the requirements.txt into the container
COPY requirements.txt /app/

# Install the dependencies specified in requirements.txt
RUN pip install --no-cache-dir -r requirements.txt

COPY . /app

CMD ["local_deployment/entry.sh"]
COPY entry.sh /entry.sh
RUN chmod +x /entry.sh
ENTRYPOINT ["/entry.sh"]
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -34,7 +34,7 @@ drop-database: # Delete project's postgres database
fi

build: # Build docker containers for local execution
docker build --no-cache -f local_deployment/Dockerfile -t local_deployment .
docker build --no-cache -f Dockerfile -t local_deployment .
docker compose build

collectstatic: # Run Django collectstatic
@@ -136,6 +136,6 @@ setup_local: # Set up the local environment
@echo "$(COLOUR_GREEN)Running initial setup for local environment...$(COLOUR_NONE)"
$(MAKE) first-use
$(MAKE) start
$(MAKE) migrate
$(MAKE) rebuild_cache
#$(MAKE) migrate
#$(MAKE) rebuild_cache
@echo "$(COLOUR_GREEN)Local setup complete.$(COLOUR_NONE)"
12 changes: 9 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -27,7 +27,9 @@ services:
web:
build:
context: .
dockerfile: local_deployment/Dockerfile
cache_from:
- fbr/application:latest
image: fbr/application:latest
env_file:
- local.env
ports:
@@ -49,7 +51,9 @@ services:
celery-worker:
build:
context: .
dockerfile: local_deployment/Dockerfile
cache_from:
- fbr/application:latest
image: fbr/application:latest
command: celery --app fbr.celery_app worker --task-events --loglevel INFO
entrypoint: ''
volumes:
@@ -72,7 +76,9 @@ services:
celery-beats:
build:
context: .
dockerfile: local_deployment/Dockerfile
cache_from:
- fbr/application:latest
image: fbr/application:latest
command: celery --app fbr.celery_app beat --loglevel INFO
entrypoint: ''
volumes:
3 changes: 0 additions & 3 deletions local_deployment/entry.sh → entry.sh
Original file line number Diff line number Diff line change
@@ -6,9 +6,6 @@ npm install
echo "Bundling WebPack"
npm run build

# Activate the Poetry virtual environment
. "$(poetry env info --path)/bin/activate"

echo "Collecting Static Files"
python manage.py collectstatic --noinput

89 changes: 89 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
amqp==5.3.1 ; python_version >= "3.12" and python_version < "4.0"
asgiref==3.8.1 ; python_version >= "3.12" and python_version < "4.0"
backoff==2.2.1 ; python_version >= "3.12" and python_version < "4.0"
billiard==4.2.1 ; python_version >= "3.12" and python_version < "4.0"
boto3==1.35.77 ; python_version >= "3.12" and python_version < "4.0"
botocore==1.35.77 ; python_version >= "3.12" and python_version < "4.0"
brotli==1.1.0 ; python_version >= "3.12" and python_version < "4.0"
celery==5.4.0 ; python_version >= "3.12" and python_version < "4.0"
celery[redis]==5.4.0 ; python_version >= "3.12" and python_version < "4.0"
certifi==2024.8.30 ; python_version >= "3.12" and python_version < "4.0"
cfgv==3.4.0 ; python_version >= "3.12" and python_version < "4"
charset-normalizer==3.4.0 ; python_version >= "3.12" and python_version < "4.0"
click-didyoumean==0.3.1 ; python_version >= "3.12" and python_version < "4.0"
click-plugins==1.1.1 ; python_version >= "3.12" and python_version < "4.0"
click-repl==0.3.0 ; python_version >= "3.12" and python_version < "4.0"
click==8.1.7 ; python_version >= "3.12" and python_version < "4.0"
colorama==0.4.6 ; python_version >= "3.12" and python_version < "4.0" and platform_system == "Windows"
config==0.5.1 ; python_version >= "3.12" and python_version < "4.0"
cron-descriptor==1.4.5 ; python_version >= "3.12" and python_version < "4.0"
dbt-copilot-python==0.2.2 ; python_version >= "3.12" and python_version < "4.0"
deprecated==1.2.15 ; python_version >= "3.12" and python_version < "4.0"
distlib==0.3.9 ; python_version >= "3.12" and python_version < "4"
dj-database-url==2.3.0 ; python_version >= "3.12" and python_version < "4.0"
django-celery-beat==2.7.0 ; python_version >= "3.12" and python_version < "4.0"
django-environ==0.11.2 ; python_version >= "3.12" and python_version < "4"
django-log-formatter-asim==0.0.4 ; python_version >= "3.12" and python_version < "4"
django-timezone-field==7.0 ; python_version >= "3.12" and python_version < "4.0"
django-webpack-loader==3.1.1 ; python_version >= "3.12" and python_version < "4.0"
django==4.2.17 ; python_version >= "3.12" and python_version < "4.0"
djangorestframework==3.15.2 ; python_version >= "3.12" and python_version < "4.0"
filelock==3.16.1 ; python_version >= "3.12" and python_version < "4"
googleapis-common-protos==1.66.0 ; python_version >= "3.12" and python_version < "4.0"
grpcio==1.68.1 ; python_version >= "3.12" and python_version < "4.0"
gunicorn==22.0.0 ; python_version >= "3.12" and python_version < "4.0"
identify==2.6.3 ; python_version >= "3.12" and python_version < "4"
idna==3.10 ; python_version >= "3.12" and python_version < "4.0"
importlib-metadata==6.11.0 ; python_version >= "3.12" and python_version < "4.0"
jinja2==3.1.4 ; python_version >= "3.12" and python_version < "4.0"
jmespath==1.0.1 ; python_version >= "3.12" and python_version < "4.0"
kombu==5.4.2 ; python_version >= "3.12" and python_version < "4.0"
kombu[redis]==5.4.2 ; python_version >= "3.12" and python_version < "4.0"
markupsafe==3.0.2 ; python_version >= "3.12" and python_version < "4.0"
nodeenv==1.9.1 ; python_version >= "3.12" and python_version < "4"
numpy==2.2.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-api==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-distro==0.43b0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-common==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-grpc==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-exporter-otlp-proto-http==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-exporter-otlp==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-instrumentation-wsgi==0.43b0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-instrumentation==0.43b0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-propagator-aws-xray==1.0.2 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-proto==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-sdk-extension-aws==2.0.2 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-sdk==1.22.0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-semantic-conventions==0.43b0 ; python_version >= "3.12" and python_version < "4.0"
opentelemetry-util-http==0.43b0 ; python_version >= "3.12" and python_version < "4.0"
packaging==24.2 ; python_version >= "3.12" and python_version < "4.0"
pandas==2.2.3 ; python_version >= "3.12" and python_version < "4.0"
platformdirs==4.3.6 ; python_version >= "3.12" and python_version < "4"
pre-commit==3.8.0 ; python_version >= "3.12" and python_version < "4"
prompt-toolkit==3.0.48 ; python_version >= "3.12" and python_version < "4.0"
protobuf==4.25.5 ; python_version >= "3.12" and python_version < "4.0"
psycopg-c==3.2.3 ; python_version >= "3.12" and python_version < "4.0"
psycopg2-binary==2.9.10 ; python_version >= "3.12" and python_version < "4.0"
psycopg[c]==3.2.3 ; python_version >= "3.12" and python_version < "4.0"
python-crontab==3.2.0 ; python_version >= "3.12" and python_version < "4.0"
python-dateutil==2.9.0.post0 ; python_version >= "3.12" and python_version < "4.0"
python-environ==0.4.54 ; python_version >= "3.12" and python_version < "4.0"
pytz==2024.2 ; python_version >= "3.12" and python_version < "4.0"
pyyaml==6.0.2 ; python_version >= "3.12" and python_version < "4"
redis==5.2.1 ; python_version >= "3.12" and python_version < "4.0"
requests==2.32.3 ; python_version >= "3.12" and python_version < "4.0"
s3transfer==0.10.4 ; python_version >= "3.12" and python_version < "4.0"
sentry-sdk==2.19.2 ; python_version >= "3.12" and python_version < "4.0"
setuptools==75.6.0 ; python_version >= "3.12" and python_version < "4.0"
six==1.17.0 ; python_version >= "3.12" and python_version < "4.0"
sqlparse==0.5.2 ; python_version >= "3.12" and python_version < "4.0"
typing-extensions==4.12.2 ; python_version >= "3.12" and python_version < "4.0"
tzdata==2024.2 ; python_version >= "3.12" and python_version < "4.0"
urllib3==2.2.3 ; python_version >= "3.12" and python_version < "4.0"
utils==1.0.2 ; python_version >= "3.12" and python_version < "4.0"
vine==5.1.0 ; python_version >= "3.12" and python_version < "4.0"
virtualenv==20.28.0 ; python_version >= "3.12" and python_version < "4"
wcwidth==0.2.13 ; python_version >= "3.12" and python_version < "4.0"
whitenoise[brotli]==6.8.2 ; python_version >= "3.12" and python_version < "4.0"
wrapt==1.17.0 ; python_version >= "3.12" and python_version < "4.0"
zipp==3.21.0 ; python_version >= "3.12" and python_version < "4.0"