From f6daa0dd81e2a64b498fcd5e319df084770011c0 Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Thu, 11 Jul 2024 13:34:16 +0100 Subject: [PATCH 1/9] fs-4462 adding in shared dependencies --- .github/workflows/publish.yml | 9 +- python-flask-dev/requirements-db.in | 0 {python-flask-dev => python-flask}/Dockerfile | 0 python-flask/requirements-db.in | 18 ++ python-flask/requirements-db.txt | 203 ++++++++++++++++++ .../requirements-dev.in | 3 +- .../requirements-dev.txt | 0 .../requirements-frontend.in | 13 +- .../requirements-frontend.txt | 9 +- .../requirements.in | 0 .../requirements.txt | 0 {python-flask-dev => python-flask}/tags | 0 12 files changed, 241 insertions(+), 14 deletions(-) delete mode 100644 python-flask-dev/requirements-db.in rename {python-flask-dev => python-flask}/Dockerfile (100%) create mode 100644 python-flask/requirements-db.in create mode 100644 python-flask/requirements-db.txt rename {python-flask-dev => python-flask}/requirements-dev.in (92%) rename {python-flask-dev => python-flask}/requirements-dev.txt (100%) rename {python-flask-dev => python-flask}/requirements-frontend.in (69%) rename {python-flask-dev => python-flask}/requirements-frontend.txt (90%) rename {python-flask-dev => python-flask}/requirements.in (100%) rename {python-flask-dev => python-flask}/requirements.txt (100%) rename {python-flask-dev => python-flask}/tags (100%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a07d2fd..1ad1046 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,11 +10,11 @@ jobs: docker-build: strategy: matrix: - flavour: ["flask", "frontend"] + flavour: ["flask", "frontend", "db"] runs-on: ubuntu-latest defaults: run: - working-directory: ./python-flask-dev + working-directory: ./python-flask steps: - name: Checkout repo uses: actions/checkout@v4 @@ -38,6 +38,7 @@ jobs: type=raw,value=${{env.PYTHON_FLASK_DEV_PYTHON}},enable=${{ github.ref == format('refs/heads/{0}', 'main')}} type=raw,value=${{env.PYTHON_FLASK_DEV_FLASK}},enable=${{ github.ref == format('refs/heads/{0}', 'main') }} type=raw,value=${{env.PYTHON_FLASK_DEV_FRONTEND}},enable=${{ (github.ref == format('refs/heads/{0}', 'main')) && (matrix.flavour == 'frontend') }} + type=raw,value=${{env.PYTHON_FLASK_DEV_DB}},enable=${{ (github.ref == format('refs/heads/{0}', 'main')) && (matrix.flavour == 'db') }} type=ref,event=branch - name: Log in to the Container registry @@ -50,9 +51,9 @@ jobs: - name: Build and push docker image uses: docker/build-push-action@v4 with: - context: ./python-flask-dev/ + context: ./python-flask/ tags: ${{ steps.metadata.outputs.tags}} labels: ${{ steps.metadata.outputs.labels }} push: true - file: ./python-flask-dev/Dockerfile + file: ./python-flask/Dockerfile target: ${{matrix.flavour}} \ No newline at end of file diff --git a/python-flask-dev/requirements-db.in b/python-flask-dev/requirements-db.in deleted file mode 100644 index e69de29..0000000 diff --git a/python-flask-dev/Dockerfile b/python-flask/Dockerfile similarity index 100% rename from python-flask-dev/Dockerfile rename to python-flask/Dockerfile diff --git a/python-flask/requirements-db.in b/python-flask/requirements-db.in new file mode 100644 index 0000000..c2d4c09 --- /dev/null +++ b/python-flask/requirements-db.in @@ -0,0 +1,18 @@ +#----------------------------------- +# Database +#----------------------------------- +SQLAlchemy[mypy]>=2.0.30 +Flask-SQLAlchemy==3.1.1 +Flask-Migrate==4.0.7 +sqlalchemy-utils==0.41.2 +sqlalchemy_json==0.7.0 +psycopg2-binary==2.9.9 +uvicorn==0.30.1 +marshmallow-sqlalchemy==1.0.0 +#----------------------------------- +# Connexion APIs +#----------------------------------- +connexion[flask,swagger-ui,uvicorn] +swagger-ui-bundle==1.1.0 +openapi-spec-validator +prance \ No newline at end of file diff --git a/python-flask/requirements-db.txt b/python-flask/requirements-db.txt new file mode 100644 index 0000000..6a6d832 --- /dev/null +++ b/python-flask/requirements-db.txt @@ -0,0 +1,203 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile requirements-db.in +# +a2wsgi==1.10.6 + # via connexion +alembic==1.13.2 + # via flask-migrate +anyio==4.4.0 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via + # connexion + # flask +attrs==23.2.0 + # via + # jsonschema + # referencing +blinker==1.8.2 + # via flask +certifi==2024.7.4 + # via + # httpcore + # httpx + # requests +chardet==5.2.0 + # via prance +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # flask + # uvicorn +connexion[flask,swagger-ui,uvicorn]==3.1.0 + # via -r requirements-db.in +exceptiongroup==1.2.1 + # via anyio +flask[async]==3.0.3 + # via + # connexion + # flask-migrate + # flask-sqlalchemy +flask-migrate==4.0.7 + # via -r requirements-db.in +flask-sqlalchemy==3.1.1 + # via + # -r requirements-db.in + # flask-migrate +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.5 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via connexion +idna==3.7 + # via + # anyio + # httpx + # requests +inflection==0.5.1 + # via connexion +itsdangerous==2.2.0 + # via flask +jinja2==3.1.4 + # via + # connexion + # flask + # swagger-ui-bundle +jsonschema==4.23.0 + # via + # connexion + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.3.3 + # via openapi-spec-validator +jsonschema-specifications==2023.12.1 + # via + # jsonschema + # openapi-schema-validator +lazy-object-proxy==1.10.0 + # via openapi-spec-validator +mako==1.3.5 + # via alembic +markupsafe==2.1.5 + # via + # jinja2 + # mako + # werkzeug +marshmallow==3.21.3 + # via marshmallow-sqlalchemy +marshmallow-sqlalchemy==1.0.0 + # via -r requirements-db.in +mypy==1.10.1 + # via sqlalchemy +mypy-extensions==1.0.0 + # via mypy +openapi-schema-validator==0.6.2 + # via openapi-spec-validator +openapi-spec-validator==0.7.1 + # via -r requirements-db.in +packaging==24.1 + # via + # marshmallow + # prance +pathable==0.4.3 + # via jsonschema-path +prance==23.6.21.0 + # via -r requirements-db.in +psycopg2-binary==2.9.9 + # via -r requirements-db.in +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.9 + # via connexion +pyyaml==6.0.1 + # via + # connexion + # jsonschema-path + # uvicorn +referencing==0.35.1 + # via + # jsonschema + # jsonschema-path + # jsonschema-specifications +requests==2.32.3 + # via + # connexion + # jsonschema-path + # prance +rfc3339-validator==0.1.4 + # via openapi-schema-validator +rpds-py==0.19.0 + # via + # jsonschema + # referencing +ruamel-yaml==0.18.6 + # via prance +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +six==1.16.0 + # via + # prance + # rfc3339-validator +sniffio==1.3.1 + # via + # anyio + # httpx +sqlalchemy[mypy]==2.0.31 + # via + # -r requirements-db.in + # alembic + # flask-sqlalchemy + # marshmallow-sqlalchemy + # sqlalchemy + # sqlalchemy-json + # sqlalchemy-utils +sqlalchemy-json==0.7.0 + # via -r requirements-db.in +sqlalchemy-utils==0.41.2 + # via -r requirements-db.in +starlette==0.37.2 + # via connexion +swagger-ui-bundle==1.1.0 + # via + # -r requirements-db.in + # connexion +tomli==2.0.1 + # via mypy +typing-extensions==4.12.2 + # via + # a2wsgi + # alembic + # anyio + # asgiref + # connexion + # mypy + # sqlalchemy + # uvicorn +urllib3==2.2.2 + # via requests +uvicorn[standard]==0.30.1 + # via + # -r requirements-db.in + # connexion +uvloop==0.19.0 + # via uvicorn +watchfiles==0.22.0 + # via uvicorn +websockets==12.0 + # via uvicorn +werkzeug==3.0.3 + # via + # connexion + # flask diff --git a/python-flask-dev/requirements-dev.in b/python-flask/requirements-dev.in similarity index 92% rename from python-flask-dev/requirements-dev.in rename to python-flask/requirements-dev.in index 801722a..077836c 100644 --- a/python-flask-dev/requirements-dev.in +++ b/python-flask/requirements-dev.in @@ -3,7 +3,8 @@ #----------------------------------- pytest==8.2.2 -pytest-mock==3.12.0 +pytest-mock==3.14.0 +pytest-html>=3.2.0 pytest-env==1.1.3 #----------------------------------- diff --git a/python-flask-dev/requirements-dev.txt b/python-flask/requirements-dev.txt similarity index 100% rename from python-flask-dev/requirements-dev.txt rename to python-flask/requirements-dev.txt diff --git a/python-flask-dev/requirements-frontend.in b/python-flask/requirements-frontend.in similarity index 69% rename from python-flask-dev/requirements-frontend.in rename to python-flask/requirements-frontend.in index c0eb9a1..7b13c89 100644 --- a/python-flask-dev/requirements-frontend.in +++ b/python-flask/requirements-frontend.in @@ -5,6 +5,16 @@ requests==2.32.3 #----------------------------------- flask-talisman==1.1.0 +#----------------------------------- +# Flask Version +#----------------------------------- +Flask==3.0.3 + +#----------------------------------- +# Security +#----------------------------------- +flask-talisman==1.1.0 + #----------------------------------- # Static Assets #----------------------------------- @@ -23,5 +33,4 @@ Flask-Babel>=2.0.0 #----------------------------------- # Forms #----------------------------------- -Flask-WTF==1.2.1 -email-validator==2.1.1 \ No newline at end of file +Flask-WTF==1.2.1 \ No newline at end of file diff --git a/python-flask-dev/requirements-frontend.txt b/python-flask/requirements-frontend.txt similarity index 90% rename from python-flask-dev/requirements-frontend.txt rename to python-flask/requirements-frontend.txt index da73055..c3729ff 100644 --- a/python-flask-dev/requirements-frontend.txt +++ b/python-flask/requirements-frontend.txt @@ -18,12 +18,9 @@ click==8.1.7 # via flask cssmin==0.2.0 # via -r requirements-frontend.in -dnspython==2.6.1 - # via email-validator -email-validator==2.1.1 - # via -r requirements-frontend.in flask==3.0.3 # via + # -r requirements-frontend.in # flask-assets # flask-babel # flask-compress @@ -41,9 +38,7 @@ flask-wtf==1.2.1 govuk-frontend-jinja==3.1.0 # via -r requirements-frontend.in idna==3.7 - # via - # email-validator - # requests + # via requests itsdangerous==2.2.0 # via # flask diff --git a/python-flask-dev/requirements.in b/python-flask/requirements.in similarity index 100% rename from python-flask-dev/requirements.in rename to python-flask/requirements.in diff --git a/python-flask-dev/requirements.txt b/python-flask/requirements.txt similarity index 100% rename from python-flask-dev/requirements.txt rename to python-flask/requirements.txt diff --git a/python-flask-dev/tags b/python-flask/tags similarity index 100% rename from python-flask-dev/tags rename to python-flask/tags From 451be31f5243f21e30b7bd7295063add06762b7e Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Thu, 11 Jul 2024 13:35:40 +0100 Subject: [PATCH 2/9] add db to dockerfile --- python-flask/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python-flask/Dockerfile b/python-flask/Dockerfile index 859ab9e..92bfbd6 100644 --- a/python-flask/Dockerfile +++ b/python-flask/Dockerfile @@ -23,10 +23,10 @@ RUN pip install --upgrade pip && pip install -r requirements-frontend.txt # -------------------------------------------------- -# Will hold config for building DB base image +# Base image to build db images on - includes sqwl alchemy and connexion # -------------------------------------------------- -# FROM flask as db -# WORKDIR /app -# COPY requirements-db.txt requirements-db.txt -# RUN pip install --upgrade pip && pip install -r requirements-db.txt +FROM flask as db +WORKDIR /app +COPY requirements-db.txt requirements-db.txt +RUN pip install --upgrade pip && pip install -r requirements-db.txt From 031f70a14aec5532c3cc65d5f9b129942a3b5edb Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 11:39:25 +0100 Subject: [PATCH 3/9] adding pre-commit --- python-flask/requirements-dev.in | 1 + python-flask/requirements-dev.txt | 32 +++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/python-flask/requirements-dev.in b/python-flask/requirements-dev.in index 077836c..0b460d2 100644 --- a/python-flask/requirements-dev.in +++ b/python-flask/requirements-dev.in @@ -11,6 +11,7 @@ pytest-env==1.1.3 # Dev Utils #----------------------------------- invoke==2.2.0 +pre-commit==3.7.1 #----------------------------------- # Code Quality diff --git a/python-flask/requirements-dev.txt b/python-flask/requirements-dev.txt index 5317e09..bd568e9 100644 --- a/python-flask/requirements-dev.txt +++ b/python-flask/requirements-dev.txt @@ -8,24 +8,38 @@ beautifulsoup4==4.12.2 # via -r requirements-dev.in black==24.3.0 # via -r requirements-dev.in +cfgv==3.4.0 + # via pre-commit click==8.1.7 # via black +distlib==0.3.8 + # via virtualenv exceptiongroup==1.2.1 # via pytest +filelock==3.15.4 + # via virtualenv flake8==7.0.0 # via # -r requirements-dev.in # flake8-pyproject flake8-pyproject==1.2.3 # via -r requirements-dev.in +identify==2.6.0 + # via pre-commit iniconfig==2.0.0 # via pytest invoke==2.2.0 # via -r requirements-dev.in +jinja2==3.1.4 + # via pytest-html +markupsafe==2.1.5 + # via jinja2 mccabe==0.7.0 # via flake8 mypy-extensions==1.0.0 # via black +nodeenv==1.9.1 + # via pre-commit packaging==24.1 # via # black @@ -33,9 +47,13 @@ packaging==24.1 pathspec==0.12.1 # via black platformdirs==4.2.2 - # via black + # via + # black + # virtualenv pluggy==1.5.0 # via pytest +pre-commit==3.7.1 + # via -r requirements-dev.in pycodestyle==2.11.1 # via flake8 pyflakes==3.2.0 @@ -44,11 +62,19 @@ pytest==8.2.2 # via # -r requirements-dev.in # pytest-env + # pytest-html + # pytest-metadata # pytest-mock pytest-env==1.1.3 # via -r requirements-dev.in -pytest-mock==3.12.0 +pytest-html==4.1.1 + # via -r requirements-dev.in +pytest-metadata==3.1.1 + # via pytest-html +pytest-mock==3.14.0 # via -r requirements-dev.in +pyyaml==6.0.1 + # via pre-commit soupsieve==2.5 # via beautifulsoup4 tomli==2.0.1 @@ -59,3 +85,5 @@ tomli==2.0.1 # pytest-env typing-extensions==4.12.2 # via black +virtualenv==20.26.3 + # via pre-commit From 8e09712c0bd564e57c148abb73235adbc03466a5 Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 13:28:50 +0100 Subject: [PATCH 4/9] install lsb_release --- python-flask/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/python-flask/Dockerfile b/python-flask/Dockerfile index 92bfbd6..a47ed2a 100644 --- a/python-flask/Dockerfile +++ b/python-flask/Dockerfile @@ -30,3 +30,4 @@ FROM flask as db WORKDIR /app COPY requirements-db.txt requirements-db.txt RUN pip install --upgrade pip && pip install -r requirements-db.txt +RUN apt-get install -y lsb-release && apt-get clean all From 6dd1d2a045b5cdc4033d40ad010deb98336e6960 Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 13:38:22 +0100 Subject: [PATCH 5/9] lsb release --- python-flask/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-flask/Dockerfile b/python-flask/Dockerfile index a47ed2a..81e3616 100644 --- a/python-flask/Dockerfile +++ b/python-flask/Dockerfile @@ -29,5 +29,6 @@ RUN pip install --upgrade pip && pip install -r requirements-frontend.txt FROM flask as db WORKDIR /app COPY requirements-db.txt requirements-db.txt +RUN apt-get update && apt-get install -y lsb-release && apt-get clean all RUN pip install --upgrade pip && pip install -r requirements-db.txt -RUN apt-get install -y lsb-release && apt-get clean all +# RUN apt-get install -y lsb-release && apt-get clean all From cf20f6ad836fae5243677c8f84ad2b354089317f Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 13:45:51 +0100 Subject: [PATCH 6/9] remove cache --- python-flask/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python-flask/Dockerfile b/python-flask/Dockerfile index 81e3616..4835cb6 100644 --- a/python-flask/Dockerfile +++ b/python-flask/Dockerfile @@ -23,7 +23,7 @@ RUN pip install --upgrade pip && pip install -r requirements-frontend.txt # -------------------------------------------------- -# Base image to build db images on - includes sqwl alchemy and connexion +# Base image to build db images on - includes sql alchemy and connexion # -------------------------------------------------- FROM flask as db @@ -31,4 +31,3 @@ WORKDIR /app COPY requirements-db.txt requirements-db.txt RUN apt-get update && apt-get install -y lsb-release && apt-get clean all RUN pip install --upgrade pip && pip install -r requirements-db.txt -# RUN apt-get install -y lsb-release && apt-get clean all From 3c3241906ad0a4d59e18dbf6bd76ac7da4e6c76a Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 14:09:32 +0100 Subject: [PATCH 7/9] BS version --- python-flask/requirements-dev.in | 2 +- python-flask/requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python-flask/requirements-dev.in b/python-flask/requirements-dev.in index 0b460d2..50d3415 100644 --- a/python-flask/requirements-dev.in +++ b/python-flask/requirements-dev.in @@ -23,4 +23,4 @@ black==24.3.0 #----------------------------------- # Other Dev Dependencies #----------------------------------- -beautifulsoup4==4.12.2 \ No newline at end of file +beautifulsoup4==4.12.3 \ No newline at end of file diff --git a/python-flask/requirements-dev.txt b/python-flask/requirements-dev.txt index bd568e9..f46fed9 100644 --- a/python-flask/requirements-dev.txt +++ b/python-flask/requirements-dev.txt @@ -4,7 +4,7 @@ # # pip-compile requirements-dev.in # -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via -r requirements-dev.in black==24.3.0 # via -r requirements-dev.in From 4e66b69893036fdb9e1504973bdecea864ffb602 Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 14:09:55 +0100 Subject: [PATCH 8/9] remove lsb --- python-flask/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/python-flask/Dockerfile b/python-flask/Dockerfile index 4835cb6..15df2c6 100644 --- a/python-flask/Dockerfile +++ b/python-flask/Dockerfile @@ -29,5 +29,4 @@ RUN pip install --upgrade pip && pip install -r requirements-frontend.txt FROM flask as db WORKDIR /app COPY requirements-db.txt requirements-db.txt -RUN apt-get update && apt-get install -y lsb-release && apt-get clean all RUN pip install --upgrade pip && pip install -r requirements-db.txt From 0fcbee0cc870ec8fe26d84cbd7001e4fe1ff2207 Mon Sep 17 00:00:00 2001 From: Sarah Sloan Date: Fri, 12 Jul 2024 14:18:24 +0100 Subject: [PATCH 9/9] add db tag --- .github/workflows/publish.yml | 1 + README.md | 4 ++-- python-flask/tags | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1ad1046..c62b30e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,6 +26,7 @@ jobs: echo "PYTHON_FLASK_DEV_PYTHON=$PYTHON_FLASK_DEV_PY" >> $GITHUB_ENV echo "PYTHON_FLASK_DEV_FLASK=$PYTHON_FLASK_DEV_FLASK" >> $GITHUB_ENV echo "PYTHON_FLASK_DEV_FRONTEND=$PYTHON_FLASK_DEV_JINJA" >> $GITHUB_ENV + echo "PYTHON_FLASK_DEV_DB=$PYTHON_FLASK_DEV_SQL_ALCHEMY" >> $GITHUB_ENV - name: Docker metadata id: metadata diff --git a/README.md b/README.md index ba95bb6..28da2de 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Base images and requirements definitions for development of funding service desi ## Python Flask Dev Contains requirements files for different versions of a python flask app. The [Dockerfile](./python-flask-dev/Dockerfile) contains the following build targets: -- `flask`: Developer image based on python 3.10 bullseye, with Flask 3.0.3 installed on top, plus everything in [requirements-dev.txt](./python-flask-dev/requirements-dev.txt) +- `flask`: Developer image based on python 3.10 bullseye, with Flask 3.0.3 installed on top, plus everything in [requirements-dev.txt](./python-flask/requirements-dev.txt) - `frontend`: Base on `flask` above, with everything from [requirements-frontend.txt](./python-flask-dev/requirements-frontend.txt) installed ## Workflows @@ -11,6 +11,6 @@ Contains requirements files for different versions of a python flask app. The [D # Future Improvements - Make the workflow just use all tags in the tags file, not hard code them? -- If we add other base images besides [python-flask-dev](./python-flask-dev/), make the workflow use a matrix approach so it builds all of these? +- If we add other base images besides [python-flask](./python-flask/), make the workflow use a matrix approach so it builds all of these? - Are we using the right tag names? - Stop using pip-compile and use rye? diff --git a/python-flask/tags b/python-flask/tags index dd3ad9d..d4236d8 100644 --- a/python-flask/tags +++ b/python-flask/tags @@ -1,3 +1,4 @@ PYTHON_FLASK_DEV_FLASK=flask-3.0.3 PYTHON_FLASK_DEV_JINJA=jinja-3.1.0 -PYTHON_FLASK_DEV_PY=3.10-bullseye \ No newline at end of file +PYTHON_FLASK_DEV_PY=3.10-bullseye +PYTHON_FLASK_DEV_SQL_ALCHEMY=sqlalchemy-2.0.30 \ No newline at end of file