[build] move requirements to pyproject.toml (mindee#1031)

.github/workflows/builds.yml

@@ -30,13 +30,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'pytorch'
         name: Cache python modules (PT)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'tensorflow'
         name: Install package (TF)
         run: |

.github/workflows/doc-status.yml

@@ -6,10 +6,10 @@ jobs:
   see-page-build-payload:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.8
           architecture: x64
       - name: check status
         run: |

.github/workflows/docker.yml

@@ -17,20 +17,32 @@ jobs:
         run: docker run doctr-tf-py3.8-slim python -c 'import doctr'
 
   pytest-api:
-    runs-on: ubuntu-latest
-    env:
-        # Port for web container
-        # See docker-compose.yml
-        PORT: 8002
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.8]
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Install poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.1.13
+      - name: Lock the requirements
+        run: |
+          cd api
+          make lock
       - name: Build & run docker
-        run: docker-compose up -d --build
+        run: cd api && docker-compose up -d --build
       - name: Ping server
-        run: wget --spider --tries=12 http://localhost:$PORT/docs
-      - name: Install dependencies in docker
-        run: |
-          docker-compose exec -T web python -m pip install --upgrade pip
-          docker-compose exec -T web pip install -r requirements-dev.txt
+        run: wget --spider --tries=12 http://localhost:8080/docs
       - name: Run docker test
-        run: docker-compose exec -T web pytest .
+        run: |
+          docker cp api/requirements-dev.txt api_web_1:/app/requirements-dev.txt
+          docker-compose -f api/docker-compose.yml exec -T web pip install -r requirements-dev.txt
+          docker cp api/tests api_web_1:/app/tests
+          docker-compose -f api/docker-compose.yml exec -T web pytest tests/

.github/workflows/docs.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: [3.7]
+        python: [3.8]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -23,9 +23,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-docs
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/main.yml

@@ -43,9 +43,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-tests
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -78,9 +76,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-tests
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -113,9 +109,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-tests
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pull_requests.yml

@@ -12,15 +12,13 @@ jobs:
       - name: Set up Python 3.7
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.8
           architecture: x64
       - name: Cache python modules
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-docs
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: [3.7]
+        python: [3.8]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
@@ -24,7 +24,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -50,11 +50,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: [3.7]
+        python: [3.8]
     needs: pypi-publish
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}

.github/workflows/scripts.yml

@@ -30,13 +30,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'pytorch'
         name: Cache python modules (PT)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'tensorflow'
         name: Install package (TF)
         run: |
@@ -76,13 +76,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'pytorch'
         name: Cache python modules (PT)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'tensorflow'
         name: Install package (TF)
         run: |

.github/workflows/style.yml

@@ -82,11 +82,11 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[all] --upgrade
+          pip install -e .[dev] --upgrade
           pip install mypy
       - name: Run mypy
         run: |

Dockerfile

@@ -4,7 +4,7 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 
 
-COPY ./requirements.txt /tmp/requirements.txt
+COPY ./pyproject.toml /tmp/pyproject.toml
 COPY ./README.md /tmp/README.md
 COPY ./setup.py /tmp/setup.py
 COPY ./doctr /tmp/doctr

Dockerfile-api → api/Dockerfile

@@ -8,27 +8,17 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONPATH "${PYTHONPATH}:/app"
 
 # copy requirements file
-COPY api/requirements.txt /app/api-requirements.txt
-COPY ./requirements.txt /tmp/requirements.txt
+COPY requirements.txt  /app/requirements.txt
+COPY Makefile /app/Makefile
 
 RUN apt-get update \
     && apt-get install --no-install-recommends ffmpeg libsm6 libxext6 -y \
     && pip install --upgrade pip setuptools wheel \
-    && pip install -r /app/api-requirements.txt \
-    && pip install -r /tmp/requirements.txt \
+    && pip install -r /app/requirements.txt \
     && pip cache purge \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /root/.cache/pip
 
-# install doctr
-COPY ./README.md /tmp/README.md
-COPY ./setup.py /tmp/setup.py
-COPY ./doctr /tmp/doctr
-
-RUN pip install -e /tmp/.[tf] \
-    && pip cache purge \
-    && rm -rf /root/.cache/pip
-
 # copy project
-COPY api /app
+COPY app /app/app

api/Makefile

@@ -0,0 +1,25 @@
+# api setup is borrowed from https://github.com/frgfm/Holocron/blob/main/api
+
+.PHONY: lock run stop test
+# Pin the dependencies
+lock:
+	poetry lock
+	poetry export -f requirements.txt --without-hashes --output requirements.txt
+	poetry export -f requirements.txt --without-hashes --dev --output requirements-dev.txt
+
+# Run the docker
+run:
+	docker-compose up -d --build
+
+# Run the docker
+stop:
+	docker-compose down
+
+# Run tests for the library
+test:
+	docker-compose up -d --build
+	docker cp requirements-dev.txt api_web_1:/app/requirements-dev.txt
+	docker-compose exec -T web pip install -r requirements-dev.txt
+	docker cp tests api_web_1:/app/tests
+	docker-compose exec -T web pytest tests/
+	docker-compose down

api/README.md

@@ -2,7 +2,7 @@
 
 ## Installation
 
-You will only need to install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/get-docker/). The container environment will be self-sufficient and install the remaining dependencies on its own.
+You will only need to install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git), [Docker](https://docs.docker.com/get-docker/) and [poetry](https://python-poetry.org/docs/#installation). The container environment will be self-sufficient and install the remaining dependencies on its own.
 
 ## Usage
 
@@ -12,16 +12,17 @@ You will need to clone the repository first:
 ```shell
 git clone https://github.com/mindee/doctr.git
 ```
-then from the repo root folder, you can start your container:
+then change in the api folder, where you can start your container:
 
 ```shell
-PORT=8050 docker-compose up -d --build
+make lock
+make run
 ```
-Once completed, your [FastAPI](https://fastapi.tiangolo.com/) server should be running on port 8050 (feel free to change this in the previous command).
+Once completed, your [FastAPI](https://fastapi.tiangolo.com/) server should be running on port 8080.
 
 ### Documentation and swagger
 
-FastAPI comes with many advantages including speed and OpenAPI features. For instance, once your server is running, you can access the automatically built documentation and swagger in your browser at: http://localhost:8050/docs
+FastAPI comes with many advantages including speed and OpenAPI features. For instance, once your server is running, you can access the automatically built documentation and swagger in your browser at: http://localhost:8080/docs
 
 
 ### Using the routes
@@ -39,7 +40,7 @@ with this snippet:
 import requests
 with open('/path/to/your/img.jpg', 'rb') as f:
     data = f.read()
-print(requests.post("http://localhost:8050/detection", files={'file': data}).json())
+print(requests.post("http://localhost:8080/detection", files={'file': data}).json())
 ```
 
 should yield
@@ -60,7 +61,7 @@ with this snippet:
 import requests
 with open('/path/to/your/img.jpg', 'rb') as f:
     data = f.read()
-print(requests.post("http://localhost:8050/recognition", files={'file': data}).json())
+print(requests.post("http://localhost:8080/recognition", files={'file': data}).json())
 ```
 
 should yield
@@ -80,7 +81,7 @@ with this snippet:
 import requests
 with open('/path/to/your/img.jpg', 'rb') as f:
     data = f.read()
-print(requests.post("http://localhost:8050/ocr", files={'file': data}).json())
+print(requests.post("http://localhost:8080/ocr", files={'file': data}).json())
 ```
 
 should yield

docker-compose.yml → api/docker-compose.yml

@@ -4,9 +4,7 @@ services:
   web:
     build:
       context: .
-      dockerfile: Dockerfile-api
+      dockerfile: Dockerfile
     command: uvicorn app.main:app --reload --workers 1 --host 0.0.0.0 --port 8080
-    volumes:
-      - ./api/:/usr/src/app/
     ports:
-      - ${PORT}:8080
+      - 8080:8080

api/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["poetry>=1.0"]
+build-backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "doctr-api"
+version = "0.5.2a0"
+description = "Backend template for your OCR API with docTR"
+authors = ["Mindee <[email protected]>"]
+license = "Apache-2.0"
+
+[tool.poetry.dependencies]
+python = ">=3.8,<3.11"
+tensorflow = ">=2.9.0,<3.0.0"
+tensorflow-addons = ">=0.17.1"
+python-doctr = ">=0.2.0"
+# Fastapi: minimum version required to avoid pydantic error
+# cf. https://github.com/tiangolo/fastapi/issues/4168
+fastapi = ">=0.73.0"
+uvicorn = ">=0.11.1"
+python-multipart = "==0.0.5"
+
+[tool.poetry.dev-dependencies]
+pytest = ">=5.3.2,<8.0.0"
+pytest-asyncio = ">=0.14.0,<1.0.0"
+httpx = ">=0.23.0"
+requests = "^2.20.0"