merge wiith mindee

h2oai · Sep 7, 2022 · 8ec8d7f · 8ec8d7f
2 parents 4450fa0 + 1cc073d
commit 8ec8d7f
Show file tree

Hide file tree

Showing 212 changed files with 5,017 additions and 4,489 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,5 +1,5 @@
 [flake8]
 max-line-length = 120
-ignore = E402, E265, F403, W503, W504, E731
+ignore = E203, E402, E265, F403, W503, W504, E731
 exclude = .git, venv*, build
 per-file-ignores = **/__init__.py:F401
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    open-pull-requests-limit: 10
+    target-branch: "main"
+    labels: ["topic: build"]
+    schedule:
+      interval: weekly
+      day: sunday
+    reviewers:
+      - "frgfm"
+      - "charlesmindee"
+      - "felixdittrich92"
diff --git a/.github/verify_pr_labels.py b/.github/verify_pr_labels.py
@@ -1,3 +1,8 @@
+# Copyright (C) 2021-2022, Mindee.
+
+# This program is licensed under the Apache License 2.0.
+# See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
+
 """
 Borrowed & adapted from https://github.com/pytorch/vision/blob/main/.github/process_commit.py
 This script finds the merger responsible for labeling a PR by a commit SHA. It is used by the workflow in
@@ -41,8 +46,8 @@
     "topic: docker",
 }
 
-GH_ORG = 'mindee'
-GH_REPO = 'doctr'
+GH_ORG = "mindee"
+GH_REPO = "doctr"
 
 
 def query_repo(cmd: str, *, accept) -> Any:
@@ -67,10 +72,12 @@ def main(args):
 
 def parse_args():
     import argparse
-    parser = argparse.ArgumentParser(description='PR label checker',
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('pr', type=int, help='PR number')
+    parser = argparse.ArgumentParser(
+        description="PR label checker", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument("pr", type=int, help="PR number")
     args = parser.parse_args()
 
     return args

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
@@ -30,13 +30,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'pytorch'
         name: Cache python modules (PT)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'tensorflow'
         name: Install package (TF)
         run: |

diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml
@@ -14,6 +14,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python: [3.8]
+        framework: [tensorflow, pytorch]
     steps:
       - if: matrix.os == 'macos-latest'
         name: Install MacOS prerequisites
@@ -24,22 +25,48 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
-      - name: Cache python modules
+      - if: matrix.framework == 'tensorflow'
+        name: Cache python modules (TF)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('demo/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-      - name: Install dependencies
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('demo/tf-requirements.txt') }}
+      - if: matrix.framework == 'pytorch'
+        name: Cache python modules (PT)
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('demo/pt-requirements.txt') }}
+
+      - if: matrix.framework == 'tensorflow'
+        name: Install dependencies (TF)
         run: |
           python -m pip install --upgrade pip
           pip install -e .[tf] --upgrade
-          pip install -r demo/requirements.txt
+          pip install -r demo/tf-requirements.txt
+      - if: matrix.framework == 'pytorch'
+        name: Install dependencies (PT)
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[torch] --upgrade
+          pip install -r demo/pt-requirements.txt
 
-      - name: Run demo
+      - if: matrix.framework == 'tensorflow'
+        name: Run demo (TF)
+        env:
+            USE_TF: 1
         run: |
           streamlit --version
           screen -dm streamlit run demo/app.py
           sleep 10
           curl http://localhost:8501/docs
+      - if: matrix.framework == 'pytorch'
+        name: Run demo (PT)
+        env:
+            USE_TORCH: 1
+        run: |
+          streamlit --version
+          screen -dm streamlit run demo/app.py
+          sleep 10
+          curl http://localhost:8501/docs
+
diff --git a/.github/workflows/doc-status.yml b/.github/workflows/doc-status.yml
@@ -6,10 +6,10 @@ jobs:
   see-page-build-payload:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7
+          python-version: 3.8
           architecture: x64
       - name: check status
         run: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -17,20 +17,32 @@ jobs:
         run: docker run doctr-tf-py3.8-slim python -c 'import doctr'
 
   pytest-api:
-    runs-on: ubuntu-latest
-    env:
-        # Port for web container
-        # See docker-compose.yml
-        PORT: 8002
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.8]
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Install poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.1.13
+      - name: Lock the requirements
+        run: |
+          cd api
+          make lock
       - name: Build & run docker
-        run: docker-compose up -d --build
+        run: cd api && docker-compose up -d --build
       - name: Ping server
-        run: wget --spider --tries=12 http://localhost:$PORT/docs
-      - name: Install dependencies in docker
-        run: |
-          docker-compose exec -T web python -m pip install --upgrade pip
-          docker-compose exec -T web pip install -r requirements-dev.txt
+        run: wget --spider --tries=12 http://localhost:8080/docs
       - name: Run docker test
-        run: docker-compose exec -T web pytest .
+        run: |
+          docker cp api/requirements-dev.txt api_web_1:/app/requirements-dev.txt
+          docker-compose -f api/docker-compose.yml exec -T web pip install -r requirements-dev.txt
+          docker cp api/tests api_web_1:/app/tests
+          docker-compose -f api/docker-compose.yml exec -T web pytest tests/
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: [3.7]
+        python: [3.8]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -23,9 +23,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-docs
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -7,6 +7,25 @@ on:
     branches: main
 
 jobs:
+  test-headers:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+      - name: Validate headers
+        # Credits to: https://github.com/frgfm/validate-python-headers
+        uses: frgfm/validate-python-headers@main
+        with:
+          license: 'Apache-2.0'
+          owner: 'Mindee'
+          starting-year: 2021
+          folders: '.github,api/app,demo,docs,doctr,references,scripts'
+          ignores: 'version.py,__init__.py'
+
   pytest-common:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -24,9 +43,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-tests
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -59,9 +76,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-tests
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -94,9 +109,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}-${{ hashFiles('tests/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-tests
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml
@@ -12,15 +12,13 @@ jobs:
       - name: Set up Python 3.7
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.8
           architecture: x64
       - name: Cache python modules
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-docs
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: [3.7]
+        python: [3.8]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
@@ -24,7 +24,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -50,11 +50,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: [3.7]
+        python: [3.8]
     needs: pypi-publish
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7
+      - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}

diff --git a/.github/workflows/scripts.yml b/.github/workflows/scripts.yml
@@ -30,13 +30,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'pytorch'
         name: Cache python modules (PT)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'tensorflow'
         name: Install package (TF)
         run: |
@@ -76,13 +76,13 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'pytorch'
         name: Cache python modules (PT)
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements-pt.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - if: matrix.framework == 'tensorflow'
         name: Install package (TF)
         run: |

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -26,6 +26,25 @@ jobs:
           flake8 --version
           flake8
 
+  black:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.8]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Run black
+        run: |
+          pip install black
+          black --version
+          black --check --diff .
+
   isort:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -63,11 +82,11 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[all] --upgrade
+          pip install -e .[dev] --upgrade
           pip install mypy
       - name: Run mypy
         run: |