From 6978c085fb54af54652acfa1a3ac4e91aec88c7a Mon Sep 17 00:00:00 2001
From: "Bater.Makhabel" <bater.makhabel@gmail.com>
Date: Wed, 29 Mar 2023 19:45:55 +0400
Subject: [PATCH] Create python-app.yml

---
 .github/workflows/python-app.yml | 315 +++++++++++++++++++++++++++++++
 1 file changed, 315 insertions(+)
 create mode 100644 .github/workflows/python-app.yml

diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
new file mode 100644
index 000000000..a426c5f56
--- /dev/null
+++ b/.github/workflows/python-app.yml
@@ -0,0 +1,315 @@
+name: Python Build
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      tmate_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # https://github.com/actions/setup-python/issues/544#issuecomment-1333893110
+        # Python 3.6 is removed from workflow matrices now.
+        # One alternative is to support 3.6 but duplicate all the workflow steps in
+        # another ubuntu-20.04. But that would result in a full copy of all the
+        # workflow steps, causing many problems in future upgrade.
+        # Afaik, there is no good way to avoid copying right now: https://stackoverflow.com/a/71570847
+        python-version: [ 3.7 ]
+        torch-version: [ 1.5.0, 1.6.0 ]
+        tensorflow-version: [ 1.15.0 ]
+        include:
+          - python-version: 3.8
+            torch-version: 1.7.1
+            tensorflow-version: 2.2.0
+          - python-version: 3.8
+            torch-version: 1.8.1
+            tensorflow-version: 2.2.0
+          - python-version: 3.9
+            torch-version: 1.7.1
+            tensorflow-version: 2.5.0
+          - python-version: 3.9
+            torch-version: 1.8.1
+            tensorflow-version: 2.5.0
+    env:
+      notebook-details-dep: "huggingface nltk"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Install dependencies
+        run: |
+          sudo apt-get install -y libsndfile1-dev
+          python -m pip install --progress-bar off --upgrade pip
+          pip install --progress-bar off Django django-guardian
+          pip install --progress-bar off black==22.12.0
+          python -m pip install ipykernel
+          python -m ipykernel install --user
+      - name: Format check with Black
+        run: |
+          black --line-length 80 --check forte/
+      - name: Obtain Stave Database Examples
+        run: |
+          git clone https://github.com/asyml/stave.git
+          cd stave/simple-backend
+          python manage.py migrate
+          cat sample_sql/*.sql | sqlite3 db.sqlite3
+          cd ../..
+          # Simply keep the database file but remove the repo.
+          cp stave/simple-backend/db.sqlite3 .
+          rm -rf stave
+      - name: Install Forte
+        run: |
+          pip install --progress-bar off .[data_aug,ir,remote,audio_ext,stave,models,test,wikipedia,nlp,extractor,payload]
+      - name: Install deep learning frameworks
+        run: |
+          pip install --progress-bar off torch==${{ matrix.torch-version }}
+          pip install --progress-bar off tensorflow==${{ matrix.tensorflow-version }}
+      - name: Install linting tools after regular installation
+        # Remember to update pre commit hook (pre-commit-config.yaml) if the following is updated.
+        # Better run after other installations (e.g. tensorflow), since those packages may ask for different versions.
+        run: |
+          pip install --progress-bar off pylint==2.14.5 flake8==5.0.4 mypy==0.991 pytest==7.2.0 black==22.12.0
+          pip install --progress-bar off types-PyYAML==5.4.8 types-typed-ast==1.4.4 types-requests==2.25.6 types-dataclasses==0.1.7
+          pip install --progress-bar off --upgrade typing-extensions
+          pip install --progress-bar off coverage codecov
+      - name: Build ontology
+        run: |
+          ./scripts/build_ontology_specs.sh
+      - name: Setup tmate session
+        # Add a tmate step after most of the installation.
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.tmate_enabled }}
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 forte/ examples/ ft/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
+          # Test the rest but ignore the tests/ folder The GitHub editor is 127 chars wide
+          flake8 forte/ examples/ ft/ scripts/ --ignore E203,W503 --count --max-complexity=10 --max-line-length=127 --statistics
+      - name: Lint with pylint
+        run: |
+          pip show typing-extensions
+          pylint forte/
+      - name: Lint main code with mypy when torch version is not 1.5.0 and python is 3.9
+        run: |
+          if [[ ${{ matrix.torch-version }} != "1.5.0" && ${{ matrix.python-version }} == "3.9" ]]; then mypy forte --check-untyped-defs; fi
+      - name: Test with pytest and run coverage
+        run: |
+          coverage run -m pytest tests  --ignore=tests/forte/notebooks --ignore=tests/forte/forte_backbone_test.py
+          coverage run --append -m pytest --doctest-modules forte
+      - name: Upload coverage
+        run: |
+          codecov
+      - name: Test notebook tutorials with installing Fortex
+        if: ${{ contains(fromJson('["1.5.0", "1.7.1"]'), matrix.torch-version) }}
+        run: |
+          git clone https://github.com/asyml/forte-wrappers.git
+          cd forte-wrappers
+          for d in ${{ env.notebook-details-dep }}; do pip install "src/"$d; done
+          cd ..
+          if [ ${{ matrix.torch-version }} == "1.5.0" ]
+          then
+            coverage run -m pytest tests/forte/notebooks  --ignore=tests/forte/notebooks/ocr_test.py --ignore=tests/forte/notebooks/tutorial_MT_with_forte_test.py
+          fi
+          if [ ${{ matrix.torch-version }} == "1.7.1" ]
+          then
+            coverage run -m pytest tests/forte/notebooks/tutorial_MT_with_forte_test.py
+          fi
+  test_ocr_example:
+    runs-on: ubuntu-latest
+    env:
+      python-version: 3.7
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ env.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: ocr test
+        run: |
+          sudo apt-get install -y automake ca-certificates g++ git libtool libleptonica-dev make pkg-config
+          sudo apt install tesseract-ocr
+          pip install --progress-bar off .[ocr_tutorial,test]
+          python tests/forte/notebooks/ocr_test.py
+  test_backbone:
+    runs-on: ubuntu-latest
+    env:
+      python-version: 3.7
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ env.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Test backbone Forte import test
+        run: |
+          # Try to install Forte backbone only and test basic imports.
+          pip install --progress-bar off .
+          python tests/forte/forte_backbone_test.py
+  test_modules:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ 3.7 ]
+        torch-version: [ 1.5.0 ]
+        tensorflow-version: [ 1.15.0 ]
+        modules:
+          - { module: "data_aug", test_file: "tests/forte/processors/data_augment"}
+          - { module: "ir",test_file: "tests/forte/processors/ir/"}
+          - { module: "remote", test_file: "tests/forte/remote_processor_test.py"}
+          - { module: "audio_ext",test_file: "tests/forte/data/readers/audio_reader_test.py"}
+          - { module: "stave",test_file: "tests/forte/data/readers/stave_reader_test.py tests/forte/processors/stave_processor_test.py"}
+          - { module: "models", test_file: "tests/forte/models"}
+          - { module: "wikipedia", test_file: "tests/forte/datasets/wikipedia"}
+          - { module: "nlp",test_file: "tests/forte/processors/subword_tokenizer_test.py tests/forte/processors/pretrained_encoder_processors_test.py"}
+          - { module: "extractor",test_file: "tests/forte/train_preprocessor_test.py forte/data/extractors tests/forte/data/data_pack_dataset_test.py tests/forte/data/converter/converter_test.py"}
+          - { module: "payload", test_file: "tests/forte/utils/payload_decorator_test.py"}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install forte
+        run: |
+          pip install --progress-bar off .
+      - name: install dependencies
+        run: |
+          sudo apt-get install -y libsndfile1-dev
+          python -m pip install --progress-bar off --upgrade pip
+          pip install --progress-bar off Django django-guardian
+      - name: Obtain Stave Database Examples
+        run: |
+          git clone https://github.com/asyml/stave.git
+          cd stave/simple-backend
+          python manage.py migrate
+          cat sample_sql/*.sql | sqlite3 db.sqlite3
+          cd ../..
+          # Simply keep the database file but remove the repo.
+          cp stave/simple-backend/db.sqlite3 .
+          rm -rf stave
+      - name: Test modules need extra packages
+        run: |
+          pip install --progress-bar off .[test]
+          pip install --progress-bar off .[${{ matrix.modules.module }}]
+          pip install --progress-bar off torch==${{ matrix.torch-version }}
+          pip install --progress-bar off pytest==5.1.3
+      - name: Setup tmate session
+        # Add a tmate step before the actual testing.
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.tmate_enabled }}
+      - name: Run the test
+        run: |
+          pytest ${{ matrix.modules.test_file }}
+  docs:
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      python-version: 3.7
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ env.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Install dependencies
+        run: |
+          python -m pip install --progress-bar off --upgrade pip
+          sudo apt-get install pandoc
+          pip install --progress-bar off -r requirements.txt
+          pip install --progress-bar off -r docs/requirements.txt
+          git clone https://github.com/asyml/texar-pytorch.git
+          cd texar-pytorch
+          pip install --progress-bar off .
+          cd ..
+          rm -rf texar-pytorch
+      - name: Build Docs
+        run: |
+          cd docs
+          sphinx-build -W -b html -d _build/doctrees . _build/html
+          sphinx-build -W -b spelling -d _build/doctrees . _build/spelling
+          cd ..
+  dispatch:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.repository == 'asyml/forte' && github.ref == 'refs/heads/master'
+    steps:
+      - name: Repository Dispatch
+        uses: peter-evans/repository-dispatch@v1
+        with:
+          token: ${{ secrets.REPO_DISPATCH_PAT_HECTOR }}
+          repository: asyml/forte-wrappers
+          event-type: trigger-forte-wrappers
+
+  readme:
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      python-version: 3.9
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ env.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python-version }}
+
+      - name: Test README.md when python version is 3.9
+        run: |
+          pip install mkcodes
+          pip install --progress-bar off .
+          pip install --progress-bar off forte.spacy nltk
+          mkcodes --github --output tests/temp_readme_test.py README.md
+          python tests/temp_readme_test.py
+          rm tests/temp_readme_test.py