build_and_test.yml

# GitHub Action workflow to build and run filecrawler's tests
#
name: Test Build

on:
    push:
      branches: [ main ]
    pull_request:
      branches: [ main ]

env:
  TEST_TAG: filecrawler:test
  LATEST_TAG: filecrawler:latest

jobs:
  lint:
    name: Check syntaxs errors and warnings
    runs-on: ubuntu-latest
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    steps:
      - name: Checkout filecrawler
        uses: actions/checkout@v3

      - name: Setup Python 3.8
        uses: actions/setup-python@v3
        with:
          python-version: 3.8

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install flake8

      - name: Check syntax errors
        run: |
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics

      - name: Check PEP8 warnings
        run: |
          flake8 . --count --ignore=E1,E2,E3,E501,W291,W293 --exit-zero --max-complexity=65 --max-line-length=127 --statistics

  test:
    name: Run unit tests and build wheel
    needs: lint
    runs-on: ${{ matrix.os }}
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
        experimental: [false]
        os: [ubuntu-latest, macos-latest]
        #os: [ubuntu-latest, macos-latest, windows-latest]
        include:
          - python-version: "3.11"
            experimental: true
            os: ubuntu-latest
          - python-version: "3.11"
            experimental: true
            os: macos-latest
    continue-on-error: ${{ matrix.experimental }}

    steps:
      - name: Checkout filecrawler
        uses: actions/checkout@v3

      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Homebrew (MacOS)
        if: matrix.os == 'macos-latest'
        run: |
          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
          brew install libmagic

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip wheel
          python -m pip install -r tests/requirements-test.txt

      - name: Install filecrawler
        run: |
          pip install .

      - name: Run unit tests
        run: |
          pytest -s tests/tests.py

      - name: Build artifact
        run: |
          python setup.py sdist

  docker_test:
    name: Check docker build and run
    needs: test
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build and export to Docker
        uses: docker/build-push-action@v4
        with:
          context: .
          load: true
          push: false
          tags: ${{ env.TEST_TAG }}

      - name: Test
        run: |
          docker run --rm ${{ env.TEST_TAG }}

      #- name: Build and push
      #  uses: docker/build-push-action@v4
      #  with:
      #    context: .
      #    platforms: linux/amd64,linux/arm64
      #    push: false
      #    tags: ${{ env.LATEST_TAG }}

  publish:
    name: Publish on Test Environment
    needs: docker_test
    runs-on: ubuntu-latest
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9"]
        experimental: [false]
    continue-on-error: ${{ matrix.experimental }}

    steps:
      - name: Checkout filecrawler
        uses: actions/checkout@v3

      - name: Install dependencies
        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: wget curl jq
          version: 1.0

      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r tests/requirements-test.txt

      - name: Change Package version
        run: |
          oldv=$( grep '__version__' filecrawler/__meta__.py | grep -oE '[0-9\.]+')
          current=$(date '+%Y%m%d%H%M%S')
          meta=$(cat filecrawler/__meta__.py | sed "s/__version__.*/__version__ = '"${oldv}"-"${current}"'/")
          echo "$meta" > filecrawler/__meta__.py

      - name: Get tika server .jar
        run: |
          VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0')
          SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1")
          wget -nv -O "./filecrawler/libs/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar"
          echo "${SERVER_HASH} ./filecrawler/libs/bin/tika-server.jar" | sha1sum -c - || exit 1

      - name: Install filecrawler
        run: |
          pip install .

      - name: Build artifact
        run: |
          python setup.py sdist

      - name: Publish package in Test Environment
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.test_pypi_password }}
          repository_url: https://test.pypi.org/legacy/
          skip_existing: true