Change SQLIte parser behaviour #157
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Action workflow to build and run filecrawler's tests | |
| # | |
| name: Test Build | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths-ignore: | |
| - .gitignore | |
| - README.md | |
| - LICENSE | |
| - TODO | |
| - Dockerfile | |
| - filecrawler/__meta__.py | |
| - .github/workflows/build_and_publish.yml | |
| pull_request: | |
| branches: [ main ] | |
| paths-ignore: | |
| - .gitignore | |
| - README.md | |
| - LICENSE | |
| - TODO | |
| - Dockerfile | |
| - filecrawler/__meta__.py | |
| - .github/workflows/build_and_publish.yml | |
| env: | |
| TEST_TAG: filecrawler:test | |
| LATEST_TAG: filecrawler:latest | |
| jobs: | |
| lint: | |
| name: Check syntaxs errors and warnings | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python 3.8 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.8 | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install flake8 | |
| - name: Check syntax errors | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Check PEP8 warnings | |
| run: | | |
| flake8 . --count --ignore=E1,E2,E3,E501,W291,W293 --exit-zero --max-complexity=65 --max-line-length=127 --statistics | |
| binutils_dependencies: | |
| name: Build external dependencies | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Install dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq zip | |
| version: 1.0 | |
| - name: Create temp path | |
| run: | | |
| mkdir -p /tmp/filecrawler/bin | |
| - name: Get tika server .jar | |
| run: | | |
| VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0') | |
| SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1") | |
| wget -nv -O "/tmp/filecrawler/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar" | |
| echo "${SERVER_HASH} /tmp/filecrawler/bin/tika-server.jar" | sha1sum -c - || exit 1 | |
| - name: Get JADX | |
| run: | | |
| VER=$(curl -s "https://api.github.com/repos/skylot/jadx/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') | |
| wget -nv -O "/tmp/jadx.zip" "https://github.com/skylot/jadx/releases/download/v$VER/jadx-$VER.zip" | |
| mkdir -p /tmp/jadx/ | |
| unzip -o /tmp/jadx.zip -d /tmp/jadx/ | |
| FILE=$(find /tmp/jadx/ -name "jadx*.jar") | |
| mv "$FILE" "/tmp/filecrawler/bin/jadx.jar" | |
| - name: Get JADX | |
| run: | | |
| VER=$(curl -s "https://api.github.com/repos/iBotPeaches/Apktool/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') | |
| wget -nv -O "/tmp/apktool.jar" "https://github.com/iBotPeaches/Apktool/releases/download/v$VER/apktool_$VER.jar" | |
| mv "/tmp/apktool.jar" "/tmp/filecrawler/bin/" | |
| - name: Create bin zip | |
| run: | | |
| zip -j -5 /tmp/filecrawler_extra_libs.zip /tmp/filecrawler/bin/* | |
| - name: Upload binutils file | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: filecrawler_extra_libs.zip | |
| path: /tmp/filecrawler_extra_libs.zip | |
| retention-days: 3 | |
| test: | |
| name: Run unit tests and build wheel | |
| needs: binutils_dependencies | |
| runs-on: ${{ matrix.os }} | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10"] | |
| experimental: [false] | |
| os: [ubuntu-latest, macos-latest] | |
| #os: [ubuntu-latest, macos-latest, windows-latest] | |
| include: | |
| - python-version: "3.11" | |
| experimental: true | |
| os: ubuntu-latest | |
| - python-version: "3.11" | |
| experimental: true | |
| os: macos-latest | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Homebrew (MacOS) | |
| if: matrix.os == 'macos-latest' | |
| run: | | |
| NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" | |
| brew install libmagic unzip | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel | |
| python -m pip install -r tests/requirements-test.txt | |
| - name: Download arfactory | |
| uses: actions/download-artifact@v3 | |
| id: download | |
| with: | |
| name: filecrawler_extra_libs.zip | |
| path: /tmp/ | |
| - name: Copy binaries | |
| run: | | |
| mkdir -p ./filecrawler/libs/bin/ | |
| unzip -o ${{steps.download.outputs.download-path}}/filecrawler_extra_libs.zip -d ./filecrawler/libs/bin/ | |
| echo "" > ./filecrawler/libs/bin/__init__.py | |
| - name: Install filecrawler | |
| run: | | |
| pip install . | |
| - name: Run unit tests | |
| run: | | |
| pytest -s tests/tests.py | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| docker_test: | |
| name: Check docker build and run | |
| needs: test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and export to Docker | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| load: true | |
| push: false | |
| tags: ${{ env.TEST_TAG }} | |
| - name: Test | |
| run: | | |
| docker run --rm ${{ env.TEST_TAG }} -h | |
| #- name: Build and push | |
| # uses: docker/build-push-action@v4 | |
| # with: | |
| # context: . | |
| # platforms: linux/amd64,linux/arm64 | |
| # push: false | |
| # tags: ${{ env.LATEST_TAG }} | |
| publish: | |
| name: Publish on Test Environment | |
| needs: docker_test | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9"] | |
| experimental: [false] | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq | |
| version: 1.0 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r tests/requirements-test.txt | |
| - name: Change Package version | |
| run: | | |
| oldv=$( grep '__version__' filecrawler/__meta__.py | grep -oE '[0-9\.]+') | |
| current=$(date '+%Y%m%d%H%M%S') | |
| meta=$(cat filecrawler/__meta__.py | sed "s/__version__.*/__version__ = '"${oldv}"-"${current}"'/") | |
| echo "$meta" > filecrawler/__meta__.py | |
| - name: Install filecrawler | |
| run: | | |
| pip install . | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| - name: Publish package in Test Environment | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.test_pypi_password }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| skip_existing: true |