Skip to content
CI

rewrite kafka connector #2079

Sign in to view logs

rewrite kafka connector

rewrite kafka connector #2079

Workflow file for this run

.github/workflows/ci.yml at 724f31d
name: CI
# only run tests for pull requests cause no file has to be changed without review
# open -> open the pull request
# synchronize -> push to branch of pull request
on:
pull_request:
types: [opened, synchronize]
jobs:
build-pex:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ansible
pip install virtualenv
pip install wheel
pip install pex
- name: Repack confluent-kafka wheel
run: |
rm -rf tmp_pip_cache &&
mkdir tmp_pip_cache &&
cd tmp_pip_cache &&
python -m pip download $(cat ../requirements.txt | grep confluent-kafka) &&
unzip * &&
rm *.whl &&
python -m wheel pack .
- name: Build Pex File
run: |
pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache
- name: Upload PEX
uses: actions/upload-artifact@v3
with:
name: Logprep
path: logprep.pex
test:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: |
pip install --upgrade pip wheel
pip install -r requirements_dev.txt
- name: Perform unit tests
env:
PYTEST_ADDOPTS: "--color=yes"
run: |
pytest -vv tests/unit
- name: Perform acceptance tests
env:
PYTEST_ADDOPTS: "--color=yes"
run: |
pytest -vv tests/acceptance
build-docs:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Install dependencies
run: |
sudo apt-get update && sudo apt-get -y install pandoc
pip install --upgrade pip wheel
pip install -r requirements_dev.txt
pip install -r doc/requirements.txt
- name: build docs
run: |
cd doc
sphinx-apidoc -fT -o source/module_reference ../logprep
make clean html
code-quality:
runs-on: ubuntu-22.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
- name: Get changed python files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
**/*.py
- name: Install dependencies
run: |
pip install --upgrade pip wheel
pip install -r requirements_dev.txt
- name: check black formating
run: |
black --check --diff --config ./pyproject.toml .
- name: lint changed and added files
if: steps.changed-files.outputs.all_changed_files
run: |
pylint --rcfile=.pylintrc --fail-under 9.5 ${{ steps.changed-files.outputs.all_changed_files }}
- name: Run tests and collect coverage
run: pytest tests/unit --cov=logprep --cov-report=xml
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v2
- name: Check semgrep rules
if: steps.changed-files.outputs.all_changed_files
run: semgrep -c .semgrep_rules -c r/python --error --skip-unknown-extensions ${{ steps.changed-files.outputs.all_changed_files }}
containerbuild:
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
runs-on: ubuntu-latest
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build images
uses: docker/build-push-action@v3
with:
push: true # Will only build if this is not here
build-args: |
LOGPREP_VERSION=dev
PYTHON_VERSION=${{ matrix.python-version }}
tags: |
ghcr.io/fkie-cad/logprep:py${{ matrix.python-version }}-${{ github.head_ref }}