match test data to check data compatibility #312
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: wipac ci/cd | |
| on: [push] | |
| env: | |
| CI_TEST: true | |
| TEST_JSON_DIR: tests/resources | |
| jobs: | |
| py-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.versions.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: versions | |
| uses: WIPACrepo/[email protected] | |
| flake8: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| - uses: WIPACrepo/[email protected] | |
| mypy: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - uses: WIPACrepo/[email protected] | |
| py-setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # dependabot can't access normal secrets | |
| # & don't run non-branch triggers (like tags) | |
| # & we don't want to trigger an update on PR's merge to main/master/default (which is a branch) | |
| # IOW: only for non-dependabot branches | |
| - if: | | |
| github.actor != 'dependabot[bot]' && | |
| github.ref_type == 'branch' && | |
| format('refs/heads/{0}', github.event.repository.default_branch) != github.ref | |
| name: checkout (only for non-dependabot non-default branches) | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - if: | | |
| github.actor != 'dependabot[bot]' && | |
| github.ref_type == 'branch' && | |
| format('refs/heads/{0}', github.event.repository.default_branch) != github.ref | |
| name: wipac-dev-py-setup-action (only for non-dependabot non-default branches) | |
| uses: WIPACrepo/[email protected] | |
| with: | |
| base-keywords: "WIPAC IceCube" | |
| unit-tests: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - name: run | |
| run: | | |
| pip install --upgrade pip wheel setuptools | |
| pip install .[tests] | |
| export OIDC_CLIENT_SECRETS=$(realpath resources/dummy_client_secrets_for_web_app.json) | |
| pytest -vvv tests/unit | |
| test-build-docker: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| file: Dockerfile | |
| tags: moudash:local | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| integration_test_ingest_type: [ | |
| xlsx, | |
| mongodump_v2, | |
| mongodump_v3, | |
| ] | |
| services: | |
| mongo: | |
| image: mongo:3 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: docker/setup-buildx-action@v2 | |
| - uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| cache-from: type=gha | |
| # cache-to: type=gha,mode=min | |
| file: Dockerfile | |
| tags: moudash:local | |
| load: true | |
| - name: mongo prep | |
| run: | | |
| sudo apt update | |
| sudo apt install wget curl gnupg2 software-properties-common apt-transport-https ca-certificates lsb-release | |
| curl -fsSL https://www.mongodb.org/static/pgp/server-6.0.asc|sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/mongodb-6.gpg | |
| echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list | |
| sudo apt update | |
| sudo apt install mongodb-org mongodb-database-tools | |
| set -x | |
| # ingest data | |
| if [ "${{ matrix.integration_test_ingest_type }}" = "mongodump_v2" ]; then | |
| if [[ ! $(grep -o 'LIVE_COLLECTION' $TEST_JSON_DIR/v2-mo-supplemental.json) ]]; then | |
| exit 1 # the ingested collection must be LIVE_COLLECTION | |
| fi | |
| mongoimport -d mo-supplemental -c LIVE_COLLECTION --type json --file $TEST_JSON_DIR/v2-mo-supplemental.json | |
| mongoimport -d mo -c LIVE_COLLECTION --jsonArray --type json --file $TEST_JSON_DIR/v2-mo.json | |
| # -> v3 includes v2 data | |
| elif [ "${{ matrix.integration_test_ingest_type }}" = "mongodump_v3" ]; then | |
| sed -i 's/LIVE_COLLECTION/100/g' $TEST_JSON_DIR/v2-mo-supplemental.json | |
| mongoimport -d mo-supplemental -c 100 --type json --file $TEST_JSON_DIR/v2-mo-supplemental.json | |
| mongoimport -d mo -c 100 --type json --jsonArray --file $TEST_JSON_DIR/v2-mo.json | |
| if [[ ! $(grep -o 'LIVE_COLLECTION' $TEST_JSON_DIR/v3-mo-supplemental.json) ]]; then | |
| exit 1 # the ingested collection must be LIVE_COLLECTION | |
| fi | |
| mongoimport -d mo-supplemental -c LIVE_COLLECTION --type json --file $TEST_JSON_DIR/v3-mo-supplemental.json | |
| mongoimport -d mo -c LIVE_COLLECTION --jsonArray --type json --file $TEST_JSON_DIR/v3-mo.json | |
| fi | |
| - name: run | |
| run: | | |
| # rest server (background) | |
| docker run --network="host" --rm -i --name rest \ | |
| --env CI_TEST=true \ | |
| moudash:local \ | |
| python -m rest_server --override-krs-insts ./resources/dummy-krs-data.json \ | |
| & | |
| sleep 30 | |
| # make test script | |
| DIR="test-script-dir" | |
| mkdir $DIR | |
| echo "#!/bin/bash" >> $DIR/test-script.sh | |
| echo "set -xe" >> $DIR/test-script.sh | |
| echo "pip install .[tests]" >> $DIR/test-script.sh | |
| echo "python -m pytest -vvv tests/integration --exitfirst" >> $DIR/test-script.sh | |
| chmod +x $DIR/test-script.sh | |
| cat $DIR/test-script.sh | |
| # test | |
| docker run --network="host" --rm -i --name test \ | |
| --env CI_TEST=true \ | |
| --env OIDC_CLIENT_SECRETS=resources/dummy_client_secrets_for_web_app.json \ | |
| --env INTEGRATION_TEST_INGEST_TYPE=${{ matrix.integration_test_ingest_type }} \ | |
| --mount type=bind,source=$(realpath $DIR),target=/local/$DIR \ | |
| moudash:local \ | |
| /local/$DIR/test-script.sh | |
| - name: mongodump | |
| if: always() | |
| run: | | |
| set -x | |
| out_dir="./mongodump-jsons" | |
| tmp_file="temp.js" | |
| echo "print('_ ' + db.getCollectionNames())" > $tmp_file | |
| for database in mo mo-supplemental; do | |
| echo "dumping $database..." | |
| for col in $(mongosh $database $tmp_file | grep '_' | awk '{print $2}' | tr ',' ' '); do | |
| mongoexport -d $database -c $col -o "$out_dir/exp_${database}_${col}.json" | |
| done | |
| done | |
| more $out_dir/* | cat | |
| - name: dump rest logs | |
| if: always() | |
| run: | | |
| docker logs rest || true | |
| - name: dump mongo logs | |
| if: always() | |
| run: | | |
| docker logs "${{ job.services.mongo.id }}" || true | |
| release: | |
| # only run on main/master/default | |
| if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref | |
| needs: [flake8, mypy, py-setup, unit-tests, integration-tests] | |
| runs-on: ubuntu-latest | |
| concurrency: release | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - name: Python Semantic Release | |
| uses: python-semantic-release/[email protected] | |
| with: | |
| github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| # repository_username: __token__ | |
| # repository_password: ${{ secrets.PYPI_TOKEN }} |