diff --git a/.github/workflows/quality.yaml b/.github/workflows/quality.yaml new file mode 100644 index 000000000..4abaff2a4 --- /dev/null +++ b/.github/workflows/quality.yaml @@ -0,0 +1,30 @@ +name: Quality + +on: + push: + branches: + - main + - v*-release + pull_request: + branches: + - main + +jobs: + + check_code_quality: + name: Check code quality + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install ruff==v0.1.2 + - name: Code quality + run: | + make quality diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 000000000..ceb68e17a --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,43 @@ +name: Quality + +on: + push: + branches: + - main + - v*-release + pull_request: + branches: + - main + +jobs: + run_tests: + name: Run tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + lfs: 'true' + ref: ${{ github.event.pull_request.head.sha }} # we want to test against our branch not against a merge commit + - name: Setup Python environment + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + - name: Install lighteval in editable mode + run: | + pip install -e .[accelerate] + - name: Get cached files + uses: actions/cache@v2 + id: get-cache + with: + path: "cache" + key: test-cache-HF + - name: Test + run: | # PYTHONPATH="${PYTHONPATH}:src" HF_DATASETS_CACHE="cache/datasets" HF_HOME="cache/models" + python -m pytest --disable-pytest-warnings + - name: Write cache + uses: actions/cache@v2 + with: + path: "cache" + key: test-cache-HF diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..e50483f1e --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +.PHONY: style format + + +style: + ruff format . + ruff check --fix . + + +quality: + ruff format --check . + ruff check . diff --git a/tests/reference_scores/harness_metrics.json b/tests/reference_scores/harness_metrics.json new file mode 100644 index 000000000..a6c506f34 --- /dev/null +++ b/tests/reference_scores/harness_metrics.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1965f0b9c66cfe1b1f3cc380a80949e32eab92ae8eac079c0339506ce827093 +size 48373142 diff --git a/tests/reference_scores/harness_prompts.json b/tests/reference_scores/harness_prompts.json new file mode 100644 index 000000000..5aef61305 --- /dev/null +++ b/tests/reference_scores/harness_prompts.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65d1d1bfb86c291b80e4117845385c6af0d965dc0af5e7a05774c820b772ba98 +size 20246116