From b5d70c94e779c214866e699f7963070aff6aebb7 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:54:32 +0800 Subject: [PATCH 1/3] add github action Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .github/workflows/monitor-tests-changes.yml | 143 ++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 .github/workflows/monitor-tests-changes.yml diff --git a/.github/workflows/monitor-tests-changes.yml b/.github/workflows/monitor-tests-changes.yml new file mode 100644 index 00000000000..8e3ede4bc7c --- /dev/null +++ b/.github/workflows/monitor-tests-changes.yml @@ -0,0 +1,143 @@ +name: Monitor Tests Changes + +on: + pull_request: + types: [opened, synchronize] + paths: + - 'tests/**' + +jobs: + create-issue: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: read + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get PR information + id: pr-info + uses: actions/github-script@v6 + with: + script: | + const pr = context.payload.pull_request; + const prNumber = pr.number; + const prTitle = pr.title; + const prAuthor = pr.user.login; + const prUrl = pr.html_url; + + core.setOutput('pr_number', prNumber); + core.setOutput('pr_title', prTitle); + core.setOutput('pr_author', prAuthor); + core.setOutput('pr_url', prUrl); + + - name: Find commits modifying tests + id: find-commits + run: | + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + + echo "Finding commits that modify the tests folder:" + COMMITS_JSON=$(git log --pretty=format:'{"hash":"%H","short_hash":"%h","author":"%an","subject":"%s","date":"%ad","body":"%b"}' $BASE_SHA..$HEAD_SHA -- tests/) + + # Convert each commit to JSON array format + COMMITS_ARRAY="[" + while IFS= read -r line; do + if [ ! -z "$line" ]; then + COMMITS_ARRAY="${COMMITS_ARRAY}${line}," + fi + done <<< "$COMMITS_JSON" + COMMITS_ARRAY="${COMMITS_ARRAY%,}]" # Remove the last comma and add closing bracket + + # Handle empty array case + if [ "$COMMITS_ARRAY" = "]" ]; then + COMMITS_ARRAY="[]" + fi + + echo "commits_json<> $GITHUB_OUTPUT + echo "$COMMITS_ARRAY" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Create issues for each commit + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const prNumber = '${{ steps.pr-info.outputs.pr_number }}'; + const prTitle = '${{ steps.pr-info.outputs.pr_title }}'; + const prAuthor = '${{ steps.pr-info.outputs.pr_author }}'; + const prUrl = '${{ steps.pr-info.outputs.pr_url }}'; + + let commitsJson = '${{ steps.find-commits.outputs.commits_json }}'; + const commits = JSON.parse(commitsJson); + + if (commits.length === 0) { + console.log("No commits modifying the tests folder were found."); + return; + } + + // Create an issue for each commit + for (const commit of commits) { + const commitUrl = `${context.payload.repository.html_url}/commit/${commit.hash}`; + + // Get the files changed in this commit in the tests folder + const filesChangedResult = await github.rest.repos.getCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: commit.hash + }); + + // Filter for files in the tests folder + const testFilesChanged = filesChangedResult.data.files + .filter(file => file.filename.startsWith('tests/')) + .map(file => { + return { + filename: file.filename, + status: file.status, + additions: file.additions, + deletions: file.deletions, + url: file.blob_url + }; + }); + + // Create file changes section + let filesSection = ''; + if (testFilesChanged.length > 0) { + filesSection = '## Files Changed\n\n'; + testFilesChanged.forEach(file => { + filesSection += `- [${file.filename}](${file.url}) (${file.status}, +${file.additions}, -${file.deletions})\n`; + }); + } + + // Create the issue + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `[Tests] ${commit.subject}`, + body: ` + # Tests Folder Modification Alert + + A commit in Pull Request [#${prNumber}: ${prTitle}](${prUrl}) has modified files in the \`tests\` folder. + + ## Commit Details + + - **Commit:** [${commit.short_hash}](${commitUrl}) + - **Author:** ${commit.author} + - **Date:** ${commit.date} + - **Message:** ${commit.subject} + + ${commit.body ? `**Description:**\n\n\`\`\`\n${commit.body}\n\`\`\`\n\n` : ''} + + ${filesSection} + + Please review these changes promptly. + ` + }); + + console.log(`Created issue for commit ${commit.short_hash}: ${commit.subject}`); + } From a90dfa1cc0ae129ed2c20d05fc0703b71de68524 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:58:56 +0800 Subject: [PATCH 2/3] add 1 test Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/defs/accuracy/test_llm_api.py b/tests/integration/defs/accuracy/test_llm_api.py index 9eea9c6a57a..e5330d18253 100644 --- a/tests/integration/defs/accuracy/test_llm_api.py +++ b/tests/integration/defs/accuracy/test_llm_api.py @@ -177,3 +177,13 @@ def test_fp8(self): extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) task = MMLU(self.MODEL_NAME) task.evaluate(llm) + + @skip_pre_ada + def test_fp8_cp(self): + quant_config = QuantConfig(QuantAlgo.FP8) + with LLM(self.MODEL_PATH, quant_config=quant_config) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) From 2d9e1487f31dcd26126bfcac16133d217b5e14ea Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:59:27 +0800 Subject: [PATCH 3/3] add 2 test Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 13a225b4650..d4dc47274b3 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -475,3 +475,9 @@ def test_auto_dtype(self): task = CnnDailymail(self.MODEL_NAME) task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + + def test_auto_dtype_cp(self): + with LLM(self.MODEL_PATH) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS)