From c9b2b9af6c68bbeef8e04a37124116399a2f6891 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 5 Sep 2025 01:14:54 +1200 Subject: [PATCH 1/6] Automatically print the GKE experiment details as a PR comment --- ci/ci_trial_build.py | 20 +++++++++++++++++++- ci/request_pr_exp.py | 12 ++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/ci/ci_trial_build.py b/ci/ci_trial_build.py index a840539d2d..9f08c46b5e 100644 --- a/ci/ci_trial_build.py +++ b/ci/ci_trial_build.py @@ -71,7 +71,25 @@ def exec_command_from_github(pull_request_number): command = ['-p', str(pull_request_number)] + command command = [c for c in command if c] logging.info('Command: %s.', command) - return request_pr_exp.main(command) + links = request_pr_exp.main(command) + try: + if isinstance(links, tuple) and len(links) == 5: + gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link = links + if all([ + gke_job_name, gke_job_link, report_link, bucket_link, bucket_gs_link + ]): + github_obj = github.Github() + repo = github_obj.get_repo('google/oss-fuzz-gen') + issue = repo.get_issue(pull_request_number) + body = (f"Requesting a GKE experiment named {gke_job_name}:\n" + f"JOB: {gke_job_link}\n" + f"REPORT: {report_link}\n" + f"BUCKET: {bucket_link}\n" + f"BUCKET GS: `{bucket_gs_link}`\n") + issue.create_comment(body) + except Exception as e: + logging.warning('Failed to post PR comment: %s', e) + return links def main(): diff --git a/ci/request_pr_exp.py b/ci/request_pr_exp.py index a24b27d42f..cb90ae3149 100644 --- a/ci/request_pr_exp.py +++ b/ci/request_pr_exp.py @@ -280,7 +280,8 @@ def _remove_existing_job_bucket(gke_job_name: str, bucket_link: str, logging.error('STDERR:\n %s', del_bucket.stderr.decode('utf-8')) -def _prepare_experiment_info(args: argparse.Namespace) -> tuple[str, str, str]: +def _prepare_experiment_info( + args: argparse.Namespace) -> tuple[str, str, str, str, str]: """ Prepares and logs the key experiment information for easier accesses. """ @@ -320,7 +321,7 @@ def _prepare_experiment_info(args: argparse.Namespace) -> tuple[str, str, str]: bucket_link, bucket_gs_link, ) - return gke_job_name, bucket_link, bucket_gs_link + return gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link def _get_gke_credential(args: argparse.Namespace): @@ -384,11 +385,14 @@ def _request_experiment(substituted_file_path: str): def main(cmd=None): """The main function.""" args = _parse_args(cmd) - gke_job_name, bucket_link, bucket_gs_link = _prepare_experiment_info(args) + gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link = _prepare_experiment_info( + args) _get_gke_credential(args) _remove_existing_job_bucket(gke_job_name, bucket_link, bucket_gs_link) _request_experiment(_fill_template(args)) + return gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link if __name__ == "__main__": - sys.exit(main()) + main() + sys.exit(0) From 319126a6852c03381b173743f1de425b37528863 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 5 Sep 2025 13:14:50 +1200 Subject: [PATCH 2/6] Wire github token in --- ci/ci_trial_build.py | 22 +++++++++++++--------- ci/cloudbuild.yaml | 1 + 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ci/ci_trial_build.py b/ci/ci_trial_build.py index 9f08c46b5e..4904f03d26 100644 --- a/ci/ci_trial_build.py +++ b/ci/ci_trial_build.py @@ -78,15 +78,19 @@ def exec_command_from_github(pull_request_number): if all([ gke_job_name, gke_job_link, report_link, bucket_link, bucket_gs_link ]): - github_obj = github.Github() - repo = github_obj.get_repo('google/oss-fuzz-gen') - issue = repo.get_issue(pull_request_number) - body = (f"Requesting a GKE experiment named {gke_job_name}:\n" - f"JOB: {gke_job_link}\n" - f"REPORT: {report_link}\n" - f"BUCKET: {bucket_link}\n" - f"BUCKET GS: `{bucket_gs_link}`\n") - issue.create_comment(body) + token = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN') + if not token: + logging.warning('GITHUB_TOKEN not set; skipping PR comment.') + else: + github_obj = github.Github(token) + repo = github_obj.get_repo('google/oss-fuzz-gen') + issue = repo.get_issue(pull_request_number) + body = (f"Requesting a GKE experiment named {gke_job_name}:\n" + f"JOB: {gke_job_link}\n" + f"REPORT: {report_link}\n" + f"BUCKET: {bucket_link}\n" + f"BUCKET GS: `{bucket_gs_link}`\n") + issue.create_comment(body) except Exception as e: logging.warning('Failed to post PR comment: %s', e) return links diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index e6cfb3db00..7afed32fa5 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -25,6 +25,7 @@ steps: - 'PULL_REQUEST_NUMBER=${_PR_NUMBER}' - 'BRANCH=${_HEAD_BRANCH}' - 'REPO=${_HEAD_REPO_URL}' + - 'GITHUB_TOKEN=${_GITHUB_TOKEN}' images: - us-central1-docker.pkg.dev/oss-fuzz-base/oss-fuzz-gen/ci logsBucket: gs://oss-fuzz-gen-ci-logs From 33ad4800380b9bbbc7dfafeb88f094bfaf2e1c41 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Thu, 9 Oct 2025 20:42:36 +1300 Subject: [PATCH 3/6] Add auto-printing as a GitHub workflow --- .github/workflows/pr-experiment-commenter.yml | 80 +++++++++++++++++++ ci/ci_trial_build.py | 26 +----- ci/cloudbuild.yaml | 1 - ci/request_pr_exp.py | 48 +++++++---- 4 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/pr-experiment-commenter.yml diff --git a/.github/workflows/pr-experiment-commenter.yml b/.github/workflows/pr-experiment-commenter.yml new file mode 100644 index 0000000000..8491bb5863 --- /dev/null +++ b/.github/workflows/pr-experiment-commenter.yml @@ -0,0 +1,80 @@ +name: PR Experiment Commenter + +on: + issue_comment: + types: [created] + +jobs: + comment-experiment-links: + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/gcbrun exp ') + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - name: Parse gcbrun command and post experiment links + uses: actions/github-script@v7 + with: + script: | + const comment = context.payload.comment.body; + const prNumber = context.payload.issue.number; + + const TRIGGER_COMMAND = '/gcbrun'; + const TRIAL_BUILD_COMMAND_STR = `${TRIGGER_COMMAND} exp `; + + if (!comment.startsWith(TRIAL_BUILD_COMMAND_STR)) { + console.log('Not a valid /gcbrun exp command'); + return; + } + + const args = comment.slice(TRIAL_BUILD_COMMAND_STR.length).trim().split(/\s+/); + console.log('Parsed args:', args); + + let nameSuffix = ''; + let benchmarkSet = 'comparison'; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '-n' && i + 1 < args.length) { + nameSuffix = args[i + 1]; + } else if (args[i] === '-b' && i + 1 < args.length) { + benchmarkSet = args[i + 1]; + } + } + + if (!nameSuffix) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: 'Error: Missing required `-n` (name suffix) parameter in /gcbrun command.' + }); + return; + } + + const now = new Date(); + const date = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; // YYYY-MM-DD local + const experimentName = `${prNumber}-${nameSuffix}`; + const gkeJobName = `ofg-pr-${experimentName}`; + + const basePath = `${date}-${prNumber}-${nameSuffix}-${benchmarkSet}`; + const gkeJobLink = `https://console.cloud.google.com/kubernetes/job/us-central1-c/llm-experiment/default/ofg-pr-${experimentName}`; + const reportLink = `https://llm-exp.oss-fuzz.com/Result-reports/ofg-pr/${basePath}/index.html`; + const bucketLink = `https://console.cloud.google.com/storage/browser/oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; + const bucketGsLink = `gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; + + const body = `Requested GKE Job: ${gkeJobName} + + JOB: ${gkeJobLink} + REPORT: ${reportLink} + BUCKET: ${bucketLink} + BUCKET (GS): ${bucketGsLink}`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + + console.log(`Posted experiment links for PR #${prNumber}, job: ${gkeJobName}`); \ No newline at end of file diff --git a/ci/ci_trial_build.py b/ci/ci_trial_build.py index 4904f03d26..58120a1351 100644 --- a/ci/ci_trial_build.py +++ b/ci/ci_trial_build.py @@ -71,29 +71,7 @@ def exec_command_from_github(pull_request_number): command = ['-p', str(pull_request_number)] + command command = [c for c in command if c] logging.info('Command: %s.', command) - links = request_pr_exp.main(command) - try: - if isinstance(links, tuple) and len(links) == 5: - gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link = links - if all([ - gke_job_name, gke_job_link, report_link, bucket_link, bucket_gs_link - ]): - token = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN') - if not token: - logging.warning('GITHUB_TOKEN not set; skipping PR comment.') - else: - github_obj = github.Github(token) - repo = github_obj.get_repo('google/oss-fuzz-gen') - issue = repo.get_issue(pull_request_number) - body = (f"Requesting a GKE experiment named {gke_job_name}:\n" - f"JOB: {gke_job_link}\n" - f"REPORT: {report_link}\n" - f"BUCKET: {bucket_link}\n" - f"BUCKET GS: `{bucket_gs_link}`\n") - issue.create_comment(body) - except Exception as e: - logging.warning('Failed to post PR comment: %s', e) - return links + return request_pr_exp.main(command) def main(): @@ -107,4 +85,4 @@ def main(): if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) \ No newline at end of file diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index 7afed32fa5..e6cfb3db00 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -25,7 +25,6 @@ steps: - 'PULL_REQUEST_NUMBER=${_PR_NUMBER}' - 'BRANCH=${_HEAD_BRANCH}' - 'REPO=${_HEAD_REPO_URL}' - - 'GITHUB_TOKEN=${_GITHUB_TOKEN}' images: - us-central1-docker.pkg.dev/oss-fuzz-base/oss-fuzz-gen/ci logsBucket: gs://oss-fuzz-gen-ci-logs diff --git a/ci/request_pr_exp.py b/ci/request_pr_exp.py index cb90ae3149..3d71c31d7f 100644 --- a/ci/request_pr_exp.py +++ b/ci/request_pr_exp.py @@ -69,17 +69,41 @@ DEFAULT_VERTEX_AI_LOCATION = 'us-central1' VERTEX_AI_LOCATIONS = { 'vertex_ai_gemini-pro': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-ultra': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-1-5': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-1-5-chat': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-2-flash': - 'europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west4,europe-west8,europe-west9,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('europe-central2,europe-north1,europe-southwest1,europe-west1,' + 'europe-west4,europe-west8,europe-west9,us-central1,us-east1,' + 'us-east4,us-east5,us-south1,us-west1,us-west4'), 'vertex_ai_gemini-2-flash-chat': - 'europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west4,europe-west8,europe-west9,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4' + ('europe-central2,europe-north1,europe-southwest1,europe-west1,' + 'europe-west4,europe-west8,europe-west9,us-central1,us-east1,' + 'us-east4,us-east5,us-south1,us-west1,us-west4') } @@ -280,8 +304,7 @@ def _remove_existing_job_bucket(gke_job_name: str, bucket_link: str, logging.error('STDERR:\n %s', del_bucket.stderr.decode('utf-8')) -def _prepare_experiment_info( - args: argparse.Namespace) -> tuple[str, str, str, str, str]: +def _prepare_experiment_info(args: argparse.Namespace) -> tuple[str, str, str]: """ Prepares and logs the key experiment information for easier accesses. """ @@ -321,7 +344,7 @@ def _prepare_experiment_info( bucket_link, bucket_gs_link, ) - return gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link + return gke_job_name, bucket_link, bucket_gs_link def _get_gke_credential(args: argparse.Namespace): @@ -385,14 +408,11 @@ def _request_experiment(substituted_file_path: str): def main(cmd=None): """The main function.""" args = _parse_args(cmd) - gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link = _prepare_experiment_info( - args) + gke_job_name, bucket_link, bucket_gs_link = _prepare_experiment_info(args) _get_gke_credential(args) _remove_existing_job_bucket(gke_job_name, bucket_link, bucket_gs_link) _request_experiment(_fill_template(args)) - return gke_job_name, bucket_link, bucket_gs_link, gke_job_link, report_link if __name__ == "__main__": - main() - sys.exit(0) + sys.exit(main()) From 0649069e260bdfba31e5e4e75b4753c373303c3e Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 10 Oct 2025 13:36:53 +1300 Subject: [PATCH 4/6] Move PR commenter script to the folder --- .github/workflows/pr-experiment-commenter.yml | 69 +++---------------- ci/ci_trial_build.py | 2 +- ci/pr_exp_script.js | 65 +++++++++++++++++ 3 files changed, 74 insertions(+), 62 deletions(-) create mode 100644 ci/pr_exp_script.js diff --git a/.github/workflows/pr-experiment-commenter.yml b/.github/workflows/pr-experiment-commenter.yml index 8491bb5863..e2b3cab9cf 100644 --- a/.github/workflows/pr-experiment-commenter.yml +++ b/.github/workflows/pr-experiment-commenter.yml @@ -13,68 +13,15 @@ jobs: pull-requests: write steps: + - name: Check out the repository to the runner + uses: actions/checkout@v4 - name: Parse gcbrun command and post experiment links uses: actions/github-script@v7 with: script: | - const comment = context.payload.comment.body; - const prNumber = context.payload.issue.number; - - const TRIGGER_COMMAND = '/gcbrun'; - const TRIAL_BUILD_COMMAND_STR = `${TRIGGER_COMMAND} exp `; - - if (!comment.startsWith(TRIAL_BUILD_COMMAND_STR)) { - console.log('Not a valid /gcbrun exp command'); - return; - } - - const args = comment.slice(TRIAL_BUILD_COMMAND_STR.length).trim().split(/\s+/); - console.log('Parsed args:', args); - - let nameSuffix = ''; - let benchmarkSet = 'comparison'; - - for (let i = 0; i < args.length; i++) { - if (args[i] === '-n' && i + 1 < args.length) { - nameSuffix = args[i + 1]; - } else if (args[i] === '-b' && i + 1 < args.length) { - benchmarkSet = args[i + 1]; - } - } - - if (!nameSuffix) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: 'Error: Missing required `-n` (name suffix) parameter in /gcbrun command.' - }); - return; - } - - const now = new Date(); - const date = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; // YYYY-MM-DD local - const experimentName = `${prNumber}-${nameSuffix}`; - const gkeJobName = `ofg-pr-${experimentName}`; - - const basePath = `${date}-${prNumber}-${nameSuffix}-${benchmarkSet}`; - const gkeJobLink = `https://console.cloud.google.com/kubernetes/job/us-central1-c/llm-experiment/default/ofg-pr-${experimentName}`; - const reportLink = `https://llm-exp.oss-fuzz.com/Result-reports/ofg-pr/${basePath}/index.html`; - const bucketLink = `https://console.cloud.google.com/storage/browser/oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; - const bucketGsLink = `gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; - - const body = `Requested GKE Job: ${gkeJobName} - - JOB: ${gkeJobLink} - REPORT: ${reportLink} - BUCKET: ${bucketLink} - BUCKET (GS): ${bucketGsLink}`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: body - }); - - console.log(`Posted experiment links for PR #${prNumber}, job: ${gkeJobName}`); \ No newline at end of file + const fs = require('fs'); + const path = require('path'); + const scriptPath = path.join(process.cwd(), 'ci', 'pr_exp_script.js'); + const code = fs.readFileSync(scriptPath, 'utf8'); + eval(code); + await runPrExperimentCommenter({ github, context }); diff --git a/ci/ci_trial_build.py b/ci/ci_trial_build.py index 58120a1351..a840539d2d 100644 --- a/ci/ci_trial_build.py +++ b/ci/ci_trial_build.py @@ -85,4 +85,4 @@ def main(): if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/ci/pr_exp_script.js b/ci/pr_exp_script.js new file mode 100644 index 0000000000..c87213408e --- /dev/null +++ b/ci/pr_exp_script.js @@ -0,0 +1,65 @@ +async function runPrExperimentCommenter({ github, context }) { + const comment = context.payload.comment.body; + const prNumber = context.payload.issue.number; + + const TRIGGER_COMMAND = '/gcbrun'; + const TRIAL_BUILD_COMMAND_STR = `${TRIGGER_COMMAND} exp `; + + if (!comment.startsWith(TRIAL_BUILD_COMMAND_STR)) { + console.log('Not a valid /gcbrun exp command'); + return; + } + + const args = comment.slice(TRIAL_BUILD_COMMAND_STR.length).trim().split(/\s+/); + console.log('Parsed args:', args); + + let nameSuffix = ''; + let benchmarkSet = 'comparison'; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '-n' && i + 1 < args.length) { + nameSuffix = args[i + 1]; + } else if (args[i] === '-b' && i + 1 < args.length) { + benchmarkSet = args[i + 1]; + } + } + + if (!nameSuffix) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: 'Error: Missing required `-n` (name suffix) parameter in /gcbrun command.' + }); + return; + } + + const now = new Date(); + const date = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; // YYYY-MM-DD local + const experimentName = `${prNumber}-${nameSuffix}`; + const gkeJobName = `ofg-pr-${experimentName}`; + + const basePath = `${date}-${prNumber}-${nameSuffix}-${benchmarkSet}`; + const gkeJobLink = `https://console.cloud.google.com/kubernetes/job/us-central1-c/llm-experiment/default/ofg-pr-${experimentName}`; + const reportLink = `https://llm-exp.oss-fuzz.com/Result-reports/ofg-pr/${basePath}/index.html`; + const bucketLink = `https://console.cloud.google.com/storage/browser/oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; + const bucketGsLink = `gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; + + const body = `Requested GKE Job: ${gkeJobName} + + JOB: ${gkeJobLink} + REPORT: ${reportLink} + BUCKET: ${bucketLink} + BUCKET (GS): ${bucketGsLink}`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + + console.log(`Posted experiment links for PR #${prNumber}, job: ${gkeJobName}`); +} + + From ca68d4f06c053f6dd56917c82ca9c120aaa3b313 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 10 Oct 2025 13:36:53 +1300 Subject: [PATCH 5/6] Move PR commenter script to the ci folder --- ci/pr_exp_script.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/pr_exp_script.js b/ci/pr_exp_script.js index c87213408e..4443df2a07 100644 --- a/ci/pr_exp_script.js +++ b/ci/pr_exp_script.js @@ -61,5 +61,3 @@ async function runPrExperimentCommenter({ github, context }) { console.log(`Posted experiment links for PR #${prNumber}, job: ${gkeJobName}`); } - - From 892f3f3ffab71b143d12b062c96b0ebd0a4240ec Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 10 Oct 2025 14:01:14 +1300 Subject: [PATCH 6/6] Refactor with docstring comments and modularity --- .github/workflows/pr-experiment-commenter.yml | 2 +- ci/pr_exp_comment.js | 140 ++++++++++++++++++ ci/pr_exp_script.js | 63 -------- 3 files changed, 141 insertions(+), 64 deletions(-) create mode 100644 ci/pr_exp_comment.js delete mode 100644 ci/pr_exp_script.js diff --git a/.github/workflows/pr-experiment-commenter.yml b/.github/workflows/pr-experiment-commenter.yml index e2b3cab9cf..0a130b5cc4 100644 --- a/.github/workflows/pr-experiment-commenter.yml +++ b/.github/workflows/pr-experiment-commenter.yml @@ -21,7 +21,7 @@ jobs: script: | const fs = require('fs'); const path = require('path'); - const scriptPath = path.join(process.cwd(), 'ci', 'pr_exp_script.js'); + const scriptPath = path.join(process.cwd(), 'ci', 'pr_exp_comment.js'); const code = fs.readFileSync(scriptPath, 'utf8'); eval(code); await runPrExperimentCommenter({ github, context }); diff --git a/ci/pr_exp_comment.js b/ci/pr_exp_comment.js new file mode 100644 index 0000000000..eda38f77db --- /dev/null +++ b/ci/pr_exp_comment.js @@ -0,0 +1,140 @@ +const DEFAULT_CLUSTER = 'llm-experiment'; +const LARGE_CLUSTER = 'llm-experiment-large'; +const DEFAULT_LOCATION = 'us-central1-c'; +const LARGE_LOCATION = 'us-central1'; +const BENCHMARK_SET = 'comparison'; + +const TRIGGER_COMMAND = '/gcbrun'; +const TRIAL_BUILD_COMMAND_STR = `${TRIGGER_COMMAND} exp `; +const SKIP_COMMAND_STR = `${TRIGGER_COMMAND} skip`; + +const PR_LINK_PREFIX = 'https://github.com/google/oss-fuzz-gen/pull'; +const JOB_LINK_PREFIX = 'https://console.cloud.google.com/kubernetes/job///default'; +const REPORT_LINK_PREFIX = 'https://llm-exp.oss-fuzz.com/Result-reports/ofg-pr'; +const BUCKET_LINK_PREFIX = 'https://console.cloud.google.com/storage/browser/oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr'; +const BUCKET_GS_LINK_PREFIX = 'gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr'; + +/** + * Parses command arguments from /gcbrun exp comment. + * @param {string} comment - The comment body + * @returns {Object|null} Parsed args {nameSuffix, benchmarkSet} or null if invalid + */ +function parseGcbrunArgs(comment) { + if (!comment.startsWith(TRIAL_BUILD_COMMAND_STR)) { + return null; + } + + const args = comment.slice(TRIAL_BUILD_COMMAND_STR.length).trim().split(/\s+/); + console.log('Parsed args:', args); + + let nameSuffix = ''; + let benchmarkSet = BENCHMARK_SET; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '-n' && i + 1 < args.length) { + nameSuffix = args[i + 1]; + } else if (args[i] === '-b' && i + 1 < args.length) { + benchmarkSet = args[i + 1]; + } + } + + return { nameSuffix, benchmarkSet }; +} + +/** + * Prepares and generates the key experiment information. + * @param {number} prId - Pull request ID + * @param {string} nameSuffix - Name suffix for the experiment + * @param {string} benchmarkSet - Benchmark set name + * @returns {Object} Generated experiment info with links and job name + */ +function prepareExperimentInfo(prId, nameSuffix, benchmarkSet) { + const experimentName = `${prId}-${nameSuffix}`; + + const gkeJobName = `ofg-pr-${experimentName}`; + + const now = new Date(); + const date = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; + + // Base path for reports and buckets + const basePath = `${date}-${prId}-${nameSuffix}-${benchmarkSet}`; + + // GKE job link + let gkeJobLink = `${JOB_LINK_PREFIX}/ofg-pr-${experimentName}`; + gkeJobLink = gkeJobLink.replace('', DEFAULT_LOCATION); + gkeJobLink = gkeJobLink.replace('', DEFAULT_CLUSTER); + + // PR link + const prLink = `${PR_LINK_PREFIX}/${prId}`; + + // Report link + const reportLink = `${REPORT_LINK_PREFIX}/${basePath}/index.html`; + + // Bucket links + const bucketLink = `${BUCKET_LINK_PREFIX}/${basePath}`; + const bucketGsLink = `${BUCKET_GS_LINK_PREFIX}/${basePath}`; + + return { + gkeJobName, + gkeJobLink, + prLink, + reportLink, + bucketLink, + bucketGsLink + }; +} + +/** + * Parses /gcbrun exp commands from PR comments and posts experiment links. + * @param {Object} params - Parameters object + * @param {Object} params.github - GitHub API client from actions/github-script + * @param {Object} params.context - GitHub Actions context with event payload + */ +async function runPrExperimentCommenter({ github, context }) { + const comment = context.payload.comment.body; + const prNumber = context.payload.issue.number; + + const parsedArgs = parseGcbrunArgs(comment); + if (!parsedArgs) { + console.log('Not a valid /gcbrun exp command'); + return; + } + + const { nameSuffix, benchmarkSet } = parsedArgs; + if (!nameSuffix) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: 'Error: Missing required `-n` (name suffix) parameter in /gcbrun command.' + }); + return; + } + + const experimentInfo = prepareExperimentInfo(prNumber, nameSuffix, benchmarkSet); + + console.log( + `Requesting a GKE experiment named ${experimentInfo.gkeJobName}:\n` + + `PR: ${experimentInfo.prLink}\n` + + `JOB: ${experimentInfo.gkeJobLink}\n` + + `REPORT: ${experimentInfo.reportLink}\n` + + `BUCKET: ${experimentInfo.bucketLink}\n` + + `BUCKET GS: ${experimentInfo.bucketGsLink}` + ); + + const body = `Requested GKE Job: ${experimentInfo.gkeJobName} + + JOB: ${experimentInfo.gkeJobLink} + REPORT: ${experimentInfo.reportLink} + BUCKET: ${experimentInfo.bucketLink} + BUCKET (GS): ${experimentInfo.bucketGsLink}`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + + console.log(`Posted experiment links for PR #${prNumber}, job: ${experimentInfo.gkeJobName}`); +} diff --git a/ci/pr_exp_script.js b/ci/pr_exp_script.js deleted file mode 100644 index 4443df2a07..0000000000 --- a/ci/pr_exp_script.js +++ /dev/null @@ -1,63 +0,0 @@ -async function runPrExperimentCommenter({ github, context }) { - const comment = context.payload.comment.body; - const prNumber = context.payload.issue.number; - - const TRIGGER_COMMAND = '/gcbrun'; - const TRIAL_BUILD_COMMAND_STR = `${TRIGGER_COMMAND} exp `; - - if (!comment.startsWith(TRIAL_BUILD_COMMAND_STR)) { - console.log('Not a valid /gcbrun exp command'); - return; - } - - const args = comment.slice(TRIAL_BUILD_COMMAND_STR.length).trim().split(/\s+/); - console.log('Parsed args:', args); - - let nameSuffix = ''; - let benchmarkSet = 'comparison'; - - for (let i = 0; i < args.length; i++) { - if (args[i] === '-n' && i + 1 < args.length) { - nameSuffix = args[i + 1]; - } else if (args[i] === '-b' && i + 1 < args.length) { - benchmarkSet = args[i + 1]; - } - } - - if (!nameSuffix) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: 'Error: Missing required `-n` (name suffix) parameter in /gcbrun command.' - }); - return; - } - - const now = new Date(); - const date = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; // YYYY-MM-DD local - const experimentName = `${prNumber}-${nameSuffix}`; - const gkeJobName = `ofg-pr-${experimentName}`; - - const basePath = `${date}-${prNumber}-${nameSuffix}-${benchmarkSet}`; - const gkeJobLink = `https://console.cloud.google.com/kubernetes/job/us-central1-c/llm-experiment/default/ofg-pr-${experimentName}`; - const reportLink = `https://llm-exp.oss-fuzz.com/Result-reports/ofg-pr/${basePath}/index.html`; - const bucketLink = `https://console.cloud.google.com/storage/browser/oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; - const bucketGsLink = `gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr/${basePath}`; - - const body = `Requested GKE Job: ${gkeJobName} - - JOB: ${gkeJobLink} - REPORT: ${reportLink} - BUCKET: ${bucketLink} - BUCKET (GS): ${bucketGsLink}`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: body - }); - - console.log(`Posted experiment links for PR #${prNumber}, job: ${gkeJobName}`); -}