-
Notifications
You must be signed in to change notification settings - Fork 39
149 lines (121 loc) · 4.52 KB
/
benchmark.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
name: Benchmark
on:
issue_comment:
types: created
jobs:
benchmark_base:
# On pull requests and if the comment starts with `/run-benchmark`
if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark')
runs-on: self-hosted
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
ref: main
- name: Get base branch commit ID
id: get_base_commit
run: echo "BASE_COMMIT_ID=$(git rev-parse HEAD)" > base_commit_id.txt
- name: Upload base commit ID
uses: actions/upload-artifact@v4
with:
name: base-commit-id
path: base_commit_id.txt
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Create virtual environment
run: python -m venv bitblas_benchmark
- name: Activate virtual environment and install dependencies
run: |
source bitblas_benchmark/bin/activate
python -m pip install --upgrade pip
if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi
- name: Install project in wheel mode
run: |
source bitblas_benchmark/bin/activate
python -m pip install .
- name: Matmul Benchmark
run: |
source bitblas_benchmark/bin/activate
cd benchmark/operators
python ./benchmark_ops_matmul.py
benchmark_head:
# On pull requests and if the comment starts with `/run-benchmark`
if: github.event.issue.pull_request != null && startsWith(github.event.comment.body, '/run-benchmark')
runs-on: self-hosted
needs: [benchmark_base]
steps:
- name: Checkout PR branch code
uses: actions/checkout@v2
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.ref }}
- name: Get PR branch commit ID
id: get_pr_commit
run: echo "PR_COMMIT_ID=$(git rev-parse HEAD)" > pr_commit_id.txt
- name: Upload PR commit ID
uses: actions/upload-artifact@v4
with:
name: pr-commit-id
path: pr_commit_id.txt
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Create virtual environment
run: python -m venv bitblas_benchmark
- name: Activate virtual environment and install dependencies
run: |
source bitblas_benchmark/bin/activate
python -m pip install --upgrade pip
if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi
- name: Install project in wheel mode
run: |
source bitblas_benchmark/bin/activate
python -m pip install .
- name: Matmul Benchmark
run: |
source bitblas_benchmark/bin/activate
cd benchmark/operators
python ./benchmark_ops_matmul.py
benchmark_compare:
if: github.event.issue.pull_request != null && contains(github.event.comment.body, '/run-benchmark')
needs: [benchmark_base, benchmark_head]
runs-on: self-hosted
steps:
- name: Download commit IDs
uses: actions/[email protected]
with:
name: base-commit-id
path: .
- name: Download PR commit ID
uses: actions/[email protected]
with:
name: pr-commit-id
path: .
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Create virtual environment
run: python -m venv bitblas_benchmark
- name: Activate virtual environment and install dependencies
run: |
source bitblas_benchmark/bin/activate
python -m pip install --upgrade pip
if [ -f requirements-dev.txt ]; then python -m pip install -r requirements-dev.txt; fi
- name: Compare benchmark results
run: |
source bitblas_benchmark/bin/activate
cd benchmark/operators
python ./compare_benchmark.py --base $(cat base_commit_id.txt) --head $(cat pr_commit_id.txt) 2>&1 | tee compare_results.txt
- name: Authenticate GitHub CLI
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token
- name: Post benchmark results
run: |
cat compare_results.txt
gh pr comment ${{ github.event.issue.number }} --body "$(cat compare_results.txt)"