From 7a36385dddfac719bb48566557abcafccdb8b141 Mon Sep 17 00:00:00 2001 From: willie Date: Mon, 25 Mar 2024 19:06:15 +0100 Subject: [PATCH 1/7] feat: Add initial benchmarking setup Will execute on pull requests, and run all benchmarks against master and post a comparison as a PR comment --- .github/workflows/benchmarks.yaml | 79 ++++++++++++++++ pom.xml | 26 ++++++ .../jmh/basic/BasicBlogBenchmark.java | 93 +++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 .github/workflows/benchmarks.yaml create mode 100644 src/test/java/org/apache/ibatis/benchmarks/jmh/basic/BasicBlogBenchmark.java diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 00000000000..4635c41d822 --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,79 @@ +name: Benchmarks +on: + pull_request: + branches: + - master + +concurrency: + group: ${{ format('{0}-{1}', github.workflow_ref, github.head_ref) }} + cancel-in-progress: true + +jobs: + benchmarks: + if: github.repository_owner == 'mybatis' + permissions: + contents: read + pull-requests: write # for benchmark comment + runs-on: ubuntu-latest + env: + COMMENT_FILE: ${{ github.workspace }}/benchmark-comment.md + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - id: get-sha + run: | + # getting exact master sha at the moment to reference it in the comment + echo "sha=$( curl -u "u:${{ github.token }}" https://api.github.com/repos/${{ github.repository }}/git/ref/heads/${{ github.base_ref }} | jq .object.sha | tr -d '"' )" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + ref: ${{ steps.get-sha.outputs.sha }} + path: benchmark-master + + - uses: actions/checkout@v4 + with: + path: benchmark-pull-request + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: zulu + + - name: Run benchmarks (pull-request) + working-directory: benchmark-pull-request + run: | + ./mvnw jmh:benchmark -Dlicense.skip=true + + echo '> [!NOTE]' >> ${{ env.COMMENT_FILE }} + echo '> These results are affected by shared workloads on GitHub runners. Use the results only to detect possible regressions, but always rerun on more stable machine before making any conclusions!' >> ${{ env.COMMENT_FILE }} + echo '### Benchmark results (pull-request, ${{ github.event.pull_request.head.sha }})' >> ${{ env.COMMENT_FILE }} + echo '```text' >> ${{ env.COMMENT_FILE }} + cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} + echo '```' >> ${{ env.COMMENT_FILE }} + +# todo: to be uncomented after this is on master +# - name: Run benchmarks (master) +# working-directory: benchmark-master +# run: | +# ./mvnw jmh:benchmark -Dlicense.skip=true +# +# echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} +# echo '```text' >> ${{ env.COMMENT_FILE }} +# cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} +# echo '```' >> ${{ env.COMMENT_FILE }} + + - name: Find benchmark results comment + uses: peter-evans/find-comment@v3 + id: benchmark-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Benchmark results + + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.benchmark-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body-path: ${{ env.COMMENT_FILE }} + edit-mode: replace \ No newline at end of file diff --git a/pom.xml b/pom.xml index dbca1c98c3d..b64e7062bfd 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,8 @@ 5.11.0 12.6.1.jre11 1.19.7 + 1.37 + 0.2.2 @@ -351,6 +353,20 @@ ${log4j.version} test + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + test + @@ -455,6 +471,16 @@ + + pw.krejci + jmh-maven-plugin + ${jmh.plugin.version} + + ${project.build.directory}/benchmark-results.txt + text + 1 + + diff --git a/src/test/java/org/apache/ibatis/benchmarks/jmh/basic/BasicBlogBenchmark.java b/src/test/java/org/apache/ibatis/benchmarks/jmh/basic/BasicBlogBenchmark.java new file mode 100644 index 00000000000..2829c6baec3 --- /dev/null +++ b/src/test/java/org/apache/ibatis/benchmarks/jmh/basic/BasicBlogBenchmark.java @@ -0,0 +1,93 @@ +/* + * Copyright 2009-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ibatis.benchmarks.jmh.basic; + +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import org.apache.ibatis.BaseDataTest; +import org.apache.ibatis.binding.BoundAuthorMapper; +import org.apache.ibatis.binding.BoundBlogMapper; +import org.apache.ibatis.domain.blog.Author; +import org.apache.ibatis.domain.blog.Blog; +import org.apache.ibatis.domain.blog.Post; +import org.apache.ibatis.mapping.Environment; +import org.apache.ibatis.session.Configuration; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.SqlSessionFactory; +import org.apache.ibatis.session.SqlSessionFactoryBuilder; +import org.apache.ibatis.transaction.TransactionFactory; +import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +@Fork(1) +@Warmup(iterations = 1) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +public class BasicBlogBenchmark { + + @State(Scope.Benchmark) + public static class SessionFactoryState { + + private SqlSessionFactory sqlSessionFactory; + + @Setup + public void setup() throws Exception { + DataSource dataSource = BaseDataTest.createBlogDataSource(); + BaseDataTest.runScript(dataSource, BaseDataTest.BLOG_DDL); + BaseDataTest.runScript(dataSource, BaseDataTest.BLOG_DATA); + + TransactionFactory transactionFactory = new JdbcTransactionFactory(); + Environment environment = new Environment("Production", transactionFactory, dataSource); + Configuration configuration = new Configuration(environment); + configuration.getTypeAliasRegistry().registerAlias(Blog.class); + configuration.getTypeAliasRegistry().registerAlias(Post.class); + configuration.getTypeAliasRegistry().registerAlias(Author.class); + configuration.addMapper(BoundBlogMapper.class); + configuration.addMapper(BoundAuthorMapper.class); + sqlSessionFactory = new SqlSessionFactoryBuilder().build(configuration); + } + + public SqlSessionFactory getSqlSessionFactory() { + return sqlSessionFactory; + } + } + + @Benchmark + public Blog retrieveSingleBlogUsingConstructorWithResultMap(SessionFactoryState sessionFactoryState) { + try (SqlSession sqlSession = sessionFactoryState.getSqlSessionFactory().openSession()) { + final BoundBlogMapper mapper = sqlSession.getMapper(BoundBlogMapper.class); + return mapper.selectBlogUsingConstructorWithResultMap(1); + } + } + + @Benchmark + public Blog retrieveSingleBlog(SessionFactoryState sessionFactoryState) { + try (SqlSession sqlSession = sessionFactoryState.getSqlSessionFactory().openSession()) { + final BoundBlogMapper mapper = sqlSession.getMapper(BoundBlogMapper.class); + return mapper.selectBlog(1); + } + } +} From 3d0c2725966c9c63f9773e9842d97502b7a194f2 Mon Sep 17 00:00:00 2001 From: willie Date: Mon, 25 Mar 2024 19:13:23 +0100 Subject: [PATCH 2/7] feat: Add check for master branch too --- .github/workflows/benchmarks.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 4635c41d822..51ede597845 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -51,16 +51,15 @@ jobs: cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} echo '```' >> ${{ env.COMMENT_FILE }} -# todo: to be uncomented after this is on master -# - name: Run benchmarks (master) -# working-directory: benchmark-master -# run: | -# ./mvnw jmh:benchmark -Dlicense.skip=true -# -# echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} -# echo '```text' >> ${{ env.COMMENT_FILE }} -# cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} -# echo '```' >> ${{ env.COMMENT_FILE }} + - name: Run benchmarks (master) + working-directory: benchmark-master + run: | + ./mvnw jmh:benchmark -Dlicense.skip=true + + echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} + echo '```text' >> ${{ env.COMMENT_FILE }} + cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} + echo '```' >> ${{ env.COMMENT_FILE }} - name: Find benchmark results comment uses: peter-evans/find-comment@v3 From 1339c93d34e870d0794ec6de460841adee5a903c Mon Sep 17 00:00:00 2001 From: willie Date: Mon, 25 Mar 2024 19:18:57 +0100 Subject: [PATCH 3/7] feat: Fix pull request demo --- .github/workflows/benchmarks.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 51ede597845..52a5ed849a9 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -42,7 +42,7 @@ jobs: - name: Run benchmarks (pull-request) working-directory: benchmark-pull-request run: | - ./mvnw jmh:benchmark -Dlicense.skip=true + ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true echo '> [!NOTE]' >> ${{ env.COMMENT_FILE }} echo '> These results are affected by shared workloads on GitHub runners. Use the results only to detect possible regressions, but always rerun on more stable machine before making any conclusions!' >> ${{ env.COMMENT_FILE }} @@ -51,15 +51,16 @@ jobs: cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} echo '```' >> ${{ env.COMMENT_FILE }} - - name: Run benchmarks (master) - working-directory: benchmark-master - run: | - ./mvnw jmh:benchmark -Dlicense.skip=true - - echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} - echo '```text' >> ${{ env.COMMENT_FILE }} - cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} - echo '```' >> ${{ env.COMMENT_FILE }} +# todo: skip until this is in master (demo on pull request) +# - name: Run benchmarks (master) +# working-directory: benchmark-master +# run: | +# ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true +# +# echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} +# echo '```text' >> ${{ env.COMMENT_FILE }} +# cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} +# echo '```' >> ${{ env.COMMENT_FILE }} - name: Find benchmark results comment uses: peter-evans/find-comment@v3 From 9fe050185ddeffdce5b138a35bb5ddab98014597 Mon Sep 17 00:00:00 2001 From: willie Date: Mon, 25 Mar 2024 19:35:52 +0100 Subject: [PATCH 4/7] feat: Use pull request target, so we can comment on the PR see: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target --- .github/workflows/benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 52a5ed849a9..52234e725e9 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -1,6 +1,6 @@ name: Benchmarks on: - pull_request: + pull_request_target: branches: - master From 37adbf7c20abb0d96ad25045d5e33ded9ec96993 Mon Sep 17 00:00:00 2001 From: willie Date: Mon, 25 Mar 2024 19:39:28 +0100 Subject: [PATCH 5/7] feat: Because we changed the event that triggers, we don't need to do this in two steps --- .github/workflows/benchmarks.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 52234e725e9..bf3184e61b5 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -51,16 +51,15 @@ jobs: cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} echo '```' >> ${{ env.COMMENT_FILE }} -# todo: skip until this is in master (demo on pull request) -# - name: Run benchmarks (master) -# working-directory: benchmark-master -# run: | -# ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true -# -# echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} -# echo '```text' >> ${{ env.COMMENT_FILE }} -# cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} -# echo '```' >> ${{ env.COMMENT_FILE }} + - name: Run benchmarks (master) + working-directory: benchmark-master + run: | + ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true + + echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} + echo '```text' >> ${{ env.COMMENT_FILE }} + cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} + echo '```' >> ${{ env.COMMENT_FILE }} - name: Find benchmark results comment uses: peter-evans/find-comment@v3 From f8d25aa66365c98789bb83a768f5109109465730 Mon Sep 17 00:00:00 2001 From: willie Date: Wed, 27 Mar 2024 10:25:19 +0100 Subject: [PATCH 6/7] feat: Add gc memory profiling --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index b64e7062bfd..cf094adbeb1 100644 --- a/pom.xml +++ b/pom.xml @@ -479,6 +479,7 @@ ${project.build.directory}/benchmark-results.txt text 1 + gc From cefe77b22f357fe61db93dc0bfd6362a1ae6507b Mon Sep 17 00:00:00 2001 From: Willie Scholtz Date: Mon, 15 Apr 2024 10:36:19 +0200 Subject: [PATCH 7/7] fix: use correct sha when checking out --- .github/workflows/benchmarks.yaml | 39 +++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index bf3184e61b5..79a7519f133 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -19,18 +19,20 @@ jobs: COMMENT_FILE: ${{ github.workspace }}/benchmark-comment.md GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - id: get-sha + - id: get-master-sha run: | # getting exact master sha at the moment to reference it in the comment echo "sha=$( curl -u "u:${{ github.token }}" https://api.github.com/repos/${{ github.repository }}/git/ref/heads/${{ github.base_ref }} | jq .object.sha | tr -d '"' )" >> "$GITHUB_OUTPUT" - uses: actions/checkout@v4 with: - ref: ${{ steps.get-sha.outputs.sha }} + ref: ${{ steps.get-master-sha.outputs.sha }} path: benchmark-master - uses: actions/checkout@v4 with: + # https://github.com/actions/checkout/issues/518#issuecomment-890401887 + ref: "${{ github.event.pull_request.merge_commit_sha }}" path: benchmark-pull-request - name: Set up JDK @@ -41,25 +43,28 @@ jobs: - name: Run benchmarks (pull-request) working-directory: benchmark-pull-request - run: | - ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true - - echo '> [!NOTE]' >> ${{ env.COMMENT_FILE }} - echo '> These results are affected by shared workloads on GitHub runners. Use the results only to detect possible regressions, but always rerun on more stable machine before making any conclusions!' >> ${{ env.COMMENT_FILE }} - echo '### Benchmark results (pull-request, ${{ github.event.pull_request.head.sha }})' >> ${{ env.COMMENT_FILE }} - echo '```text' >> ${{ env.COMMENT_FILE }} - cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} - echo '```' >> ${{ env.COMMENT_FILE }} + run: ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true - name: Run benchmarks (master) working-directory: benchmark-master - run: | - ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true + run: ./mvnw -B -V --no-transfer-progress jmh:benchmark -Dlicense.skip=true - echo '### Benchmark results (${{ github.base_ref }}, ${{ steps.get-sha.outputs.sha }})' >> ${{ env.COMMENT_FILE }} - echo '```text' >> ${{ env.COMMENT_FILE }} - cat target/benchmark-results.txt >> ${{ env.COMMENT_FILE }} - echo '```' >> ${{ env.COMMENT_FILE }} + - name: Compose comment content + run: | + cat <> ${{ env.COMMENT_FILE }} + > [!NOTE] + > These results are affected by shared workloads on GitHub runners. Use the results only to detect possible regressions, but always rerun on more stable machine before making any conclusions! + + ### Benchmark results (pull-request, ${{ github.event.pull_request.head.sha }}) + \`\`\`text + $(cat benchmark-pull-request/target/benchmark-results.txt) + \`\`\` + + ### Benchmark results (${{ github.base_ref }}, ${{ steps.get-master-sha.outputs.sha }}) + \`\`\`text + $(cat benchmark-master/target/benchmark-results.txt) + \`\`\` + EOF - name: Find benchmark results comment uses: peter-evans/find-comment@v3