Skip to content

Commit

Permalink
[improve][ci] Detect test thread leaks in CI builds and add tooling f…
Browse files Browse the repository at this point in the history
…or resource leak investigation
  • Loading branch information
lhotari committed Oct 22, 2023
1 parent 7b1b455 commit 59f3f09
Show file tree
Hide file tree
Showing 9 changed files with 404 additions and 28 deletions.
66 changes: 53 additions & 13 deletions .github/workflows/pulsar-ci-flaky.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,20 @@ on:
- '17'
- '21'
default: '17'

trace_test_resource_cleanup:
description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
required: true
type: choice
options:
- 'off'
- 'on'
- 'full'
default: 'off'
thread_leak_detector_wait_millis:
description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
required: true
type: number
default: 10000

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
Expand Down Expand Up @@ -134,6 +147,10 @@ jobs:
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
runs-on: ubuntu-22.04
timeout-minutes: 100
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
Expand All @@ -144,6 +161,10 @@ jobs:
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Clean Disk when tracing test resource cleanup
if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
uses: ./.github/actions/clean-disk

- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
Expand Down Expand Up @@ -185,8 +206,26 @@ jobs:
if: ${{ always() }}
uses: ./.github/actions/copy-test-reports

- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'

- name: Report detected thread leaks
if: ${{ always() }}
run: |
if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
cd "$THREAD_LEAK_DETECTOR_DIR"
for file in threadleak*.txt; do
cat "$file" | awk '{ if($0 != "") { print "::warning::"$0 }}'
done
fi
- name: Create Jacoco reports
if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
continue-on-error: true
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
cd $GITHUB_WORKSPACE/target
Expand All @@ -199,33 +238,34 @@ jobs:
name: Jacoco-coverage-report-flaky
path: target/jacoco_test_coverage_report_flaky.zip
retention-days: 3
if-no-files-found: ignore

- name: Upload to Codecov
if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
uses: ./.github/actions/upload-coverage
continue-on-error: true
with:
flags: unittests

- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'

- name: Upload Surefire reports
uses: actions/upload-artifact@v3
if: ${{ !success() }}
if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
with:
name: Unit-BROKER_FLAKY-surefire-reports
path: surefire-reports
retention-days: 7
if-no-files-found: ignore

- name: Upload possible heap dump
- name: Upload possible heap dump, core dump or crash files
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: Unit-BROKER_FLAKY-heapdump
path: /tmp/*.hprof
name: Unit-BROKER_FLAKY-dumps
path: |
/tmp/*.hprof
**/hs_err_*.log
**/core.*
${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
retention-days: 7
if-no-files-found: ignore
if-no-files-found: ignore
36 changes: 35 additions & 1 deletion .github/workflows/pulsar-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,20 @@ on:
- '17'
- '21'
default: '17'
trace_test_resource_cleanup:
description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
required: true
type: choice
options:
- 'off'
- 'on'
- 'full'
default: 'off'
thread_leak_detector_wait_millis:
description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
required: true
type: number
default: 10000

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
Expand Down Expand Up @@ -209,6 +223,10 @@ jobs:
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
runs-on: ubuntu-22.04
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: ['preconditions', 'build-and-license-check']
Expand Down Expand Up @@ -251,6 +269,10 @@ jobs:
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Clean Disk when tracing test resource cleanup
if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
uses: ./.github/actions/clean-disk

- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
Expand Down Expand Up @@ -312,9 +334,19 @@ jobs:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'

- name: Report detected thread leaks
if: ${{ always() }}
run: |
if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
cd "$THREAD_LEAK_DETECTOR_DIR"
for file in threadleak*.txt; do
cat "$file" | awk '{ if($0 != "") { print "::warning::"$0 }}'
done
fi
- name: Upload Surefire reports
uses: actions/upload-artifact@v3
if: ${{ !success() }}
if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
with:
name: Unit-${{ matrix.group }}-surefire-reports
path: surefire-reports
Expand All @@ -329,6 +361,8 @@ jobs:
/tmp/*.hprof
**/hs_err_*.log
**/core.*
${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
retention-days: 7
if-no-files-found: ignore

Expand Down
53 changes: 53 additions & 0 deletions buildtools/src/main/java/org/apache/pulsar/tests/HeapDumpUtil.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.pulsar.tests;

import com.sun.management.HotSpotDiagnosticMXBean;
import java.io.File;
import java.lang.management.ManagementFactory;
import javax.management.MBeanServer;

public class HeapDumpUtil {
private static final String HOTSPOT_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic";

// Utility method to get the HotSpotDiagnosticMXBean
private static HotSpotDiagnosticMXBean getHotSpotDiagnosticMXBean() {
try {
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
return ManagementFactory.newPlatformMXBeanProxy(server, HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class);
} catch (Exception e) {
throw new RuntimeException(e);
}
}

/**
* Dump the heap of the JVM.
*
* @param file the system-dependent filename
* @param liveObjects if true dump only live objects i.e. objects that are reachable from others
*/
public static void dumpHeap(File file, boolean liveObjects) {
try {
HotSpotDiagnosticMXBean hotspotMBean = getHotSpotDiagnosticMXBean();
hotspotMBean.dumpHeap(file.getAbsolutePath(), liveObjects);
} catch (Exception e) {
throw new RuntimeException("Error generating heap dump", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.pulsar.tests;

import java.lang.management.ManagementFactory;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import javax.management.JMException;
import javax.management.ObjectName;

public class HeapHistogramUtil {
public static String buildHeapHistogram() {
StringBuilder dump = new StringBuilder();
dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now())));
dump.append("\n\n");
try {
dump.append(callDiagnosticCommand("gcHeapInfo"));
} catch (Exception e) {
e.printStackTrace();
return null;
}
dump.append("\n");
try {
dump.append(callDiagnosticCommand("gcClassHistogram"));
} catch (Exception e) {
e.printStackTrace();
return null;
}
return dump.toString();
}

/**
* Calls a diagnostic commands.
* The available operations are similar to what the jcmd commandline tool has,
* however the naming of the operations are different. The "help" operation can be used
* to find out the available operations. For example, the jcmd command "Thread.print" maps
* to "threadPrint" operation name.
*/
static String callDiagnosticCommand(String operationName, String... args)
throws JMException {
return (String) ManagementFactory.getPlatformMBeanServer()
.invoke(new ObjectName("com.sun.management:type=DiagnosticCommand"),
operationName, new Object[] {args}, new String[] {String[].class.getName()});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import java.lang.management.MonitorInfo;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import javax.management.JMException;
Expand Down Expand Up @@ -65,7 +65,7 @@ static String buildThreadDump() {
// fallback to using JMX for creating the thread dump
StringBuilder dump = new StringBuilder();

dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(LocalDateTime.now())));
dump.append(String.format("Timestamp: %s", DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now())));
dump.append("\n\n");

Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
Expand Down
Loading

0 comments on commit 59f3f09

Please sign in to comment.