Skip to content

Thread Sanitizer analysis #3797

Thread Sanitizer analysis

Thread Sanitizer analysis #3797

name: Thread Sanitizer analysis
on:
workflow_dispatch:
inputs:
fastdds_branch:
description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)'
required: true
default: 'master'
optional_cmake_args:
description: 'Optional CMake Compilation Flags'
required: false
type: string
optional_ctest_args:
description: 'Optional CTest Testing Flags'
required: false
type: string
pull_request:
types:
- review_requested
paths-ignore:
- '**.md'
- '**.txt'
- '!**/CMakeLists.txt'
schedule:
- cron: '0 1 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
ubuntu-sanitizer-run:
name: Sanitizer Evaluation
if: ${{ (
!contains(github.event.pull_request.labels.*.name, 'skip-ci') &&
!contains(github.event.pull_request.labels.*.name, 'no-test') &&
!contains(github.event.pull_request.labels.*.name, 'conflicts')
) }}
runs-on: ubuntu-22.04
env:
FASTDDS_BRANCH: ${{ github.head_ref || github.event.inputs.fastdds_branch || 'master' }}
TSAN_OPTIONS: second_deadlock_stack=1 history_size=7 memory_limit_mb=5000
# GCC 11.3 (Ubuntu Jammy default) produces several false positives regarding timed synchronization protocols
# These issues were fixed in GCC 12 so we upgrade to that version.
CC: gcc-12
CXX: g++-12
steps:
- name: Add ci-pending label if PR
if: ${{ github.event_name == 'pull_request' }}
uses: eProsima/eProsima-CI/external/add_labels@v0
with:
labels: ci-pending
number: ${{ github.event.number }}
repo: eProsima/Fast-DDS
# https://github.com/actions/runner-images/issues/9491
- name: Fix kernel mmap rnd bits
run: sudo sysctl vm.mmap_rnd_bits=28
- name: Install apt packages
uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0
with:
packages: curl grep libasio-dev libtinyxml2-dev python3 python3-pip software-properties-common wget
- name: Get minimum supported version of CMake
uses: eProsima/eProsima-CI/external/get-cmake@v0
with:
cmakeVersion: '3.22.6'
- name: Setup CCache
uses: eProsima/eProsima-CI/external/setup-ccache-action@v0
- name: Install colcon
uses: eProsima/eProsima-CI/ubuntu/install_colcon@v0
- name: Install Python dependencies
uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0
with:
packages: vcstool setuptools gcovr tomark xmltodict jsondiff pandas
- name: Get fastrtps.repos file
uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@v0
with:
source_repository: eProsima/Fast-DDS
source_repository_branch: ${{ env.FASTDDS_BRANCH }}
file_name: fastrtps.repos
file_result: fastrtps.repos
- name: Fetch Fast DDS & dependencies
uses: eProsima/eProsima-CI/ubuntu/vcs_import@v0
with:
vcs_repos_file: fastrtps.repos
destination_workspace: src
- name: Checkout Fast DDS branch
run: |
cd ./src/fastrtps
git checkout ${{ env.FASTDDS_BRANCH }}
- name: Fetch Fast DDS CI dependencies
uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0
with:
vcs_repos_file: ${{ github.workspace }}/src/fastrtps/.github/workflows/config/ci.repos
destination_workspace: src
skip_existing: 'true'
- name: Build workspace
run: |
cat src/fastrtps/.github/workflows/config/tsan_colcon.meta
colcon build \
--event-handlers=console_direct+ \
--metas src/fastrtps/.github/workflows/config/tsan_colcon.meta \
--cmake-args -DCMAKE_BUILD_TYPE=Debug ${{ inputs.optional_cmake_args }}
- name: Run tests Fast DDS
run: |
source install/setup.bash && \
colcon test \
--packages-select fastrtps \
--event-handlers=console_direct+ \
--return-code-on-test-failure \
--ctest-args \
--timeout 30 \
-V -E DDSSQLFilterValueTests ${{ inputs.optional_ctest_args }}
continue-on-error: true
- name: Upload Logs
uses: actions/upload-artifact@v3
with:
name: tsan-logs
path: log/
if: always()
- name: Process sanitizer reports
id: report_summary
shell: pwsh
continue-on-error: true
run: |
# Create a dir for the exports
$exports = New-Item -ItemType Directory -Path ./exports
# Move to the reports dir
pushd ./log/latest_test/fastrtps
# Install the report parser module
Find-Module -Repository PSGallery -Name SanReportParser | Install-Module -Scope CurrentUser -Force
# Parse the report files
$rp = Show-Tsan -Path ./stdout_stderr.log
# filter duplicates
$rp = $rp | group md5hash | % { $_.group[0] }
# Export raw data
$rp | Export-CliXML (Join-Path $exports all_reports.xml)
# Group the reports by issue
$g = $rp | group fuzzhash
# Split up deadlocks and race reports
$gd = $rp | ? type -match dead | group fuzzhash
$gr = $rp | ? type -match race | group fuzzhash
# Simplified deadlock summary (only one representative report and tests associated)
$sd = $gd | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, `
@{l="échantillon";e={$_.group[0].report}}, @{l="tests"; `
e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}}
# Simplified race summary (only one representative report and tests associated)
$sr = $gr | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, `
@{l="échantillon";e={$_.group[0].report}}, @{l="tests"; `
e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}}
# Export simplified summaries
$sd, $sr | Export-Clixml (Join-Path $exports summary_data.xml)
# Export CSV summary of frequencies
$sd | select fuzzhash, count | Export-CSV -Path (Join-Path $exports deadlocks.csv)
$sr | select fuzzhash, count | Export-CSV -Path (Join-Path $exports races.csv)
# Export test mappings
& {$sd; $sr} | Sort-Object { [int]$_.fuzzhash} | % { $fuzzy=$_.fuzzhash;$_.tests |
select @{l="fuzzhash";e={$fuzzy}}, @{l="test";e={$_}}} |
Export-Csv (Join-Path $exports issue_test_map.csv)
# Keep a file per issue
$dir = New-Item -ItemType Directory -Path (Join-Path $exports reports)
& {$sd; $sr} | % { $_.échantillon | Out-File (Join-Path $dir "$($_.fuzzhash).tsan") }
# Create a summary table
@{Type="Deadlock";Failed=$sd.count;Hashes=$sd.fuzzhash},
@{Type="Data race";Failed=$sr.count;Hashes=$sr.fuzzhash} |
% { $_.Summary = $_.Hashes | select -First 5 | Join-String -Separator ", "
if ($_.Hashes.count -gt 5 ) {
$_.Summary += ", ..." }; $_ } |
% { [PSCustomObject]$_} |
New-MDTable -Columns ([ordered]@{Failed=$null;Type=$null;Summary=$null}) |
Out-File $Env:GITHUB_STEP_SUMMARY
# The step fails on new deadlocks
$LASTEXITCODE=$sd.count
- name: Archive sanitation results
uses: actions/upload-artifact@v3
with:
name: sanitation-report
path: exports/*
- name: Check on failures
if: ${{ steps.report_summary.outcome == 'failure' }}
shell: pwsh
run: |
Write-Host ${{ steps.report_summary.outcome }}
exit 1