Skip to content

Commit

Permalink
Support Metadata like Threadid (#40)
Browse files Browse the repository at this point in the history 
* Support Meta

* Add test that we enforce meta

* backwards compatibility: support non-meta profiles

* Add corner-cases tests for manually constructed profiles; ensure we catch every sample

Fix the bugs in handling those cases :)

* Bump to version 3.1.0: Support meta!

* Disable meta test for v1.6

---------

Co-authored-by: Nathan Daly <[email protected]>
  • Loading branch information
@vchuravy @NHDaly
vchuravy and NHDaly authored Nov 16, 2023
1 parent 6e9f56c commit 30785d3
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 25 deletions.
6 changes: 3 additions & 3 deletions Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "PProf"
uuid = "e4faabce-9ead-11e9-39d9-4379958e3056"
authors = ["Valentin Churavy <[email protected]>", "Nathan Daly <[email protected]>"]
version = "3.0.0"
version = "3.1.0"

[deps]
AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
Expand All @@ -17,18 +17,18 @@ pprof_jll = "cf2c5f97-e748-59fa-a03f-dda3c62118cb"

[compat]
AbstractTrees = "0.3, 0.4"
CodecZlib = "0.7"
EnumX = "1"
FlameGraphs = "0.2, 1"
OrderedCollections = "1.1"
ProgressMeter = "1.7"
ProtoBuf = "1"
julia = "1.6"
pprof_jll = "0.1, 1"
CodecZlib = "0.7"

[extras]
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[targets]
Expand Down
88 changes: 69 additions & 19 deletions src/PProf.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ clear

include(joinpath("..", "lib", "perftools", "perftools.jl"))

import .perftools.profiles: ValueType, Sample, Function, Location, Line
import .perftools.profiles: ValueType, Sample, Function,
Location, Line, Label
const PProfile = perftools.profiles.Profile

const proc = Ref{Union{Base.Process, Nothing}}(nothing)
Expand Down Expand Up @@ -96,14 +97,16 @@ function pprof(data::Union{Nothing, Vector{UInt}} = nothing,
keep_frames::Union{Nothing, AbstractString} = nothing,
ui_relative_percentages::Bool = true,
)
has_meta = false
if data === nothing
data = if isdefined(Profile, :has_meta)
copy(Profile.fetch(include_meta = false))
has_meta = true
copy(Profile.fetch(include_meta = true))
else
copy(Profile.fetch())
end
elseif isdefined(Profile, :has_meta) && Profile.has_meta(data)
data = Profile.strip_meta(data)
elseif isdefined(Profile, :has_meta)
has_meta = Profile.has_meta(data)
end
lookup = lidict
if lookup === nothing
Expand All @@ -122,6 +125,8 @@ function pprof(data::Union{Nothing, Vector{UInt}} = nothing,
enter!(string) = _enter!(string_table, string)
enter!(::Nothing) = _enter!(string_table, "nothing")
ValueType!(_type, unit) = ValueType(enter!(_type), enter!(unit))
Label!(key, value, unit) = Label(key = enter!(key), num = value, num_unit = enter!(unit))
Label!(key, value) = Label(key = enter!(key), str = enter!(string(value)))

# Setup:
enter!("") # NOTE: pprof requires first entry to be ""
Expand All @@ -136,35 +141,71 @@ function pprof(data::Union{Nothing, Vector{UInt}} = nothing,

sample_type = [
ValueType!("events", "count"), # Mandatory
ValueType!("stack_depth", "count")
]

period_type = ValueType!("cpu", "nanoseconds")
drop_frames = isnothing(drop_frames) ? 0 : enter!(drop_frames)
keep_frames = isnothing(keep_frames) ? 0 : enter!(keep_frames)
# start decoding backtraces
location_id = Vector{eltype(data)}()
lastwaszero = true

for ip in data
# ip == 0x0 is the sentinel value for finishing a backtrace, therefore finising a sample
if ip == 0
# All samples get the same value for CPU profiles.
value = [
1, # events
]

lastwaszero = true # (Legacy: used when has_meta = false)

# The Profile data buffer is a big array, with each sample appended one after the other.
# Each sample now looks like this:
# | ip | ip | ip | meta1 | meta2 | meta3 | meta4| 0x0 | 0x0 |
# We iterate backwards, starting from the end, so that we don't encounter the metadata
# and mistake it for more ip addresses. For each sample, we skip the zeros, consume the
# metadata, then continue scanning the ip addresses, and when we hit another end of a
# block, we finish the sample we just consumed.
idx = length(data)
meta = nothing
while idx > 0
# We handle the very first sample after the loop.
if has_meta && Profile.is_block_end(data, idx)
if meta !== nothing
# Finish last block
push!(samples, Sample(;location_id = reverse!(location_id), value = value, label = meta))
location_id = Vector{eltype(data)}()
end

# Consume all of the metadata entries in the buffer, and then position the IP
# at the idx for the actual ip.
thread_sleeping = data[idx - Profile.META_OFFSET_SLEEPSTATE] - 1 # "Sleeping" is recorded as 1 or 2, to avoid 0s, which indicate end-of-block.
cpu_cycle_clock = data[idx - Profile.META_OFFSET_CPUCYCLECLOCK]
taskid = data[idx - Profile.META_OFFSET_TASKID]
threadid = data[idx - Profile.META_OFFSET_THREADID]

meta = Label[
Label!("thread_sleeping", thread_sleeping != 0),
Label!("cycle_clock", cpu_cycle_clock, "nanoseconds"),
Label!("taskid", taskid),
Label!("threadid", threadid),
]
idx -= (Profile.nmeta + 2) # skip all the metas, plus the 2 nulls that end a block.
continue
elseif !has_meta && data[idx] == 0
# Avoid creating empty samples
# ip == 0x0 is the sentinel value for finishing a backtrace (when meta is disabled), therefore finising a sample
# On some platforms, we sometimes get two 0s in a row for some reason...
if lastwaszero
@assert length(location_id) == 0
continue
else
# Finish last block
push!(samples, Sample(;location_id = reverse!(location_id), value = value))
location_id = Vector{eltype(data)}()
lastwaszero = true
end

# End of sample
value = [
1, # events
length(location_id), # stack_depth
]
push!(samples, Sample(;location_id, value))
location_id = Vector{eltype(data)}()
lastwaszero = true
idx -= 1
continue
end
ip = data[idx]
idx -= 1
lastwaszero = false

# A backtrace consists of a set of IP (Instruction Pointers), each IP points
Expand Down Expand Up @@ -245,6 +286,15 @@ function pprof(data::Union{Nothing, Vector{UInt}} = nothing,
push!(location_id, ip)
end
end
if length(data) > 0
# Finish the very last sample
if has_meta
push!(samples, Sample(;location_id = reverse!(location_id), value = value, label = meta))
else
push!(samples, Sample(;location_id = reverse!(location_id), value = value))
end
location_id = Vector{eltype(data)}()
end

# If from_c=false funcs and locs should NOT contain C functions
prof = PProfile(
Expand Down
67 changes: 64 additions & 3 deletions test/PProf.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,44 @@ function load_prof_proto(file)
open(io->decode(ProtoDecoder(GzipDecompressorStream(io)), PProf.perftools.profiles.Profile), file, "r")
end

const HAS_META = isdefined(Profile, :has_meta)
@testset "Corner Cases" begin
@testset "non-meta profile" begin

@testset "0 sample profile" begin
prof = load_prof_proto(pprof(UInt64[], out=tempname(), web=false))
@test length(prof.sample) == 0
end
@testset "1 sample profile" begin
prof = load_prof_proto(pprof(UInt64[0xdeadbeef,0], out=tempname(), web=false))
@test length(prof.sample) == 1
end

@testset "2 sample, 1 location profile" begin
prof = load_prof_proto(pprof(UInt64[0xdeadbeef,0, 0xdeadbeef, 0], out=tempname(), web=false))
@test length(prof.sample) == 2
@test length(prof.location) == 1
end
end
if HAS_META
@testset "with-meta profile" begin
@testset "1 sample profile" begin
data = UInt64[0xdeadbeef, 1, 1, 1, 1, 0, 0]
prof = load_prof_proto(pprof(data, out=tempname(), web=false))
@test length(prof.sample) == 1
end

@testset "2 sample 1 location profile" begin
data = UInt64[0xdeadbeef, 1, 1, 1, 1, 0, 0, 0xdeadbeef, 1, 1, 1, 1, 0, 0]
prof = load_prof_proto(pprof(data, out=tempname(), web=false))
@test length(prof.sample) == 2
@test length(prof.location) == 1
end
end
end
end


@testset "with_c" begin
Profile.clear()

Expand All @@ -74,12 +112,34 @@ end
end
sleep(2)
end
for i in 1:2
@testset for i in 1:4
if i == 1
data = Profile.fetch()
if !HAS_META
continue
end
data = Profile.fetch(include_meta = true)
args = (data,)
elseif i == 2
if !HAS_META
continue
end
data,lidict = Profile.retrieve(include_meta = true)
args = (data, lidict)
elseif i == 3
# Ensure we are backwards compatible with older, non-meta profiles
if HAS_META
data = Profile.fetch(include_meta = false)
else
data = Profile.fetch()
end
args = (data,)
else
data,lidict = Profile.retrieve()
# Ensure we are backwards compatible with older, non-meta profiles
if HAS_META
data,lidict = Profile.retrieve(include_meta = false)
else
data,lidict = Profile.retrieve()
end
args = (data, lidict)
end

Expand Down Expand Up @@ -135,6 +195,7 @@ end

@testset "subprocess refresh" begin

PProf.kill()
@pprof foo(10000, 5, [])

current_proc = PProf.proc[]
Expand Down

2 comments on commit 30785d3

@NHDaly
Copy link
Member

@NHDaly NHDaly commented on 30785d3 Nov 16, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register()

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/95461

Tip: Release Notes

Did you know you can add release notes too? Just add markdown formatted text underneath the comment after the text
"Release notes:" and it will be added to the registry PR, and if TagBot is installed it will also be added to the
release that TagBot creates. i.e.

@JuliaRegistrator register

Release notes:

## Breaking changes

- blah

To add them here just re-invoke and the PR will be updated.

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v3.1.0 -m "<description of version>" 30785d319c05411a18d8ee234b4ba2180a4cd732
git push origin v3.1.0

Please sign in to comment.