Sampling service (#302)

* sampling_daemon

* Added correct dependencies.

* sampling daemon synched

* updated sampling daemon

* comments

* minor changes

* PR comments addressed

* ze make updated

---------

Co-authored-by: Solomon Bekele <[email protected]>
Co-authored-by: Brice Videau <[email protected]>
Co-authored-by: sbekele <[email protected]>

xprof/xprof.rb.in

@@ -368,7 +368,7 @@ end
 
 def sampling?
   return false unless OPTIONS[:sample]
-  env_fetch_first('LTTNG_UST_SAMPLING_MASTER_ONLY', default: '1') == '0' || mpi_local_master?
+  mpi_local_master?
 end
 
 def env_tracers
@@ -732,9 +732,51 @@ def gm_rename_folder
   thapi_trace_dir_root
 end
 
+class SamplingDaemon
+  SIGRTMIN = 34
+  SIG_SAMPLING_READY = SIGRTMIN
+  SIG_SAMPLING_FINISH = SIGRTMIN + 1
+
+  attr_reader :pid
+
+  def initialize
+    @pid = nil
+  end
+
+  def start(parent_pid)
+    return unless sampling?
+
+    daemon_path = "#{__dir__}/sampling_daemon"
+    raise "No sampling_daemon binary found at #{daemon_path}" unless File.exist?(daemon_path)
+
+    @pid = spawn("#{daemon_path} #{parent_pid}")
+    Process.detach(@pid)
+
+    wait_for_ready_signal
+  end
+
+  def finalize
+    return unless @pid
+
+    Process.kill(SIG_SAMPLING_FINISH, @pid)
+
+    wait_for_ready_signal
+  end
+
+  private
+
+  def wait_for_ready_signal
+    received_ready = false
+    Signal.trap(SIG_SAMPLING_READY) do
+      received_ready = true
+    end
+    sleep(0.1) until received_ready # Wait until READY signal is received
+  end
+end
+
 # Start, Stop lttng, amd do the on-node analsysis
 def trace_and_on_node_processing(usr_argv)
-  def teardown_lttng(syncd, pids)
+  def teardown_lttng(syncd, sampling_daemon, pids)
     # We need to be sure that all the local ranks are finished
     syncd.local_barrier('waiting_for_application_ending')
 
@@ -754,9 +796,11 @@ def trace_and_on_node_processing(usr_argv)
     end
     # we can kill the session daemon
     lm_lttng_kill_sessiond
+    sampling_daemon&.finalize
   end
 
   SyncDaemon.open do |syncd|
+    sampling_daemon = nil
     # Load Tracers and APILoaders Lib
     backends, h = env_tracers
 
@@ -767,19 +811,24 @@ def trace_and_on_node_processing(usr_argv)
     pids = if mpi_local_master?
              lm_setup_lttng(backends)
              lm_babeltrace(backends) if OPTIONS[:archive]
-            end
-
+           end
+
+    if sampling?
+      sampling_daemon = SamplingDaemon.new
+      sampling_daemon&.start(Process.pid)
+    end
+
     syncd.local_barrier('waiting_for_lttng_setup')
 
     # Launch User Command
     begin
       XprofExitCode.update(launch_usr_bin(h, usr_argv), usr_argv.join(' '))
     rescue Errno::ENOENT
-      teardown_lttng(syncd, pids)
+      teardown_lttng(syncd, sampling_daemon, pids)
       raise
     end
 
-    teardown_lttng(syncd, pids)
+    teardown_lttng(syncd, sampling_daemon, pids)
     return unless mpi_local_master?
 
     # Preprocess trace

ze/Makefile.am

@@ -106,6 +106,7 @@ EXTRA_DIST += \
 	ze_model.rb \
 	gen_babeltrace_ze_model.rb
 
+
 ZE_PROBES = $(ZE_NAMESPACES:=_tracepoints) $(ZE_STRUCTS_NAMESPACES:=_tracepoints)
 ZE_PROBES_TP = $(ZE_PROBES:=.tp)
 ZE_PROBES_INCL = $(ZE_PROBES:=.h)
@@ -148,14 +149,26 @@ BUILT_SOURCES = \
 	$(ZE_PROBES_INCL) \
 	$(ZE_STATIC_PROBES_INCL)
 
+bin_PROGRAMS = sampling_daemon
+
+sampling_daemon_SOURCES = sampling_daemon.c
+
+nodist_sampling_daemon_SOURCES = \
+	$(ZE_PROBES_INCL) \
+	$(ZE_STATIC_PROBES_INCL)
+
+sampling_daemon_CPPFLAGS = -I$(top_srcdir)/utils -I$(top_srcdir)/utils/include -I$(top_srcdir)/sampling -I$(top_srcdir)/ze/include -I./
+sampling_daemon_CFLAGS = -Wall -Wextra $(WERROR) $(LTTNG_UST_CFLAGS)
+sampling_daemon_LDADD = libzetracepoints.la -ldl -lpthread $(LTTNG_UST_LIBS) ../sampling/libThapiSampling.la
+
 tracer_ze.c: $(srcdir)/gen_ze.rb $(srcdir)/tracer_ze_helpers.include.c $(srcdir)/ze.h.include $(ZE_MODEL) $(ZE_PROBES_INCL) $(ZE_STATIC_PROBES_INCL)
 	SRC_DIR=$(srcdir) $(RUBY) $< > $@
 
 EXTRA_DIST += \
 	gen_ze.rb \
 	tracer_ze_helpers.include.c
 
-CLEANFILES += tracer_ze.c
+CLEANFILES += tracer_ze.c sampling_daemon
 
 bin_SCRIPTS = \
 	tracer_ze.sh