scx_rusty: set up AMD IBS performance counter

Author: etsal

scheds/rust/scx_rusty/src/bpf/main.bpf.c

@@ -198,6 +198,7 @@ struct {
 	__uint(map_flags, 0);
 } node_data SEC(".maps");
 
+
 struct lock_wrapper {
 	struct bpf_spin_lock lock;
 };
@@ -572,16 +573,45 @@ static void refresh_tune_params(void)
 	}
 }
 
+void *bpf_cast_to_kern_ctx(void *) __ksym;
+
 /*
  * Performance counter callback.
  */
-
 SEC("perf_event")
-int drain_counters(void *ctx)
+int read_sample(struct bpf_perf_event_data_kern __kptr *arg)
 {
-	scx_bpf_error("HIT");
-	bpf_printk("Callback detected");
-	return 1;
+	struct bpf_perf_event_data_kern *ctx, a;
+	struct perf_sample_data data;
+	union perf_mem_data_src data_src;
+	int ret;
+
+	ctx = bpf_cast_to_kern_ctx(arg);
+
+	if ((ret = bpf_probe_read_kernel(&a, sizeof(a), ctx))) {
+		scx_bpf_error("[0] %s: bpf_probe_read_kernel failed", __func__);
+		return -EACCES;
+	}
+
+	if ((ret = bpf_probe_read_kernel(&data, sizeof(data), a.data))) {
+		scx_bpf_error("%s: bpf_probe_read_kernel failed", __func__);
+		return -EACCES;
+	}
+
+	data_src = ctx->data->data_src;
+	if (!ctx->data->sample_flags || data_src.mem_op == 1)
+		return 0;
+
+	bpf_printk("%s\t(0x%lx,0x%lx,0x%lx) [%llx, %llx]",
+			data_src.mem_op == 2 ? "STORE" : (data_src.mem_op == 4 ? "LOAD" : "UNKNOWN") ,
+			data_src.mem_lvl_num,
+			data_src.mem_snoop,
+			data_src.mem_remote,
+			ctx->data->phys_addr,
+			ctx->data->addr
+			);
+
+	return 0;
 }
 
 static u64 min(u64 a, u64 b)

scheds/rust/scx_rusty/src/main.rs

@@ -351,6 +351,7 @@ struct Scheduler<'a> {
 
     tuner: Tuner,
     stats_server: StatsServer<StatsCtx, (StatsCtx, ClusterStats)>,
+    _pefds: Vec<(i32, libbpf_rs::Link)>,
 }
 
 impl<'a> Scheduler<'a> {
@@ -448,10 +449,13 @@ impl<'a> Scheduler<'a> {
         // Attach.
         let mut skel = scx_ops_load!(skel, rusty, uei)?;
 
-        let (pefd, _link) = init_perf_counters(&mut skel, &0)?;
-        println!("Got perf file descriptor {}", pefd);
+        let mut pefds: Vec<(i32, libbpf_rs::Link)> = vec![];
+        for i in 0..32 {
+            pefds.push(init_perf_counters(&mut skel, &i)?);
+        }
 
         let struct_ops = Some(scx_ops_attach!(skel, rusty)?);
+
         let stats_server = StatsServer::new(stats::server_data()).launch()?;
 
         for (id, dom) in domains.doms().iter() {
@@ -489,6 +493,7 @@ impl<'a> Scheduler<'a> {
                 opts.slice_us_overutil * 1000,
             )?,
             stats_server,
+            _pefds: pefds,
         })
     }
 

scheds/rust/scx_rusty/src/perf.rs

@@ -10,18 +10,83 @@ use std::mem;
 
 // Expanded from systing's code: www.github.com/josefbacik/systing
 
-const PERF_TYPE_HARDWARE: u32 = 0x0;
-const PERF_TYPE_SOFTWARE: u32 = 0x1;
-const PERF_TYPE_RAW: u32 = 0x3;
-const PERF_TYPE_IBS: u32 = 0xb;
-
-const PERF_COUNT_HW_CPU_CYCLES: u64 = 0;
-const PERF_COUNT_HW_CACHE_REFERENCES: u64 = 2;
-const PERF_COUNT_HW_CACHE_MISSES: u64 = 3;
-const PERF_COUNT_HW_STALLED_CYCLES_FRONTEND: u64 = 7;
-const PERF_COUNT_HW_STALLED_CYCLES_BACKEND: u64 = 8;
-
-const PERF_COUNT_SW_CPU_CLOCK: u64 = 0;
+const _PERF_TYPE_HARDWARE: u32 = 0x0;
+const _PERF_TYPE_SOFTWARE: u32 = 0x1;
+const _PERF_TYPE_RAW: u32 = 0x3;
+const _PERF_TYPE_AMD_IBS: u32 = 0xb;
+
+const _PERF_COUNT_HW_CPU_CYCLES: u64 = 0;
+const _PERF_COUNT_HW_CACHE_REFERENCES: u64 = 2;
+const _PERF_COUNT_HW_CACHE_MISSES: u64 = 3;
+const _PERF_COUNT_HW_STALLED_CYCLES_FRONTEND: u64 = 7;
+const _PERF_COUNT_HW_STALLED_CYCLES_BACKEND: u64 = 8;
+
+const _PERF_COUNT_SW_CPU_CLOCK: u64 = 0;
+
+// WARNING: These are not guaranteed to be correct because the layout of the bitfield
+// in the perf_sample_attr C struct that contains them is not guaranteed by the C standard.
+const _PERF_SAMPLE_FLAG_DISABLED: u64 = 1 << 0;
+const _PERF_SAMPLE_FLAG_INHERIT: u64 = 1 << 1;
+const _PERF_SAMPLE_FLAG_PINNED: u64 = 1 << 2;
+const _PERF_SAMPLE_FLAG_EXCLUSIVE: u64 = 1 << 3;
+const _PERF_SAMPLE_FLAG_EXCLUDE_USER: u64 = 1 << 4;
+const _PERF_SAMPLE_FLAG_EXCLUDE_KERNEL: u64 = 1 << 5;
+const _PERF_SAMPLE_FLAG_EXCLUDE_HV: u64 = 1 << 6;
+const _PERF_SAMPLE_FLAG_EXCLUDE_IDLE: u64 = 1 << 7;
+const _PERF_SAMPLE_FLAG_MMAP: u64 = 1 << 8;
+const _PERF_SAMPLE_FLAG_COMM: u64 = 1 << 9;
+const _PERF_SAMPLE_FLAG_FREQ: u64 = 1 << 10;
+const _PERF_SAMPLE_FLAG_INHERIT_STAT: u64 = 1 << 11;
+const _PERF_SAMPLE_FLAG_ENABLE_ON_EXEC: u64 = 1 << 12;
+const _PERF_SAMPLE_FLAG_TASK: u64 = 1 << 13;
+const _PERF_SAMPLE_FLAG_WATERMARK: u64 = 1 << 14;
+const _PERF_SAMPLE_FLAG_PRECISE_IP: u64 = 1 << 15;
+const _PERF_SAMPLE_FLAG_MMAP_DATA: u64 = 1 << 17;
+const _PERF_SAMPLE_FLAG_ID_ALL: u64 = 1 << 18;
+const _PERF_SAMPLE_FLAG_EXCLUDE_HOST: u64 = 1 << 19;
+const _PERF_SAMPLE_FLAG_EXCLUDE_GUEST: u64 = 1 << 20;
+const _PERF_SAMPLE_FLAG_EXCLUDE_CALLCHAIN_KERNEL: u64 = 1 << 21;
+const _PERF_SAMPLE_FLAG_EXCLUDE_CALLCHAIN_USER: u64 = 1 << 22;
+const _PERF_SAMPLE_FLAG_MMAP2: u64 = 1 << 23;
+const _PERF_SAMPLE_FLAG_COMM_EXEC: u64 = 1 << 24;
+const _PERF_SAMPLE_FLAG_USE_CLOCKID: u64 = 1 << 25;
+const _PERF_SAMPLE_FLAG_WRITE_BACKWARD: u64 = 1 << 26;
+const _PERF_SAMPLE_FLAG_NAMESPACES: u64 = 1 << 27;
+const _PERF_SAMPLE_FLAG_KSYMBOL: u64 = 1 << 28;
+const _PERF_SAMPLE_FLAG_BPF_SYMBOL: u64 = 1 << 29;
+const _PERF_SAMPLE_FLAG_AUX_OUTPUT: u64 = 1 << 30;
+const _PERF_SAMPLE_FLAG_CGROUP: u64 = 1 << 31;
+const _PERF_SAMPLE_FLAG_TEXT_POKE: u64 = 1 << 32;
+const _PERF_SAMPLE_FLAG_BUILD_ID: u64 = 1 << 33;
+const _PERF_SAMPLE_FLAG_INHERIT_THREAD: u64 = 1 << 34;
+const _PERF_SAMPLE_FLAG_REMOVE_ON_EXEC: u64 = 1 << 35;
+const _PERF_SAMPLE_FLAG_SIGTRAP: u64 = 1 << 36;
+
+const _PERF_SAMPLE_IP: u64 = 1 << 0;
+const _PERF_SAMPLE_TID: u64 = 1 << 1;
+const _PERF_SAMPLE_TIME: u64 = 1 << 2;
+const _PERF_SAMPLE_ADDR: u64 = 1 << 3;
+const _PERF_SAMPLE_READ: u64 = 1 << 4;
+const _PERF_SAMPLE_CALLCHAIN: u64 = 1 << 5;
+const _PERF_SAMPLE_ID: u64 = 1 << 6;
+const _PERF_SAMPLE_CPU: u64 = 1 << 7;
+const _PERF_SAMPLE_PERIOD: u64 = 1 << 8;
+const _PERF_SAMPLE_STREAM_ID: u64 = 1 << 9;
+const _PERF_SAMPLE_RAW: u64 = 1 << 10;
+const _PERF_SAMPLE_BRANCH_STACK: u64 = 1 << 11;
+const _PERF_SAMPLE_REGS_USER: u64 = 1 << 12;
+const _PERF_SAMPLE_STACK_USER: u64 = 1 << 13;
+const _PERF_SAMPLE_WEIGHT: u64 = 1 << 14;
+const _PERF_SAMPLE_DATA_SRC: u64 = 1 << 15;
+const _PERF_SAMPLE_IDENTIFIER: u64 = 1 << 16;
+const _PERF_SAMPLE_TRANSACTION: u64 = 1 << 17;
+const _PERF_SAMPLE_REGS_INTR: u64 = 1 << 18;
+const _PERF_SAMPLE_PHYS_ADDR: u64 = 1 << 19;
+const _PERF_SAMPLE_PHYS_AUX: u64 = 1 << 20;
+const _PERF_SAMPLE_PHYS_CGROUP: u64 = 1 << 21;
+const _PERF_SAMPLE_DATA_PAGE_SIZE: u64 = 1 << 22;
+const _PERF_SAMPLE_CODE_PAGE_SIZE: u64 = 1 << 23;
+const _PERF_SAMPLE_WEIGHT_STRUCT: u64 = 1 << 24;
 
 #[repr(C)]
 union sample_un {
@@ -141,19 +206,24 @@ pub fn init_perf_counters(skel: &mut BpfSkel, cpu: &i32) -> Result<(i32, libbpf_
         )
     };
 
-    attr._type = PERF_TYPE_HARDWARE;
+    /* 
+     * XXX Discover the counter instead of hardcoding it.
+     * Afterwards we can use any counter we care for.
+     */
+    attr._type = _PERF_TYPE_AMD_IBS;
     attr.size = mem::size_of::<perf_event_attr>() as u32;
-    attr.config = PERF_COUNT_HW_CPU_CYCLES;
+    attr.config = 0;
+    attr.sample_type = _PERF_SAMPLE_CPU | _PERF_SAMPLE_IP | _PERF_SAMPLE_TID | _PERF_SAMPLE_DATA_SRC | _PERF_SAMPLE_PHYS_ADDR | _PERF_SAMPLE_ADDR;
     attr.sample.sample_period = 1000;
-    attr.flags = 0;
+    attr.flags = 3 * _PERF_SAMPLE_FLAG_PRECISE_IP;
 
     let pefd = perf_event_open(attr.as_ref(), -1, *cpu, -1, 0) as i32;
     if pefd == -1 {
         let os_error = io::Error::last_os_error();
         return Err(libbpf_rs::Error::from(os_error));
     }
 
-    let link = skel.progs.drain_counters.attach_perf_event_with_opts(pefd);
+    let link = skel.progs.read_sample.attach_perf_event_with_opts(pefd);
 
     Ok((pefd, link?))
 }