Merge pull request #1683 from multics69/lavd-pelt2

scx_lavd: Calculate scaled and invariant CPU utilization.

Author: multics69

scheds/rust/scx_lavd/src/bpf/intf.h

@@ -64,7 +64,9 @@ enum {
  */
 struct sys_stat {
 	u64	last_update_clk;
-	u64	util;		/* average of the CPU utilization */
+	u64	avg_util;	/* average of the CPU utilization */
+	u64	avg_sc_util;	/* average of the scaled CPU utilization,
+				   which is capacity and frequency invariant */
 
 	u64	avg_svc_time;	/* average service time per task */
 	u64	nr_queued_task;
@@ -112,7 +114,7 @@ struct task_ctx {
 	u64	run_freq;		/* scheduling frequency in a second */
 	u64	wait_freq;		/* waiting frequency in a second */
 	u64	wake_freq;		/* waking-up frequency in a second */
-	u64	svc_time;		/* total CPU time consumed for this task */
+	u64	svc_time;		/* total CPU time consumed for this task scaled by task's weight */
 	u64	dsq_id;			/* DSQ id where a task run for statistics */
 
 	/*
@@ -143,6 +145,7 @@ struct task_ctx_x {
 	u16	static_prio;	/* nice priority */
 	u32	cpu_id;		/* where a task ran */
 	u64	cpu_util;	/* cpu utilization in [0..100] */
+	u64	cpu_sutil;	/* scaled cpu utilization in [0..100] */
 	u32	thr_perf_cri;	/* performance criticality threshold */
 	u32	avg_lat_cri;	/* average latency criticality */
 	u32	nr_active;	/* number of active cores */

scheds/rust/scx_lavd/src/bpf/introspec.bpf.c

@@ -37,6 +37,7 @@ int submit_task_ctx(struct task_struct *p, struct task_ctx *taskc, u32 cpu_id)
 	__builtin_memcpy_inline(m->taskc_x.comm, p->comm, TASK_COMM_LEN);
 	m->taskc_x.static_prio = get_nice_prio(p);
 	m->taskc_x.cpu_util = s2p(cpuc->avg_util);
+	m->taskc_x.cpu_sutil = s2p(cpuc->avg_sc_util);
 	m->taskc_x.cpu_id = cpu_id;
 	m->taskc_x.avg_lat_cri = sys_stat.avg_lat_cri;
 	m->taskc_x.thr_perf_cri = sys_stat.thr_perf_cri;

scheds/rust/scx_lavd/src/bpf/lavd.bpf.h

@@ -96,13 +96,17 @@ struct cpu_ctx {
 	 */
 	volatile u32	avg_util;	/* average of the CPU utilization */
 	volatile u32	cur_util;	/* CPU utilization of the current interval */
+	volatile u32	avg_sc_util;	/* average of the scaled CPU utilization, which is capacity and frequency invariant. */
+	volatile u32	cur_sc_util;	/* the scaled CPU utilization of the current interval, which is capacity and frequency invariant. */
 	volatile u64	idle_total;	/* total idle time so far */
 	volatile u64	idle_start_clk;	/* when the CPU becomes idle */
 
 	/*
 	 * Information used to keep track of load
 	 */
-	volatile u64	tot_svc_time;	/* total service time on a CPU */
+	volatile u64	tot_svc_time;	/* total service time on a CPU scaled by tasks' weights */
+	volatile u64	tot_sc_time;	/* total scaled CPU time, which is capacity and frequency invariant. */
+	volatile u64	cpu_release_clk; /* when the CPU is taken by higher-priority scheduler class */
 
 	/*
 	 * Information used to keep track of latency criticality

scheds/rust/scx_lavd/src/bpf/main.bpf.c

@@ -673,6 +673,12 @@ static void update_stat_for_stopping(struct task_struct *p,
 	if (READ_ONCE(cur_svc_time) < taskc->svc_time)
 		WRITE_ONCE(cur_svc_time, taskc->svc_time);
 
+	/*
+	 * Increase total scaled CPU time of this CPU,
+	 * whcih is capacity and frequency invariant.
+	 */
+	cpuc->tot_sc_time += scale_cap_freq(task_runtime, cpuc->cpu_id);
+
 	/*
 	 * Reset task's lock and futex boost count
 	 * for a lock holder to be boosted only once.
@@ -1317,20 +1323,52 @@ void BPF_STRUCT_OPS(lavd_set_cpumask, struct task_struct *p,
 	set_on_core_type(taskc, cpumask);
 }
 
-void BPF_STRUCT_OPS(lavd_cpu_release, s32 cpu,
-		    struct scx_cpu_release_args *args)
+void BPF_STRUCT_OPS(lavd_cpu_acquire, s32 cpu,
+		    struct scx_cpu_acquire_args *args)
 {
 	struct cpu_ctx *cpuc;
+	u64 dur, scaled_dur;
+
+	cpuc = get_cpu_ctx_id(cpu);
+	if (!cpuc) {
+		scx_bpf_error("Failed to lookup cpu_ctx %d", cpu);
+		return;
+	}
 
 	/*
-	 * When a CPU is released to serve higher priority scheduler class,
-	 * reset the CPU's preemption information so it cannot be a victim.
+	 * When regaining control of a CPU under the higher priority scheduler
+	 * class, measure how much time the higher priority scheduler class
+	 * used -- i.e., [lavd_cpu_release, lavd_cpu_acquire]. This will be
+	 * used to calculate capacity-invariant and frequency-invariant CPU
+	 * utilization.
+	 */
+	dur = time_delta(scx_bpf_now(), cpuc->cpu_release_clk);
+	scaled_dur = scale_cap_freq(dur, cpu);
+	cpuc->tot_sc_time += scaled_dur;
+
+	/*
+	 * The higher-priority scheduler class could change the CPU frequency,
+	 * so let's keep track of the frequency when we gain the CPU control.
+	 * This helps to make the frequency update decision.
 	 */
+	cpuc->cpuperf_cur = scx_bpf_cpuperf_cur(cpu);
+}
+
+void BPF_STRUCT_OPS(lavd_cpu_release, s32 cpu,
+		    struct scx_cpu_release_args *args)
+{
+	struct cpu_ctx *cpuc;
+
 	cpuc = get_cpu_ctx_id(cpu);
 	if (!cpuc) {
 		scx_bpf_error("Failed to lookup cpu_ctx %d", cpu);
 		return;
 	}
+
+	/*
+	 * When a CPU is released to serve higher priority scheduler class,
+	 * reset the CPU's preemption information so it cannot be a victim.
+	 */
 	reset_cpu_preemption_info(cpuc, true);
 
 	/*
@@ -1343,6 +1381,13 @@ void BPF_STRUCT_OPS(lavd_cpu_release, s32 cpu,
 	 * the target properly after regaining the control.
 	 */
 	reset_cpuperf_target(cpuc);
+
+	/*
+	 * Keep track of when the higher-priority scheduler class takes
+	 * the CPUto calculate capacity-invariant and frequency-invariant
+	 * CPU utilization.
+	 */
+	cpuc->cpu_release_clk = scx_bpf_now();
 }
 
 void BPF_STRUCT_OPS(lavd_enable, struct task_struct *p)
@@ -1566,6 +1611,7 @@ static s32 init_per_cpu_ctx(u64 now)
 		cpuc->stopping_tm_est_ns = SCX_SLICE_INF;
 		cpuc->online_clk = now;
 		cpuc->offline_clk = now;
+		cpuc->cpu_release_clk = now;
 		cpuc->is_online = bpf_cpumask_test_cpu(cpu, online_cpumask);
 		cpuc->capacity = get_cpuperf_cap(cpu);
 		cpuc->cpdom_poll_pos = cpu % LAVD_CPDOM_MAX_NR;
@@ -1745,6 +1791,7 @@ SCX_OPS_DEFINE(lavd_ops,
 	       .cpu_offline		= (void *)lavd_cpu_offline,
 	       .update_idle		= (void *)lavd_update_idle,
 	       .set_cpumask		= (void *)lavd_set_cpumask,
+	       .cpu_acquire		= (void *)lavd_cpu_acquire,
 	       .cpu_release		= (void *)lavd_cpu_release,
 	       .enable			= (void *)lavd_enable,
 	       .init_task		= (void *)lavd_init_task,

scheds/rust/scx_lavd/src/bpf/power.bpf.c

@@ -60,7 +60,7 @@ static u64 calc_nr_active_cpus(void)
 	/*
 	 * nr_active = ceil(nr_cpus_onln * cpu_util * per_core_max_util)
 	 */
-	nr_active  = ((nr_cpus_onln * sys_stat.util) << LAVD_SHIFT) + p2s(50);
+	nr_active  = ((nr_cpus_onln * sys_stat.avg_util) << LAVD_SHIFT) + p2s(50);
 	nr_active /= (LAVD_CC_PER_CORE_MAX_CTUIL << LAVD_SHIFT);
 
 	/*
@@ -299,23 +299,23 @@ static int do_autopilot(void)
 	 * performance is not required. We run the scheduler in powersave mode
 	 * to save energy consumption.
 	 */
-	if (sys_stat.util <= LAVD_AP_LOW_UTIL)
-		return do_set_power_profile(LAVD_PM_POWERSAVE, sys_stat.util);
+	if (sys_stat.avg_util <= LAVD_AP_LOW_UTIL)
+		return do_set_power_profile(LAVD_PM_POWERSAVE, sys_stat.avg_util);
 
 	/*
 	 * If the CPU utiulization is moderate (say > 5%, <= 30%), we run the
 	 * scheduler in balanced mode. Actually, balanced mode can save energy
 	 * consumption only under moderate CPU load.
 	 */
-	if (sys_stat.util <= LAVD_AP_HIGH_UTIL)
-		return do_set_power_profile(LAVD_PM_BALANCED, sys_stat.util);
+	if (sys_stat.avg_util <= LAVD_AP_HIGH_UTIL)
+		return do_set_power_profile(LAVD_PM_BALANCED, sys_stat.avg_util);
 
 	/*
 	 * If the CPU utilization is high enough (say > 30%), we run the
 	 * scheduler in performance mode. The system indeed needs perrformance
 	 * also there is little energy benefit even under balanced mode anyway.
 	 */
-	return do_set_power_profile(LAVD_PM_PERFORMANCE, sys_stat.util);
+	return do_set_power_profile(LAVD_PM_PERFORMANCE, sys_stat.avg_util);
 }
 
 static void update_thr_perf_cri(void)
@@ -521,6 +521,21 @@ static u16 get_cputurbo_cap(void)
 	return turbo_cap;
 }
 
+static u64 scale_cap_freq(u64 dur, s32 cpu)
+{
+	u64 cap, freq, scaled_dur;
+
+	/*
+	 * Scale the duration by CPU capacity and frequency, so calculate
+	 * capacity-invariant and frequency-invariant time duration.
+	 */
+	cap = get_cpuperf_cap(cpu);
+	freq = scx_bpf_cpuperf_cur(cpu);
+	scaled_dur = (dur * cap * freq) / (LAVD_SCALE * LAVD_SCALE);
+
+	return scaled_dur;
+}
+
 static void init_autopilot_low_util(void)
 {
 	if (nr_cpus_big < nr_cpus_onln) {

scheds/rust/scx_lavd/src/bpf/sys_stat.bpf.c

@@ -29,6 +29,7 @@ struct sys_stat_ctx {
 	u64		idle_total;
 	u64		compute_total;
 	u64		tot_svc_time;
+	u64		tot_sc_time;
 	u64		nr_queued_task;
 	s32		max_lat_cri;
 	s32		avg_lat_cri;
@@ -47,6 +48,7 @@ struct sys_stat_ctx {
 	u64		sum_perf_cri;
 	u32		thr_perf_cri;
 	u32		cur_util;
+	u32		cur_sc_util;
 	u32		nr_violation;
 };
 
@@ -134,6 +136,20 @@ static void collect_sys_stat(struct sys_stat_ctx *c)
 		c->tot_svc_time += cpuc->tot_svc_time;
 		cpuc->tot_svc_time = 0;
 
+		/*
+		 * Update scaled CPU utilization,
+		 * which is capacity and frequency invariant.
+		 */
+		cpuc->cur_sc_util = (cpuc->tot_sc_time << LAVD_SHIFT) / c->duration;
+		cpuc->avg_sc_util = calc_avg(cpuc->avg_sc_util, cpuc->cur_sc_util);
+
+		/*
+		 * Accumulate cpus' scaled loads,
+		 * whcih is capacity and frequency invariant.
+		 */
+		c->tot_sc_time += cpuc->tot_sc_time;
+		cpuc->tot_sc_time = 0;
+
 		/*
 		 * Accumulate statistics.
 		 */
@@ -208,7 +224,7 @@ static void collect_sys_stat(struct sys_stat_ctx *c)
 		if (c->duration > cpuc->idle_total)
 			compute = c->duration - cpuc->idle_total;
 
-		cpuc->cur_util = (compute * LAVD_SCALE) / c->duration;
+		cpuc->cur_util = (compute << LAVD_SHIFT) / c->duration;
 		cpuc->avg_util = calc_avg(cpuc->avg_util, cpuc->cur_util);
 
 		if (cpuc->turbo_core) {
@@ -247,7 +263,8 @@ static void calc_sys_stat(struct sys_stat_ctx *c)
 		c->compute_total = c->duration_total - c->idle_total;
 	else
 		c->compute_total = 0;
-	c->cur_util = (c->compute_total * LAVD_SCALE) / c->duration_total;
+	c->cur_util = (c->compute_total << LAVD_SHIFT) / c->duration_total;
+	c->cur_sc_util = (c->tot_sc_time << LAVD_SHIFT) / c->duration_total;
 
 	if (c->nr_sched == 0) {
 		/*
@@ -272,7 +289,8 @@ static void calc_sys_stat(struct sys_stat_ctx *c)
 	/*
 	 * Update the CPU utilization to the next version.
 	 */
-	sys_stat.util = calc_avg(sys_stat.util, c->cur_util);
+	sys_stat.avg_util = calc_avg(sys_stat.avg_util, c->cur_util);
+	sys_stat.avg_sc_util = calc_avg(sys_stat.avg_sc_util, c->cur_sc_util);
 	sys_stat.max_lat_cri = calc_avg32(sys_stat.max_lat_cri, c->max_lat_cri);
 	sys_stat.avg_lat_cri = calc_avg32(sys_stat.avg_lat_cri, c->avg_lat_cri);
 	sys_stat.thr_lat_cri = sys_stat.max_lat_cri - ((sys_stat.max_lat_cri -

scheds/rust/scx_lavd/src/main.rs

@@ -733,6 +733,7 @@ impl<'a> Scheduler<'a> {
             thr_perf_cri: tx.thr_perf_cri,
             cpuperf_cur: tx.cpuperf_cur,
             cpu_util: tx.cpu_util,
+            cpu_sutil: tx.cpu_sutil,
             nr_active: tx.nr_active,
         }) {
             Ok(()) | Err(TrySendError::Full(_)) => 0,

scheds/rust/scx_lavd/src/stats.rs

@@ -157,8 +157,10 @@ pub struct SchedSample {
     pub thr_perf_cri: u32,
     #[stat(desc = "Target performance level of this CPU")]
     pub cpuperf_cur: u32,
-    #[stat(desc = "CPU utilization of this particular CPU")]
+    #[stat(desc = "CPU utilization of this CPU")]
     pub cpu_util: u64,
+    #[stat(desc = "Scaled CPU utilization of this CPU")]
+    pub cpu_sutil: u64,
     #[stat(desc = "Number of active CPUs when core compaction is enabled")]
     pub nr_active: u32,
 }
@@ -167,7 +169,7 @@ impl SchedSample {
     pub fn format_header<W: Write>(w: &mut W) -> Result<()> {
         writeln!(
             w,
-            "\x1b[93m| {:6} | {:7} | {:17} | {:5} | {:4} | {:8} | {:8} | {:7} | {:8} | {:7} | {:9} | {:9} | {:9} | {:9} | {:8} | {:8} | {:8} | {:8} | {:6} |\x1b[0m",
+            "\x1b[93m| {:6} | {:7} | {:17} | {:5} | {:4} | {:8} | {:8} | {:7} | {:8} | {:7} | {:9} | {:9} | {:9} | {:9} | {:8} | {:8} | {:8} | {:8} | {:8} | {:6} |\x1b[0m",
             "MSEQ",
             "PID",
             "COMM",
@@ -186,6 +188,7 @@ impl SchedSample {
             "THR_PC",
             "CPUFREQ",
             "CPU_UTIL",
+            "CPU_SUTIL",
             "NR_ACT",
         )?;
         Ok(())
@@ -198,7 +201,7 @@ impl SchedSample {
 
         writeln!(
             w,
-            "| {:6} | {:7} | {:17} | {:5} | {:4} | {:8} | {:8} | {:7} | {:8} | {:7} | {:9} | {:9} | {:9} | {:9} | {:8} | {:8} | {:8} | {:8} | {:6} |",
+            "| {:6} | {:7} | {:17} | {:5} | {:4} | {:8} | {:8} | {:7} | {:8} | {:7} | {:9} | {:9} | {:9} | {:9} | {:8} | {:8} | {:8} | {:8} | {:8} | {:6} |",
             self.mseq,
             self.pid,
             self.comm,
@@ -217,6 +220,7 @@ impl SchedSample {
             self.thr_perf_cri,
             self.cpuperf_cur,
             self.cpu_util,
+            self.cpu_sutil,
             self.nr_active,
         )?;
         Ok(())