Add target to runtime benchmarks

Author: Kobzol

collector/src/bin/collector.rs

@@ -121,14 +121,21 @@ struct RuntimeBenchmarkConfig {
     runtime_suite: BenchmarkSuite,
     filter: RuntimeBenchmarkFilter,
     iterations: u32,
+    target: Target,
 }
 
 impl RuntimeBenchmarkConfig {
-    fn new(suite: BenchmarkSuite, filter: RuntimeBenchmarkFilter, iterations: u32) -> Self {
+    fn new(
+        suite: BenchmarkSuite,
+        filter: RuntimeBenchmarkFilter,
+        iterations: u32,
+        target: Target,
+    ) -> Self {
         Self {
             runtime_suite: suite.filter(&filter),
             filter,
             iterations,
+            target,
         }
     }
 }
@@ -860,8 +867,9 @@ fn main_result() -> anyhow::Result<i32> {
             purge,
         } => {
             log_db(&db);
-            let toolchain =
-                get_local_toolchain_for_runtime_benchmarks(&local, &require_host_target_tuple())?;
+
+            let host_tuple = require_host_target_tuple();
+            let toolchain = get_local_toolchain_for_runtime_benchmarks(&local, &host_tuple)?;
             let pool = Pool::open(&db.db);
 
             let isolation_mode = if no_isolate {
@@ -899,6 +907,7 @@ fn main_result() -> anyhow::Result<i32> {
                 runtime_suite,
                 RuntimeBenchmarkFilter::new(local.exclude, local.include),
                 iterations,
+                Target::from_str(&host_tuple).expect("Found unexpected host target"),
             );
             rt.block_on(run_benchmarks(conn.as_mut(), shared, None, Some(config)))?;
             Ok(0)
@@ -1189,6 +1198,7 @@ fn main_result() -> anyhow::Result<i32> {
                             runtime_suite,
                             filter: RuntimeBenchmarkFilter::keep_all(),
                             iterations: DEFAULT_RUNTIME_ITERATIONS,
+                            target: Target::default(),
                         };
                         let shared = SharedBenchmarkConfig {
                             artifact_id,
@@ -1806,6 +1816,7 @@ async fn create_benchmark_configs(
             runtime_suite,
             filter: RuntimeBenchmarkFilter::keep_all(),
             iterations: DEFAULT_RUNTIME_ITERATIONS,
+            target: job.target().into(),
         })
     } else {
         None
@@ -2232,6 +2243,7 @@ async fn run_benchmarks(
             &collector,
             runtime.filter,
             runtime.iterations,
+            runtime.target,
         )
         .await
         .context("Runtime benchmarks failed")
@@ -2306,6 +2318,7 @@ async fn bench_published_artifact(
             runtime_suite,
             RuntimeBenchmarkFilter::keep_all(),
             DEFAULT_RUNTIME_ITERATIONS,
+            Target::default(),
         )),
     )
     .await

collector/src/runtime/mod.rs

@@ -20,6 +20,7 @@ use crate::{command_output, CollectorCtx};
 mod benchmark;
 mod profile;
 
+use crate::compile::benchmark::target::Target;
 pub use benchmark::RuntimeCompilationOpts;
 pub use profile::{profile_runtime, RuntimeProfiler};
 
@@ -35,6 +36,7 @@ pub async fn bench_runtime(
     collector: &CollectorCtx,
     filter: RuntimeBenchmarkFilter,
     iterations: u32,
+    target: Target,
 ) -> anyhow::Result<()> {
     let filtered = suite.filtered_benchmark_count(&filter);
     println!("Executing {filtered} benchmarks\n");
@@ -74,6 +76,7 @@ pub async fn bench_runtime(
                             tx.conn(),
                             collector.artifact_row_id,
                             &rustc_perf_version,
+                            target,
                             result,
                         )
                         .await;
@@ -126,13 +129,15 @@ async fn record_stats(
     conn: &dyn Connection,
     artifact_id: ArtifactIdNumber,
     rustc_perf_version: &str,
+    target: Target,
     result: BenchmarkResult,
 ) {
     async fn record<'a>(
         conn: &'a dyn Connection,
         artifact_id: ArtifactIdNumber,
         collection_id: CollectionId,
         result: &'a BenchmarkResult,
+        target: Target,
         value: Option<u64>,
         metric: &'a str,
     ) {
@@ -142,6 +147,7 @@ async fn record_stats(
                 artifact_id,
                 &result.name,
                 metric,
+                target.into(),
                 value as f64,
             )
             .await;
@@ -156,6 +162,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.instructions,
             "instructions:u",
         )
@@ -165,6 +172,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.cycles,
             "cycles:u",
         )
@@ -174,6 +182,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.branch_misses,
             "branch-misses",
         )
@@ -183,6 +192,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             stat.cache_misses,
             "cache-misses",
         )
@@ -192,6 +202,7 @@ async fn record_stats(
             artifact_id,
             collection_id,
             &result,
+            target,
             Some(stat.wall_time.as_nanos() as u64),
             "wall-time",
         )

database/src/bin/import-sqlite.rs

@@ -82,7 +82,7 @@ async fn main() {
             }
         }
 
-        for (&(benchmark, metric), id) in sqlite_idx.runtime_statistic_descriptions() {
+        for (&(benchmark, target, metric), id) in sqlite_idx.runtime_statistic_descriptions() {
             let stat = sqlite_conn
                 .get_runtime_pstats(&[id], &[Some(sqlite_aid)])
                 .await
@@ -92,7 +92,14 @@ async fn main() {
                 .unwrap();
             if let Some(stat) = stat {
                 postgres_conn
-                    .record_runtime_statistic(cid, postgres_aid, &benchmark, metric.as_str(), stat)
+                    .record_runtime_statistic(
+                        cid,
+                        postgres_aid,
+                        &benchmark,
+                        metric.as_str(),
+                        target,
+                        stat,
+                    )
                     .await;
             }
         }

database/src/lib.rs

@@ -522,7 +522,7 @@ pub struct Index {
     /// For legacy reasons called `pstat_series` in the database, and so the name is kept here.
     pstat_series: Indexed<(Benchmark, Profile, Scenario, CodegenBackend, Target, Metric)>,
     /// Id lookup of runtime stat description ids
-    runtime_pstat_series: Indexed<(Benchmark, Metric)>,
+    runtime_pstat_series: Indexed<(Benchmark, Target, Metric)>,
 }
 
 /// An index lookup
@@ -725,7 +725,7 @@ impl Index {
         self.runtime_pstat_series
             .map
             .keys()
-            .map(|(_, metric)| metric)
+            .map(|(_, _, metric)| metric)
             .collect::<std::collections::HashSet<_>>()
             .into_iter()
             .map(|s| s.to_string())
@@ -752,7 +752,7 @@ impl Index {
 
     pub fn runtime_statistic_descriptions(
         &self,
-    ) -> impl Iterator<Item = (&(Benchmark, Metric), StatisticalDescriptionId)> + '_ {
+    ) -> impl Iterator<Item = (&(Benchmark, Target, Metric), StatisticalDescriptionId)> + '_ {
         self.runtime_pstat_series
             .map
             .iter()

database/src/pool.rs

@@ -64,6 +64,7 @@ pub trait Connection: Send + Sync {
         artifact: ArtifactIdNumber,
         benchmark: &str,
         metric: &str,
+        target: Target,
         value: f64,
     );
     /// Records a self-profile artifact in S3.

database/src/pool/postgres.rs

@@ -432,6 +432,12 @@ static MIGRATIONS: &[&str] = &[
             benchmark_set
         );
     "#,
+    // Add target to runtime_pstat_series
+    r#"
+    ALTER TABLE runtime_pstat_series ADD target TEXT NOT NULL DEFAULT 'x86_64-unknown-linux-gnu';
+    ALTER TABLE runtime_pstat_series DROP CONSTRAINT runtime_pstat_series_benchmark_metric_key;
+    ALTER TABLE runtime_pstat_series ADD CONSTRAINT runtime_test_case UNIQUE(benchmark, target, metric);
+    "#,
 ];
 
 #[async_trait::async_trait]
@@ -660,8 +666,8 @@ impl PostgresConnection {
                     select name, category
                     from benchmark
                 ").await.unwrap(),
-                insert_runtime_pstat_series: conn.prepare("insert into runtime_pstat_series (benchmark, metric) VALUES ($1, $2) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
-                select_runtime_pstat_series: conn.prepare("select id from runtime_pstat_series where benchmark = $1 and metric = $2").await.unwrap(),
+                insert_runtime_pstat_series: conn.prepare("insert into runtime_pstat_series (benchmark, target, metric) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(),
+                select_runtime_pstat_series: conn.prepare("select id from runtime_pstat_series where benchmark = $1 and target = $2 and metric = $3").await.unwrap(),
                 insert_runtime_pstat: conn
                     .prepare("insert into runtime_pstat (series, aid, cid, value) VALUES ($1, $2, $3, $4)")
                     .await
@@ -883,7 +889,7 @@ where
             runtime_pstat_series: self
                 .conn()
                 .query(
-                    "select id, benchmark, metric from runtime_pstat_series;",
+                    "select id, benchmark, target, metric from runtime_pstat_series;",
                     &[],
                 )
                 .await
@@ -893,8 +899,9 @@ where
                     (
                         row.get::<_, i32>(0) as u32,
                         (
-                            row.get::<_, String>(1).as_str().into(),
-                            row.get::<_, String>(2).as_str().into(),
+                            row.get::<_, &str>(1).into(),
+                            Target::from_str(row.get::<_, &str>(2)).unwrap(),
+                            row.get::<_, &str>(3).into(),
                         ),
                     )
                 })
@@ -1123,13 +1130,15 @@ where
         artifact: ArtifactIdNumber,
         benchmark: &str,
         metric: &str,
+        target: Target,
         value: f64,
     ) {
+        let target = target.to_string();
         let sid = self
             .conn()
             .query_opt(
                 &self.statements().select_runtime_pstat_series,
-                &[&benchmark, &metric],
+                &[&benchmark, &target, &metric],
             )
             .await
             .unwrap();
@@ -1139,14 +1148,14 @@ where
                 self.conn()
                     .query_opt(
                         &self.statements().insert_runtime_pstat_series,
-                        &[&benchmark, &metric],
+                        &[&benchmark, &target, &metric],
                     )
                     .await
                     .unwrap();
                 self.conn()
                     .query_one(
                         &self.statements().select_runtime_pstat_series,
-                        &[&benchmark, &metric],
+                        &[&benchmark, &target, &metric],
                     )
                     .await
                     .unwrap()

database/src/pool/sqlite.rs

@@ -431,6 +431,21 @@ static MIGRATIONS: &[Migration] = &[
         CREATE INDEX error_artifact_idx ON error(aid);
         "#,
     ),
+    // Add runtime target as a unique constraint, defaulting to 'x86_64-unknown-linux-gnu'
+    Migration::without_foreign_key_constraints(
+        r#"
+        create table runtime_pstat_series_with_target(
+            id integer primary key not null,
+            benchmark text not null,
+            target text not null default 'x86_64-unknown-linux-gnu',
+            metric text not null,
+            UNIQUE(benchmark, target, metric)
+        );
+        insert into runtime_pstat_series_with_target select id, benchmark, 'x86_64-unknown-linux-gnu', metric from runtime_pstat_series;
+        drop table runtime_pstat_series;
+        alter table runtime_pstat_series_with_target rename to runtime_pstat_series;
+    "#,
+    ),
 ];
 
 #[async_trait::async_trait]
@@ -579,14 +594,15 @@ impl Connection for SqliteConnection {
             .collect();
         let runtime_pstat_series = self
             .raw()
-            .prepare("select id, benchmark, metric from runtime_pstat_series;")
+            .prepare("select id, benchmark, target, metric from runtime_pstat_series;")
             .unwrap()
             .query_map(params![], |row| {
                 Ok((
                     row.get::<_, i32>(0)? as u32,
                     (
                         row.get::<_, String>(1)?.as_str().into(),
-                        row.get::<_, String>(2)?.as_str().into(),
+                        Target::from_str(row.get::<_, String>(2)?.as_str()).unwrap(),
+                        row.get::<_, String>(3)?.as_str().into(),
                     ),
                 ))
             })
@@ -750,19 +766,21 @@ impl Connection for SqliteConnection {
         artifact: ArtifactIdNumber,
         benchmark: &str,
         metric: &str,
+        target: Target,
         value: f64,
     ) {
+        let target = target.to_string();
         self.raw_ref()
             .execute(
-                "insert or ignore into runtime_pstat_series (benchmark, metric) VALUES (?, ?)",
-                params![&benchmark, &metric,],
+                "insert or ignore into runtime_pstat_series (benchmark, target, metric) VALUES (?, ?, ?)",
+                params![&benchmark, &target, &metric],
             )
             .unwrap();
         let sid: i32 = self
             .raw_ref()
             .query_row(
-                "select id from runtime_pstat_series where benchmark = ? and metric = ?",
-                params![&benchmark, &metric,],
+                "select id from runtime_pstat_series where benchmark = ? and target = ? and metric = ?",
+                params![&benchmark, &target, &metric],
                 |r| r.get(0),
             )
             .unwrap();