feat(state sync): record latency of p2p state requests (#12323)

Analogous to STATE_SYNC_EXTERNAL_PARTS_REQUEST_DELAY metric for
centralized state sync.

chain/client/src/metrics.rs

@@ -482,6 +482,16 @@ pub(crate) static STATE_SYNC_PARTS_TOTAL: LazyLock<IntGaugeVec> = LazyLock::new(
     .unwrap()
 });
 
+pub(crate) static STATE_SYNC_P2P_REQUEST_DELAY: LazyLock<HistogramVec> = LazyLock::new(|| {
+    try_create_histogram_vec(
+        "near_state_sync_p2p_request_delay_sec",
+        "Latency of state requests to peers",
+        &["shard_id", "type"],
+        Some(exponential_buckets(0.001, 2.0, 20).unwrap()),
+    )
+    .unwrap()
+});
+
 pub(crate) static STATE_SYNC_EXTERNAL_PARTS_REQUEST_DELAY: LazyLock<HistogramVec> =
     LazyLock::new(|| {
         try_create_histogram_vec(

chain/client/src/sync/state/network.rs

@@ -1,5 +1,6 @@
 use super::task_tracker::TaskHandle;
 use super::StateSyncDownloadSource;
+use crate::metrics;
 use crate::sync::state::util::increment_download_count;
 use futures::future::BoxFuture;
 use futures::FutureExt;
@@ -180,6 +181,10 @@ impl StateSyncDownloadSourcePeer {
             PartIdOrHeader::Header => "header",
         };
 
+        let _timer = metrics::STATE_SYNC_P2P_REQUEST_DELAY
+            .with_label_values(&[&key.shard_id.to_string(), &typ])
+            .start_timer();
+
         handle.set_status("Sending network request");
         match request_sender.send_async(network_request).await {
             Ok(response) => {