From e0ea6672637c128a470f9b91026bd143f2e86ab6 Mon Sep 17 00:00:00 2001 From: Andrew Yuan Date: Fri, 24 Oct 2025 15:00:11 -0700 Subject: [PATCH 1/6] Gate worker heartbeating behind namespace capabilities --- crates/sdk-core/src/lib.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/sdk-core/src/lib.rs b/crates/sdk-core/src/lib.rs index 1832f4a9c..f13c9c370 100644 --- a/crates/sdk-core/src/lib.rs +++ b/crates/sdk-core/src/lib.rs @@ -113,12 +113,31 @@ where worker_config.versioning_strategy.clone(), )); + let heartbeat_interval = if runtime.heartbeat_interval.is_some() { + let describe_namespace = runtime + .tokio_handle() + .block_on(client_bag.describe_namespace())?; + if let Some(namespace_info) = describe_namespace.namespace_info + && let Some(capabilities) = namespace_info.capabilities + && capabilities.worker_heartbeats + { + runtime.heartbeat_interval + } else { + warn!( + "Worker heartbeating configured for runtime, but server version does not support it." + ); + None + } + } else { + None + }; + Worker::new( worker_config.clone(), sticky_q, client_bag.clone(), Some(&runtime.telemetry), - runtime.heartbeat_interval, + heartbeat_interval, ) } From 13e92ff6adc7502fd1aa36275af6eac88904d5f7 Mon Sep 17 00:00:00 2001 From: Andrew Yuan Date: Mon, 27 Oct 2025 10:19:34 -0700 Subject: [PATCH 2/6] Lazily check namespace capabilities in worker heartbeater task --- crates/sdk-core/src/lib.rs | 21 +-------------------- crates/sdk-core/src/worker/heartbeat.rs | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/sdk-core/src/lib.rs b/crates/sdk-core/src/lib.rs index f13c9c370..1832f4a9c 100644 --- a/crates/sdk-core/src/lib.rs +++ b/crates/sdk-core/src/lib.rs @@ -113,31 +113,12 @@ where worker_config.versioning_strategy.clone(), )); - let heartbeat_interval = if runtime.heartbeat_interval.is_some() { - let describe_namespace = runtime - .tokio_handle() - .block_on(client_bag.describe_namespace())?; - if let Some(namespace_info) = describe_namespace.namespace_info - && let Some(capabilities) = namespace_info.capabilities - && capabilities.worker_heartbeats - { - runtime.heartbeat_interval - } else { - warn!( - "Worker heartbeating configured for runtime, but server version does not support it." - ); - None - } - } else { - None - }; - Worker::new( worker_config.clone(), sticky_q, client_bag.clone(), Some(&runtime.telemetry), - heartbeat_interval, + runtime.heartbeat_interval, ) } diff --git a/crates/sdk-core/src/worker/heartbeat.rs b/crates/sdk-core/src/worker/heartbeat.rs index d6198710e..6181aeb15 100644 --- a/crates/sdk-core/src/worker/heartbeat.rs +++ b/crates/sdk-core/src/worker/heartbeat.rs @@ -67,6 +67,25 @@ impl SharedNamespaceWorker { let heartbeat_map_clone = heartbeat_map.clone(); tokio::spawn(async move { + match client_clone.describe_namespace().await { + Ok(namespace_resp) => { + if namespace_resp + .namespace_info + .and_then(|info| info.capabilities) + .map(|caps| caps.worker_heartbeats) + != Some(true) + { + warn!( + "Worker heartbeating configured for runtime, but server version does not support it." + ); + return; + } + } + Err(e) => { + warn!(error=?e, "Network error while describing namespace for heartbeat capabilities"); + return; + } + } let mut ticker = tokio::time::interval(heartbeat_interval); loop { tokio::select! { From 7052b8b4a4eecfdc506ab544b945373b0d3f9479 Mon Sep 17 00:00:00 2001 From: Andrew Yuan Date: Mon, 27 Oct 2025 10:40:10 -0700 Subject: [PATCH 3/6] Shutdown worker whenever we return early --- crates/sdk-core/src/worker/heartbeat.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/sdk-core/src/worker/heartbeat.rs b/crates/sdk-core/src/worker/heartbeat.rs index 6181aeb15..25bffc47c 100644 --- a/crates/sdk-core/src/worker/heartbeat.rs +++ b/crates/sdk-core/src/worker/heartbeat.rs @@ -78,11 +78,13 @@ impl SharedNamespaceWorker { warn!( "Worker heartbeating configured for runtime, but server version does not support it." ); + worker.shutdown().await; return; } } Err(e) => { warn!(error=?e, "Network error while describing namespace for heartbeat capabilities"); + worker.shutdown().await; return; } } @@ -101,6 +103,7 @@ impl SharedNamespaceWorker { } if let Err(e) = client_clone.record_worker_heartbeat(namespace_clone.clone(), hb_to_send).await { if matches!(e.code(), tonic::Code::Unimplemented) { + worker.shutdown().await; return; } warn!(error=?e, "Network error while sending worker heartbeat"); From 9e0fc3a02756187b2414ccc1fd90b3253864bf10 Mon Sep 17 00:00:00 2001 From: Andrew Yuan Date: Mon, 27 Oct 2025 17:55:33 -0700 Subject: [PATCH 4/6] Mock describe namespace for heartbeating unit test --- crates/sdk-core/src/worker/heartbeat.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/sdk-core/src/worker/heartbeat.rs b/crates/sdk-core/src/worker/heartbeat.rs index 25bffc47c..6beaeba3c 100644 --- a/crates/sdk-core/src/worker/heartbeat.rs +++ b/crates/sdk-core/src/worker/heartbeat.rs @@ -167,7 +167,10 @@ mod tests { time::Duration, }; use temporalio_common::{ - protos::temporal::api::workflowservice::v1::RecordWorkerHeartbeatResponse, + protos::temporal::api::namespace::v1::{NamespaceInfo, namespace_info::Capabilities}, + protos::temporal::api::workflowservice::v1::{ + DescribeNamespaceResponse, RecordWorkerHeartbeatResponse, + }, worker::PollerBehavior, }; @@ -202,6 +205,18 @@ mod tests { Ok(RecordWorkerHeartbeatResponse {}) }, ); + mock.expect_describe_namespace().returning(move || { + Ok(DescribeNamespaceResponse { + namespace_info: Some(NamespaceInfo { + capabilities: Some(Capabilities { + worker_heartbeats: true, + ..Capabilities::default() + }), + ..NamespaceInfo::default() + }), + ..DescribeNamespaceResponse::default() + }) + }); let config = test_worker_cfg() .activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize)) From e7499eb1c48cc76ac0c76c2539f621a30b2d916f Mon Sep 17 00:00:00 2001 From: Andrew Yuan Date: Thu, 30 Oct 2025 09:55:32 -0700 Subject: [PATCH 5/6] use v1.29.1 branch for new server patch --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 94997223c..54a858816 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,6 @@ [env] # This temporarily overrides the version of the CLI used for integration tests, locally and in CI -#CLI_VERSION_OVERRIDE = "v1.4.1-cloud-v1-29-0-139-2.0" +CLI_VERSION_OVERRIDE = "server-1-29-1" [alias] # Not sure why --all-features doesn't work From 70b9ad01d3adcbd805bc29ba33a86844b3693d1e Mon Sep 17 00:00:00 2001 From: Andrew Yuan Date: Thu, 30 Oct 2025 13:28:18 -0700 Subject: [PATCH 6/6] Revert "use v1.29.1 branch for new server patch" This reverts commit e7499eb1c48cc76ac0c76c2539f621a30b2d916f. --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 54a858816..94997223c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,6 @@ [env] # This temporarily overrides the version of the CLI used for integration tests, locally and in CI -CLI_VERSION_OVERRIDE = "server-1-29-1" +#CLI_VERSION_OVERRIDE = "v1.4.1-cloud-v1-29-0-139-2.0" [alias] # Not sure why --all-features doesn't work