diff --git a/tembo-operator/src/dedicated_networking.rs b/tembo-operator/src/dedicated_networking.rs index d6ab522d7..37f3a2598 100644 --- a/tembo-operator/src/dedicated_networking.rs +++ b/tembo-operator/src/dedicated_networking.rs @@ -15,12 +15,12 @@ use tracing::{debug, error, info}; /// Reconcile dedicated networking resources for the CoreDB instance. /// /// This function handles the creation, update, or deletion of Kubernetes resources -/// required for dedicated networking, including services, network policies, and -/// ingress routes. +/// required for dedicated networking, including services and network policies /// /// # Parameters /// - `cdb`: The CoreDB custom resource instance. /// - `ctx`: The operator context containing the Kubernetes client and other configurations. +/// - `basedomain`: The base domain for the service. pub async fn reconcile_dedicated_networking( cdb: &CoreDB, ctx: Arc, @@ -52,7 +52,7 @@ pub async fn reconcile_dedicated_networking( "Reconciling network policies for dedicated networking in namespace: {}", ns ); - reconcile_dedicated_networking_network_policies(cdb.clone(), client.clone(), &ns) + reconcile_dedicated_network_policies(cdb.clone(), client.clone(), &ns) .await .map_err(|e| { error!("Failed to reconcile network policies: {:?}", e); @@ -108,12 +108,26 @@ pub async fn reconcile_dedicated_networking( e })?; } + + debug!( + "Completed reconciliation of dedicated networking for CoreDB instance: {} in namespace: {}", + cdb.name_any(), + ns + ); } else { debug!( - "Dedicated networking is disabled. Deleting services and ingress routes for CoreDB instance: {}", + "Dedicated networking is disabled for CoreDB instance: {}", cdb.name_any() ); + // Handling the case if dedicatedNetworking is disabled + delete_dedicated_networking_policies(client.clone(), &ns, &cdb.name_any()) + .await + .map_err(|e| { + error!("Failed to delete network policies: {:?}", e); + e + })?; + delete_dedicated_networking_service(client.clone(), &ns, &cdb.name_any(), false) .await .map_err(|e| { @@ -134,6 +148,14 @@ pub async fn reconcile_dedicated_networking( cdb.name_any() ); + // Handling the case even if dedicatedNetworking is not present in the spec + delete_dedicated_networking_policies(client.clone(), &ns, &cdb.name_any()) + .await + .map_err(|e| { + error!("Failed to delete network policies: {:?}", e); + e + })?; + delete_dedicated_networking_service(client.clone(), &ns, &cdb.name_any(), false) .await .map_err(|e| { @@ -149,11 +171,6 @@ pub async fn reconcile_dedicated_networking( })?; } - debug!( - "Completed reconciliation of dedicated networking for CoreDB instance: {} in namespace: {}", - cdb.name_any(), - ns - ); Ok(()) } @@ -163,11 +180,10 @@ pub async fn reconcile_dedicated_networking( /// to the CoreDB pods. /// /// # Parameters +/// - `cdb`: The CoreDB Spec /// - `client`: The Kubernetes client. /// - `namespace`: The namespace in which to apply the network policy. -/// - `cdb_name`: The name of the CoreDB instance. -/// - `cidr`: The CIDR block to allow traffic from. -async fn reconcile_dedicated_networking_network_policies( +async fn reconcile_dedicated_network_policies( cdb: CoreDB, client: Client, namespace: &str, @@ -243,17 +259,74 @@ async fn reconcile_dedicated_networking_network_policies( Ok(()) } +/// Delete the NetworkPolicy resource for dedicated networking. +/// +/// This function deletes the NetworkPolicy resource associated with the CoreDB instance. +/// +/// # Parameters +/// - `client`: The Kubernetes client. +/// - `namespace`: The namespace in which to delete the network policy. +/// - `cdb_name`: The name of the CoreDB instance. +async fn delete_dedicated_networking_policies( + client: Client, + namespace: &str, + cdb_name: &str, +) -> Result<(), OperatorError> { + let np_api: Api = Api::namespaced(client, namespace); + let policy_name = format!("{}-allow-nlb", cdb_name); + + debug!( + "Checking if network policy: {} exists in namespace: {} for deletion", + policy_name, namespace + ); + + if np_api.get(&policy_name).await.is_ok() { + info!( + "Network policy: {} exists in namespace: {}. Proceeding with deletion.", + policy_name, namespace + ); + + np_api + .delete(&policy_name, &Default::default()) + .await + .map_err(|e| { + error!( + "Failed to delete network policy: {} in namespace: {}. Error: {}", + policy_name, namespace, e + ); + OperatorError::NetworkPolicyError(format!( + "Failed to delete network policy: {:?}", + e + )) + })?; + + info!( + "Successfully deleted network policy: {} in namespace: {}", + policy_name, namespace + ); + } else { + debug!( + "Network policy: {} does not exist in namespace: {}. Skipping deletion.", + policy_name, namespace + ); + } + + Ok(()) +} + /// Reconcile the Service resource for dedicated networking. /// -/// This function creates or deletes a Service resource for the primary or standby service +/// This function creates or updates a Service resource for the primary or standby service /// based on the dedicated networking configuration. /// /// # Parameters +/// - `cdb`: The CoreDB custom resource instance. /// - `client`: The Kubernetes client. -/// - `namespace`: The namespace in which to create the service. -/// - `cdb_name`: The name of the CoreDB instance. +/// - `namespace`: The namespace in which to create or update the service. /// - `is_public`: Whether the service is public or private. /// - `is_standby`: Whether the service is for a standby (read-only) instance. +/// - `service_type`: The type of the Kubernetes Service (e.g., "LoadBalancer", "ClusterIP"). +/// - `basedomain`: The base domain for the service. async fn reconcile_dedicated_networking_service( cdb: &CoreDB, client: Client, diff --git a/tembo-operator/tests/integration_tests.rs b/tembo-operator/tests/integration_tests.rs index 6e799888b..a484388ab 100644 --- a/tembo-operator/tests/integration_tests.rs +++ b/tembo-operator/tests/integration_tests.rs @@ -1901,9 +1901,11 @@ mod test { #[tokio::test] #[ignore] async fn test_networking() { + // Initialize the Kubernetes client let client = kube_client().await; let state = State::default(); + // Configurations let mut rng = rand::thread_rng(); let suffix = rng.gen_range(1000..10000); let name = &format!("test-dedicated-networking-{}", suffix.clone()); @@ -1917,8 +1919,10 @@ mod test { let kind = "CoreDB"; let replicas = 2; + // Create a pod we can use to run commands in the cluster let pods: Api = Api::namespaced(client.clone(), &namespace); + // Apply a basic configuration of CoreDB println!("Creating CoreDB resource {}", name); let _test_metric_decr = format!("coredb_integration_test_{}", suffix.clone()); let coredbs: Api = Api::namespaced(client.clone(), &namespace); @@ -1936,6 +1940,7 @@ mod test { let patch = Patch::Apply(&coredb_json); let _coredb_resource = coredbs.patch(name, ¶ms, &patch).await.unwrap(); + // Wait for Pod to be created let pod_name = format!("{}-1", name); let _check_for_pod = tokio::time::timeout( Duration::from_secs(TIMEOUT_SECONDS_START_POD), @@ -1952,6 +1957,7 @@ mod test { let ing_route_tcp_name = format!("{}-rw-0", name); let ingress_route_tcp_api: Api = Api::namespaced(client.clone(), &namespace); + // Get the ingress route tcp let ing_route_tcp = ingress_route_tcp_api .get(&ing_route_tcp_name) .await @@ -1964,11 +1970,14 @@ mod test { .expect("Ingress route has no services")[0] .name .clone(); + // Assert the ingress route tcp service points to coredb service + // The coredb service is named the same as the coredb resource assert_eq!(&service_name, format!("{}-rw", name).as_str()); let ing_route_tcp_name = format!("{}-ro-0", name); let ingress_route_tcp_api: Api = Api::namespaced(client.clone(), &namespace); + // Get the ingress route tcp let ing_route_tcp = ingress_route_tcp_api .get(&ing_route_tcp_name) .await @@ -1981,6 +1990,8 @@ mod test { .expect("Ingress route has no services")[0] .name .clone(); + // Assert the ingress route tcp service points to coredb service + // The coredb service is named the same as the coredb resource assert_eq!(&service_name, format!("{}-ro", name).as_str()); let coredb_json = serde_json::json!({ @@ -1997,11 +2008,13 @@ mod test { let patch = Patch::Merge(&coredb_json); let _coredb_resource = coredbs.patch(name, ¶ms, &patch).await.unwrap(); + // extra domains should be created almost right away, within a few milliseconds tokio::time::sleep(Duration::from_secs(5)).await; let ing_route_tcp_name = format!("extra-{}-rw", name); let ingress_route_tcp_api: Api = Api::namespaced(client.clone(), &namespace); + // Get the ingress route tcp let ing_route_tcp = ingress_route_tcp_api .get(&ing_route_tcp_name) .await @@ -2014,6 +2027,8 @@ mod test { .expect("Ingress route has no services")[0] .name .clone(); + // Assert the ingress route tcp service points to coredb service + // The coredb service is named the same as the coredb resource assert_eq!(&service_name, format!("{}-rw", name).as_str()); let matcher = ing_route_tcp.spec.routes[0].r#match.clone(); assert_eq!( @@ -2035,8 +2050,10 @@ mod test { let patch = Patch::Merge(&coredb_json); let _coredb_resource = coredbs.patch(name, ¶ms, &patch).await.unwrap(); + // extra domains should be created almost right away, within a few milliseconds tokio::time::sleep(Duration::from_secs(5)).await; + // Get the ingress route tcp let ing_route_tcp = ingress_route_tcp_api .get(&ing_route_tcp_name) .await @@ -2049,10 +2066,13 @@ mod test { .expect("Ingress route has no services")[0] .name .clone(); + // Assert the ingress route tcp service points to coredb service + // The coredb service is named the same as the coredb resource assert_eq!(&service_name, format!("{}-rw", name).as_str()); let matcher = ing_route_tcp.spec.routes[0].r#match.clone(); assert_eq!(matcher, "HostSNI(`new-domain.com`)"); + // Check that a middleware was applied let middlewares = ing_route_tcp.spec.routes[0].middlewares.clone().unwrap(); assert_eq!(middlewares.len(), 1); @@ -2071,6 +2091,7 @@ mod test { let patch = Patch::Apply(&coredb_json); let _coredb_resource = coredbs.patch(name, ¶ms, &patch).await.unwrap(); + // wait for ingress route tcp to be deleted let mut i = 0; loop { tokio::time::sleep(Duration::from_secs(5)).await; @@ -2080,7 +2101,9 @@ mod test { } i += 1; } + // Get the ingress route tcp let ing_route_tcp = ingress_route_tcp_api.get(&ing_route_tcp_name).await; + // Should be deleted assert!(ing_route_tcp.is_err()); // Enable Dedicated Networking Test @@ -2223,6 +2246,7 @@ mod test { }); println!("CoreDB resource deleted {}", name); + // Delete namespace let _ = delete_namespace(client.clone(), &namespace).await; }