From ab718d2d844159be06330c430124d79b7d64f08d Mon Sep 17 00:00:00 2001 From: Sviatoslav Boichuk Date: Wed, 17 Jul 2024 09:47:13 +0300 Subject: [PATCH] Utilize TLS connection for Redis & PostgeSQL --- Cargo.lock | 95 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 41 ++++++++-------- README.md | 34 ++++++++----- run_cgw.sh | 8 +++- src/cgw_db_accessor.rs | 23 +++++++-- src/cgw_errors.rs | 3 ++ src/cgw_remote_discovery.rs | 32 ++++++++++--- src/cgw_tls.rs | 64 +++++++++++++++++++++++++ 8 files changed, 256 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83b543c..051f193 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -273,6 +273,22 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bcder" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627747a6774aab38beb35990d88309481378558875a41da1a4b2e373c906ef0" +dependencies = [ + "bytes", + "smallvec", +] + [[package]] name = "bindgen" version = "0.69.4" @@ -415,6 +431,12 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "convert_case" version = "0.4.0" @@ -471,6 +493,16 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der-parser" version = "9.0.0" @@ -1457,6 +1489,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pem" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +dependencies = [ + "base64 0.22.1", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2132,6 +2174,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -2169,6 +2220,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2394,6 +2455,20 @@ dependencies = [ "whoami", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab" +dependencies = [ + "ring", + "rustls 0.23.10", + "tokio", + "tokio-postgres", + "tokio-rustls 0.26.0", + "x509-certificate", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -2665,6 +2740,7 @@ dependencies = [ "tokio", "tokio-pg-mapper", "tokio-postgres", + "tokio-postgres-rustls", "tokio-rustls 0.26.0", "tokio-stream", "tokio-tungstenite 0.23.0", @@ -3080,6 +3156,25 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "x509-certificate" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66534846dec7a11d7c50a74b7cdb208b9a581cad890b7866430d438455847c85" +dependencies = [ + "bcder", + "bytes", + "chrono", + "der", + "hex", + "pem", + "ring", + "signature", + "spki", + "thiserror", + "zeroize", +] + [[package]] name = "x509-parser" version = "0.16.0" diff --git a/Cargo.toml b/Cargo.toml index bf63ef0..a2f67e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,41 +5,42 @@ edition = "2021" [dependencies] serde = { version = "1.0.144", features = ["derive"] } -serde_json = "1.0.85" -env_logger = "0.11.3" -log = "0.4.20" +serde_json = { version = "1.0.85" } +env_logger = { version = "0.11.3" } +log = { version = "0.4.20" } tokio = { version = "1.34.0", features = ["full"] } tokio-stream = { version = "0.1.15", features = ["full"] } tokio-tungstenite = { version = "0.23.0" } -tokio-rustls = "0.26.0" +tokio-rustls = { version = "0.26.0" } tokio-postgres = { version = "0.7.10", features = ["with-eui48-1"] } -tokio-pg-mapper = "0.2.0" +tokio-postgres-rustls = { version = "0.12.0" } +tokio-pg-mapper = { version = "0.2.0" } tungstenite = { version = "0.23.0" } futures-util = { version = "0.3.0", default-features = false } -futures-channel = "0.3.0" +futures-channel = { version = "0.3.0" } futures-executor = { version = "0.3.0", optional = true } -futures = "0.3.0" -rlimit = "0.10.1" -tonic = "0.11.0" -prost = "0.12" -rdkafka = "0.36.2" +futures = { version = "0.3.0" } +rlimit = { version = "0.10.1" } +tonic = { version = "0.11.0" } +prost = { version = "0.12" } +rdkafka = { version = "0.36.2" } eui48 = { version = "1.1.0", features = ["serde"] } uuid = { version = "1.6.1", features = ["serde"] } redis = { version = "0.25.3", features = [ "tokio-rustls-comp", "tls-rustls-insecure", ] } -warp = "0.3.7" +warp = { version = "0.3.7" } prometheus = { version = "0.13.4", features = ["process"] } -lazy_static = "1.4.0" +lazy_static = { version = "1.4.0" } petgraph = { version = "0.6.4", features = ["stable_graph"] } -flate2 = "1.0.28" -base64 = "0.22.0" -rustls-pemfile = "2.1.2" -rustls-pki-types = "1.7.0" -x509-parser = "0.16.0" -chrono = "0.4.38" -derive_more = "0.99.17" +flate2 = { version = "1.0.28" } +base64 = { version = "0.22.0" } +rustls-pemfile = { version = "2.1.2" } +rustls-pki-types = { version = "1.7.0" } +x509-parser = { version = "0.16.0" } +chrono = { version = "0.4.38" } +derive_more = { version = "0.99.17" } [build-dependencies] tonic-build = "0.11.0" diff --git a/README.md b/README.md index 4103186..4385417 100644 --- a/README.md +++ b/README.md @@ -80,38 +80,44 @@ CGW_LOG_LEVEL - Log level to start CGW application with (debug, info) CGW_METRICS_PORT - PORT of metrics to connect to CGW_CERTS_PATH - Path to certificates located on host machine CGW_ALLOW_CERT_MISMATCH - Allow client certificate CN and device MAC address mismatch (used for OWLS) +CGW_NB_INFRA_CERTS_DIR - Path to NB infrastructure (Redis, PostgreSQL)certificates located on host machine ``` Example of properly configured list of env variables to start CGW: ```console $ export | grep CGW -declare -x CGW_DB_HOST="localhost" # PSQL server is located at the local host +declare -x CGW_DB_HOST="localhost" # PSQL server is located at the local host declare -x CGW_DB_PORT="5432" -declare -x CGW_DB_USERNAME="cgw" # PSQL login credentials (username) default 'cgw' will be used -declare -x CGW_DB_PASS="123" # PSQL login credentials (password) default '123' will be used -declare -x CGW_GRPC_LISTENING_IP="127.0.0.1" # Local default subnet is 127.0.0.1/24 +declare -x CGW_DB_USERNAME="cgw" # PSQL login credentials (username) default 'cgw' will be used +declare -x CGW_DB_PASS="123" # PSQL login credentials (password) default '123' will be used +declare -x CGW_GRPC_LISTENING_IP="127.0.0.1" # Local default subnet is 127.0.0.1/24 declare -x CGW_GRPC_LISTENING_PORT="50051" declare -x CGW_GRPC_PUBLIC_HOST="localhost" declare -x CGW_GRPC_PUBLIC_PORT="50051" declare -x CGW_ID="0" -declare -x CGW_KAFKA_HOST="localhost" # Kafka is located at the local host +declare -x CGW_KAFKA_HOST="localhost" # Kafka is located at the local host declare -x CGW_KAFKA_PORT="9092" declare -x CGW_LOG_LEVEL="debug" -declare -x CGW_REDIS_HOST="localhost" # Redis server can be found at the local host +declare -x CGW_REDIS_HOST="localhost" # Redis server can be found at the local host declare -x CGW_REDIS_PORT="6379" -declare -x CGW_REDIS_USERNAME="cgw" # REDIS login credentials (username) - optional -declare -x CGW_REDIS_PASSWORD="123" # REDIS login credentials (password) - optional +declare -x CGW_REDIS_USERNAME="cgw" # REDIS login credentials (username) - optional +declare -x CGW_REDIS_PASSWORD="123" # REDIS login credentials (password) - optional declare -x CGW_METRICS_PORT="8080" -declare -x CGW_WSS_IP="0.0.0.0" # Accept WSS connections at all interfaces / subnets +declare -x CGW_WSS_IP="0.0.0.0" # Accept WSS connections at all interfaces / subnets declare -x CGW_WSS_PORT="15002" declare -x CGW_WSS_CAS="cas.pem" declare -x CGW_WSS_CERT="cert.pem" declare -x CGW_WSS_KEY="key.pem" -declare -x CGW_CERTS_PATH="/etc/ssl/certs" # Path to certificates located on host machine -declare -x CGW_ALLOW_CERT_MISMATCH="no" # Allow client certificate CN and device MAC address mismatch +declare -x CGW_CERTS_PATH="/etc/ssl/certs" # Path to certificates located on host machine +declare -x CGW_ALLOW_CERT_MISMATCH="no" # Allow client certificate CN and device MAC address mismatch +declare -x CGW_NB_INFRA_CERTS_PATH="/etc/nb_infra_certs" ``` # Certificates -The CGW uses a number of certificates to provide security. +The CGW uses two different sets of certificate configuration: +1. AP/Switch connectivity (southbound) +2. Infrastructure connectivity (northbound) + +The AP/Switch connectivity uses a number of certificates to provide security (mTLS). There are 2 types of certificates required for a normal deployment: 1. Server certificates 2. Client certificates @@ -123,3 +129,7 @@ There are several environment variable to configure certificates path and names 2. CGW_WSS_KEY - CGW WSS Private Key 3. CGW_WSS_CAS - Chain certificates to validate client (root/issuer) 4. CGW_CERTS_PATH - path to certificates located on host machine + +The infrastructure connectivity use root certs store - the directory with trusted certificates +The environemt variable to configure certificates path: +1. CGW_NB_INFRA_CERTS_PATH - path to certificates located on host machine diff --git a/run_cgw.sh b/run_cgw.sh index f831b77..58adff8 100755 --- a/run_cgw.sh +++ b/run_cgw.sh @@ -36,6 +36,7 @@ DEFAULT_REDIS_PORT=6379 DEFAULT_METRICS_PORT=8080 CONTAINTER_CERTS_VOLUME="/etc/cgw/certs" +CONTAINTER_NB_INFRA_CERTS_VOLUME="/etc/cgw/nb_infra/certs" DEFAULT_ALLOW_CERT_MISMATCH="no" @@ -65,6 +66,7 @@ export CGW_REDIS_PORT="${CGW_REDIS_PORT:-$DEFAULT_REDIS_PORT}" export CGW_METRICS_PORT="${CGW_METRICS_PORT:-$DEFAULT_METRICS_PORT}" export CGW_CERTS_PATH="${CGW_CERTS_PATH:-$DEFAULT_CERTS_PATH}" export CGW_ALLOW_CERT_MISMATCH="${CGW_ALLOW_CERT_MISMATCH:-$DEFAULT_ALLOW_CERT_MISMATCH}" +export CGW_NB_INFRA_CERTS_PATH="${CGW_NB_INFRA_CERTS_PATH:-$DEFAULT_CERTS_PATH}" if [ -z "${!CGW_REDIS_USERNAME}" ]; then export CGW_REDIS_USERNAME="${CGW_REDIS_USERNAME}" @@ -92,10 +94,12 @@ echo "CGW REDIS HOST/PORT : $CGW_REDIS_HOST:$CGW_REDIS_PORT" echo "CGW METRICS PORT : $CGW_METRICS_PORT" echo "CGW CERTS PATH : $CGW_CERTS_PATH" echo "CGW ALLOW CERT MISMATCH : $CGW_ALLOW_CERT_MISMATCH" +echo "CGW NB INFRA CERTS PATH : $CGW_NB_INFRA_CERTS_PATH" docker run \ - --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ - -v $CGW_CERTS_PATH:$CONTAINTER_CERTS_VOLUME \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + -v $CGW_CERTS_PATH:$CONTAINTER_CERTS_VOLUME \ + -v $CGW_NB_INFRA_CERTS_PATH:$CONTAINTER_NB_INFRA_CERTS_VOLUME \ -e CGW_LOG_LEVEL \ -e CGW_ID \ -e CGW_WSS_IP \ diff --git a/src/cgw_db_accessor.rs b/src/cgw_db_accessor.rs index 46e4d15..6e4f3a9 100644 --- a/src/cgw_db_accessor.rs +++ b/src/cgw_db_accessor.rs @@ -1,5 +1,6 @@ use crate::cgw_app_args::CGWDBArgs; +use crate::cgw_tls::cgw_tls_create_db_connect; use crate::{ cgw_errors::{Error, Result}, cgw_metrics::{CGWMetrics, CGWMetricsHealthComponent, CGWMetricsHealthComponentStatus}, @@ -7,7 +8,7 @@ use crate::{ use eui48::MacAddress; -use tokio_postgres::{row::Row, Client, NoTls}; +use tokio_postgres::{row::Row, Client}; #[derive(Clone)] pub struct CGWDBInfra { @@ -53,19 +54,33 @@ pub struct CGWDBAccessor { impl CGWDBAccessor { pub async fn new(db_args: &CGWDBArgs) -> Result { let conn_str = format!( - "host={host} port={port} user={user} dbname={db} password={pass} connect_timeout=10", + "sslmode={sslmode} host={host} port={port} user={user} dbname={db} password={pass} connect_timeout=10", host = db_args.db_host, port = db_args.db_port, user = db_args.db_username, db = db_args.db_name, - pass = db_args.db_password + pass = db_args.db_password, + sslmode = "require", ); debug!( "Trying to connect to remote db ({}:{})...\nConn args {}", db_args.db_host, db_args.db_port, conn_str ); - let (client, connection) = match tokio_postgres::connect(&conn_str, NoTls).await { + let tls = match cgw_tls_create_db_connect().await { + Ok(tls_connect) => tls_connect, + Err(e) => { + error!( + "Failed to build TLS connection with remote DB, reason: {}", + e.to_string() + ); + return Err(Error::DbAccessor( + "Failed to build TLS connection with remote DB", + )); + } + }; + + let (client, connection) = match tokio_postgres::connect(&conn_str, tls).await { Ok((cl, conn)) => (cl, conn), Err(e) => { error!("Failed to establish connection with DB, reason: {:?}", e); diff --git a/src/cgw_errors.rs b/src/cgw_errors.rs index 22b80b9..8bdcf6b 100644 --- a/src/cgw_errors.rs +++ b/src/cgw_errors.rs @@ -41,6 +41,9 @@ pub enum Error { #[from] TokioSync(tokio::sync::TryLockError), + #[from] + Tokiofs(tokio::fs::ReadDir), + #[from] IpAddressParse(std::net::AddrParseError), diff --git a/src/cgw_remote_discovery.rs b/src/cgw_remote_discovery.rs index f87ef1e..a66b5dc 100644 --- a/src/cgw_remote_discovery.rs +++ b/src/cgw_remote_discovery.rs @@ -9,6 +9,7 @@ use crate::{ CGWMetricsHealthComponentStatus, }, cgw_remote_client::CGWRemoteClient, + cgw_tls::cgw_read_root_certs_dir, AppArgs, }; @@ -16,11 +17,12 @@ use std::{ collections::HashMap, net::{Ipv4Addr, SocketAddr}, sync::Arc, + time::Duration, }; use redis::{ aio::MultiplexedConnection, Client, ConnectionInfo, RedisConnectionInfo, RedisResult, - ToRedisArgs, + TlsCertificates, ToRedisArgs, }; use eui48::MacAddress; @@ -147,9 +149,14 @@ pub struct CGWRemoteDiscovery { local_shard_id: i32, } -fn cgw_create_redis_client(redis_args: &CGWRedisArgs) -> Result { +async fn cgw_create_redis_client(redis_args: &CGWRedisArgs) -> Result { let redis_client_info = ConnectionInfo { - addr: redis::ConnectionAddr::Tcp(redis_args.redis_host.clone(), redis_args.redis_port), + addr: redis::ConnectionAddr::TcpTls { + host: redis_args.redis_host.clone(), + port: redis_args.redis_port, + insecure: true, + tls_params: None, + }, redis: RedisConnectionInfo { username: redis_args.redis_username.clone(), password: redis_args.redis_password.clone(), @@ -157,7 +164,14 @@ fn cgw_create_redis_client(redis_args: &CGWRedisArgs) -> Result { }, }; - match redis::Client::open(redis_client_info) { + let root_cert = cgw_read_root_certs_dir().await.ok(); + + let tls_certs: TlsCertificates = TlsCertificates { + client_tls: None, + root_cert, + }; + + match redis::Client::build_with_tls(redis_client_info, tls_certs) { Ok(client) => Ok(client), Err(e) => Err(Error::Redis(format!("Failed to start Redis Client: {}", e))), } @@ -170,7 +184,7 @@ impl CGWRemoteDiscovery { app_args.redis_args.redis_host, app_args.redis_args.redis_port ); - let redis_client = match cgw_create_redis_client(&app_args.redis_args) { + let redis_client = match cgw_create_redis_client(&app_args.redis_args).await { Ok(c) => c, Err(e) => { error!( @@ -181,7 +195,13 @@ impl CGWRemoteDiscovery { } }; - let redis_client = match redis_client.get_multiplexed_tokio_connection().await { + let redis_client = match redis_client + .get_multiplexed_tokio_connection_with_response_timeouts( + Duration::from_secs(1), + Duration::from_secs(5), + ) + .await + { Ok(conn) => conn, Err(e) => { error!( diff --git a/src/cgw_tls.rs b/src/cgw_tls.rs index 821abf1..302f095 100644 --- a/src/cgw_tls.rs +++ b/src/cgw_tls.rs @@ -3,8 +3,13 @@ use crate::cgw_errors::{collect_results, Error, Result}; use eui48::MacAddress; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use std::fs; +use std::io::BufRead; +use std::path::Path; use std::{fs::File, io::BufReader, str::FromStr, sync::Arc}; use tokio::net::TcpStream; +use tokio_postgres_rustls::MakeRustlsConnect; +use tokio_rustls::rustls; use tokio_rustls::{ rustls::{server::WebPkiClientVerifier, RootCertStore, ServerConfig}, server::TlsStream, @@ -13,6 +18,7 @@ use tokio_rustls::{ use x509_parser::parse_x509_certificate; const CGW_TLS_CERTIFICATES_PATH: &str = "/etc/cgw/certs"; +const CGW_TLS_NB_INFRA_CERTS_PATH: &str = "/etc/cgw/nb_infra/certs"; pub async fn cgw_tls_read_certs(cert_file: &str) -> Result>> { let file = match File::open(cert_file) { @@ -103,6 +109,7 @@ pub async fn cgw_tls_get_cn_from_stream(stream: &TlsStream) -> Result Err(Error::Tls("Failed to read peer comman name!".to_string())) } + pub async fn cgw_tls_create_acceptor(wss_args: &CGWWSSArgs) -> Result { // Read root/issuer certs. let cas_path = format!("{}/{}", CGW_TLS_CERTIFICATES_PATH, wss_args.wss_cas); @@ -162,3 +169,60 @@ pub async fn cgw_tls_create_acceptor(wss_args: &CGWWSSArgs) -> Result Result> { + let mut certs_vec = Vec::new(); + + // Read the directory entries + for entry in fs::read_dir(Path::new(CGW_TLS_NB_INFRA_CERTS_PATH))? { + let entry = entry?; + let path = entry.path(); + + // Check if the entry is a file and has a .crt extension (or other extensions if needed) + if path.is_file() { + let extension = path.extension().and_then(|ext| ext.to_str()); + if extension == Some("crt") || extension == Some("pem") { + if let Ok(md) = path.metadata() { + if !md.is_symlink() { + let cert_contents = fs::read(path)?; + certs_vec.extend(cert_contents); + } + } + } + } + } + + Ok(certs_vec) +} + +pub async fn cgw_get_root_certs_store() -> Result { + let certs = cgw_read_root_certs_dir().await?; + + let buf = &mut certs.as_slice() as &mut dyn BufRead; + let certs = rustls_pemfile::certs(buf); + let mut root_cert_store = rustls::RootCertStore::empty(); + for result in certs { + let _r = root_cert_store.add(result.unwrap()); + } + + println!("Number of certs read: {}", root_cert_store.len()); + + Ok(root_cert_store) +} + +pub async fn cgw_tls_create_db_connect() -> Result { + let root_store = match cgw_get_root_certs_store().await { + Ok(certs) => certs, + Err(e) => { + error!("{}", e.to_string()); + return Err(e); + } + }; + + // Create the client certs verifier. + let config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + Ok(tokio_postgres_rustls::MakeRustlsConnect::new(config)) +}