Replaced reqwest dependency with manual https calls with tokio_rustls.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "firewall"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["Jerome David <[email protected]>"]
 categories = ["web-programming::http-server"]
@@ -9,12 +9,12 @@ keywords = ["firewall", "cidr", "cloudflare"]
 license = "MIT"
 repository = "https://github.com/programingjd/firewall"
 include = [
-  "Cargo.toml", "src", "examples", "LICENSE", "README.md"
+    "Cargo.toml", "src", "examples", "LICENSE", "README.md"
 ]
 
 [dependencies.rustls]
 optional = true
-version = "0.22"
+version = "0.23"
 default-features = false
 features = ["ring"]
 
@@ -24,11 +24,29 @@ version = "0.10"
 default-features = true
 features = ["v111"]
 
-[dependencies.reqwest]
+[dependencies.tokio]
 optional = true
-version = "0.11"
+version = "1.36"
 default-features = false
-features = ["rustls-tls"]
+features = ["net", "rt"]
+
+[dependencies.tokio-rustls]
+optional = true
+version = "0.26"
+default-features = false
+features = ["ring"]
+
+[dependencies.webpki-roots]
+optional = true
+version = "0.26"
+default-features = false
+features = []
+
+[dependencies.read_until_slice]
+optional = true
+version = "0.1"
+default-features = false
+features = []
 
 [dependencies.cidr]
 optional = true
@@ -43,16 +61,16 @@ builder = ["dep:cidr"]
 rustls = ["dep:rustls"]
 openssl = ["dep:openssl"]
 
-cloudflare = ["builder","dep:reqwest"]
-all = ["cloudflare","rustls","openssl"]
+cloudflare = ["builder", "dep:tokio", "dep:tokio-rustls", "dep:webpki-roots", "dep:read_until_slice"]
+all = ["cloudflare", "rustls", "openssl"]
 
 [dev-dependencies.tokio]
-version = "1.35"
+version = "1.36"
 default-features = false
 features = ["net", "io-util", "rt-multi-thread", "macros"]
 
 [dev-dependencies.tokio-rustls]
-version = "0.25"
+version = "0.26"
 default-features = false
 features = ["ring"]
 

src/cloudflare/mod.rs

@@ -1,8 +1,17 @@
 use crate::builder::Firewall;
 use crate::cloudflare::errors::Error;
 use cidr::IpCidr;
-use reqwest::Client;
-use std::str::FromStr;
+use read_until_slice::AsyncBufReadUntilSliceExt;
+use rustls::pki_types::ServerName;
+use rustls::{ClientConfig, RootCertStore};
+use std::io::BufRead;
+use std::str::{from_utf8, FromStr};
+use std::sync::Arc;
+use tokio::io::{AsyncReadExt, AsyncWriteExt, BufStream};
+use tokio::net::TcpStream;
+use tokio_rustls::client::TlsStream;
+use tokio_rustls::TlsConnector;
+use webpki_roots::TLS_SERVER_ROOTS;
 
 pub mod errors;
 
@@ -16,31 +25,119 @@ impl Firewall {
 }
 
 pub async fn fetch_cloudflare_ip_ranges() -> Result<Vec<IpCidr>> {
-    let client = Client::new();
-    let ips_v4 = client
-        .get("https://www.cloudflare.com/ips-v4/")
-        .send()
+    let root_store = RootCertStore::from_iter(TLS_SERVER_ROOTS.iter().cloned());
+    let config = ClientConfig::builder()
+        .with_root_certificates(root_store)
+        .with_no_client_auth();
+    let connector = TlsConnector::from(Arc::new(config));
+    let tcp_stream = TcpStream::connect("www.cloudflare.com:443")
         .await
-        .map_err(|_| Error::IpRangeFetchError)?
-        .text()
+        .map_err(|_| Error::IpRangeFetchError)?;
+    let stream = connector
+        .connect(
+            ServerName::try_from("www.cloudflare.com").unwrap(),
+            tcp_stream,
+        )
+        .await
+        .map_err(|_| Error::IpRangeFetchError)?;
+    let mut stream = BufStream::new(stream);
+    stream
+        .write_all(
+            b"\
+        GET /ips-v4/ HTTP/1.1\r\n\
+        Host: www.cloudflare.com\r\n\
+        Accept-Encoding: identity\r\n\
+        Accept: text/plain\r\n\
+        Connection: keep-alive\r\n\
+        Cache-Control: no-cache\r\n\
+        User-Agent: firewall\r\n\
+        \r\n\
+    ",
+        )
+        .await
+        .map_err(|_| Error::IpRangeFetchError)?;
+    stream.flush().await.map_err(|_| Error::IpRangeFetchError)?;
+    let mut ips_v4 = vec![];
+    let length = content_length(&mut stream, &mut ips_v4)
+        .await
+        .ok_or(Error::IpRangeFetchError)?;
+    ips_v4.resize(length, 0u8);
+    stream
+        .read_exact(&mut ips_v4)
+        .await
+        .map_err(|_| Error::IpRangeFetchError)?;
+    stream
+        .write_all(
+            b"\
+        GET /ips-v6/ HTTP/1.1\r\n\
+        Host: www.cloudflare.com\r\n\
+        Accept-Encoding: identity\r\n\
+        Accept: text/plain\r\n\
+        Connection: close\r\n\
+        Cache-Control: no-cache\r\n\
+        User-Agent: firewall\r\n\
+        \r\n\
+    ",
+        )
         .await
         .map_err(|_| Error::IpRangeFetchError)?;
-    let ips_v6 = client
-        .get("https://www.cloudflare.com/ips-v6/")
-        .send()
+    stream.flush().await.map_err(|_| Error::IpRangeFetchError)?;
+    let mut ips_v6 = vec![];
+    let content_length = content_length(&mut stream, &mut ips_v6)
         .await
-        .map_err(|_| Error::IpRangeFetchError)?
-        .text()
+        .ok_or(Error::IpRangeFetchError)?;
+    ips_v6.clear();
+    stream
+        .read_to_end(&mut ips_v6)
         .await
         .map_err(|_| Error::IpRangeFetchError)?;
+    assert_eq!(ips_v6.len(), content_length);
     ips_v4
         .lines()
         .chain(ips_v6.lines())
-        .filter(|&line| !line.is_empty())
-        .map(|line| IpCidr::from_str(line).map_err(|_| Error::IpRangeFetchError))
+        .filter_map(|line| {
+            if let Ok(line) = line {
+                if line.is_empty() {
+                    None
+                } else {
+                    Some(line)
+                }
+            } else {
+                None
+            }
+        })
+        .map(|line| IpCidr::from_str(line.as_str()).map_err(|_| Error::IpRangeFetchError))
         .collect()
 }
 
+async fn content_length(
+    stream: &mut BufStream<TlsStream<TcpStream>>,
+    buf: &mut Vec<u8>,
+) -> Option<usize> {
+    stream.read_until_slice(b"\r\n\r\n", buf).await.ok()?;
+    assert!(buf.starts_with(b"HTTP/1.1 200 OK\r\n"));
+    let indices = buf
+        .windows(2)
+        .enumerate()
+        .filter_map(|it| if it.1 == b"\r\n" { Some(it.0) } else { None })
+        .collect::<Vec<_>>();
+    indices.windows(2).find_map(|it| {
+        let mut split = buf[it[0] + 2..it[1]].split(|&it| it == b':');
+        if let Some(key) = split.next() {
+            if key.eq_ignore_ascii_case(b"content-length") {
+                split
+                    .next()
+                    .and_then(|it| from_utf8(it).ok())
+                    .and_then(|it| it.trim().parse::<usize>().ok())
+            } else {
+                None
+            }
+        } else {
+            None
+        }
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;