enhancement(enrichment_tables): Changes to support GeoIP Anonymous IP…

… database (#20946) * Adding files to support Anonymous IP database * Addressing bugs discovered in testing * Adding changelog fragment * spellcheck Signed-off-by: Jesse Szwedko <[email protected]> * cargo fmt Signed-off-by: Jesse Szwedko <[email protected]> --------- Signed-off-by: Jesse Szwedko <[email protected]> Co-authored-by: Jesse Szwedko <[email protected]>
vectordotdev · Jul 26, 2024 · 7f286c4 · 7f286c4
1 parent 96de0ac
commit 7f286c4
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 11 deletions.
diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt
@@ -95,6 +95,7 @@
 ^\Qlib/vector-core/tests/data/fixtures/lookup/quoted\E$
 ^\Qlib/vector-lookup/tests/fixtures/lookup/quoted\E$
 ^\Qlib/vector-vrl/tests/resources/protobuf_descriptor_set.desc\E$
+^\Qlib/vector-vrl/tests/resources/public_suffix_list.dat\E$
 ^\Qlib/vrl/lookup/tests/fixtures/lookup/quoted\E$
 ^\Qlib/vrl/stdlib/benches/benches.rs\E$
 ^\Qlib/vrl/stdlib/src/encode_percent.rs\E$

diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -17,6 +17,7 @@ amzn
 anchore
 androideabi
 andy
+anonymousip
 ansicpg
 anumber
 anycondition
@@ -60,7 +61,6 @@ awscli
 awsec
 awslabs
 axum
-Aziz
 azureresourceid
 babim
 badunit
@@ -315,10 +315,8 @@ endler
 enduml
 eni
 enp
-Ensar
 enumdecl
 enumflags
-ENVARS
 envsubst
 EOIG
 EOL'ed
@@ -730,7 +728,6 @@ nbase
 ndarray
 ndjson
 nearline
-neuronull
 newcerts
 newrelix
 nextest
@@ -863,7 +860,6 @@ protoc
 protofbuf
 protosizer
 Prt
-psl
 psv
 publickey
 purgecss
@@ -946,12 +942,10 @@ samerole
 sameuser
 sandboxed
 sandboxing
-Saraj
 sccache
 schemaless
 schemars
 schoen
-scl
 sda
 sdata
 SDID
@@ -976,7 +970,6 @@ shortcode
 shortstat
 should've
 shutsdown
-Sichert
 SIEM
 sighup
 Signsoff
@@ -1040,7 +1033,6 @@ stdduration
 stdlog
 Steensen
 stephenwakely
-strat
 strconv
 streamsink
 strng
@@ -1123,7 +1115,6 @@ Tomola
 tonydanza
 toolbars
 toolchains
-toolset
 toor
 topdir
 topojson

diff --git a/changelog.d/20946_support_geoip_anonymousip_database.enhancement.md b/changelog.d/20946_support_geoip_anonymousip_database.enhancement.md
@@ -0,0 +1,3 @@
+Adds support for the GeoIP2-Anonymous-IP MaxMind database when using enrichment_tables.geoip
+
+authors: publicfacingusername
diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs
@@ -7,7 +7,7 @@
 use std::{collections::BTreeMap, fs, net::IpAddr, sync::Arc, time::SystemTime};
 
 use maxminddb::{
-    geoip2::{City, ConnectionType, Isp},
+    geoip2::{AnonymousIp, City, ConnectionType, Isp},
     MaxMindDBError, Reader,
 };
 use ordered_float::NotNan;
@@ -26,6 +26,7 @@ pub enum DatabaseKind {
     Isp,
     ConnectionType,
     City,
+    AnonymousIp,
 }
 
 impl TryFrom<&str> for DatabaseKind {
@@ -37,6 +38,7 @@ impl TryFrom<&str> for DatabaseKind {
             "GeoIP2-ISP" => Ok(Self::Isp),
             "GeoIP2-Connection-Type" => Ok(Self::ConnectionType),
             "GeoIP2-City" | "GeoLite2-City" => Ok(Self::City),
+            "GeoIP2-Anonymous-IP" => Ok(Self::AnonymousIp),
             _ => Err(()),
         }
     }
@@ -128,6 +130,7 @@ impl Geoip {
             DatabaseKind::Asn | DatabaseKind::Isp => dbreader.lookup::<Isp>(ip).map(|_| ()),
             DatabaseKind::ConnectionType => dbreader.lookup::<ConnectionType>(ip).map(|_| ()),
             DatabaseKind::City => dbreader.lookup::<City>(ip).map(|_| ()),
+            DatabaseKind::AnonymousIp => dbreader.lookup::<AnonymousIp>(ip).map(|_| ()),
         };
 
         match result {
@@ -225,6 +228,16 @@ impl Geoip {
 
                 add_field!("connection_type", data.connection_type);
             }
+            DatabaseKind::AnonymousIp => {
+                let data = self.dbreader.lookup::<AnonymousIp>(ip).ok()?;
+
+                add_field!("is_anonymous", data.is_anonymous);
+                add_field!("is_anonymous_vpn", data.is_anonymous_vpn);
+                add_field!("is_hosting_provider", data.is_hosting_provider);
+                add_field!("is_public_proxy", data.is_public_proxy);
+                add_field!("is_residential_proxy", data.is_residential_proxy);
+                add_field!("is_tor_exit_node", data.is_tor_exit_node);
+            }
         }
 
         Some(map)
@@ -461,6 +474,27 @@ mod tests {
 
         assert!(result.is_err());
     }
+    #[test]
+    fn anonymous_ip_lookup() {
+        let values = find("101.99.92.179", "tests/data/GeoIP2-Anonymous-IP-Test.mmdb").unwrap();
+
+        let mut expected = ObjectMap::new();
+        expected.insert("is_anonymous".into(), true.into());
+        expected.insert("is_anonymous_vpn".into(), true.into());
+        expected.insert("is_hosting_provider".into(), true.into());
+        expected.insert("is_tor_exit_node".into(), true.into());
+        expected.insert("is_public_proxy".into(), Value::Null);
+        expected.insert("is_residential_proxy".into(), Value::Null);
+
+        assert_eq!(values, expected);
+    }
+
+    #[test]
+    fn anonymous_ip_lookup_no_results() {
+        let values = find("10.1.12.1", "tests/data/GeoIP2-Anonymous-IP-Test.mmdb");
+
+        assert!(values.is_none());
+    }
 
     fn find(ip: &str, database: &str) -> Option<ObjectMap> {
         find_select(ip, database, None)

diff --git a/tests/data/GeoIP2-Anonymous-IP-Test.mmdb b/tests/data/GeoIP2-Anonymous-IP-Test.mmdb
diff --git a/website/cue/reference/configuration.cue b/website/cue/reference/configuration.cue
@@ -221,6 +221,8 @@ configuration: {
 						* [GeoIP2-ISP.mmdb](\(urls.maxmind_geoip2_isp)) (paid) — Determine the Internet
 							Service Provider (ISP), organization name, and autonomous system organization
 							and number associated with an IP address.
+						* [GeoIP2-Anonymous-IP.mmdb](\(urls.maxmind_geoip2_anonymous_ip)) (paid) — Determine
+							proxy, VPN, hosting, and other anonymous IP addresses.	
 
 						The database file should be in the [MaxMind DB file format](\(urls.maxmind_db_file_format)).
 

diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue
@@ -352,6 +352,7 @@ urls: {
 	maxmind_geoip2:                             "https://dev.maxmind.com/geoip/geoip2/downloadable"
 	maxmind_geoip2_city:                        "https://www.maxmind.com/en/geoip2-city"
 	maxmind_geoip2_isp:                         "https://www.maxmind.com/en/geoip2-isp-database"
+	maxmind_geoip2_anonymous_ip:                "https://www.maxmind.com/en/geoip-anonymous-ip-database"
 	maxmind_geolite2_asn:                       "https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access"
 	maxmind_geolite2_city:                      "https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access"
 	memory_safety:                              "\(wikipedia)/wiki/Memory_safety"