From 57f09ecbc99083380220d75e9ad0e43652712142 Mon Sep 17 00:00:00 2001 From: Metin de Vreugd Date: Sun, 8 Jul 2018 01:17:40 +0200 Subject: [PATCH 1/4] Add support for GeoIP2 anonymous database --- .../java/org/logstash/filters/Fields.java | 9 ++++ .../org/logstash/filters/GeoIPFilter.java | 50 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/main/java/org/logstash/filters/Fields.java b/src/main/java/org/logstash/filters/Fields.java index 1d38599..055a146 100644 --- a/src/main/java/org/logstash/filters/Fields.java +++ b/src/main/java/org/logstash/filters/Fields.java @@ -26,6 +26,11 @@ enum Fields { AUTONOMOUS_SYSTEM_NUMBER("asn"), AUTONOMOUS_SYSTEM_ORGANIZATION("as_org"), + ANONYMOUS_IS_ANONYMOUS("is_anonymous"), + ANONYMOUS_IS_VPN("is_anonymous_vpn"), + ANONYMOUS_IS_HOSTING_PROVIDER("is_hosting_provider"), + ANONYMOUS_IS_PUBLIC_PROXY("is_public_proxy"), + ANONYMOUS_IS_TOR_EXIT_NODE("is_tor_exit_node"), CITY_NAME("city_name"), COUNTRY_NAME("country_name"), CONTINENT_CODE("continent_code"), @@ -70,6 +75,10 @@ public String fieldName() { static final EnumSet DEFAULT_ASN_LITE_FIELDS = EnumSet.of(Fields.IP, Fields.AUTONOMOUS_SYSTEM_NUMBER, Fields.AUTONOMOUS_SYSTEM_ORGANIZATION); + static final EnumSet DEFAULT_ANONYMOUS_FIELDS = EnumSet.of(Fields.IP, Fields.ANONYMOUS_IS_ANONYMOUS, + Fields.ANONYMOUS_IS_VPN, Fields.ANONYMOUS_IS_HOSTING_PROVIDER, Fields.ANONYMOUS_IS_PUBLIC_PROXY, + Fields.ANONYMOUS_IS_TOR_EXIT_NODE); + public static Fields parseField(String value) { try { return valueOf(value.toUpperCase(Locale.ROOT)); diff --git a/src/main/java/org/logstash/filters/GeoIPFilter.java b/src/main/java/org/logstash/filters/GeoIPFilter.java index f23b9ad..e7aa8c2 100644 --- a/src/main/java/org/logstash/filters/GeoIPFilter.java +++ b/src/main/java/org/logstash/filters/GeoIPFilter.java @@ -56,6 +56,7 @@ public class GeoIPFilter { private static final String CITY_SOUTH_AMERICA_DB_TYPE = "GeoIP2-City-South-America"; private static final String COUNTRY_DB_TYPE = "GeoIP2-Country"; private static final String ISP_DB_TYPE = "GeoIP2-ISP"; + private static final String ANONYMOUS_DB_TYPE = "GeoIP2-Anonymous-IP"; private final String sourceField; private final String targetField; @@ -99,6 +100,9 @@ private Set createDesiredFields(List fields) { case ASN_LITE_DB_TYPE: desiredFields = Fields.DEFAULT_ASN_LITE_FIELDS; break; + case ANONYMOUS_DB_TYPE: + desiredFields = Fields.DEFAULT_ANONYMOUS_DB_TYPE; + break; } } else { for (String fieldName : fields) { @@ -153,6 +157,8 @@ public boolean handleEvent(RubyEvent rubyEvent) { case ISP_DB_TYPE: geoData = retrieveIspGeoData(ipAddress); break; + case ANONYMOUS_DB_TYPE: + geoData = retrieveAnonymousData(ipAddress); default: throw new IllegalStateException("Unsupported database type " + databaseReader.getMetadata().getDatabaseType() + ""); } @@ -401,4 +407,48 @@ private Map retrieveAsnGeoData(InetAddress ipAddress) throws Geo return geoData; } + + private Map retrieveAnonymousData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + AnonymousIpResponse response = databaseReader.anonymousIp(ipAddress); + Map geoData = new HashMap<>(); + for (Fields desiredField : this.desiredFields) { + switch (desiredField) { + case IP: + geoData.put(Fields.IP.fieldName(), ipAddress.getHostAddress()); + break; + case ANONYMOUS_IS_ANONOYMOUS: + Boolean is_anonoymous = response.isAnonymous(); + if (is_anonoymous != null) { + geoData.put(Fields.ANONYMOUS_IS_ANONYMOUS.fieldName(), is_anonoymous); + } + break; + case ANONYMOUS_IS_VPN: + Boolean is_vpn = response.isAnonymousVpn(); + if (is_vpn != null) { + geoData.put(Fields.ANONYMOUS_IS_VPN.fieldName(), is_vpn); + } + break; + case ANONYMOUS_IS_HOSTING_PROVIDER: + Boolean is_hosting_provider = response.isHostingProvider(); + if (is_hosting_provider != null) { + geoData.put(Fields.ANONYMOUS_IS_HOSTING_PROVIDER.fieldName(), is_hosting_provider); + } + break; + case ANONYMOUS_IS_PUBLIC_PROXY: + Boolean is_public_proxy = response.isPublicProxy(); + if (is_public_proxy != null) { + geoData.put(Fields.ANONYMOUS_IS_PUBLIC_PROXY.fieldName(), is_public_proxy); + } + break; + case ANONYMOUS_IS_TOR_EXIT_NODE: + Boolean is_tor_exit_node = response.isTorExitNode(); + if (is_tor_exit_node != null) { + geoData.put(Fields.ANONYMOUS_IS_TOR_EXIT_NODE.fieldName(), is_tor_exit_node); + } + break; + } + } + + return geoData; + } } From d2fe0e361b6ccb15dfcb41a5402486908c27bfd8 Mon Sep 17 00:00:00 2001 From: Metin de Vreugd Date: Sun, 8 Jul 2018 01:29:52 +0200 Subject: [PATCH 2/4] Typo fix --- src/main/java/org/logstash/filters/GeoIPFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/logstash/filters/GeoIPFilter.java b/src/main/java/org/logstash/filters/GeoIPFilter.java index e7aa8c2..8e8bec2 100644 --- a/src/main/java/org/logstash/filters/GeoIPFilter.java +++ b/src/main/java/org/logstash/filters/GeoIPFilter.java @@ -416,7 +416,7 @@ private Map retrieveAnonymousData(InetAddress ipAddress) throws case IP: geoData.put(Fields.IP.fieldName(), ipAddress.getHostAddress()); break; - case ANONYMOUS_IS_ANONOYMOUS: + case ANONYMOUS_IS_ANONYMOUS: Boolean is_anonoymous = response.isAnonymous(); if (is_anonoymous != null) { geoData.put(Fields.ANONYMOUS_IS_ANONYMOUS.fieldName(), is_anonoymous); From 4f33ae50dc1e9fc934de4e9c2e28f72e612b094e Mon Sep 17 00:00:00 2001 From: Metin de Vreugd Date: Sun, 8 Jul 2018 01:39:23 +0200 Subject: [PATCH 3/4] Import and typo fix --- src/main/java/org/logstash/filters/GeoIPFilter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/logstash/filters/GeoIPFilter.java b/src/main/java/org/logstash/filters/GeoIPFilter.java index 8e8bec2..c9fc03e 100644 --- a/src/main/java/org/logstash/filters/GeoIPFilter.java +++ b/src/main/java/org/logstash/filters/GeoIPFilter.java @@ -26,6 +26,7 @@ import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.geoip2.model.AnonymousIpResponse; import com.maxmind.geoip2.record.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -101,7 +102,7 @@ private Set createDesiredFields(List fields) { desiredFields = Fields.DEFAULT_ASN_LITE_FIELDS; break; case ANONYMOUS_DB_TYPE: - desiredFields = Fields.DEFAULT_ANONYMOUS_DB_TYPE; + desiredFields = Fields.DEFAULT_ANONYMOUS_FIELDS; break; } } else { From 34a69d53bd5bcda1c10c02dc2d11c3bbe7f7c5e0 Mon Sep 17 00:00:00 2001 From: Metin de Vreugd Date: Sun, 8 Jul 2018 23:49:05 +0200 Subject: [PATCH 4/4] update GeoLite2 sha --- vendor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor.json b/vendor.json index 9802c55..892be8b 100644 --- a/vendor.json +++ b/vendor.json @@ -1,7 +1,7 @@ [ { "url": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz", - "sha1": "faccb3c92fd5bee0261e6e7640a79c7e37624d16" + "sha1": "6e9bcfac392052725463550b3bbaafc91920caba" }, { "url": "https://s3.amazonaws.com/download.elasticsearch.org/logstash/maxmind/GeoLite2-ASN.mmdb",