feat: add new certspotter integration module

eredotpkfr · Oct 3, 2024 · eafeb26 · eafeb26
1 parent ca1343a
commit eafeb26
Show file tree

Hide file tree

Showing 21 changed files with 281 additions and 65 deletions.
diff --git a/.env.template b/.env.template
@@ -3,3 +3,4 @@ SUBSCAN_BINARYEDGE_APIKEY=bar
 SUBSCAN_BUFFEROVER_APIKEY=baz
 SUBSCAN_BUILTWITH_APIKEY=foo
 SUBSCAN_CENSYS_APIKEY=bar
+SUBSCAN_CERTSPOTTER_APIKEY=baz
diff --git a/src/bin/subscan.rs b/src/bin/subscan.rs
@@ -17,7 +17,7 @@ async fn main() {
         let mut module = item.lock().await;
         let requester = module.requester().await.unwrap();
 
-        if module.name().await != "Censys" {
+        if module.name().await != "certspotter" {
             continue;
         }
 

diff --git a/src/cache.rs b/src/cache.rs
@@ -2,7 +2,9 @@ use crate::{
     enums::SubscanModuleDispatcher,
     modules::{
         engines::{bing, duckduckgo, google, yahoo},
-        integrations::{alienvault, anubis, bevigil, binaryedge, bufferover, builtwith, censys},
+        integrations::{
+            alienvault, anubis, bevigil, binaryedge, bufferover, builtwith, censys, certspotter,
+        },
     },
 };
 use lazy_static::lazy_static;
@@ -22,10 +24,11 @@ lazy_static! {
         Mutex::new(alienvault::AlienVault::dispatcher()),
         Mutex::new(anubis::Anubis::dispatcher()),
         Mutex::new(bevigil::Bevigil::dispatcher()),
-        Mutex::new(binaryedge::Binaryedge::dispatcher()),
-        Mutex::new(bufferover::Bufferover::dispatcher()),
-        Mutex::new(builtwith::Builtwith::dispatcher()),
+        Mutex::new(binaryedge::BinaryEdge::dispatcher()),
+        Mutex::new(bufferover::BufferOver::dispatcher()),
+        Mutex::new(builtwith::BuiltWith::dispatcher()),
         Mutex::new(censys::Censys::dispatcher()),
+        Mutex::new(certspotter::CertSpotter::dispatcher()),
     ];
 }
 

diff --git a/src/modules/engines/bing.rs b/src/modules/engines/bing.rs
@@ -6,7 +6,7 @@ use crate::{
 };
 use reqwest::Url;
 
-pub const BING_MODULE_NAME: &str = "Bing";
+pub const BING_MODULE_NAME: &str = "bing";
 pub const BING_SEARCH_URL: &str = "https://www.bing.com/search";
 pub const BING_SEARCH_PARAM: &str = "q";
 pub const BING_CITE_TAG: &str = "cite";
@@ -18,7 +18,7 @@ pub const BING_CITE_TAG: &str = "cite";
 ///
 /// | Property           | Value                         |
 /// |:------------------:|:-----------------------------:|
-/// | Module Name        | `Bing`                        |
+/// | Module Name        | `bing`                        |
 /// | Search URL         | <https://www.bing.com/search> |
 /// | Search Param       | `q`                           |
 /// | Subdomain Selector | `cite`                        |

diff --git a/src/modules/engines/duckduckgo.rs b/src/modules/engines/duckduckgo.rs
@@ -6,7 +6,7 @@ use crate::{
 };
 use reqwest::Url;
 
-pub const DUCKDUCKGO_MODULE_NAME: &str = "DuckDuckGo";
+pub const DUCKDUCKGO_MODULE_NAME: &str = "duckduckgo";
 pub const DUCKDUCKGO_SEARCH_URL: &str = "https://duckduckgo.com";
 pub const DUCKDUCKGO_SEARCH_PARAM: &str = "q";
 pub const DUCKDUCKGO_CITE_TAG: &str = "article > div > div > a > span:first-child";
@@ -18,7 +18,7 @@ pub const DUCKDUCKGO_CITE_TAG: &str = "article > div > div > a > span:first-chil
 ///
 /// | Property           | Value                                        |
 /// |:------------------:|:--------------------------------------------:|
-/// | Module Name        | `DuckDuckGo`                                 |
+/// | Module Name        | `duckduckgo`                                 |
 /// | Search URL         | <https://duckduckgo.com>                     |
 /// | Search Param       | `q`                                          |
 /// | Subdomain Selector | `article > div > div > a > span:first-child` |

diff --git a/src/modules/engines/google.rs b/src/modules/engines/google.rs
@@ -6,7 +6,7 @@ use crate::{
 };
 use reqwest::Url;
 
-pub const GOOGLE_MODULE_NAME: &str = "Google";
+pub const GOOGLE_MODULE_NAME: &str = "google";
 pub const GOOGLE_SEARCH_URL: &str = "https://www.google.com/search";
 pub const GOOGLE_SEARCH_PARAM: &str = "q";
 pub const GOOGLE_CITE_TAG: &str = "cite";
@@ -18,7 +18,7 @@ pub const GOOGLE_CITE_TAG: &str = "cite";
 ///
 /// | Property           | Value                           |
 /// |:------------------:|:-------------------------------:|
-/// | Module Name        | `Google`                        |
+/// | Module Name        | `google`                        |
 /// | Search URL         | <https://www.google.com/search> |
 /// | Search Param       | `q`                             |
 /// | Subdomain Selector | `cite`                          |

diff --git a/src/modules/engines/yahoo.rs b/src/modules/engines/yahoo.rs
@@ -6,7 +6,7 @@ use crate::{
 };
 use reqwest::Url;
 
-pub const YAHOO_MODULE_NAME: &str = "Yahoo";
+pub const YAHOO_MODULE_NAME: &str = "yahoo";
 pub const YAHOO_SEARCH_URL: &str = "https://search.yahoo.com/search";
 pub const YAHOO_SEARCH_PARAM: &str = "p";
 pub const YAHOO_CITE_TAG: &str = "ol > li > div > div > h3 > a > span";
@@ -18,7 +18,7 @@ pub const YAHOO_CITE_TAG: &str = "ol > li > div > div > h3 > a > span";
 ///
 /// | Property           | Value                                 |
 /// |:------------------:|:-------------------------------------:|
-/// | Module Name        | `Yahoo`                               |
+/// | Module Name        | `yahoo`                               |
 /// | Search URL         | <https://search.yahoo.com/search>     |
 /// | Search Param       | `p`                                   |
 /// | Subdomain Selector | `ol > li > div > div > h3 > a > span` |

diff --git a/src/modules/integrations/alienvault.rs b/src/modules/integrations/alienvault.rs
@@ -10,13 +10,19 @@ use crate::{
 use reqwest::Url;
 use serde_json::Value;
 
-pub const ALIENVAULT_MODULE_NAME: &str = "AlienVault";
+pub const ALIENVAULT_MODULE_NAME: &str = "alienvault";
 pub const ALIENVAULT_URL: &str = "https://otx.alienvault.com/api/v1/indicators/domain";
 
-/// Alienvault API integration module
+/// `AlienVault` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
+///
+/// | Property           | Value                           |
+/// |:------------------:|:-------------------------------:|
+/// | Module Name        | `alienvault`                    |
+/// | Doc URL            | <https://otx.alienvault.com>    |
+/// | Authentication     | [`APIAuthMethod::NoAuth`]       |
 pub struct AlienVault {}
 
 impl AlienVault {
@@ -36,14 +42,14 @@ impl AlienVault {
         generic.into()
     }
 
-    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
-        None
-    }
-
     pub fn get_query_url(domain: &str) -> String {
         format!("{ALIENVAULT_URL}/{domain}/passive_dns")
     }
 
+    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
+        None
+    }
+
     pub fn extract(content: Value, _domain: String) -> BTreeSet<Subdomain> {
         if let Some(passives) = content["passive_dns"].as_array() {
             let filter = |item: &Value| Some(item["hostname"].as_str()?.to_string());

diff --git a/src/modules/integrations/anubis.rs b/src/modules/integrations/anubis.rs
@@ -9,13 +9,19 @@ use reqwest::Url;
 use serde_json::Value;
 use std::collections::BTreeSet;
 
-pub const ANUBIS_MODULE_NAME: &str = "Anubis";
+pub const ANUBIS_MODULE_NAME: &str = "anubis";
 pub const ANUBIS_URL: &str = "https://jonlu.ca/anubis/subdomains";
 
-/// Anubis API integration module
+/// `Anubis` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
+///
+/// | Property           | Value                           |
+/// |:------------------:|:-------------------------------:|
+/// | Module Name        | `anubis`                        |
+/// | Doc URL            | <https://jonlu.ca/anubis>       |
+/// | Authentication     | [`APIAuthMethod::NoAuth`]       |
 pub struct Anubis {}
 
 impl Anubis {
@@ -35,14 +41,14 @@ impl Anubis {
         generic.into()
     }
 
-    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
-        None
-    }
-
     pub fn get_query_url(domain: &str) -> String {
         format!("{ANUBIS_URL}/{domain}")
     }
 
+    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
+        None
+    }
+
     pub fn extract(content: Value, _domain: String) -> BTreeSet<Subdomain> {
         if let Some(subs) = content.as_array() {
             let filter = |item: &Value| Some(item.as_str()?.to_string());

diff --git a/src/modules/integrations/bevigil.rs b/src/modules/integrations/bevigil.rs
@@ -9,13 +9,19 @@ use reqwest::Url;
 use serde_json::Value;
 use std::collections::BTreeSet;
 
-pub const BEVIGIL_MODULE_NAME: &str = "Bevigil";
+pub const BEVIGIL_MODULE_NAME: &str = "bevigil";
 pub const BEVIGIL_URL: &str = "https://osint.bevigil.com/api";
 
-/// Bevigil API integration module
+/// `Bevigil` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
+///
+/// | Property           | Value                             |
+/// |:------------------:|:---------------------------------:|
+/// | Module Name        | `bevigil`                         |
+/// | Doc URL            | <https://bevigil.com>             |
+/// | Authentication     | [`APIAuthMethod::APIKeyAsHeader`] |
 pub struct Bevigil {}
 
 impl Bevigil {
@@ -35,14 +41,14 @@ impl Bevigil {
         generic.into()
     }
 
-    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
-        None
-    }
-
     pub fn get_query_url(domain: &str) -> String {
         format!("{BEVIGIL_URL}/{domain}/subdomains")
     }
 
+    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
+        None
+    }
+
     pub fn extract(content: Value, _domain: String) -> BTreeSet<Subdomain> {
         if let Some(subs) = content["subdomains"].as_array() {
             let filter = |item: &Value| Some(item.as_str()?.to_string());

diff --git a/src/modules/integrations/binaryedge.rs b/src/modules/integrations/binaryedge.rs
@@ -9,16 +9,22 @@ use reqwest::Url;
 use serde_json::Value;
 use std::collections::BTreeSet;
 
-pub const BINARYEDGE_MODULE_NAME: &str = "Binaryedge";
+pub const BINARYEDGE_MODULE_NAME: &str = "binaryedge";
 pub const BINARYEDGE_URL: &str = "https://api.binaryedge.io/v2/query/domains/subdomain";
 
-/// Binaryedge API integration module
+/// `BinaryEdge` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
-pub struct Binaryedge {}
+///
+/// | Property           | Value                             |
+/// |:------------------:|:---------------------------------:|
+/// | Module Name        | `binaryedge`                      |
+/// | Doc URL            | <https://www.binaryedge.io>       |
+/// | Authentication     | [`APIAuthMethod::APIKeyAsHeader`] |
+pub struct BinaryEdge {}
 
-impl Binaryedge {
+impl BinaryEdge {
     pub fn dispatcher() -> SubscanModuleDispatcher {
         let requester: RequesterDispatcher = HTTPClient::default().into();
         let extractor: JSONExtractor = JSONExtractor::new(Box::new(Self::extract));

diff --git a/src/modules/integrations/bufferover.rs b/src/modules/integrations/bufferover.rs
@@ -11,16 +11,22 @@ use reqwest::Url;
 use serde_json::Value;
 use std::collections::BTreeSet;
 
-pub const BUFFEROVER_MODULE_NAME: &str = "Bufferover";
+pub const BUFFEROVER_MODULE_NAME: &str = "bufferover";
 pub const BUFFEROVER_URL: &str = "https://tls.bufferover.run";
 
-/// Bufferover API integration module
+/// `BufferOver` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
-pub struct Bufferover {}
+///
+/// | Property           | Value                             |
+/// |:------------------:|:---------------------------------:|
+/// | Module Name        | `bufferover`                      |
+/// | Doc URL            | <https://tls.bufferover.run>      |
+/// | Authentication     | [`APIAuthMethod::APIKeyAsHeader`] |
+pub struct BufferOver {}
 
-impl Bufferover {
+impl BufferOver {
     pub fn dispatcher() -> SubscanModuleDispatcher {
         let requester: RequesterDispatcher = HTTPClient::default().into();
         let extractor: JSONExtractor = JSONExtractor::new(Box::new(Self::extract));
@@ -37,14 +43,14 @@ impl Bufferover {
         generic.into()
     }
 
-    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
-        None
-    }
-
     pub fn get_query_url(domain: &str) -> String {
         format!("{BUFFEROVER_URL}/dns?q={domain}")
     }
 
+    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
+        None
+    }
+
     pub fn extract(content: Value, domain: String) -> BTreeSet<Subdomain> {
         let mut subs = BTreeSet::new();
         let pattern = generate_subdomain_regex(domain).unwrap();

diff --git a/src/modules/integrations/builtwith.rs b/src/modules/integrations/builtwith.rs
@@ -9,16 +9,22 @@ use reqwest::Url;
 use serde_json::Value;
 use std::collections::BTreeSet;
 
-pub const BUILTWITH_MODULE_NAME: &str = "Builtwith";
+pub const BUILTWITH_MODULE_NAME: &str = "builtwith";
 pub const BUILTWITH_URL: &str = "https://api.builtwith.com/v21/api.json";
 
-/// Builtwith API integration module
+/// `BuiltWith` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
-pub struct Builtwith {}
+///
+/// | Property           | Value                                 |
+/// |:------------------:|:-------------------------------------:|
+/// | Module Name        | `builtwith`                           |
+/// | Doc URL            | <https://api.builtwith.com>           |
+/// | Authentication     | [`APIAuthMethod::APIKeyAsQueryParam`] |
+pub struct BuiltWith {}
 
-impl Builtwith {
+impl BuiltWith {
     pub fn dispatcher() -> SubscanModuleDispatcher {
         let requester: RequesterDispatcher = HTTPClient::default().into();
         let extractor: JSONExtractor = JSONExtractor::new(Box::new(Self::extract));
@@ -35,10 +41,6 @@ impl Builtwith {
         generic.into()
     }
 
-    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
-        None
-    }
-
     pub fn get_query_url(domain: &str) -> String {
         let params = &[
             ("HIDETEXT", "yes"),
@@ -55,6 +57,10 @@ impl Builtwith {
         url.unwrap().to_string()
     }
 
+    pub fn get_next_url(_url: Url, _content: Value) -> Option<Url> {
+        None
+    }
+
     pub fn extract(content: Value, domain: String) -> BTreeSet<Subdomain> {
         let mut subs = BTreeSet::new();
 

diff --git a/src/modules/integrations/censys.rs b/src/modules/integrations/censys.rs
@@ -11,13 +11,19 @@ use reqwest::Url;
 use serde_json::Value;
 use std::collections::BTreeSet;
 
-pub const CENSYS_MODULE_NAME: &str = "Censys";
+pub const CENSYS_MODULE_NAME: &str = "censys";
 pub const CENSYS_URL: &str = "https://search.censys.io/api/v2/certificates/search";
 
-/// Censys API integration module
+/// `Censys` API integration module
 ///
 /// It uses [`GenericAPIIntegrationModule`] its own inner
 /// here are the configurations
+///
+/// | Property           | Value                             |
+/// |:------------------:|:---------------------------------:|
+/// | Module Name        | `censys`                          |
+/// | Doc URL            | <https://search.censys.io>        |
+/// | Authentication     | [`APIAuthMethod::APIKeyAsHeader`] |
 pub struct Censys {}
 
 impl Censys {