From 7addc2a0bfc6b1798afe5c3d5fcd2e39bcc36eef Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:46:35 +0100 Subject: [PATCH 1/6] fix(external_tools): incorrect Naabu regex --- web/fixtures/external_tools.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index a5435cd3a..1e7c41363 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -70,7 +70,7 @@ version_lookup_command: naabu -version update_command: go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest install_command: go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest - version_match_regex: ' (\d+\.)?(\d+\.)?(\*|\d+)' + version_match_regex: '(\b\d+\.\d+\.\d+\b)' is_default: true is_subdomain_gathering: false is_github_cloned: false From b33b61ccdec396f38ed46e2c1e852dbb752f327c Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:03:55 +0100 Subject: [PATCH 2/6] feat(external_tools): remove deprecated Gauplus --- web/Dockerfile | 1 - web/fixtures/external_tools.yaml | 33 ++++++++------------------------ web/reNgine/tasks.py | 3 --- 3 files changed, 8 insertions(+), 29 deletions(-) diff --git a/web/Dockerfile b/web/Dockerfile index ff1665768..b3719588a 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -67,7 +67,6 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 # Download Go packages -RUN go install -v -v github.com/bp0lr/gauplus@latest RUN go install -v github.com/jaeles-project/gospider@latest RUN go install -v github.com/tomnomnom/gf@latest RUN go install -v github.com/tomnomnom/unfurl@latest diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index 1e7c41363..15328a32e 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -171,23 +171,6 @@ subdomain_gathering_command: null - model: scanEngine.installedexternaltool pk: 10 - fields: - logo_url: null - name: gauplus - description: Get all URLs - github_url: https://github.com/bp0lr/gauplus - license_url: https://github.com/bp0lr/gauplus/blob/main/LICENSE - version_lookup_command: gauplus -version - update_command: go install -v -v github.com/bp0lr/gauplus@latest - install_command: go install -v -v github.com/bp0lr/gauplus@latest - version_match_regex: '[vV]*(\d+\.)?(\d+\.)?(\*|\d+)' - is_default: true - is_subdomain_gathering: false - is_github_cloned: true - github_clone_path: /usr/src/github/gauplus - subdomain_gathering_command: null -- model: scanEngine.installedexternaltool - pk: 11 fields: logo_url: null name: OneForAll @@ -204,7 +187,7 @@ github_clone_path: /usr/src/github/OneForAll subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 12 + pk: 11 fields: logo_url: https://raw.githubusercontent.com/laramies/theHarvester/master/theHarvester-logo.png name: theHarvester @@ -225,7 +208,7 @@ github_clone_path: /usr/src/github/theHarvester subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 13 + pk: 12 fields: logo_url: null name: ctfr @@ -244,7 +227,7 @@ github_clone_path: /usr/src/github/ctfr subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 14 + pk: 13 fields: logo_url: https://user-images.githubusercontent.com/8293321/174841003-01a62bad-2ecf-4874-89c4-efa53dd56884.png name: tlsx @@ -261,7 +244,7 @@ github_clone_path: null subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 15 + pk: 14 fields: logo_url: https://avatars.githubusercontent.com/u/79084675?v=4 name: netlas @@ -278,7 +261,7 @@ github_clone_path: null subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 16 + pk: 15 fields: logo_url: https://github.githubassets.com/images/icons/emoji/unicode/1f98a.png name: dalfox @@ -296,7 +279,7 @@ github_clone_path: null subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 17 + pk: 16 fields: logo_url: https://user-images.githubusercontent.com/8293321/196779266-421c79d4-643a-4f73-9b54-3da379bbac09.png name: katana @@ -313,7 +296,7 @@ github_clone_path: null subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 18 + pk: 17 fields: logo_url: https://user-images.githubusercontent.com/25837540/90128972-fc3bdf00-dd91-11ea-8c3b-0d6f4e8c6ba3.png name: crlfuzz @@ -330,7 +313,7 @@ github_clone_path: null subdomain_gathering_command: null - model: scanEngine.installedexternaltool - pk: 19 + pk: 18 fields: logo_url: null name: gau diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index 048dc5048..14f5e1faf 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -1759,7 +1759,6 @@ def fetch_url(self, urls=[], ctx={}, description=None): # Tools cmds cmd_map = { 'gau': f'gau', - 'gauplus': f'gauplus -random-agent', 'hakrawler': 'hakrawler -subs -u', 'waybackurls': 'waybackurls', 'gospider': f'gospider -S {input_path} --js -d 2 --sitemap --robots -w -r', @@ -1767,13 +1766,11 @@ def fetch_url(self, urls=[], ctx={}, description=None): } if proxy: cmd_map['gau'] += f' --proxy "{proxy}"' - cmd_map['gauplus'] += f' -p "{proxy}"' cmd_map['gospider'] += f' -p {proxy}' cmd_map['hakrawler'] += f' -proxy {proxy}' cmd_map['katana'] += f' -proxy {proxy}' if threads > 0: cmd_map['gau'] += f' --threads {threads}' - cmd_map['gauplus'] += f' -t {threads}' cmd_map['gospider'] += f' -t {threads}' cmd_map['katana'] += f' -c {threads}' if custom_header: From 1ca8db6ac01a9cd43cc0a9bb65ef394a90e59f37 Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:32:58 +0100 Subject: [PATCH 3/6] feat(external_tools): add lookup command theHarvester --- web/fixtures/external_tools.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index 15328a32e..59e4e56d5 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -198,10 +198,10 @@ names, subdomains, IPs and URLs using multiple public data sources. github_url: https://github.com/laramies/theHarvester license_url: https://github.com/laramies/theHarvester/blob/master/README/LICENSES - version_lookup_command: null + version_lookup_command: grep -oP '(?<=theHarvester )\d+\.\d+\.\d+' <<< "$(python3 theHarvester.py -h)" update_command: git pull install_command: git clone https://github.com/laramies/theHarvester - version_match_regex: (\d+\.)?(\d+\.)?(\*|\d+) + version_match_regex: (\d+\.\d+\.\d+) is_default: true is_subdomain_gathering: false is_github_cloned: true From a204e51443f1769b7f8d17c40896b00e70700cf2 Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:41:40 +0100 Subject: [PATCH 4/6] feat(external_tools): add lookup command Netlas --- web/fixtures/external_tools.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index 59e4e56d5..bf5c38c92 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -251,7 +251,7 @@ description: Non-intrusive Internet Scanner. github_url: https://github.com/netlas-io/netlas-python license_url: https://github.com/netlas-io/netlas-python/blob/master/LICENSE - version_lookup_command: null + version_lookup_command: pip3 show netlas update_command: pip3 install netlas --update install_command: pip3 install netlas version_match_regex: (\d+\.)?(\d+\.)?(\*|\d+) From e68e80bf9403ccca113b196780c0e75914a283ac Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 8 Dec 2023 11:50:43 +0530 Subject: [PATCH 5/6] update theHarvester version lookup command --- web/fixtures/external_tools.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index bf5c38c92..d3ba8ce55 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -198,10 +198,10 @@ names, subdomains, IPs and URLs using multiple public data sources. github_url: https://github.com/laramies/theHarvester license_url: https://github.com/laramies/theHarvester/blob/master/README/LICENSES - version_lookup_command: grep -oP '(?<=theHarvester )\d+\.\d+\.\d+' <<< "$(python3 theHarvester.py -h)" + version_lookup_command: cat /usr/src/github/theHarvester/theHarvester/lib/version.py update_command: git pull install_command: git clone https://github.com/laramies/theHarvester - version_match_regex: (\d+\.\d+\.\d+) + version_match_regex: \b\d+\.\d+\.\d+\b is_default: true is_subdomain_gathering: false is_github_cloned: true @@ -328,4 +328,4 @@ is_subdomain_gathering: false is_github_cloned: false github_clone_path: null - subdomain_gathering_command: null \ No newline at end of file + subdomain_gathering_command: null From d3154484c5e3fe15a65513fcb657f934e00bbbe1 Mon Sep 17 00:00:00 2001 From: Yogesh Ojha Date: Fri, 8 Dec 2023 12:14:37 +0530 Subject: [PATCH 6/6] fix oneforall version --- web/fixtures/external_tools.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/fixtures/external_tools.yaml b/web/fixtures/external_tools.yaml index d3ba8ce55..0c2994b64 100644 --- a/web/fixtures/external_tools.yaml +++ b/web/fixtures/external_tools.yaml @@ -177,10 +177,10 @@ description: A powerful subdomain integration tool. github_url: https://github.com/shmilylty/OneForAll license_url: https://github.com/shmilylty/OneForAll/blob/master/LICENSE - version_lookup_command: python3 /usr/src/github/OneForAll/oneforall.py version + version_lookup_command: cat /usr/src/github/OneForAll/oneforall.py update_command: git pull install_command: git clone https://github.com/shmilylty/OneForAll - version_match_regex: 'v(\d+\.)?(\d+\.)?(\*|\d+)' + version_match_regex: v\d+\.\d+\.\d+\b is_default: true is_subdomain_gathering: true is_github_cloned: true