From e6bf5c515999beaac5e90f8bef7a602239ef4941 Mon Sep 17 00:00:00 2001 From: Psyray Date: Wed, 13 Nov 2024 21:07:05 +0100 Subject: [PATCH] fix(tools): update configuration file paths and directory structure (#224) * refactor: update configuration file paths and directory structure - Modified the directory structure and file paths for configuration files in the Docker setup and application code. - Updated the amass configuration file path to be more organized under a dedicated directory. - Adjusted symbolic links and file handling logic to align with the new directory structure. - Enhanced the update_config function to accommodate the new file naming conventions. * fix: update GAU configuration file path and usage - Updated the path and usage of the GAU configuration file across multiple files to ensure consistency and correct file referencing. - Modified the command map in tasks.py to include the GAU configuration file path. - Adjusted the Dockerfile to copy the GAU configuration file to the correct location and removed unnecessary symbolic link creation. - Updated file path references in views.py and scanEngine/views.py to align with the new GAU configuration file path. * refactor: standardize theHarvester configuration element IDs The changes standardize the casing of the ID and name attributes for the theHarvester configuration text area across JavaScript, HTML, and Python files. This involves updating the ID from theharvester_config_text_area to theHarvester_config_text_area to ensure consistency and improve code readability. * chore: update dependencies & fix theHarvester - Updated several Go-based tool dependencies to their latest versions in the Celery Dockerfile. - Modified the command in tasks.py to remove github-code sources for theHarvester due to this bug. https://github.com/laramies/theHarvester/issues/1857 - Adjusted the column layout in the form_engine.html template to improve UI responsiveness. * fix(docker): correct s3scanner installation reference in Dockerfile Corrected the reference for the s3scanner installation in the Celery Dockerfile to fix a typo in the commit hash. --- docker/celery/Dockerfile | 28 ++++++++++--------- docker/celery/config/{.gau.toml => gau.toml} | 0 web/api/views.py | 4 +-- web/reNgine/tasks.py | 12 ++++---- .../static/scanEngine/js/custom_tools.js | 12 ++++---- .../scanEngine/_items/form_engine.html | 2 +- .../templates/scanEngine/settings/tool.html | 4 +-- web/scanEngine/views.py | 10 +++---- 8 files changed, 37 insertions(+), 35 deletions(-) rename docker/celery/config/{.gau.toml => gau.toml} (100%) diff --git a/docker/celery/Dockerfile b/docker/celery/Dockerfile index f45f1dc6..d555a2ff 100644 --- a/docker/celery/Dockerfile +++ b/docker/celery/Dockerfile @@ -112,17 +112,17 @@ RUN ARCH=$(dpkg --print-architecture) \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/tomnomnom/gf@dcd4c361f9f5ba302294ed38b8ce278e8ba69006 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/tomnomnom/unfurl@v0.4.3 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/tomnomnom/waybackurls@v0.1.0 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/httpx/cmd/httpx@v1.6.0 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@v2.6.6 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@v3.2.6 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/naabu/v2/cmd/naabu@v2.3.0 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/httpx/cmd/httpx@v1.6.9 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@v2.6.7 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@v3.3.5 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/naabu/v2/cmd/naabu@v2.3.2 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/hakluke/hakrawler@latest \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/lc/gau/v2/cmd/gau@v2.2.1 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/lc/gau/v2/cmd/gau@v2.2.4 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/owasp-amass/amass/v4/...@v4.2.0 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/ffuf/ffuf/v2@v2.1.0 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/tlsx/cmd/tlsx@v1.1.6 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/hahwul/dalfox/v2@v2.9.2 \ - && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/katana/cmd/katana@v1.1.0 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/tlsx/cmd/tlsx@v1.1.8 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/hahwul/dalfox/v2@v2.9.3 \ + && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/katana/cmd/katana@v1.1.1 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/dwisiswant0/crlfuzz/cmd/crlfuzz@v1.4.1 \ && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/sa7mon/s3scanner@c544f1cf00f70cae3f2155b24d336f515b7c598b \ && chmod 700 -R $GOPATH/pkg/* \ @@ -177,12 +177,14 @@ RUN cd $TOOLPATH/.github/OneForAll && mv /home/$USERNAME/oneforall-pyproject.tom cd /home/$USERNAME && poetry install # Create tools config files -RUN nuclei -silent && naabu -version && subfinder -version && mkdir -p /home/$USERNAME/.config/theHarvester +RUN nuclei -silent && naabu -version && subfinder -version && \ + mkdir -p /home/$USERNAME/.config/theHarvester && \ + mkdir -p /home/$USERNAME/.config/amass && \ + mkdir -p /home/$USERNAME/.config/gau COPY --chown=$USERNAME:$USERNAME ./config/the-harvester-api-keys.yaml /home/$USERNAME/.config/theHarvester/api-keys.yaml -COPY --chown=$USERNAME:$USERNAME ./config/amass.ini /home/$USERNAME/.config/amass.ini -COPY --chown=$USERNAME:$USERNAME ./config/.gau.toml /home/$USERNAME/.config/.gau.toml -RUN ln -s /home/$USERNAME/.config/.gau.toml /home/$USERNAME/.gau.toml - +COPY --chown=$USERNAME:$USERNAME ./config/amass.ini /home/$USERNAME/.config/amass/config.ini +COPY --chown=$USERNAME:$USERNAME ./config/gau.toml /home/$USERNAME/.config/gau/config.toml +RUN ln -s /home/$USERNAME/.config/theHarvester /home/$USERNAME/.theHarvester COPY ./entrypoint.sh /entrypoint.sh RUN mkdir -p /home/$USERNAME/rengine /home/$USERNAME/scan_results \ && chown -R $USERNAME:$USERNAME /home/$USERNAME/rengine \ diff --git a/docker/celery/config/.gau.toml b/docker/celery/config/gau.toml similarity index 100% rename from docker/celery/config/.gau.toml rename to docker/celery/config/gau.toml diff --git a/web/api/views.py b/web/api/views.py index bad17743..c651e84c 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -1324,7 +1324,7 @@ def get(self, request, format=None): return Response(response) if 'amass_config' in req.query_params: - path = str(Path.home() / ".config" / "amass.ini") + path = str(Path.home() / ".config" / "amass" / "config.ini") if not os.path.exists(path): run_command(f'touch {path}') response['message'] = 'File Created!' @@ -1361,7 +1361,7 @@ def get(self, request, format=None): return Response(response) if 'gau_config' in req.query_params: - path = str(Path.home() / ".config" / '.gau.toml') + path = str(Path.home() / ".config" / 'gau' / 'config.toml') if not os.path.exists(path): run_command(f'touch {path}') response['message'] = 'File Created!' diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py index a3ed329f..4e016824 100644 --- a/web/reNgine/tasks.py +++ b/web/reNgine/tasks.py @@ -422,14 +422,14 @@ def subdomain_discovery( if tool == 'amass-passive': use_amass_config = config.get(USE_AMASS_CONFIG, False) cmd = f'amass enum -passive -d {host} -o ' + str(Path(self.results_dir) / 'subdomains_amass.txt') - cmd += (' -config ' + str(Path.home() / '.config' / 'amass.ini')) if use_amass_config else '' + cmd += (' -config ' + str(Path.home() / '.config' / 'amass' / 'config.ini')) if use_amass_config else '' elif tool == 'amass-active': use_amass_config = config.get(USE_AMASS_CONFIG, False) amass_wordlist_name = config.get(AMASS_WORDLIST, AMASS_DEFAULT_WORDLIST_NAME) wordlist_path = str(Path(AMASS_DEFAULT_WORDLIST_PATH) / f'{amass_wordlist_name}.txt') cmd = f'amass enum -active -d {host} -o ' + str(Path(self.results_dir) / 'subdomains_amass_active.txt') - cmd += (' -config ' + str(Path.home() / '.config' / 'amass.ini')) if use_amass_config else '' + cmd += (' -config ' + str(Path.home() / '.config' / 'amass' / 'config.ini')) if use_amass_config else '' cmd += f' -brute -w {wordlist_path}' elif tool == 'sublist3r': @@ -1027,7 +1027,7 @@ def theHarvester(config, host, scan_history_id, activity_id, results_dir, ctx={} output_path_json = str(Path(results_dir) / 'theHarvester.json') theHarvester_dir = str(Path.home() / ".config" / 'theHarvester') history_file = str(Path(results_dir) / 'commands.txt') - cmd = f'theHarvester -d {host} -b all -f {output_path_json}' + cmd = f'theHarvester -d {host} -f {output_path_json} -b anubis,baidu,bevigil,binaryedge,bing,bingapi,bufferoverun,brave,censys,certspotter,criminalip,crtsh,dnsdumpster,duckduckgo,fullhunt,hackertarget,hunter,hunterhow,intelx,netlas,onyphe,otx,pentesttools,projectdiscovery,rapiddns,rocketreach,securityTrails,sitedossier,subdomaincenter,subdomainfinderc99,threatminer,tomba,urlscan,virustotal,yahoo,zoomeye' # Update proxies.yaml proxy_query = Proxy.objects.all() @@ -1828,11 +1828,11 @@ def fetch_url(self, urls=[], ctx={}, description=None): # Initialize command map for tools cmd_map = { - 'gau': f'gau', + 'gau': 'gau --config ' + str(Path.home() / '.config' / 'gau' / 'config.toml'), 'hakrawler': 'hakrawler -subs -u', 'waybackurls': 'waybackurls', - 'gospider': f'gospider --js -d 2 --sitemap --robots -w -r -a', - 'katana': f'katana -silent -jc -kf all -d 3 -fs rdn', + 'gospider': 'gospider --js -d 2 --sitemap --robots -w -r -a', + 'katana': 'katana -silent -jc -kf all -d 3 -fs rdn', } if proxy: cmd_map['gau'] += f' --proxy "{proxy}"' diff --git a/web/scanEngine/static/scanEngine/js/custom_tools.js b/web/scanEngine/static/scanEngine/js/custom_tools.js index 4d8cef2a..f0fa38be 100644 --- a/web/scanEngine/static/scanEngine/js/custom_tools.js +++ b/web/scanEngine/static/scanEngine/js/custom_tools.js @@ -132,17 +132,17 @@ $("#amass_config_text_area").dblclick(function() { // get theharvester config $.getJSON(`/api/getFileContents?theharvester_config&format=json`, function(data) { - $("#theharvester_config_text_area").attr("rows", 14); - $("textarea#theharvester_config_text_area").html(htmlEncode(data['content'])); + $("#theHarvester_config_text_area").attr("rows", 14); + $("textarea#theHarvester_config_text_area").html(htmlEncode(data['content'])); }).fail(function(){ - $("#theharvester_config_text_area").removeAttr("readonly"); - $("textarea#theharvester_config_text_area").html(`# Your the Harvester configuration here.`); + $("#theHarvester_config_text_area").removeAttr("readonly"); + $("textarea#theHarvester_config_text_area").html(`# Your the Harvester configuration here.`); $("#theHarvester-config-form").append(''); }); -$("#theharvester_config_text_area").dblclick(function() { +$("#theHarvester_config_text_area").dblclick(function() { if (!document.getElementById('theharvester-config-submit')) { - $("#theharvester_config_text_area").removeAttr("readonly"); + $("#theHarvester_config_text_area").removeAttr("readonly"); $("#theharvester-config-form").append(''); } }); diff --git a/web/scanEngine/templates/scanEngine/_items/form_engine.html b/web/scanEngine/templates/scanEngine/_items/form_engine.html index 96792fd4..3350dbca 100644 --- a/web/scanEngine/templates/scanEngine/_items/form_engine.html +++ b/web/scanEngine/templates/scanEngine/_items/form_engine.html @@ -110,7 +110,7 @@
Vulnerability Scanner    +

YAML Configurations

reNgine supports YAML based configurations for customizing the scan. The default settings are given below, you may choose to proceed with the default settings or configure it according to your choice. diff --git a/web/scanEngine/templates/scanEngine/settings/tool.html b/web/scanEngine/templates/scanEngine/settings/tool.html index 0462fafd..3920e282 100644 --- a/web/scanEngine/templates/scanEngine/settings/tool.html +++ b/web/scanEngine/templates/scanEngine/settings/tool.html @@ -65,7 +65,7 @@

Current Subfinder Configuration

Amass

This section lets you modify Amass config file. You can find more information about the Amass config here.
- Please note that only YAML config is supported. + Please note that only INI config is supported.

{% csrf_token %} @@ -159,7 +159,7 @@

theHarvester

{% csrf_token %}

Current theHarvester Configuration

- +
diff --git a/web/scanEngine/views.py b/web/scanEngine/views.py index f61d47a1..a4740f6c 100644 --- a/web/scanEngine/views.py +++ b/web/scanEngine/views.py @@ -223,9 +223,9 @@ def handle_post_request(request): 'nuclei_config_text_area': lambda r: update_config(r, 'nuclei', 'Nuclei'), 'subfinder_config_text_area': lambda r: update_config(r, 'subfinder', 'Subfinder'), 'naabu_config_text_area': lambda r: update_config(r, 'naabu', 'Naabu'), - 'amass_config_text_area': lambda r: update_config(r, 'amass', 'Amass', '.ini'), - 'theharvester_config_text_area': lambda r: update_config(r, 'theHarvester/api-keys', 'theHarvester'), - 'gau_config_text_area': lambda r: update_config(r, '.gau', 'GAU', '.toml'), + 'amass_config_text_area': lambda r: update_config(r, 'amass', 'Amass', 'config', '.ini'), + 'theHarvester_config_text_area': lambda r: update_config(r, 'theHarvester', 'theHarvester', 'api-keys', '.yaml'), + 'gau_config_text_area': lambda r: update_config(r, 'gau', 'GAU', 'config', '.toml'), } for key, handler in handlers.items(): if key in request.FILES or key in request.POST: @@ -250,8 +250,8 @@ def handle_file_upload(request, file_key, directory, expected_extension, pattern file.write(uploaded_file.read().decode("utf-8")) messages.info(request, f'{pattern_name} {uploaded_file.name[:4]} successfully uploaded') -def update_config(request, tool_name, display_name, file_extension='.yaml'): - config_path = Path.home() / '.config' / tool_name / f'config{file_extension}' +def update_config(request, tool_name, display_name, file_name='config', file_extension='.yaml'): + config_path = Path.home() / '.config' / tool_name / f'{file_name}{file_extension}' with open(config_path, "w", encoding='utf-8') as fhandle: fhandle.write(request.POST.get(f'{tool_name}_config_text_area')) messages.info(request, f'{display_name} config updated!')