Merge pull request #727 from six2dez/dev

Dev

README.md

@@ -7,12 +7,12 @@
 </h1>
 
 <p align="center">
-  <a href="https://github.com/six2dez/reconftw/releases/tag/v2.6">
-    <img src="https://img.shields.io/badge/release-v2.6-green">
+  <a href="https://github.com/six2dez/reconftw/releases/tag/v2.7">
+    <img src="https://img.shields.io/badge/release-v2.7-green">
   </a>
    </a>
-  <a href="https://www.gnu.org/licenses/gpl-3.0.en.html">
-      <img src="https://img.shields.io/badge/license-GPL3-_red.svg">
+  <a href="https://opensource.org/licenses/MIT">
+      <img src="https://img.shields.io/badge/License-MIT-yellow.svg">
   </a>
   <a href="https://twitter.com/Six2dez1">
     <img src="https://img.shields.io/badge/twitter-%40Six2dez1-blue">
@@ -59,7 +59,6 @@ So, what are you waiting for? Go! Go! Go! :boom:
     - [Perform all steps (whole recon + all attacks) a.k.a. YOLO mode](#perform-all-steps-whole-recon--all-attacks-aka-yolo-mode)
     - [Show help section](#show-help-section)
 - [Axiom Support :cloud:](#axiom-support-cloud)
-- [BBRF Support :computer:](#bbrf-support-computer)
 - [Sample video](#sample-video)
 - [:fire: Features :fire:](#fire-features-fire)
   - [Osint](#osint)
@@ -146,7 +145,7 @@ Yes! reconFTW can also be easily deployed with Terraform and Ansible to AWS, if
 
 ```yaml
 #################################################################
-#   reconFTW config file   #
+#			reconFTW config file			#
 #################################################################
 
 # General values
@@ -155,9 +154,13 @@ SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # Get current sc
 profile_shell=".$(basename $(echo $SHELL))rc" # Get current shell profile
 reconftw_version=$(git rev-parse --abbrev-ref HEAD)-$(git describe --tags) # Fetch current reconftw version
 generate_resolvers=false # Generate custom resolvers with dnsvalidator
-update_resolvers=true # Fetch and rewrite resolvers before DNS resolution
+update_resolvers=true # Fetch and rewrite resolvers from trickest/resolvers before DNS resolution
+resolvers_url="https://raw.githubusercontent.com/trickest/resolvers/main/resolvers.txt"
+resolvers_trusted_url="https://raw.githubusercontent.com/six2dez/resolvers_reconftw/main/resolvers_trusted.txt"
+fuzzing_remote_list="https://raw.githubusercontent.com/six2dez/OneListForAll/main/onelistforallmicro.txt" # Used to send to axiom(if used) on fuzzing 
 proxy_url="http://127.0.0.1:8080/" # Proxy url
 install_golang=true # Set it to false if you already have Golang configured and ready
+upgrade_tools=true
 #dir_output=/custom/output/path
 
 # Golang Vars (Comment or change on your own)
@@ -169,6 +172,8 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH
 #NOTIFY_CONFIG=~/.config/notify/provider-config.yaml # No need to define
 AMASS_CONFIG=~/.config/amass/config.ini
 GITHUB_TOKENS=${tools}/.github_tokens
+GITLAB_TOKENS=${tools}/.gitlab_tokens
+SUBGPT_COOKIE=${tools}/subgpt_cookies.json
 #CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path
 
 # APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line
@@ -189,7 +194,7 @@ GOOGLE_DORKS=true
 GITHUB_DORKS=true
 GITHUB_REPOS=true
 METADATA=true # Fetch metadata from indexed office documents
-EMAILS=true # Fetch emails from differents sites
+EMAILS=true # Fetch emails from differents sites 
 DOMAIN_INFO=true # whois info
 REVERSE_WHOIS=true # amass intel reverse whois info, takes some time
 IP_INFO=true    # Reverse IP search, geolocation and whois
@@ -206,8 +211,10 @@ SUBANALYTICS=true # Google Analytics search
 SUBBRUTE=true # DNS bruteforcing
 SUBSCRAPING=true # Subdomains extraction from web crawling
 SUBPERMUTE=true # DNS permutations
+SUBREGEXPERMUTE=true # Permutations by regex analysis
+SUBGPT=true # Permutations by BingGPT prediction
 PERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper)
-GOTATOR_FLAGS="-depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator
+GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator
 SUBTAKEOVER=false # Check subdomain takeovers, false by default cuz nuclei already check this
 SUB_RECURSIVE_PASSIVE=false # Uses a lot of API keys queries
 DEEP_RECURSIVE_PASSIVE=10 # Number of top subdomains for recursion
@@ -216,7 +223,7 @@ ZONETRANSFER=true # Check zone transfer
 S3BUCKETS=true # Check S3 buckets misconfigs
 REVERSE_IP=false # Check reverse IP subdomain search (set True if your target is CIDR/IP)
 TLS_PORTS="21,22,25,80,110,135,143,261,271,324,443,448,465,563,614,631,636,664,684,695,832,853,854,990,993,989,992,994,995,1129,1131,1184,2083,2087,2089,2096,2221,2252,2376,2381,2478,2479,2482,2484,2679,2762,3077,3078,3183,3191,3220,3269,3306,3410,3424,3471,3496,3509,3529,3539,3535,3660,36611,3713,3747,3766,3864,3885,3995,3896,4031,4036,4062,4064,4081,4083,4116,4335,4336,4536,4590,4740,4843,4849,5443,5007,5061,5321,5349,5671,5783,5868,5986,5989,5990,6209,6251,6443,6513,6514,6619,6697,6771,7202,7443,7673,7674,7677,7775,8243,8443,8991,8989,9089,9295,9318,9443,9444,9614,9802,10161,10162,11751,12013,12109,14143,15002,16995,41230,16993,20003"
-INSCOPE=false # Uses inscope tool to filter the scope, requires .scope file in reconftw folder
+INSCOPE=false # Uses inscope tool to filter the scope, requires .scope file in reconftw folder 
 
 # Web detection
 WEBPROBESIMPLE=true # Web probing on 80/443
@@ -225,12 +232,10 @@ WEBSCREENSHOT=true # Webs screenshooting
 VIRTUALHOSTS=false # Check virtualhosts by fuzzing HOST header
 NMAP_WEBPROBE=true # If disabled it will run httpx directly over subdomains list, nmap before web probing is used to increase the speed and avoid repeated requests
 UNCOMMON_PORTS_WEB="81,300,591,593,832,981,1010,1311,1099,2082,2095,2096,2480,3000,3001,3002,3003,3128,3333,4243,4567,4711,4712,4993,5000,5104,5108,5280,5281,5601,5800,6543,7000,7001,7396,7474,8000,8001,8008,8014,8042,8060,8069,8080,8081,8083,8088,8090,8091,8095,8118,8123,8172,8181,8222,8243,8280,8281,8333,8337,8443,8500,8834,8880,8888,8983,9000,9001,9043,9060,9080,9090,9091,9092,9200,9443,9502,9800,9981,10000,10250,11371,12443,15672,16080,17778,18091,18092,20720,32000,55440,55672"
-# You can change to aquatone if gowitness fails, comment the one you don't want
-AXIOM_SCREENSHOT_MODULE=webscreenshot # Choose between aquatone,gowitness,webscreenshot
 
 # Host
 FAVICON=true # Check Favicon domain discovery
-PORTSCANNER=true # Enable or disable the whole Port scanner module
+PORTSCANNER=true # Enable or disable the whole Port scanner module 
 PORTSCAN_PASSIVE=true # Port scanner with Shodan
 PORTSCAN_ACTIVE=true # Port scanner with nmap
 CDN_IP=true # Check which IPs belongs to CDN
@@ -239,8 +244,8 @@ CDN_IP=true # Check which IPs belongs to CDN
 WAF_DETECTION=true # Detect WAFs
 NUCLEICHECK=true # Enable or disable nuclei
 NUCLEI_SEVERITY="info,low,medium,high,critical" # Set templates criticity
-NUCLEI_FLAGS="-silent -t ~/nuclei-templates/ -retries 2" # Additional nuclei extra flags, don't set the severity here but the exclusions like "-etags openssh"
-NUCLEI_FLAGS_JS="-silent -tags exposure,token -severity info,low,medium,high,critical" # Additional nuclei extra flags for js secrets
+NUCLEI_FLAGS=" -silent -t $HOME/nuclei-templates/ -retries 2" # Additional nuclei extra flags, don't set the severity here but the exclusions like " -etags openssh"
+NUCLEI_FLAGS_JS=" -silent -tags exposure,token -severity info,low,medium,high,critical" # Additional nuclei extra flags for js secrets
 URL_CHECK=true # Enable or disable URL collection
 URL_CHECK_PASSIVE=true # Search for urls, passive methods from Archive, OTX, CommonCrawl, etc
 URL_CHECK_ACTIVE=true # Search for urls by crawling the websites
@@ -265,13 +270,16 @@ SSRF_CHECKS=true # SSRF checks
 CRLF_CHECKS=true # CRLF checks
 LFI=true # LFI by fuzzing
 SSTI=true # SSTI by fuzzing
-SQLI=true # Check SQLI with sqlmap
+SQLI=true # Check SQLI
+SQLMAP=true # Check SQLI with sqlmap
+GHAURI=false # Check SQLI with ghauri
 BROKENLINKS=true # Check for brokenlinks
 SPRAY=true # Performs password spraying
 COMM_INJ=true # Check for command injections with commix
 PROTO_POLLUTION=true # Check for prototype pollution flaws
 SMUGGLING=true # Check for HTTP request smuggling flaws
-WEBCACHE=true # Check for HTTP request smuggling flaws
+WEBCACHE=true # Check for Web Cache issues
+BYPASSER4XX=true # Check for 4XX bypasses
 
 # Extra features
 NOTIFICATION=false # Notification for every function
@@ -285,8 +293,9 @@ REMOVELOG=false # Delete logs after execution
 PROXY=false # Send to proxy the websites found
 SENDZIPNOTIFY=false # Send to zip the results (over notify)
 PRESERVE=true      # set to true to avoid deleting the .called_fn files on really large scans
-FFUF_FLAGS="-mc all -fc 404 -ac -sf" # Ffuf flags
-HTTPX_FLAGS="-follow-redirects -random-agent -status-code -silent -title -web-server -tech-detect -location" # Httpx flags for simple web probing
+FFUF_FLAGS=" -mc all -fc 404 -ach -sf -of json" # Ffuf flags
+HTTPX_FLAGS=" -follow-redirects -random-agent -status-code -silent -title -web-server -tech-detect -location -content-length" # Httpx flags for simple web probing
+GOWITNESS_FLAGS=" --disable-logging --timeout 5"
 
 # HTTP options
 HEADER="User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" # Default header
@@ -305,14 +314,14 @@ PUREDNS_PUBLIC_LIMIT=0 # Set between 2000 - 10000 if your router blows up, 0 mea
 PUREDNS_TRUSTED_LIMIT=400
 PUREDNS_WILDCARDTEST_LIMIT=30
 PUREDNS_WILDCARDBATCH_LIMIT=1500000
-WEBSCREENSHOT_THREADS=200
-GOWITNESS_THREADS=8
+GOWITNESS_THREADS=20
 RESOLVE_DOMAINS_THREADS=150
 PPFUZZ_THREADS=30
 DNSVALIDATOR_THREADS=200
 INTERLACE_THREADS=10
 TLSX_THREADS=1000
 XNLINKFINDER_DEPTH=3
+BYP4XX_THREADS=20
 
 # Rate limits
 HTTPX_RATELIMIT=150
@@ -342,19 +351,13 @@ resolvers_trusted=${tools}/resolvers_trusted.txt
 # AXIOM=false Uncomment only to overwrite command line flags
 AXIOM_FLEET_LAUNCH=true # Enable or disable spin up a new fleet, if false it will use the current fleet with the AXIOM_FLEET_NAME prefix
 AXIOM_FLEET_NAME="reconFTW" # Fleet's prefix name
-AXIOM_FLEET_COUNT=5 # Fleet's number
+AXIOM_FLEET_COUNT=10 # Fleet's number
 AXIOM_FLEET_REGIONS="eu-central" # Fleet's region
 AXIOM_FLEET_SHUTDOWN=true # # Enable or disable delete the fleet after the execution
 # This is a script on your reconftw host that might prep things your way...
 #AXIOM_POST_START="~/Tools/axiom_config.sh" # Useful  to send your config files to the fleet
 AXIOM_EXTRA_ARGS="" # Leave empty if you don't want to add extra arguments
-#AXIOM_EXTRA_ARGS="--rm-logs" # Example
-
-# BBRF
-BBRF_CONNECTION=false
-BBRF_SERVER=https://demo.bbrf.me/bbrf
-BBRF_USERNAME="user"
-BBRF_PASSWORD="password"
+#AXIOM_EXTRA_ARGS=" --rm-logs" # Example
 
 # TERM COLORS
 bred='\033[1;31m'
@@ -463,13 +466,6 @@ reset='\033[0m'
 - During the configuration of axiom you need to select `reconftw` as provisoner.
 - You can create your own axiom's fleet before running reconFTW or let reconFTW to create and destroy it automatically just modifying reconftw.cfg file.
 
-# BBRF Support :computer:
-
-- To add reconFTW results to your [BBRF instance](https://github.com/honoki/bbrf-server) just add IP and credentials to reconftw.cfg file section dedicated to bbrf.
-
-- During the execution of the scans the results will be added dynamically when each step ends.
-- Even you can set up locally your BBRF instance to be able to visualize your results in a fancy web UI.
-
 # Sample video
 
 ![Video](images/reconFTW.gif)
@@ -483,12 +479,12 @@ reset='\033[0m'
 - Metadata finder ([MetaFinder](https://github.com/Josue87/MetaFinder))
 - Google Dorks ([dorks_hunter](https://github.com/six2dez/dorks_hunter))
 - Github Dorks ([gitdorks_go](https://github.com/damit5/gitdorks_go))
-- GitHub org analysis ([enumerepo](https://github.com/trickest/enumerepo) and [trufflehog](https://github.com/trufflesecurity/trufflehog))
+- GitHub org analysis ([enumerepo](https://github.com/trickest/enumerepo), [trufflehog](https://github.com/trufflesecurity/trufflehog) and [gitleaks](https://github.com/gitleaks/gitleaks))
 
 ## Subdomains
 
 - Passive ([amass](https://github.com/OWASP/Amass), [subfinder](https://github.com/projectdiscovery/subfinder) and [github-subdomains](https://github.com/gwen001/github-subdomains))
-- Certificate transparency ([ctfr](https://github.com/UnaPibaGeek/ctfr))
+- Certificate transparency ([crt](https://github.com/cemulus/crt))
 - NOERROR subdomain discovery ([dnsx](https://github.com/projectdiscovery/dnsx), more info [here](https://www.securesystems.de/blog/enhancing-subdomain-enumeration-ents-and-noerror/))
 - Bruteforce ([puredns](https://github.com/d3mondev/puredns))
 - Permutations ([Gotator](https://github.com/Josue87/gotator), [ripgen](https://github.com/resyncgg/ripgen) and [regulator](https://github.com/cramppet/regulator))
@@ -508,19 +504,19 @@ reset='\033[0m'
 - CDN checker ([ipcdn](https://github.com/six2dez/ipcdn))
 - WAF checker ([wafw00f](https://github.com/EnableSecurity/wafw00f))
 - Port Scanner (Active with [nmap](https://github.com/nmap/nmap) and passive with [smap](https://github.com/s0md3v/Smap))
-- Port services vulnerability checks ([searchsploit](https://github.com/offensive-security/exploitdb))
+- Port services vulnerability checks ([vulners](https://github.com/vulnersCom/nmap-vulners))
 - Password spraying ([brutespray](https://github.com/x90skysn3k/brutespray))
 
 ## Webs
 
-- Web Prober ([httpx](https://github.com/projectdiscovery/httpx) and [unimap](https://github.com/Edu4rdSHL/unimap))
+- Web Prober ([httpx](https://github.com/projectdiscovery/httpx))
 - Web screenshoting ([webscreenshot](https://github.com/maaaaz/webscreenshot) or [gowitness](https://github.com/sensepost/gowitness))
 - Web templates scanner ([nuclei](https://github.com/projectdiscovery/nuclei) and [nuclei geeknik](https://github.com/geeknik/the-nuclei-templates.git))
 - CMS Scanner ([CMSeeK](https://github.com/Tuhinshubhra/CMSeeK))
 - Url extraction ([gau](https://github.com/lc/gau),[waymore](https://github.com/xnl-h4ck3r/waymore), [katana](https://github.com/projectdiscovery/katana), [github-endpoints](https://gist.github.com/six2dez/d1d516b606557526e9a78d7dd49cacd3) and [JSA](https://github.com/w9w/JSA))
 - URL patterns Search and filtering ([urless](https://github.com/xnl-h4ck3r/urless), [gf](https://github.com/tomnomnom/gf) and [gf-patterns](https://github.com/1ndianl33t/Gf-Patterns))
 - Favicon Real IP ([fav-up](https://github.com/pielco11/fav-up))
-- Javascript analysis ([subjs](https://github.com/lc/subjs), [JSA](https://github.com/w9w/JSA), [xnLinkFinder](https://github.com/xnl-h4ck3r/xnLinkFinder), [getjswords](https://github.com/m4ll0k/BBTz))
+- Javascript analysis ([subjs](https://github.com/lc/subjs), [JSA](https://github.com/w9w/JSA), [xnLinkFinder](https://github.com/xnl-h4ck3r/xnLinkFinder), [getjswords](https://github.com/m4ll0k/BBTz), [Mantra](https://github.com/MrEmpy/Mantra))
 - Fuzzing ([ffuf](https://github.com/ffuf/ffuf))
 - URL sorting by extension
 - Wordlist generation
@@ -653,4 +649,4 @@ If you want to contribute to this project, you can do it in multiple ways:
 
 Usage of this program for attacking targets without consent is illegal. It is the user's responsibility to obey all applicable laws. The developer assumes no liability and is not responsible for any misuse or damage caused by this program. Please use responsibly.
 
-The material contained in this repository is licensed under GNU GPLv3.
+The material contained in this repository is licensed under MIT.