hakrawler-results-parser.sh

#!/bin/sh

# she makes a folder
mkdir $1

# she sends hakrawler on his way
hakrawler -depth 5 -forms -js -linkfinder  -scope subs -urls -url https://$1 | tee $1/hkrawled.txt

cd $1

# we divide to understand.
cat hkrawled.txt | grep '\[forms\]' | sort -u  |  awk '{print $2}' >  forms.txt
cat hkrawled.txt | grep  '\[javascript\]' | sort -u    | awk '{print $2}' > javascript.txt
cat hkrawled.txt | grep '\[url\]' | sort -u  | awk '{print $2}' > urls.txt
cat hkrawled.txt | grep '\[linkfinder\]' | sort -u | awk '{print $2}' > linkfinder.txt

for link in $(cat linkfinder.txt); do echo 'https://$1'$link; done

# and a nice kick in the teeth by gospider on the assist
gospider -u "Googlebot" -o './gspider.txt' -t 5 -d 0 --sitemap --robots --other-source --include-other-source -w -s https://$1