diff --git a/.gitmodules b/.gitmodules index 70a1921d..d1f8bc29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,12 @@ path = src/detect/src/detectors/shen-ReScue url = https://github.com/davisjam/ReScue branch = SupportForVRD +[submodule "src/detect/src/detectors/mclaughlin-regulator"] + path = src/detect/src/detectors/mclaughlin-regulator + url = https://github.com/ucsb-seclab/regulator-dynamic.git +[submodule "src/detect/src/detectors/hong-EvilStrGen"] + path = src/detect/src/detectors/hong-EvilStrGen + url = https://github.com/Anonymous89813/EvilStrGen.git +[submodule "src/detect/src/detectors/li-ReDoSHunter"] + path = src/detect/src/detectors/li-ReDoSHunter + url = https://github.com/yetingli/ReDoSHunter.git diff --git a/src/detect/src/detectors/hong-EvilStrGen b/src/detect/src/detectors/hong-EvilStrGen new file mode 160000 index 00000000..a07d7a98 --- /dev/null +++ b/src/detect/src/detectors/hong-EvilStrGen @@ -0,0 +1 @@ +Subproject commit a07d7a989d9b817524846668712a4b7e038d0226 diff --git a/src/detect/src/detectors/li-ReDoSHunter b/src/detect/src/detectors/li-ReDoSHunter new file mode 160000 index 00000000..1d8d62f6 --- /dev/null +++ b/src/detect/src/detectors/li-ReDoSHunter @@ -0,0 +1 @@ +Subproject commit 1d8d62f659da56588388108b9107212ba08d96af diff --git a/src/detect/src/detectors/mclaughlin-regulator b/src/detect/src/detectors/mclaughlin-regulator new file mode 160000 index 00000000..9b40c0dc --- /dev/null +++ b/src/detect/src/detectors/mclaughlin-regulator @@ -0,0 +1 @@ +Subproject commit 9b40c0dcb89f102dd0814728fb7da938b62bab68 diff --git a/src/detect/src/drivers/query-hong-EvilStrGen.pl b/src/detect/src/drivers/query-hong-EvilStrGen.pl new file mode 100755 index 00000000..b4e2ea07 --- /dev/null +++ b/src/detect/src/drivers/query-hong-EvilStrGen.pl @@ -0,0 +1,186 @@ +#!/usr/bin/env perl +# Author: Jamie Davis +# Description: Query the shen-ReScue detector for pattern safety +# +# Dependencies: +# - VULN_REGEX_DETECTOR_ROOT must be defined + +use strict; +use warnings; + +use IPC::Cmd qw[can_run]; # Check PATH +use JSON::PP; # I/O +use Data::Dumper; +use Carp; + +# Check dependencies. +if (not defined $ENV{VULN_REGEX_DETECTOR_ROOT}) { + die "Error, VULN_REGEX_DETECTOR_ROOT must be defined\n"; +} + +my $EvilStrGenDir = "$ENV{VULN_REGEX_DETECTOR_ROOT}/src/detect/src/detectors/hong-EvilStrGen"; +if (not -d $EvilStrGenDir) { + die "Error, could not find EvilStrGenDir <$EvilStrGenDir>\n"; +} + +if (not can_run("java")) { + die "Error, cannot find 'java'\n"; +} + +# Check args. +if (scalar(@ARGV) != 1) { + die "Usage: $0 pattern.json\n"; +} + +my $patternFile = $ARGV[0]; +if (not -f $patternFile) { + die "Error, no such patternFile $patternFile\n"; +} + +# Read. +my $cont = &readFile("file"=>$patternFile); +my $pattern = decode_json($cont); +my $regex = $pattern->{pattern}; + +# Write out to a tmp file for EvilStrGen format. +my $tmpFile = "/tmp/query-hong-EvilStrGen-$$.regex"; +my $tmpOutputDir = "/tmp/query-hong-EvilStrGen-$$-output"; +my $regexEngine = 14; # Python +my $maxAttackSize = 100; +unlink $tmpFile; +unlink $tmpOutputDir; +&writeToFile("file"=>$tmpFile, "contents"=>$regex); +print STDERR "CLEANUP: $tmpFile\n"; # If we time out, the parent can clean up for us. + +# Run. +&cmd("cd $ENV{VULN_REGEX_DETECTOR_ROOT}/src/detect 2>&1"); +my $cmdString = "./src/detectors/hong-EvilStrGen/build/EvilStrGen $tmpFile $tmpOutputDir $regexEngine $maxAttackSize 0"; +my ($rc, $out) = &cmd("$cmdString 2>&1"); +unlink $tmpFile; + +# Parse to determine opinion +my $opinion = { }; + +print STDOUT &cmd("ls $tmpOutputDir"); + +# Check if EvilStrGen detected ReDoS +if ($out =~ m/writing file\n/) { + # Initialize an array to store all attack strings + my @attackStrings; + + # Iterate over all files in the output directory that match the pattern "*.txt" + opendir(my $dir, $tmpOutputDir) or die "Cannot open directory: $!"; + while (my $file = readdir($dir)) { + next unless ($file =~ /\.txt$/); # Process only .txt files + + # Read the content of each file and add it to the array + my $attackStr = &readFile("file"=>"$tmpOutputDir/$file"); + push @attackStrings, $attackStr; + } + closedir($dir); + + # Populate opinion + $opinion->{canAnalyze} = 1; + $opinion->{isSafe} = "NO"; + $opinion->{evilInput} = \@attackStrings; + +} elsif (length($out) == 0) { + $opinion->{canAnalyze} = 1; + $opinion->{isSafe} = "YES"; + $opinion->{evilInput} = []; +} else { + $opinion->{canAnalyze} = 0; + $opinion->{isSafe} = "UNKNOWN"; + $opinion->{evilInput} = ["COULD-NOT-PARSE"]; +} + +unlink $tmpOutputDir; + +&log("\n\n--------------------\n\nOutput:\n$out"); + +# Update $pattern. +$pattern->{opinion} = $opinion; + +# Emit. +print STDOUT encode_json($pattern) . "\n"; + + +##################### + +# input: ($cmd) +# output: ($rc, $out) +sub cmd { + my ($cmd) = @_; + &log("CMD: $cmd"); + my $out = `$cmd`; + return ($? >> 8, $out); +} + +sub log { + my ($msg) = @_; + print STDERR "$msg\n"; +} + +# input: %args: keys: file contents +# output: $file +sub writeToFile { + my %args = @_; + + open(my $fh, '>', $args{file}); + print $fh $args{contents}; + close $fh; + + return $args{file}; +} + +# input: ($exploitString) JSON object +# fields: separators[] pumps[] suffix +# separators and pumps have the same length +# output: ($translatedExploitString) hashref +# fields: pumpPairs[] suffix +# Each pumpPair is an object with keys: prefix pump +sub translateExploitString { + my ($es) = @_; + + if (defined($es->{separators}) and defined($es->{pumps}) and defined($es->{suffix})) { + &log("exploitString looks valid"); + } + else { + croak("Invalid exploitString: " . Dumper($es)); + } + + # Convert Weideman's format to something more sensible: + # pumpPairs[] + # suffix + + my @separators = @{$es->{separators}}; + my @pumps = @{$es->{pumps}}; + if (scalar(@separators) ne scalar(@pumps)) { + croak("Invalid exploitString: " . Dumper($es)); + } + + my @pumpPairs; + for (my $i = 0; $i < scalar(@separators); $i++) { + push @pumpPairs, { "prefix" => $separators[$i], + "pump" => $pumps[$i] + }; + } + + my $suffix = $es->{suffix}; + + return { "pumpPairs" => \@pumpPairs, + "suffix" => $suffix + }; +} + +# input: %args: keys: file +# output: $contents +sub readFile { + my %args = @_; + + open(my $FH, '<', $args{file}) or confess "Error, could not read $args{file}: $!"; + my $contents = do { local $/; <$FH> }; # localizing $? wipes the line separator char, so <> gets it all at once. + close $FH; + + return $contents; +} diff --git a/src/detect/src/drivers/query-li-ReDoSHunter.pl b/src/detect/src/drivers/query-li-ReDoSHunter.pl new file mode 100755 index 00000000..8fd65488 --- /dev/null +++ b/src/detect/src/drivers/query-li-ReDoSHunter.pl @@ -0,0 +1,186 @@ +#!/usr/bin/env perl +# Author: Jamie Davis +# Description: Query the shen-ReScue detector for pattern safety +# +# Dependencies: +# - VULN_REGEX_DETECTOR_ROOT must be defined + +use strict; +use warnings; + +use IPC::Cmd qw[can_run]; # Check PATH +use JSON::PP; # I/O +use Data::Dumper; +use File::Glob ':glob'; +use Carp; + + +# Check dependencies. +if (not defined $ENV{VULN_REGEX_DETECTOR_ROOT}) { + die "Error, VULN_REGEX_DETECTOR_ROOT must be defined\n"; +} + +my $ReDoSHunterDir = "$ENV{VULN_REGEX_DETECTOR_ROOT}/src/detect/src/detectors/li-ReDoSHunter"; +if (not -d $ReDoSHunterDir) { + die "Error, could not find ReDoSHunterDir <$ReDoSHunterDir>\n"; +} + +if (not can_run("java")) { + die "Error, cannot find 'java'\n"; +} + +# Check args. +if (scalar(@ARGV) != 1) { + die "Usage: $0 pattern.json\n"; +} + +my $patternFile = $ARGV[0]; +if (not -f $patternFile) { + die "Error, no such patternFile $patternFile\n"; +} + +# Read. +my $cont = &readFile("file"=>$patternFile); +my $pattern = decode_json($cont); +my $regex = $pattern->{pattern}; + +# Write out to a tmp file for ReDoSHunter format. +my $filePath = "$ReDoSHunterDir/data/paper_dataset"; +my $fileName = "query-li-ReDoSHunter-$$.txt"; +my $tmpFile = "$filePath/$fileName"; +unlink $tmpFile; +&writeToFile("file"=>$tmpFile, "contents"=>$regex); +print STDERR "CLEANUP: $tmpFile\n"; # If we time out, the parent can clean up for us. + +# Run ReDoSHunter +my $jvmNoDumpFlags = ""; # TODO Is there a portable way to do this? "-XXnoJrDump -XXdumpSize:none"; # Disable crash files (generated if ulimit on memory exceeded). +my $cmdString = "cd $ReDoSHunterDir && java $jvmNoDumpFlags -jar $ReDoSHunterDir/target/ReDoSHunter-1.0.0.jar $filePath $fileName"; +my ($rc, $out) = &cmd("$cmdString 2>&1"); +unlink $tmpFile; + +# Read results +my $resDir = "$ReDoSHunterDir/data/expr"; +my $result = ""; +my ($resFile) = bsd_glob("$resDir/*redos_s_java*"); +if ($resFile) { + $result = &readFile("file"=>$resFile); +} else { + die "Error: No file found containing 'redos_s_java' in $resDir\n"; +} +unlink glob("$resDir/*") or warn "Could not delete files in $resDir: $!"; + +# Parse to determine opinion +print STDOUT "Result: $result\n"; +my $opinion = { }; +if ($result =~ m/.*RESULT-TRUE.*/s) { + $opinion->{canAnalyze} = 1; + $opinion->{isSafe} = "NO"; + + # Capture attack sstrings + my @attack_strings; + while ($result =~ /AttackStringļ¼š(.+?)\s*$/gm) { + push @attack_strings, $1; # Capture the attack string content + } + $opinion->{evilInput} = \@attack_strings; +} elsif ($result =~ m/.*RESULT-FALSE.*/s) { + if ($result =~ m/.*TIME-OUT.*/s) { + $opinion->{canAnalyze} = 0; + $opinion->{isSafe} = "UNKNOWN"; + $opinion->{evilInput} = ["TIME-OUT"]; + } else { + $opinion->{canAnalyze} = 1; + $opinion->{isSafe} = "YES"; + } +} else { + $opinion->{canAnalyze} = 0; + $opinion->{isSafe} = "UNKNOWN"; + $opinion->{evilInput} = ["COULD-NOT-PARSE"]; +} + +&log("\n\n--------------------\n\nOutput:\n$out"); + +# Update $pattern. +$pattern->{opinion} = $opinion; + +# Emit. +print STDOUT encode_json($pattern) . "\n"; + +##################### + +# input: ($cmd) +# output: ($rc, $out) +sub cmd { + my ($cmd) = @_; + &log("CMD: $cmd"); + my $out = `$cmd`; + return ($? >> 8, $out); +} + +sub log { + my ($msg) = @_; + print STDERR "$msg\n"; +} + +# input: %args: keys: file contents +# output: $file +sub writeToFile { + my %args = @_; + + open(my $fh, '>', $args{file}); + print $fh $args{contents}; + close $fh; + + return $args{file}; +} + +# input: ($exploitString) JSON object +# fields: separators[] pumps[] suffix +# separators and pumps have the same length +# output: ($translatedExploitString) hashref +# fields: pumpPairs[] suffix +# Each pumpPair is an object with keys: prefix pump +sub translateExploitString { + my ($es) = @_; + + if (defined($es->{separators}) and defined($es->{pumps}) and defined($es->{suffix})) { + &log("exploitString looks valid"); + } + else { + croak("Invalid exploitString: " . Dumper($es)); + } + + # Convert Weideman's format to something more sensible: + # pumpPairs[] + # suffix + + my @separators = @{$es->{separators}}; + my @pumps = @{$es->{pumps}}; + if (scalar(@separators) ne scalar(@pumps)) { + croak("Invalid exploitString: " . Dumper($es)); + } + + my @pumpPairs; + for (my $i = 0; $i < scalar(@separators); $i++) { + push @pumpPairs, { "prefix" => $separators[$i], + "pump" => $pumps[$i] + }; + } + + my $suffix = $es->{suffix}; + + return { "pumpPairs" => \@pumpPairs, + "suffix" => $suffix + }; +} + +# input: %args: keys: file +# output: $contents +sub readFile { + my %args = @_; + + open(my $FH, '<', $args{file}) or confess "Error, could not read $args{file}: $!"; + my $contents = do { local $/; <$FH> }; # localizing $? wipes the line separator char, so <> gets it all at once. + close $FH; + + return $contents; +} diff --git a/src/detect/src/drivers/query-mclaughlin-regulator.pl b/src/detect/src/drivers/query-mclaughlin-regulator.pl new file mode 100755 index 00000000..586646f7 --- /dev/null +++ b/src/detect/src/drivers/query-mclaughlin-regulator.pl @@ -0,0 +1,172 @@ +#!/usr/bin/env perl +# Author: Jamie Davis +# Description: Query the shen-ReScue detector for pattern safety +# +# Dependencies: +# - VULN_REGEX_DETECTOR_ROOT must be defined + +use strict; +use warnings; + +use IPC::Cmd qw[can_run]; # Check PATH +use JSON::PP; # I/O +use Data::Dumper; +use Carp; + +# Check dependencies. +if (not defined $ENV{VULN_REGEX_DETECTOR_ROOT}) { + die "Error, VULN_REGEX_DETECTOR_ROOT must be defined\n"; +} + +my $RegulatorDir = "$ENV{VULN_REGEX_DETECTOR_ROOT}/src/detect/src/detectors/hong-EvilStrGen"; +if (not -d $RegulatorDir) { + die "Error, could not find RegulatorDir <$RegulatorDir>\n"; +} + +if (not can_run("java")) { + die "Error, cannot find 'java'\n"; +} + +# Check args. +if (scalar(@ARGV) != 1) { + die "Usage: $0 pattern.json\n"; +} + +my $patternFile = $ARGV[0]; +if (not -f $patternFile) { + die "Error, no such patternFile $patternFile\n"; +} + +# Read. +my $cont = &readFile("file"=>$patternFile); +my $pattern = decode_json($cont); +my $regex = $pattern->{pattern}; + +# Run. +&cmd("cd $ENV{VULN_REGEX_DETECTOR_ROOT}/src/detect 2>&1"); +my $cmdString = "./src/detectors/mclaughlin-regulator/fuzzer/build/fuzzer -r '$regex' -l 20 -w 1 --maxtot 1"; +my ($rc, $out) = &cmd("$cmdString 2>&1"); + +# Parse to determine opinion +my $opinion = { }; + +# Find the total steps +my $regexTotal = 0; +if ($out =~ m/Total=([0-9]+)/) { + $regexTotal = $1; +} + +my $regexMaxObs = 0; +if ($out =~ m/MaxObservation=([0-9]+)/) { + $regexMaxObs = $1; +} + +# Check if Regulator detected ReDoS +if ($regexTotal > 70 || $regexMaxObs > 1) { + $opinion->{canAnalyze} = 1; + $opinion->{isSafe} = "NO"; + + # Capture attack strings + my $attackString = ""; + if ($out =~ m/word="([^"]+)"/) { + $attackString = $1; + } + $opinion->{evilInput} = [$attackString]; +} elsif ($out =~ m/Regexp compilation failed/) { + $opinion->{canAnalyze} = 0; + $opinion->{isSafe} = "UNKNOWN"; + $opinion->{evilInput} = ["COULD-NOT-COMPILE"]; +} else { + $opinion->{canAnalyze} = 1; + $opinion->{isSafe} = "YES"; + $opinion->{evilInput} = []; +} + +&log("\n\n--------------------\n\nOutput:\n$out"); + +# Update $pattern. +$pattern->{opinion} = $opinion; + +# Emit. +print STDOUT encode_json($pattern) . "\n"; + + +##################### + +# input: ($cmd) +# output: ($rc, $out) +sub cmd { + my ($cmd) = @_; + &log("CMD: $cmd"); + my $out = `$cmd`; + return ($? >> 8, $out); +} + +sub log { + my ($msg) = @_; + print STDERR "$msg\n"; +} + +# input: %args: keys: file contents +# output: $file +sub writeToFile { + my %args = @_; + + open(my $fh, '>', $args{file}); + print $fh $args{contents}; + close $fh; + + return $args{file}; +} + +# input: ($exploitString) JSON object +# fields: separators[] pumps[] suffix +# separators and pumps have the same length +# output: ($translatedExploitString) hashref +# fields: pumpPairs[] suffix +# Each pumpPair is an object with keys: prefix pump +sub translateExploitString { + my ($es) = @_; + + if (defined($es->{separators}) and defined($es->{pumps}) and defined($es->{suffix})) { + &log("exploitString looks valid"); + } + else { + croak("Invalid exploitString: " . Dumper($es)); + } + + # Convert Weideman's format to something more sensible: + # pumpPairs[] + # suffix + + my @separators = @{$es->{separators}}; + my @pumps = @{$es->{pumps}}; + if (scalar(@separators) ne scalar(@pumps)) { + croak("Invalid exploitString: " . Dumper($es)); + } + + my @pumpPairs; + for (my $i = 0; $i < scalar(@separators); $i++) { + push @pumpPairs, { "prefix" => $separators[$i], + "pump" => $pumps[$i] + }; + } + + my $suffix = $es->{suffix}; + + return { "pumpPairs" => \@pumpPairs, + "suffix" => $suffix + }; +} + +# input: %args: keys: file +# output: $contents +sub readFile { + my %args = @_; + + open(my $FH, '<', $args{file}) or confess "Error, could not read $args{file}: $!"; + my $contents = do { local $/; <$FH> }; # localizing $? wipes the line separator char, so <> gets it all at once. + close $FH; + + return $contents; +}