diff --git a/scripts/thunderstorm-collector.bat b/scripts/thunderstorm-collector.bat index 00daae0..e7cb8f5 100644 --- a/scripts/thunderstorm-collector.bat +++ b/scripts/thunderstorm-collector.bat @@ -46,6 +46,9 @@ SET /A MAX_AGE=30 :: Debug SET DEBUG=0 +:: Source +SET SOURCE= + :: WELCOME ------------------------------------------------------- ECHO ============================================================= @@ -74,10 +77,20 @@ ECHO Cannot find curl in PATH or the current directory. Download it from https:/ ECHO If you're collecting on Windows systems older than Windows Vista, use curl version 7.46.0 from https://bintray.com/vszakats/generic/download_file?file_path=curl-7.46.0-win32-mingw.7z EXIT /b 1 :CHECKDONE -ECHO Curl has been found. We're ready to go. +ECHO Curl has been found. We're ready to go. :: COLLECTION -------------------------------------------------- +:: SOURCE +IF "%SOURCE%"=="" ( + ECHO Getting hostname + FOR /F "tokens=*" %%i IN ('hostname') DO SET SOURCE=%%i + ECHO No Source provided, using hostname=!SOURCE! +) +IF "%SOURCE%" NEQ "" ( + SET SOURCE=?source=%SOURCE% +) + :: Directory walk and upload ECHO Processing %COLLECT_DIRS% with filters MAX_SIZE: %COLLECT_MAX_SIZE% MAX_AGE: %MAX_AGE% days EXTENSIONS: %RELEVANT_EXTENSIONS% ECHO This could take a while depending on the disk size and number of files. (set DEBUG=1 to see all skips) @@ -112,7 +125,7 @@ FOR %%T IN (%COLLECT_DIRS%) DO ( :: Upload ECHO Uploading %%F .. :: We'll start the upload process in background to speed up the submission process - START /B curl -F file=@%%F -H "Content-Type: multipart/form-data" -o nul -s %URL_SCHEME%://%THUNDERSTORM_SERVER%:%THUNDERSTORM_PORT%/api/checkAsync + START /B curl -F file=@%%F -H "Content-Type: multipart/form-data" -o nul -s %URL_SCHEME%://%THUNDERSTORM_SERVER%:%THUNDERSTORM_PORT%/api/checkAsync%SOURCE% ) ) ) diff --git a/scripts/thunderstorm-collector.pl b/scripts/thunderstorm-collector.pl index 7e51c2f..122077a 100755 --- a/scripts/thunderstorm-collector.pl +++ b/scripts/thunderstorm-collector.pl @@ -12,12 +12,14 @@ # Usage examples: # $> perl thunderstorm-collector.pl -- -s thunderstorm.internal.net # $> perl thunderstorm-collector.pl -- --dir / --server thunderstorm.internal.net +# $> perl thunderstorm-collector.pl -- --dir / --server thunderstorm.internal.net --so "My Source" use warnings; use strict; use Getopt::Long; use LWP::UserAgent; use File::Spec::Functions qw( catfile ); +use Sys::Hostname; use Cwd; # module for finding the current working directory @@ -27,20 +29,32 @@ my $server = ""; my $port = 8080; my $scheme = "http"; -our $max_age = 3; # in days +my $source = ""; +our $max_age = 3000; # in days our $max_size = 10; # in megabytes our @skipElements = map { qr{$_} } ('^\/proc', '^\/mnt', '\.dat$', '\.npm'); our @hardSkips = ('/proc', '/dev', '/sys'); # Command Line Parameters -GetOptions("dir=s" => \$targetdir, # same for --dir or -d - "server=s" => \$server, # same for --server or -s - "port=i" => \$port, # same for --port or -p - "debug" => \$debug # --debug - ); +GetOptions( + "dir|d=s" => \$targetdir, # --dir or -d + "server|s=s" => \$server, # --server or -s + "port|p=i" => \$port, # --port or -p + "source|so=s" => \$source, # --source or -so + "debug" => \$debug # --debug +); + +# Use Hostname as Source if not set +if ( $source eq "" ) { + $source = hostname; +} +# Add Source to URL if available +if ( $source ne "" ) { + $source = "?source=$source"; +} # Composed Values -our $api_endpoint = "$scheme://$server:$port/api/checkAsync"; +our $api_endpoint = "$scheme://$server:$port/api/checkAsync$source"; our $current_date = time; # Stats @@ -86,7 +100,7 @@ sub processDir { if ( $debug ) { print "[DEBUG] Checking $filepath ...\n"; } } - # Characteristics + # Characteristics my $size = (stat($filepath))[7]; my $mdate = (stat($filepath))[9]; #print("SIZE: $size MDATE: $mdate\n"); diff --git a/scripts/thunderstorm-collector.ps1 b/scripts/thunderstorm-collector.ps1 index 3253cbb..751ea8f 100644 --- a/scripts/thunderstorm-collector.ps1 +++ b/scripts/thunderstorm-collector.ps1 @@ -6,7 +6,7 @@ # Date Created: 07.10.2020 # Last Modified: 07.10.2020 ################################################## - + #Requires -Version 3 <# @@ -19,6 +19,8 @@ Server name (FQDN) or IP address of your Thunderstorm instance .PARAMETER ThunderstormPort Port number on which the Thunderstorm service is listening (default: 8080) + .PARAMETER Source + Source of the submission (default: hostname of the system) .PARAMETER Folder Folder to process (default: C:\) .PARAMETER MaxAge @@ -46,43 +48,47 @@ # Parameters ---------------------------------------------------------- # ##################################################################### -param -( - [Parameter( - HelpMessage='Server name (FQDN) or IP address of your Thunderstorm instance')] - [ValidateNotNullOrEmpty()] +param +( + [Parameter( + HelpMessage='Server name (FQDN) or IP address of your Thunderstorm instance')] + [ValidateNotNullOrEmpty()] [Alias('TS')] - [string]$ThunderstormServer, + [string]$ThunderstormServer, - [Parameter(HelpMessage="Port number on which the Thunderstorm service is listening (default: 8080)")] - [ValidateNotNullOrEmpty()] - [Alias('TP')] + [Parameter(HelpMessage="Port number on which the Thunderstorm service is listening (default: 8080)")] + [ValidateNotNullOrEmpty()] + [Alias('TP')] [int]$ThunderstormPort = 8080, - [Parameter(HelpMessage="Folder to process (default: C:\)")] - [ValidateNotNullOrEmpty()] + [Parameter(HelpMessage="Source of the submission (default: hostname of the system)")] + [Alias('S')] + [string]$Source=$env:COMPUTERNAME, + + [Parameter(HelpMessage="Folder to process (default: C:\)")] + [ValidateNotNullOrEmpty()] [Alias('F')] [string]$Folder = "C:\", - - [Parameter( - HelpMessage='Select files based on the number of days in which the file has been create or modified (default: 0 = no age selection)')] - [ValidateNotNullOrEmpty()] - [Alias('MA')] - [int]$MaxAge, - - [Parameter( - HelpMessage='Select only files smaller than the given number in MegaBytes (default: 20MB) ')] - [ValidateNotNullOrEmpty()] - [Alias('MS')] - [int]$MaxSize, - - [Parameter(HelpMessage='Extensions to select for submission (default: all of them)')] - [ValidateNotNullOrEmpty()] - [Alias('E')] - [string[]]$Extensions, - - [Parameter(HelpMessage='Enables debug output and skips cleanup at the end of the scan')] - [ValidateNotNullOrEmpty()] + + [Parameter( + HelpMessage='Select files based on the number of days in which the file has been create or modified (default: 0 = no age selection)')] + [ValidateNotNullOrEmpty()] + [Alias('MA')] + [int]$MaxAge, + + [Parameter( + HelpMessage='Select only files smaller than the given number in MegaBytes (default: 20MB) ')] + [ValidateNotNullOrEmpty()] + [Alias('MS')] + [int]$MaxSize, + + [Parameter(HelpMessage='Extensions to select for submission (default: all of them)')] + [ValidateNotNullOrEmpty()] + [Alias('E')] + [string[]]$Extensions, + + [Parameter(HelpMessage='Enables debug output and skips cleanup at the end of the scan')] + [ValidateNotNullOrEmpty()] [Alias('D')] [switch]$Debugging = $False ) @@ -144,19 +150,19 @@ if ( $Args.Count -eq 0 -and $ThunderstormServer -eq "" ) { function Write-Log { param ( [Parameter(Mandatory=$True, Position=0, HelpMessage="Log entry")] - [ValidateNotNullOrEmpty()] + [ValidateNotNullOrEmpty()] [String]$Entry, - [Parameter(Position=1, HelpMessage="Log file to write into")] - [ValidateNotNullOrEmpty()] - [Alias('SS')] + [Parameter(Position=1, HelpMessage="Log file to write into")] + [ValidateNotNullOrEmpty()] + [Alias('SS')] [IO.FileInfo]$LogFile = "thunderstorm-collector.log", [Parameter(Position=3, HelpMessage="Level")] - [ValidateNotNullOrEmpty()] + [ValidateNotNullOrEmpty()] [String]$Level = "Info" ) - + # Indicator $Indicator = "[+]" if ( $Level -eq "Warning" ) { @@ -179,7 +185,7 @@ function Write-Log { } else { Write-Host "$($Indicator) $($Entry)" } - + # Log File if ( $global:NoLog -eq $False ) { "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss.fff') $($env:COMPUTERNAME): $Entry" | Out-File -FilePath $LogFile -Append @@ -219,7 +225,11 @@ if ( $AutoDetectPlatform -ne "" ) { } # URL Creation -$Url = "http://$($ThunderstormServer):$($ThunderstormPort)/api/checkAsync" +if ( $Source -ne "" ) { + Write-Log "Using Source: $($Source)" + $SourceParam = "?Source=$Source" +} +$Url = "http://$($ThunderstormServer):$($ThunderstormPort)/api/checkAsync$($SourceParam)" Write-Log "Sending to URI: $($Url)" -Level "Debug" # --------------------------------------------------------------------- @@ -227,19 +237,19 @@ Write-Log "Sending to URI: $($Url)" -Level "Debug" # --------------------------------------------------------------------- $ProgressPreference = "SilentlyContinue" try { - Get-ChildItem -Path $Folder -File -Recurse -ErrorAction SilentlyContinue | + Get-ChildItem -Path $Folder -File -Recurse -ErrorAction SilentlyContinue | ForEach-Object { # ------------------------------------------------------------- - # Filter ------------------------------------------------------ + # Filter ------------------------------------------------------ # Size Check if ( ( $_.Length / 1MB ) -gt $($MaxSize) ) { - Write-Log "$_ skipped due to size filter" -Level "Debug" + Write-Log "$_ skipped due to size filter" -Level "Debug" return } - # Age Check + # Age Check if ( $($MaxAge) -gt 0 ) { if ( $_.LastWriteTime -lt (Get-Date).AddDays(-$($MaxAge)) ) { - Write-Log "$_ skipped due to age filter" -Level "Debug" + Write-Log "$_ skipped due to age filter" -Level "Debug" return } } @@ -253,7 +263,7 @@ try { # ------------------------------------------------------------- # Submission -------------------------------------------------- - + Write-Log "Processing $($_.FullName) ..." -Level "Debug" # Reading the file data & preparing the request try { @@ -264,12 +274,12 @@ try { $fileEnc = [System.Text.Encoding]::GetEncoding('UTF-8').GetString($fileBytes); $boundary = [System.Guid]::NewGuid().ToString(); $LF = "`r`n"; - $bodyLines = ( + $bodyLines = ( "--$boundary", "Content-Disposition: form-data; name=`"file`"; filename=`"$($_.FullName)`"", "Content-Type: application/octet-stream$LF", $fileEnc, - "--$boundary--$LF" + "--$boundary--$LF" ) -join $LF # Submitting the request @@ -280,7 +290,7 @@ try { Write-Log "Submitting to Thunderstorm server: $($_.FullName) ..." -Level "Info" $Response = Invoke-WebRequest -uri $($Url) -Method Post -ContentType "multipart/form-data; boundary=`"$boundary`"" -Body $bodyLines $StatusCode = [int]$Response.StatusCode - } + } # Catch all non 200 status codes catch { $StatusCode = $_.Exception.Response.StatusCode.value__ @@ -304,8 +314,8 @@ try { } } } -} catch { - Write-Log "Unknown error during Thunderstorm Collection $_" -Level "Error" +} catch { + Write-Log "Unknown error during Thunderstorm Collection $_" -Level "Error" } # --------------------------------------------------------------------- diff --git a/scripts/thunderstorm-collector.py b/scripts/thunderstorm-collector.py index 64a0bf0..7c207c6 100755 --- a/scripts/thunderstorm-collector.py +++ b/scripts/thunderstorm-collector.py @@ -7,6 +7,7 @@ import ssl import time import uuid +import socket # Configuration schema = "http" @@ -206,6 +207,12 @@ def submit_sample(filepath): action="store_true", help="Skip TLS verification and proceed without checking.", ) + parser.add_argument( + "-S", + "--source", + default=socket.gethostname(), + help="Source identifier to be used in the Thunderstorm submission.", + ) parser.add_argument("--debug", action="store_true", help="Enable debug logging.") args = parser.parse_args() @@ -213,7 +220,11 @@ def submit_sample(filepath): if args.tls: schema = "https" - api_endpoint = "{}://{}:{}/api/checkAsync".format(schema, args.server, args.port) + source = "" + if args.source: + source = f"?source={args.source}" + + api_endpoint = "{}://{}:{}/api/checkAsync{}".format(schema, args.server, args.port, source) print("=" * 80) print(" Python Thunderstorm Collector") @@ -227,6 +238,7 @@ def submit_sample(filepath): print("Maximum Age of Files: {}".format(max_age)) print("Maximum File Size: {} MB".format(max_size)) print("Excluded directories: {}".format(", ".join(hard_skips))) + print("Source Identifier: {}".format(args.source)) if args.source else None print() print("Starting the walk at: {} ...".format(", ".join(args.dirs))) diff --git a/scripts/thunderstorm-collector.sh b/scripts/thunderstorm-collector.sh index 5e336d7..ea8cb0c 100755 --- a/scripts/thunderstorm-collector.sh +++ b/scripts/thunderstorm-collector.sh @@ -16,11 +16,16 @@ LOG_TO_CMDLINE=1 # Thunderstorm Server THUNDERSTORM_SERVER="ygdrasil.nextron" +THUNDERSTORM_PORT=8080 USE_SSL=0 ASYNC_MODE=1 +# Source +HOSTNAME=$(hostname -f) +echo "$HOSTNAME" + # Target selection -declare -a SCAN_FOLDERS=('/root' '/tmp' '/home' '/var' '/usr'); # folders to scan +declare -a SCAN_FOLDERS=('/root' '/tmp' '/home' '/var' '/usr'); # folders to scan MAX_AGE=14 MAX_FILE_SIZE=2000 # max file size to check in kilobyte, default 2 MB @@ -124,6 +129,11 @@ scheme="http" if [[ $USE_SSL -eq 1 ]]; then scheme="https" fi +source="" +if [[ -n $HOSTNAME ]]; then + source="?source=${HOSTNAME}" + echo "Source: $source" +fi # Loop over filesystem for scandir in "${SCAN_FOLDERS[@]}"; @@ -142,7 +152,7 @@ do for retry in {1..3}; do # Submit sample result=$(curl -s -X POST \ - "$scheme://$THUNDERSTORM_SERVER:8080/api/$api_endpoint" \ + "$scheme://$THUNDERSTORM_SERVER:$THUNDERSTORM_PORT/api/$api_endpoint$source" \ --form "file=@${file_path};filename=${file_path}") curl_exit=$? if [ $curl_exit -ne 0 ]; then