Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat/MC-500: include source in script uploads #6

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat/MC-500: include source in script uploads #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ba395b
feat: include source in script uploads
ersanne Sep 11, 2024
0a6c02b
fix: bat and ps1 source
ersanne Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions scripts/thunderstorm-collector.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ SET /A MAX_AGE=30
:: Debug
SET DEBUG=0

:: Source
SET SOURCE=

:: WELCOME -------------------------------------------------------

ECHO =============================================================
Expand Down Expand Up @@ -74,10 +77,20 @@ ECHO Cannot find curl in PATH or the current directory. Download it from https:/
ECHO If you're collecting on Windows systems older than Windows Vista, use curl version 7.46.0 from https://bintray.com/vszakats/generic/download_file?file_path=curl-7.46.0-win32-mingw.7z
EXIT /b 1
:CHECKDONE
ECHO Curl has been found. We're ready to go.
ECHO Curl has been found. We're ready to go.

:: COLLECTION --------------------------------------------------

:: SOURCE
IF "%SOURCE%"=="" (
ECHO Getting hostname
FOR /F "tokens=*" %%i IN ('hostname') DO SET SOURCE=%%i
ECHO No Source provided, using hostname=!SOURCE!
)
IF "%SOURCE%" NEQ "" (
SET SOURCE=?source=%SOURCE%
)

:: Directory walk and upload
ECHO Processing %COLLECT_DIRS% with filters MAX_SIZE: %COLLECT_MAX_SIZE% MAX_AGE: %MAX_AGE% days EXTENSIONS: %RELEVANT_EXTENSIONS%
ECHO This could take a while depending on the disk size and number of files. (set DEBUG=1 to see all skips)
Expand Down Expand Up @@ -112,7 +125,7 @@ FOR %%T IN (%COLLECT_DIRS%) DO (
:: Upload
ECHO Uploading %%F ..
:: We'll start the upload process in background to speed up the submission process
START /B curl -F file=@%%F -H "Content-Type: multipart/form-data" -o nul -s %URL_SCHEME%://%THUNDERSTORM_SERVER%:%THUNDERSTORM_PORT%/api/checkAsync
START /B curl -F file=@%%F -H "Content-Type: multipart/form-data" -o nul -s %URL_SCHEME%://%THUNDERSTORM_SERVER%:%THUNDERSTORM_PORT%/api/checkAsync%SOURCE%
)
)
)
Expand Down
30 changes: 22 additions & 8 deletions scripts/thunderstorm-collector.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
# Usage examples:
# $> perl thunderstorm-collector.pl -- -s thunderstorm.internal.net
# $> perl thunderstorm-collector.pl -- --dir / --server thunderstorm.internal.net
# $> perl thunderstorm-collector.pl -- --dir / --server thunderstorm.internal.net --so "My Source"

use warnings;
use strict;
use Getopt::Long;
use LWP::UserAgent;
use File::Spec::Functions qw( catfile );
use Sys::Hostname;

use Cwd; # module for finding the current working directory

Expand All @@ -27,20 +29,32 @@
my $server = "";
my $port = 8080;
my $scheme = "http";
our $max_age = 3; # in days
my $source = "";
our $max_age = 3000; # in days
our $max_size = 10; # in megabytes
our @skipElements = map { qr{$_} } ('^\/proc', '^\/mnt', '\.dat$', '\.npm');
our @hardSkips = ('/proc', '/dev', '/sys');

# Command Line Parameters
GetOptions("dir=s" => \$targetdir, # same for --dir or -d
"server=s" => \$server, # same for --server or -s
"port=i" => \$port, # same for --port or -p
"debug" => \$debug # --debug
);
GetOptions(
"dir|d=s" => \$targetdir, # --dir or -d
"server|s=s" => \$server, # --server or -s
"port|p=i" => \$port, # --port or -p
"source|so=s" => \$source, # --source or -so
"debug" => \$debug # --debug
);

# Use Hostname as Source if not set
if ( $source eq "" ) {
$source = hostname;
}
# Add Source to URL if available
if ( $source ne "" ) {
$source = "?source=$source";
}

# Composed Values
our $api_endpoint = "$scheme://$server:$port/api/checkAsync";
our $api_endpoint = "$scheme://$server:$port/api/checkAsync$source";
our $current_date = time;

# Stats
Expand Down Expand Up @@ -86,7 +100,7 @@ sub processDir {
if ( $debug ) { print "[DEBUG] Checking $filepath ...\n"; }
}

# Characteristics
# Characteristics
my $size = (stat($filepath))[7];
my $mdate = (stat($filepath))[9];
#print("SIZE: $size MDATE: $mdate\n");
Expand Down
112 changes: 61 additions & 51 deletions scripts/thunderstorm-collector.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# Date Created: 07.10.2020
# Last Modified: 07.10.2020
##################################################

#Requires -Version 3

<#
Expand All @@ -19,6 +19,8 @@
Server name (FQDN) or IP address of your Thunderstorm instance
.PARAMETER ThunderstormPort
Port number on which the Thunderstorm service is listening (default: 8080)
.PARAMETER Source
Source of the submission (default: hostname of the system)
.PARAMETER Folder
Folder to process (default: C:\)
.PARAMETER MaxAge
Expand Down Expand Up @@ -46,43 +48,47 @@
# Parameters ----------------------------------------------------------
# #####################################################################

param
(
[Parameter(
HelpMessage='Server name (FQDN) or IP address of your Thunderstorm instance')]
[ValidateNotNullOrEmpty()]
param
(
[Parameter(
HelpMessage='Server name (FQDN) or IP address of your Thunderstorm instance')]
[ValidateNotNullOrEmpty()]
[Alias('TS')]
[string]$ThunderstormServer,
[string]$ThunderstormServer,

[Parameter(HelpMessage="Port number on which the Thunderstorm service is listening (default: 8080)")]
[ValidateNotNullOrEmpty()]
[Alias('TP')]
[Parameter(HelpMessage="Port number on which the Thunderstorm service is listening (default: 8080)")]
[ValidateNotNullOrEmpty()]
[Alias('TP')]
[int]$ThunderstormPort = 8080,

[Parameter(HelpMessage="Folder to process (default: C:\)")]
[ValidateNotNullOrEmpty()]
[Parameter(HelpMessage="Source of the submission (default: hostname of the system)")]
[Alias('S')]
[string]$Source=$env:COMPUTERNAME,

[Parameter(HelpMessage="Folder to process (default: C:\)")]
[ValidateNotNullOrEmpty()]
[Alias('F')]
[string]$Folder = "C:\",
[Parameter(
HelpMessage='Select files based on the number of days in which the file has been create or modified (default: 0 = no age selection)')]
[ValidateNotNullOrEmpty()]
[Alias('MA')]
[int]$MaxAge,

[Parameter(
HelpMessage='Select only files smaller than the given number in MegaBytes (default: 20MB) ')]
[ValidateNotNullOrEmpty()]
[Alias('MS')]
[int]$MaxSize,

[Parameter(HelpMessage='Extensions to select for submission (default: all of them)')]
[ValidateNotNullOrEmpty()]
[Alias('E')]
[string[]]$Extensions,

[Parameter(HelpMessage='Enables debug output and skips cleanup at the end of the scan')]
[ValidateNotNullOrEmpty()]

[Parameter(
HelpMessage='Select files based on the number of days in which the file has been create or modified (default: 0 = no age selection)')]
[ValidateNotNullOrEmpty()]
[Alias('MA')]
[int]$MaxAge,

[Parameter(
HelpMessage='Select only files smaller than the given number in MegaBytes (default: 20MB) ')]
[ValidateNotNullOrEmpty()]
[Alias('MS')]
[int]$MaxSize,

[Parameter(HelpMessage='Extensions to select for submission (default: all of them)')]
[ValidateNotNullOrEmpty()]
[Alias('E')]
[string[]]$Extensions,

[Parameter(HelpMessage='Enables debug output and skips cleanup at the end of the scan')]
[ValidateNotNullOrEmpty()]
[Alias('D')]
[switch]$Debugging = $False
)
Expand Down Expand Up @@ -144,19 +150,19 @@ if ( $Args.Count -eq 0 -and $ThunderstormServer -eq "" ) {
function Write-Log {
param (
[Parameter(Mandatory=$True, Position=0, HelpMessage="Log entry")]
[ValidateNotNullOrEmpty()]
[ValidateNotNullOrEmpty()]
[String]$Entry,

[Parameter(Position=1, HelpMessage="Log file to write into")]
[ValidateNotNullOrEmpty()]
[Alias('SS')]
[Parameter(Position=1, HelpMessage="Log file to write into")]
[ValidateNotNullOrEmpty()]
[Alias('SS')]
[IO.FileInfo]$LogFile = "thunderstorm-collector.log",

[Parameter(Position=3, HelpMessage="Level")]
[ValidateNotNullOrEmpty()]
[ValidateNotNullOrEmpty()]
[String]$Level = "Info"
)

# Indicator
$Indicator = "[+]"
if ( $Level -eq "Warning" ) {
Expand All @@ -179,7 +185,7 @@ function Write-Log {
} else {
Write-Host "$($Indicator) $($Entry)"
}

# Log File
if ( $global:NoLog -eq $False ) {
"$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss.fff') $($env:COMPUTERNAME): $Entry" | Out-File -FilePath $LogFile -Append
Expand Down Expand Up @@ -219,27 +225,31 @@ if ( $AutoDetectPlatform -ne "" ) {
}

# URL Creation
$Url = "http://$($ThunderstormServer):$($ThunderstormPort)/api/checkAsync"
if ( $Source -ne "" ) {
Write-Log "Using Source: $($Source)"
$SourceParam = "?Source=$Source"
}
$Url = "http://$($ThunderstormServer):$($ThunderstormPort)/api/checkAsync$($SourceParam)"
Write-Log "Sending to URI: $($Url)" -Level "Debug"

# ---------------------------------------------------------------------
# Run THOR Thunderstorm Collector -------------------------------------
# ---------------------------------------------------------------------
$ProgressPreference = "SilentlyContinue"
try {
Get-ChildItem -Path $Folder -File -Recurse -ErrorAction SilentlyContinue |
Get-ChildItem -Path $Folder -File -Recurse -ErrorAction SilentlyContinue |
ForEach-Object {
# -------------------------------------------------------------
# Filter ------------------------------------------------------
# Filter ------------------------------------------------------
# Size Check
if ( ( $_.Length / 1MB ) -gt $($MaxSize) ) {
Write-Log "$_ skipped due to size filter" -Level "Debug"
Write-Log "$_ skipped due to size filter" -Level "Debug"
return
}
# Age Check
# Age Check
if ( $($MaxAge) -gt 0 ) {
if ( $_.LastWriteTime -lt (Get-Date).AddDays(-$($MaxAge)) ) {
Write-Log "$_ skipped due to age filter" -Level "Debug"
Write-Log "$_ skipped due to age filter" -Level "Debug"
return
}
}
Expand All @@ -253,7 +263,7 @@ try {

# -------------------------------------------------------------
# Submission --------------------------------------------------

Write-Log "Processing $($_.FullName) ..." -Level "Debug"
# Reading the file data & preparing the request
try {
Expand All @@ -264,12 +274,12 @@ try {
$fileEnc = [System.Text.Encoding]::GetEncoding('UTF-8').GetString($fileBytes);
$boundary = [System.Guid]::NewGuid().ToString();
$LF = "`r`n";
$bodyLines = (
$bodyLines = (
"--$boundary",
"Content-Disposition: form-data; name=`"file`"; filename=`"$($_.FullName)`"",
"Content-Type: application/octet-stream$LF",
$fileEnc,
"--$boundary--$LF"
"--$boundary--$LF"
) -join $LF

# Submitting the request
Expand All @@ -280,7 +290,7 @@ try {
Write-Log "Submitting to Thunderstorm server: $($_.FullName) ..." -Level "Info"
$Response = Invoke-WebRequest -uri $($Url) -Method Post -ContentType "multipart/form-data; boundary=`"$boundary`"" -Body $bodyLines
$StatusCode = [int]$Response.StatusCode
}
}
# Catch all non 200 status codes
catch {
$StatusCode = $_.Exception.Response.StatusCode.value__
Expand All @@ -304,8 +314,8 @@ try {
}
}
}
} catch {
Write-Log "Unknown error during Thunderstorm Collection $_" -Level "Error"
} catch {
Write-Log "Unknown error during Thunderstorm Collection $_" -Level "Error"
}

# ---------------------------------------------------------------------
Expand Down
14 changes: 13 additions & 1 deletion scripts/thunderstorm-collector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import ssl
import time
import uuid
import socket

# Configuration
schema = "http"
Expand Down Expand Up @@ -206,14 +207,24 @@ def submit_sample(filepath):
action="store_true",
help="Skip TLS verification and proceed without checking.",
)
parser.add_argument(
"-S",
"--source",
default=socket.gethostname(),
help="Source identifier to be used in the Thunderstorm submission.",
)
parser.add_argument("--debug", action="store_true", help="Enable debug logging.")

args = parser.parse_args()

if args.tls:
schema = "https"

api_endpoint = "{}://{}:{}/api/checkAsync".format(schema, args.server, args.port)
source = ""
if args.source:
source = f"?source={args.source}"

api_endpoint = "{}://{}:{}/api/checkAsync{}".format(schema, args.server, args.port, source)

print("=" * 80)
print(" Python Thunderstorm Collector")
Expand All @@ -227,6 +238,7 @@ def submit_sample(filepath):
print("Maximum Age of Files: {}".format(max_age))
print("Maximum File Size: {} MB".format(max_size))
print("Excluded directories: {}".format(", ".join(hard_skips)))
print("Source Identifier: {}".format(args.source)) if args.source else None
print()

print("Starting the walk at: {} ...".format(", ".join(args.dirs)))
Expand Down
14 changes: 12 additions & 2 deletions scripts/thunderstorm-collector.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@ LOG_TO_CMDLINE=1

# Thunderstorm Server
THUNDERSTORM_SERVER="ygdrasil.nextron"
THUNDERSTORM_PORT=8080
USE_SSL=0
ASYNC_MODE=1

# Source
HOSTNAME=$(hostname -f)
echo "$HOSTNAME"

# Target selection
declare -a SCAN_FOLDERS=('/root' '/tmp' '/home' '/var' '/usr'); # folders to scan
declare -a SCAN_FOLDERS=('/root' '/tmp' '/home' '/var' '/usr'); # folders to scan
MAX_AGE=14
MAX_FILE_SIZE=2000 # max file size to check in kilobyte, default 2 MB

Expand Down Expand Up @@ -124,6 +129,11 @@ scheme="http"
if [[ $USE_SSL -eq 1 ]]; then
scheme="https"
fi
source=""
if [[ -n $HOSTNAME ]]; then
source="?source=${HOSTNAME}"
echo "Source: $source"
fi

# Loop over filesystem
for scandir in "${SCAN_FOLDERS[@]}";
Expand All @@ -142,7 +152,7 @@ do
for retry in {1..3}; do
# Submit sample
result=$(curl -s -X POST \
"$scheme://$THUNDERSTORM_SERVER:8080/api/$api_endpoint" \
"$scheme://$THUNDERSTORM_SERVER:$THUNDERSTORM_PORT/api/$api_endpoint$source" \
--form "file=@${file_path};filename=${file_path}")
curl_exit=$?
if [ $curl_exit -ne 0 ]; then
Expand Down