diff --git a/pom.xml b/pom.xml index 59cca3f..ee9a43c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 malwarepuller MalPull - 1.0-stable + 1.1-stable jar UTF-8 diff --git a/src/main/java/concurrency/DownloadWorker.java b/src/main/java/concurrency/DownloadWorker.java new file mode 100644 index 0000000..4ae1acc --- /dev/null +++ b/src/main/java/concurrency/DownloadWorker.java @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2020 Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package concurrency; + +import endpoints.*; +import exceptions.SampleNotFoundException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import malpull.MalPull; + +/** + * A worker class that extends the runnable interface, meaning it can be + * executed as a thread. This class tries to download the given sample from any + * of the endpoints in the list. If it fails, the hash is added to the missing + * hashes list in the main class. + * + * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + */ +public class DownloadWorker implements Runnable { + + /** + * The list of endpoints iterate through in an attempt to download the hash + */ + private List endpoints; + + /** + * The path to write the file to if it is found + */ + private String path; + + /** + * The hash of the sample to download + */ + private String hash; + + /** + * The number of this worker in the queue, used to print if the sample can + * be downloaded to display the progress to the user + */ + private int count; + + /** + * The total amount of samples that are in the queue, used to display the + * progress to the user + */ + private int total; + + /** + * Creates a worker object, which can be queued for the thread pool + * + * @param endpoints the list of endpoints to attempt to download from + * @param path the location to write the file to the disk + * @param hash the hash to look for + * @param count the queue number of this worker, remains unchanged after + * creation + * @param total the total number of samples to be downloaded + */ + public DownloadWorker(List endpoints, String path, String hash, int count, int total) { + this.endpoints = endpoints; + this.path = path; + this.hash = hash; + this.count = count; + this.total = total; + } + + /** + * Downloads the sample to the given location. If the sample cannot be + * found, the hash is added to the list of missing hashes, which is printed + * at the end. + */ + @Override + public void run() { + try { + //Add the hash to the file name + String filePath = path + File.separator + hash; + //The boolean to check if the sample has been downloaded + boolean isDownloaded = false; + //Iterate through all the endpoints + for (IEndpoint endpoint : endpoints) { + //Try to dowload the file + try { + /** + * If it is already downloaded, the iteration loop should be + * broken to avoid more API calls than are required, and to + * move on to the next worker in the queue + */ + if (isDownloaded) { + break; + } + //Get the sample from the endpoint + byte[] output = endpoint.getSample(hash); + //If the output is not zero bytes + if (output.length > 0) { + //The file is written to disk + writeToDisk(output, filePath); + //A message is printed for the user + System.out.println("Wrote " + output.length + " bytes to " + filePath + " from " + endpoint.getName() + " (" + count + " / " + total + ")"); + //The boolean is set to true, causing the next iteration to break out of the loop + isDownloaded = true; + } + } catch (SampleNotFoundException e) { + /** + * The exception message can be ignored, as failure to + * download the sample results in the missing hash, but only + * if none of the configured endpoints has the hash + */ + //System.out.println(e.getMessage()); + } + } + //If the sample is not downloaded after the loop, it is missing + if (isDownloaded == false) { + //This method is thread safe + MalPull.addMissingHash(hash); + } + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + } + + /** + * Writes the given byte array to the disk to the given location + * + * @param output the data to write to the disk + * @param path the location to write the given data to + */ + private void writeToDisk(byte[] output, String path) { + try (FileOutputStream fos = new FileOutputStream(path)) { + fos.write(output); + } catch (IOException ex) { + System.out.println("An error occured when writing the sample to \"" + path + "\". Verify your permissions and try again!"); + } + } +} diff --git a/src/main/java/endpoints/GenericEndpoint.java b/src/main/java/endpoints/GenericEndpoint.java index 7e27406..0b85f13 100644 --- a/src/main/java/endpoints/GenericEndpoint.java +++ b/src/main/java/endpoints/GenericEndpoint.java @@ -31,15 +31,31 @@ public abstract class GenericEndpoint { */ protected Downloader downloader; + /** + * The name of the endpoint + */ + public String name; + /** * The base URL of the API, to which specific API actions can be appended */ protected String apiBase; - public GenericEndpoint(String apiBase) { + public GenericEndpoint(String apiBase, String name) { //Sets the apiBase variable this.apiBase = apiBase; //Initialises the downloader class downloader = new Downloader(); + //Sets the name variable + this.name = name; + } + + /** + * Gets the endpoint name + * + * @return the name of the endpoint + */ + public String getName() { + return name; } } diff --git a/src/main/java/exceptions/Error404NotFoundException.java b/src/main/java/endpoints/IEndpoint.java similarity index 64% rename from src/main/java/exceptions/Error404NotFoundException.java rename to src/main/java/endpoints/IEndpoint.java index 672854f..2ec9c39 100644 --- a/src/main/java/exceptions/Error404NotFoundException.java +++ b/src/main/java/endpoints/IEndpoint.java @@ -14,17 +14,27 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package exceptions; +package endpoints; /** - * This exception is used when the HTTP request returns a 404 (Not Found) status - * code * * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] */ -public class Error404NotFoundException extends Exception { +public interface IEndpoint { - public Error404NotFoundException() { + /** + * Gets the sample from the endpoint, based on the given hash + * + * @param hash the hash of the file to download + * @return a byte[] that contains the file's data + * @throws Exception in case any error occurs + */ + public byte[] getSample(String hash) throws Exception; - } + /** + * Gets the name of the endpoint + * + * @return + */ + public String getName(); } diff --git a/src/main/java/endpoints/Koodous.java b/src/main/java/endpoints/Koodous.java index 17677ba..4ed64f9 100644 --- a/src/main/java/endpoints/Koodous.java +++ b/src/main/java/endpoints/Koodous.java @@ -16,9 +16,6 @@ */ package endpoints; -import exceptions.Error404NotFoundException; -import exceptions.Error429TooManyRequestsException; -import exceptions.HttpConnectionFailed; import exceptions.SampleNotFoundException; import okhttp3.Request; import org.json.JSONArray; @@ -29,7 +26,7 @@ * * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] */ -public class Koodous extends GenericEndpoint { +public class Koodous extends GenericEndpoint implements IEndpoint { /** * The API key that is used to interact with the MalShare API @@ -43,7 +40,7 @@ public class Koodous extends GenericEndpoint { */ public Koodous(String key) { //Sets the apiBase variable in the abstract GenericEndpoint class - super("https://koodous.com/api/apks"); + super("https://koodous.com/api/apks", "Koodous"); //Sets the key variable this.key = key; } @@ -54,14 +51,10 @@ public Koodous(String key) { * * @param hash the hash to look for * @return the sample as a byte array - * @throws HttpConnectionFailed if no connection can be made from the - * current machine, or to the given host * @throws SampleNotFoundException if the sample cannot be found - * @throws Error404NotFoundException if the target returns a 404 status code - * @throws Error429TooManyRequestsException if the target returns a 429 - * status code */ - public byte[] getSample(String hash) throws HttpConnectionFailed, SampleNotFoundException, Error404NotFoundException, Error429TooManyRequestsException { + @Override + public byte[] getSample(String hash) throws SampleNotFoundException { //Get the SHA-256 hash via the search function of the API, as only SHA-256 hashes can be used when downloading a sample String sha256Hash = getSha256Hash(hash); //Return the API's response @@ -81,7 +74,7 @@ public byte[] getSample(String hash) throws HttpConnectionFailed, SampleNotFound * @throws Error404NotFoundException if the target returns a 404 status code * @throws Error429TooManyRequestsException if the target returns a 429 */ - private String getSha256Hash(String hash) throws HttpConnectionFailed, SampleNotFoundException, Error404NotFoundException, Error429TooManyRequestsException { + private String getSha256Hash(String hash) throws SampleNotFoundException { //Create the url String url = apiBase + "?search=" + hash + "&page=%1&page_size=%100"; //Create the requested, based on the given URL and with the required header token @@ -97,7 +90,7 @@ private String getSha256Hash(String hash) throws HttpConnectionFailed, SampleNot int count = jsonObject.optInt("count", 0); if (count == 0) { //If there are no hits, the sample is not present - throw new SampleNotFoundException(); + throw new SampleNotFoundException("Sample " + hash + " not found on Koodous!"); } //Get the results if there are any JSONArray results = jsonObject.getJSONArray("results"); @@ -115,7 +108,7 @@ private String getSha256Hash(String hash) throws HttpConnectionFailed, SampleNot * @throws Error404NotFoundException if the target returns a 404 status code * @throws Error429TooManyRequestsException if the target returns a 429 */ - private byte[] download(String hash) throws HttpConnectionFailed, Error404NotFoundException, Error429TooManyRequestsException { + private byte[] download(String hash) throws SampleNotFoundException { //Create the URL String url = apiBase + "/" + hash + "/download"; //Prepare the request with teh API token diff --git a/src/main/java/endpoints/MalShare.java b/src/main/java/endpoints/MalShare.java index 990a336..5cbc48f 100644 --- a/src/main/java/endpoints/MalShare.java +++ b/src/main/java/endpoints/MalShare.java @@ -16,10 +16,7 @@ */ package endpoints; -import exceptions.Error404NotFoundException; -import exceptions.Error429TooManyRequestsException; import exceptions.SampleNotFoundException; -import exceptions.HttpConnectionFailed; import okhttp3.Request; /** @@ -27,12 +24,7 @@ * * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] */ -public class MalShare extends GenericEndpoint { - - /** - * The API key that is used to interact with the MalShare API - */ - private String key; +public class MalShare extends GenericEndpoint implements IEndpoint { /** * Creates an object to interact with the MalShare API @@ -41,9 +33,7 @@ public class MalShare extends GenericEndpoint { */ public MalShare(String key) { //Sets the apiBase variable in the abstract GenericEndpoint class - super("https://malshare.com/api.php?api_key=" + key + "&action="); - //Sets the key variable - this.key = key; + super("https://malshare.com/api.php?api_key=" + key + "&action=", "MalShare"); } /** @@ -64,14 +54,10 @@ private String getDownloadUrl(String hash) { * * @param hash the hash to look for * @return the sample as a byte array - * @throws HttpConnectionFailed if no connection can be made from the - * current machine, or to the given host * @throws SampleNotFoundException if the sample cannot be found - * @throws Error404NotFoundException if the target returns a 404 status code - * @throws Error429TooManyRequestsException if the target returns a 429 - * status code */ - public byte[] getSample(String hash) throws HttpConnectionFailed, SampleNotFoundException, Error404NotFoundException, Error429TooManyRequestsException { + @Override + public byte[] getSample(String hash) throws SampleNotFoundException { //Gets the URL String url = getDownloadUrl(hash); //Create the request based on the URL @@ -90,7 +76,7 @@ public byte[] getSample(String hash) throws HttpConnectionFailed, SampleNotFound */ if (temp.contains("Sample not found by hash")) { //If the sample cannot be found, an exception is thrown - throw new SampleNotFoundException(); + throw new SampleNotFoundException("Sample " + hash + " not found on MalShare!"); } //Return the sample return result; diff --git a/src/main/java/endpoints/MalwareBazaar.java b/src/main/java/endpoints/MalwareBazaar.java index 851d3fc..9911b33 100644 --- a/src/main/java/endpoints/MalwareBazaar.java +++ b/src/main/java/endpoints/MalwareBazaar.java @@ -16,7 +16,6 @@ */ package endpoints; -import exceptions.HttpConnectionFailed; import exceptions.SampleNotFoundException; import okhttp3.MultipartBody; import okhttp3.RequestBody; @@ -28,7 +27,7 @@ * * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] */ -public class MalwareBazaar extends GenericEndpoint { +public class MalwareBazaar extends GenericEndpoint implements IEndpoint { /** * Creates an object to interact with the MalwareBazaar API @@ -36,7 +35,7 @@ public class MalwareBazaar extends GenericEndpoint { */ public MalwareBazaar() { //Sets the apiBase variable in the abstract GenericEndpoint class - super("https://mb-api.abuse.ch/api/v1/"); + super("https://mb-api.abuse.ch/api/v1/", "MalwareBazaar"); } /** @@ -45,16 +44,15 @@ public MalwareBazaar() { * * @param hash the hash to look for * @return the sample as a byte array - * @throws HttpConnectionFailed if no connection can be made from the - * current machine, or to the given host * @throws SampleNotFoundException if the sample cannot be found */ - public byte[] getSample(String hash) throws HttpConnectionFailed, SampleNotFoundException { + @Override + public byte[] getSample(String hash) throws SampleNotFoundException { //Get the SHA-256 hash via the search function of the API, as only SHA-256 hashes can be used when downloading a sample String sha256Hash = getSha256Hash(hash); //If it cannot be found, there is no such sample if (sha256Hash.isEmpty()) { - throw new SampleNotFoundException(); + throw new SampleNotFoundException("Sample " + hash + " not found on MalwareBazaar!"); } //If the sample exists, download it return download(sha256Hash); @@ -71,7 +69,7 @@ public byte[] getSample(String hash) throws HttpConnectionFailed, SampleNotFound * current machine, or to the given host * @throws SampleNotFoundException if the sample cannot be found */ - private String getSha256Hash(String hash) throws HttpConnectionFailed, SampleNotFoundException { + private String getSha256Hash(String hash) throws SampleNotFoundException { //Create a new request body for the HTTP POST request RequestBody requestBody = new MultipartBody.Builder() .setType(MultipartBody.FORM) @@ -90,7 +88,7 @@ private String getSha256Hash(String hash) throws HttpConnectionFailed, SampleNot return jsonObject.getJSONArray("data").getJSONObject(0).getString("sha256_hash"); } else { //If the status is not OK, the sample is not present - throw new SampleNotFoundException(); + throw new SampleNotFoundException("Sample " + hash + " not found on MalwareBazaar!"); } } @@ -102,7 +100,7 @@ private String getSha256Hash(String hash) throws HttpConnectionFailed, SampleNot * @throws HttpConnectionFailed if no connection can be made from the * current machine, or to the given host */ - private byte[] download(String hash) throws HttpConnectionFailed { + private byte[] download(String hash) throws SampleNotFoundException { //Create the request body for the HTTP POST request with the form data RequestBody requestBody = new MultipartBody.Builder() .setType(MultipartBody.FORM) diff --git a/src/main/java/endpoints/VirusTotal.java b/src/main/java/endpoints/VirusTotal.java new file mode 100644 index 0000000..d165538 --- /dev/null +++ b/src/main/java/endpoints/VirusTotal.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2020 Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package endpoints; + +import exceptions.SampleNotFoundException; +import okhttp3.Request; + +/** + * + * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + */ +public class VirusTotal extends GenericEndpoint implements IEndpoint { + + /** + * The API key for VirusTotal + */ + private String key; + + /** + * Creates an object to interact with the VirusTotal endpoint, based on API version 2 + * + * @param key + */ + public VirusTotal(String key) { + super("https://www.virustotal.com/vtapi/v2/", "VirusTotal"); + this.key = key; + } + + /** + * Download the sample from the API + * + * @param hash the SHA-256 hash of the sample to download + * @return the API's response, which is the raw file + * @throws SampleNotFoundException if the sample cannot be found + */ + @Override + public byte[] getSample(String hash) throws SampleNotFoundException { + //Create the URL + String url = apiBase + "file/download?apikey=" + key + "&hash=" + hash; + //Prepare the request with the API token + Request request = new Request.Builder() + .url(url) + .build(); + //Return the value of the direct download link + return downloader.get(request); + } +} diff --git a/src/main/java/exceptions/Error429TooManyRequestsException.java b/src/main/java/exceptions/Error429TooManyRequestsException.java deleted file mode 100644 index 501e774..0000000 --- a/src/main/java/exceptions/Error429TooManyRequestsException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2020 Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package exceptions; - -/** - * This exception is used when the HTTP request returns a 429 (Too Many - * Requests) status code - * - * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] - */ -public class Error429TooManyRequestsException extends Exception { - - public Error429TooManyRequestsException() { - - } -} diff --git a/src/main/java/exceptions/HttpConnectionFailed.java b/src/main/java/exceptions/HttpConnectionFailed.java deleted file mode 100644 index fe4edb5..0000000 --- a/src/main/java/exceptions/HttpConnectionFailed.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2020 Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package exceptions; - -/** - * This exception is thrown when the HTTP request fails due to either of two - * reasons. Firstly, the machine where this JAR is executed from, has no - * internet connection. Or, secondly, the target host is unreachable - * - * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] - */ -public class HttpConnectionFailed extends Exception { - - public HttpConnectionFailed() { - - } -} diff --git a/src/main/java/exceptions/SampleNotFoundException.java b/src/main/java/exceptions/SampleNotFoundException.java index 76e98b2..07c00fe 100644 --- a/src/main/java/exceptions/SampleNotFoundException.java +++ b/src/main/java/exceptions/SampleNotFoundException.java @@ -24,7 +24,7 @@ */ public class SampleNotFoundException extends Exception { - public SampleNotFoundException() { + public SampleNotFoundException(String message) { + super(message); } - } diff --git a/src/main/java/malpull/ArgumentHandler.java b/src/main/java/malpull/ArgumentHandler.java new file mode 100644 index 0000000..6e208be --- /dev/null +++ b/src/main/java/malpull/ArgumentHandler.java @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2020 Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package malpull; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This class parses the given arguments into an Arguments object, thereby + * simplifying the code in the main function, and splitting the sanity checks + * from the rest of the code + * + * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + */ +public class ArgumentHandler { + + /** + * Parses the arguments into an Arguments object. Stops MalPull upon the + * occurence of an exception and prints a help message for the user + * + * @param args the arguments to parse + * @return the parsed arguments + */ + public static Arguments handle(String[] args) { + //Only 4 arguments are accepted: + //java -jar malpull.jar threadCount /path/to/keys.txt /path/to/hashes.txt /path/to/write/samples/to + if (args.length != 4) { + //If that is the case, the usage should be printed + printUsage(); + //Then the system should exit + System.exit(0); + } + //Test if the thread count is a valid number + try { + Integer.parseInt(args[0]); + } catch (Exception e) { + System.out.println("Please provide a valid number for the thread count: " + args[0]); + System.exit(0); + } + //Get the thread count if its a valid number + int threadCount = Integer.parseInt(args[0]); + if (threadCount <= 0) { + System.out.println("At least 1 thread is required!"); + System.exit(0); + } + + //Get each key per line, sanity checks are performed within the loadFile function + List keys = loadFile(args[1]); + /** + * Initialise all strings as null, only changing the value if it is + * included in the keys list. Only if the value is not null, it should + * be used later on + */ + String koodousKey = null; + String malwareBazaarKey = null; + String malShareKey = null; + String virusTotalKey = null; + //Iterate through all keys, note that the endpoint prefix is case insensitive + for (String key : keys) { + if (key.toLowerCase().startsWith("koodous=".toLowerCase())) { + koodousKey = key.substring("koodous".length(), key.length()); + } else if (key.toLowerCase().startsWith("malwarebazaar=".toLowerCase())) { + malwareBazaarKey = key.substring("malwarebazaar=".length(), key.length()); + } else if (key.toLowerCase().startsWith("malshare=".toLowerCase())) { + malShareKey = key.substring("malshare=".length(), key.length()); + } else if (key.toLowerCase().startsWith("virustotal=".toLowerCase())) { + virusTotalKey = key.substring("virustotal=".length(), key.length()); + } + } + + //Create a new set to store all hashes in, as a set cannot contain duplicate strings + Set hashes = new HashSet<>(); + //Add all hashes from the file into the set, sanity checks are performed within the loadFile function + hashes.addAll(loadFile(args[2])); + + //Get the output path as a file object. The path is filtered for the home symbol + File path = new File(filterPath(args[3])); + //If it does not exist, any missing folder is created + if (path.exists() == false) { + path.mkdirs(); + } + + //Return the parsed arguments + return new Arguments(hashes, path.getAbsolutePath(), threadCount, koodousKey, malwareBazaarKey, malShareKey, virusTotalKey); + } + + /** + * Loads a file at the given path. The program exists if the file does not + * exist, if the the given location is a folder, or if an IOException occurs + * + * @param path the path to load + * @return the content of the file, one line per entry in the list + */ + private static List loadFile(String path) { + //Create the output list + List output = new ArrayList<>(); + //Create the file object based on the given path + path = filterPath(path); + File file = new File(path); + + //Perform santiy checks + if (file.isDirectory()) { + System.out.println("The file at " + file.getAbsolutePath() + " is a directory!"); + System.exit(0); + } else if (file.exists() == false) { + System.out.println("The file at " + file.getAbsolutePath() + " does not exist!"); + System.exit(0); + } + + //Read the file, line by line where one line contains one hash + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + String line; + while ((line = br.readLine()) != null) { + output.add(line); + } + } catch (IOException ex) { + System.out.println("An exception occured when reading " + file.getAbsolutePath() + ":"); + Logger.getLogger(MalPull.class.getName()).log(Level.SEVERE, null, ex); + System.exit(0); + } + return output; + } + + /** + * Replaces (if present) a starting tilde with the user's home directory + * + * @param path the path to check + * @return the corrected path + */ + private static String filterPath(String path) { + if (path.startsWith("~")) { + path = System.getProperty("user.home") + path.substring(1, path.length()); + } + return path; + } + + /** + * Prints the program's usage + */ + private static void printUsage() { + System.out.println("This tool downloads samples from MalShare, MalwareBazaar, Koodous, and VirusTotal based on given MD-5, SHA-1, or SHA-256 hashes."); + System.out.println("The sample is written to the given output directory. API Keys for any of the used services is required."); + System.out.println("Once all samples are downloaded, the hashes that couldn't be found will be listed."); + System.out.println("For detailed information on the usage of MalPull, please visit https://maxkersten.nl/wordpress/projects/malpull/#usage"); + System.out.println(""); + System.out.println("Sample usage of this program:"); + System.out.println("\t\tjava -jar /path/toMalPull.jar threadCount /path/to/keys.txt /path/to/hashes.txt /path/to/write/samples/to"); + } +} diff --git a/src/main/java/malpull/Arguments.java b/src/main/java/malpull/Arguments.java new file mode 100644 index 0000000..7ce60e6 --- /dev/null +++ b/src/main/java/malpull/Arguments.java @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2020 Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package malpull; + +import java.util.Set; + +/** + * The argument class, which contains all relevant information from the parsed + * arguments + * + * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] + */ +public class Arguments { + + /** + * The set that contains all loaded hashes + */ + private Set hashes; + + /** + * The folder to write all samples to + */ + private String outputPath; + + /** + * The amount of threads to use, minimally one + */ + private int threadCount; + + /** + * The API key for Koodous + */ + private String koodousKey; + + /** + * The API key for Malware Bazaar is not required, meaning any value will enable downloading from this service + */ + private String malwareBazaarKey; + + /** + * The API key for MalShare + */ + private String malShareKey; + + /** + * The API key for VirusTotal + */ + private String virusTotalKey; + + /** + * Creates an object that contains all parsed arguments + * @param hashes all loaded hashes + * @param outputPath the folder to write the downloads to + * @param threadCount the amount of threads to use + * @param koodousKey the API key for Koodous + * @param malwareBazaarKey the API key for Malware Bazaar + * @param malShareKey the API key for MalShare + * @param virusTotalKey the API key for VirusTotal + */ + public Arguments(Set hashes, String outputPath, int threadCount, String koodousKey, String malwareBazaarKey, String malShareKey, String virusTotalKey) { + this.hashes = hashes; + this.outputPath = outputPath; + this.threadCount = threadCount; + this.koodousKey = koodousKey; + this.malwareBazaarKey = malwareBazaarKey; + this.malShareKey = malShareKey; + this.virusTotalKey = virusTotalKey; + } + + /** + * Gets all loaded hashes without duplicates + * @return all loaded hashes + */ + public Set getHashes() { + return hashes; + } + + /** + * Get the output folder where the downloads should be written to + * @return the path to the output folder + */ + public String getOutputPath() { + return outputPath; + } + + /** + * Get the given thread count to use when downloading samples + * @return the thread count + */ + public int getThreadCount() { + return threadCount; + } + + /** + * Get the API key of Koodous, can be null if the API key is not used + * @return the API key + */ + public String getKoodousKey() { + return koodousKey; + } + + /** + * The API key of Malware Bazaar does not exist, so this value is either null or not null. Exclusion from the keys file means the service wont be used + * @return a value to see if this endpoint is to be used + */ + public String getMalwareBazaarKey() { + return malwareBazaarKey; + } + + /** + * Gets the API key of MalShare, can be null if the API key is not used + * @return the API key + */ + public String getMalShareKey() { + return malShareKey; + } + + /** + * The API key of VirusTotal, can be null if the API key is not used + * @return the API key + */ + public String getVirusTotalKey() { + return virusTotalKey; + } +} diff --git a/src/main/java/malpull/Downloader.java b/src/main/java/malpull/Downloader.java index 0f35c51..0d30bac 100644 --- a/src/main/java/malpull/Downloader.java +++ b/src/main/java/malpull/Downloader.java @@ -16,9 +16,7 @@ */ package malpull; -import exceptions.Error404NotFoundException; -import exceptions.Error429TooManyRequestsException; -import exceptions.HttpConnectionFailed; +import exceptions.SampleNotFoundException; import java.io.IOException; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -56,11 +54,9 @@ public Downloader() { * @param url the URL to contact * @param requestBody the request body with the form data * @return the data that the API returned in the form of a byte array - * @throws HttpConnectionFailed if no HTTP connection can be established - * (either because the machine that executes the JAR has no internet, or - * because the host is unreachable) + * @throws SampleNotFoundException is thrown if the sample cannot be found */ - public byte[] post(String url, RequestBody requestBody) throws HttpConnectionFailed { + public byte[] post(String url, RequestBody requestBody) throws SampleNotFoundException { //Create the request object based on the given URL, where the given request body is also used Request request = new Request.Builder() .url(url) @@ -71,13 +67,13 @@ public byte[] post(String url, RequestBody requestBody) throws HttpConnectionFai try (Response response = httpClient.newCall(request).execute()) { //Check the return value if (!response.isSuccessful()) { - throw new HttpConnectionFailed(); + throw new SampleNotFoundException("Sample not found, HTTP status code: " + response.code() + " for URL: " + request.url()); } //Return response body return response.body().bytes(); } catch (IOException e) { - throw new HttpConnectionFailed(); + throw new SampleNotFoundException("IOException in Dowlnoader.post for " + request.url()); } } @@ -87,33 +83,20 @@ public byte[] post(String url, RequestBody requestBody) throws HttpConnectionFai * @param request the request to send, which is already addressed to a * specific URL * @return the data that the API returned in the form of a byte array - * @throws HttpConnectionFailed if no HTTP connection can be established - * (either because the machine that executes the JAR has no internet, or - * because the host is unreachable) - * @throws Error404NotFoundException if the returned status code is 404 (Not - * Found) - * @throws Error429TooManyRequestsException if the returned status code is - * 429 (Too Many Requests) + * @throws SampleNotFoundException if the sample cannot be found */ - public byte[] get(Request request) throws HttpConnectionFailed, Error404NotFoundException, Error429TooManyRequestsException { + public byte[] get(Request request) throws SampleNotFoundException { //Make the call try (Response response = httpClient.newCall(request).execute()) { //Check the response code if (!response.isSuccessful()) { - switch (response.code()) { - case 404: - throw new Error404NotFoundException(); - case 429: - throw new Error429TooManyRequestsException(); - default: - throw new HttpConnectionFailed(); - } + throw new SampleNotFoundException("Sample not found, HTTP status code: " + response.code() + " for URL: " + request.url()); } //Return response body return response.body().bytes(); } catch (IOException e) { - throw new HttpConnectionFailed(); + throw new SampleNotFoundException("IOException in Dowlnoader.get for " + request.url()); } } } diff --git a/src/main/java/malpull/MalPull.java b/src/main/java/malpull/MalPull.java index c0db147..1837f26 100644 --- a/src/main/java/malpull/MalPull.java +++ b/src/main/java/malpull/MalPull.java @@ -16,15 +16,18 @@ */ package malpull; +import concurrency.DownloadWorker; +import endpoints.IEndpoint; import endpoints.Koodous; import endpoints.MalShare; import endpoints.MalwareBazaar; -import exceptions.Error404NotFoundException; -import exceptions.Error429TooManyRequestsException; -import exceptions.HttpConnectionFailed; -import exceptions.SampleNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; +import endpoints.VirusTotal; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * This class is the main class of the project, and creates and calls all @@ -36,190 +39,136 @@ * class around the OkHttp3 library that is being used to perform the HTTP * requests. * - * To compile this project, use : mvn clean compile assembly:single + * To compile this project, use: mvn clean compile assembly:single * * @author Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl] */ public class MalPull { /** - * A variable that is used to debug the program during development. If set - * to true, the argument length check is skipped, and the parameters are - * instantiated based on hard coded values within the main function. + * A list of hashes that could not be downloaded, taken from the + * deduplicated input */ - private static final boolean DEBUG = false; + private static List missingHashes = new ArrayList<>(); + /** + * Build using: + * + * mvn clean compile assembly:single + * + * @param args the command-line arguments + */ public static void main(String[] args) { + //Get the start time + long start = Instant.now().getEpochSecond(); //Show the version information printVersionInformation(); /** - * If the length of the arguments is not equal to four, not all - * information is provided. Additionally, the debug flag should be set - * to false + * Parse the arguments into a newly created object. If the arguments + * cannot be parsed properly, the error message is displayed and MalPull + * shuts down. */ - if (args.length != 4 && !DEBUG) { - //If that is the case, the usage should be printed - printUsage(); - //Then the system should exit - System.exit(0); - } + Arguments arguments = ArgumentHandler.handle(args); - /** - * Initialise local variables which are either set based on the given - * arguments, or based on hard coded values when debugging - */ - String keyMalShare; - String keyKoodous; - String hash; - String path; + //Show the input back to the user, as this helps to avoid mistakes + System.out.println("Read " + arguments.getHashes().size() + " hashes"); + System.out.println("Downloading will be done using " + arguments.getThreadCount() + " thread(s)"); + System.out.println("Output will be written to: " + arguments.getOutputPath()); + System.out.println(""); - if (DEBUG) { - //The MalShare key to use during debugging - keyMalShare = ""; - //The Koodous key to use during debugging - keyKoodous = ""; - /** - * Sample hashes to test with: - * - * Hash for MalShare: 78f29761f7f0f57a8f92e5f23d9e4d2d6465e848 - * - * Hash for MalwareBazaar: - * 094fd325049b8a9cf6d3e5ef2a6d4cc6a567d7d49c35f8bb8dd9e3c6acf3d78d - * - * Hash for Koodous: a24f14c4b95994d8440ed8f1c001a6135706716f - */ - hash = "a24f14c4b95994d8440ed8f1c001a6135706716f"; - //The path to write the sample to - path = "/home/user/test"; - } else { - //Get the keys, hash, and path from the command line arguments - keyMalShare = args[0]; - keyKoodous = args[1]; - hash = args[2]; - path = args[3]; - } + //Get all hashes in deduplicated form + Set hashes = arguments.getHashes(); - /** - * To get a sample, a try[ServiceName] function is called. This function - * uses the respective service's endpoint class to check for the - * availability of the sample. If it is present, it is download, after - * which MalwarePuller shuts down to avoid using more API calls than - * required. - */ - tryMalShare(keyMalShare, hash, path); - tryMalwareBazaar(hash, path); - tryKoodous(keyKoodous, hash, path); - } + //Get the thread count from the parsed arguments + ExecutorService executor = Executors.newFixedThreadPool(arguments.getThreadCount()); + //Keep track of the count so each thread can print what number it had comapred to the total amount of downloads + int count = 0; + //Iterate through all hashes, adding an endpoint for each of the configured endpoints + for (String hash : hashes) { + count++; + List endpoints = new ArrayList<>(); + if (arguments.getMalwareBazaarKey() != null) { + IEndpoint malwareBazaar = new MalwareBazaar(); + endpoints.add(malwareBazaar); + } + if (arguments.getMalShareKey() != null) { + IEndpoint malShare = new MalShare(arguments.getMalShareKey()); + endpoints.add(malShare); + } + if (arguments.getKoodousKey() != null) { + IEndpoint koodous = new Koodous(arguments.getKoodousKey()); + endpoints.add(koodous); + } + if (arguments.getVirusTotalKey() != null) { + IEndpoint virusTotal = new VirusTotal(arguments.getVirusTotalKey()); + endpoints.add(virusTotal); + } - /** - * Tries to get the sample from MalShare using the given API key. If it is - * present, it is written to the given path and closes MalwarePuller. - * - * @param key the API key for MalShare - * @param hash the sample's hash - * @param path the path to write the sample to - */ - private static void tryMalShare(String key, String hash, String path) { - try { - System.out.println("Searching MalShare..."); - byte[] output = new MalShare(key).getSample(hash); - System.out.println("Sample found on MalShare!"); - System.out.println("Saving the sample to " + path); - writeToDisk(output, path); - System.out.println("Wrote sample to disk!"); - System.out.println("Exiting now"); - System.exit(0); - } catch (HttpConnectionFailed ex) { - System.out.println("An error occured when contacting MalShare, verify your own internet connection, as well as the uptime of MalShare!"); - } catch (SampleNotFoundException | Error404NotFoundException ex) { - System.out.println("Sample not found on MalShare"); - } catch (Error429TooManyRequestsException ex) { - System.out.println("MalShare reports that there are too many requests with this API key, try again later!"); + //Create a download worker for the hash, with all configured endpoints embedded + DownloadWorker downloadWorker = new DownloadWorker(endpoints, arguments.getOutputPath(), hash, count, hashes.size()); + //Execute the download worker in the future, disregarding when specifically + executor.execute(downloadWorker); } - } + //Once all tasks are done, shut the executor down, meaning no new tasks can be added + executor.shutdown(); + //Wait until the executor is terminated, which only happens when all downloads are finished + while (!executor.isTerminated()) { + } + //Notify the user that all downloads are finished + System.out.println(""); + System.out.println("All downloads finished! The sample number count is not always printed ascending as the threads print the messages."); - /** - * Tries to get the sample from MalwareBazaar. If it is present, it is - * written to the given path and closes MalwarePuller. - * - * @param hash the sample's hash - * @param path the path to write the sample to - */ - private static void tryMalwareBazaar(String hash, String path) { - try { - System.out.println("Searching MalwareBazaar..."); - byte[] output = new MalwareBazaar().getSample(hash); - System.out.println("Sample found on MalwareBazaar!"); - System.out.println("Saving the sample to " + path); - writeToDisk(output, path); - System.out.println("Wrote sample to disk!"); - System.out.println("Exiting now"); - System.exit(0); - } catch (HttpConnectionFailed ex) { - System.out.println("An error occured when contacting MalwareBazaar, verify your own internet connection, as well as the uptime of MalwareBazaar!"); - } catch (SampleNotFoundException ex) { - System.out.println("Sample not found on Malware Bazaar"); + //If some hashes could not be found, these are printed + if (missingHashes.size() > 0) { + System.out.println("\n\nMissing " + missingHashes.size() + " hashes:"); + for (String missingHash : missingHashes) { + System.out.println(missingHash); + } } + //Get the time since the start of the downloading + String time = getDuration(start); + //Display the time that the download process took + System.out.println("\nDownloaded " + hashes.size() + " samples in " + time + "!"); + //Exit the program explicitly, as it sometimes remains open in some edge cases + System.exit(0); } /** - * Tries to get the sample from Koodous using the given API key. If it is - * present, it is written to the given path and closes MalwarePuller. + * Gets the duration from the given starting point until the moment this + * function is executed in the format of hh:mm:ss. * - * @param key the API key for Koodous - * @param hash the sample's hash - * @param path the path to write the sample to + * @param start the time to start in seconds from epoch */ - private static void tryKoodous(String key, String hash, String path) { - try { - System.out.println("Searching Koodous..."); - byte[] output = new Koodous(key).getSample(hash); - System.out.println("Sample found on Koodous!"); - System.out.println("Saving the sample to " + path); - writeToDisk(output, path); - System.out.println("Wrote sample to disk!"); - System.out.println("Exiting now"); - System.exit(0); - } catch (HttpConnectionFailed ex) { - System.out.println("An error occured when contacting Koodous, verify your own internet connection, as well as the uptime of Koodous!"); - } catch (SampleNotFoundException | Error404NotFoundException ex) { - System.out.println("Sample not found on Koodous"); - } catch (Error429TooManyRequestsException ex) { - System.out.println("Koodous reports that there are too many requests with this API key, try again later!"); - } + private static String getDuration(long start) { + //Get the end time + long end = Instant.now().getEpochSecond(); + //Calculate the time difference + long duration = end - start; + + //Calculate the amount of seconds + long seconds = duration % 60; + //Calculate the amount of minutes + long minutes = (duration / 60) % 60; + //Calculate the amount of hours + long hours = (duration / (60 * 60)) % 24; + //Format the times into a single string and return those + return String.format("%02d:%02d:%02d", hours, minutes, seconds); } /** - * Writes the given byte array to the disk to the given location + * As the add function is not thread safe, this synchronised function is + * used as a wrapper * - * @param output the data to write to the disk - * @param path the location to write the given data to + * @param missingHash the hash to add */ - private static void writeToDisk(byte[] output, String path) { - try (FileOutputStream fos = new FileOutputStream(path)) { - fos.write(output); - } catch (IOException ex) { - System.out.println("An error occured when writing the sample to \"" + path + "\". Verify your permissions and try again!"); - } + public static synchronized void addMissingHash(String missingHash) { + missingHashes.add(missingHash); } /** * Prints the version information, together with an additional newline */ private static void printVersionInformation() { - System.out.println("MalPull version 1.0-stable by Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl]\n"); + System.out.println("MalPull version 1.1-stable by Max 'Libra' Kersten [@LibraAnalysis, https://maxkersten.nl]\n"); } - - /** - * Prints the program's usage - */ - private static void printUsage() { - System.out.println("This tool downloads a sample from MalShare, MalwareBazaar, and Koodous based on a given MD-5, SHA-1, or SHA-256 hash."); - System.out.println("The sample is written to the given output directory. API Keys for both MalShare and Koodous are required."); - System.out.println("Once the sample is found, it is downloaded and saved, after which MalwarePuller will exit."); - System.out.println(""); - System.out.println("Sample usage of this program:"); - System.out.println("\t\tjava -jar /path/toMalwarePuller.jar malshare_api_key koodous_api_key sampleHash /path/to/save/the/sample/to"); - } - } diff --git a/target/MalPull-1.0-stable-jar-with-dependencies.jar b/target/MalPull-1.1-stable-jar-with-dependencies.jar similarity index 95% rename from target/MalPull-1.0-stable-jar-with-dependencies.jar rename to target/MalPull-1.1-stable-jar-with-dependencies.jar index e593321..497f3c9 100644 Binary files a/target/MalPull-1.0-stable-jar-with-dependencies.jar and b/target/MalPull-1.1-stable-jar-with-dependencies.jar differ