From 44a2145e82623f9875c585f911251b7420ce0efd Mon Sep 17 00:00:00 2001 From: Davide Arcuri Date: Thu, 12 Dec 2024 15:13:54 +0100 Subject: [PATCH] #1073 --- orochi/api/models.py | 10 +- orochi/api/routers/symbols.py | 133 +++++++++++++++++- orochi/templates/website/index.html | 2 +- orochi/templates/website/list_symbols.html | 29 +++- .../website/partial_isf_download.html | 2 +- .../website/partial_packages_upload.html | 2 +- orochi/website/views.py | 100 +++---------- 7 files changed, 182 insertions(+), 96 deletions(-) diff --git a/orochi/api/models.py b/orochi/api/models.py index 49b93c3d..556a1b50 100644 --- a/orochi/api/models.py +++ b/orochi/api/models.py @@ -421,10 +421,14 @@ class SymbolsBannerIn(Schema): banner: str = None -class SymbolsInfo(Schema): +class UploadFileInfo(Schema): original_name: Optional[str] = None local_folder: Optional[str] = None -class SymbolsIn(Schema): - info: Optional[List[SymbolsInfo]] = [] +class UploadFileIn(Schema): + info: Optional[List[UploadFileInfo]] = [] + + +class ISFIn(Schema): + path: str diff --git a/orochi/api/routers/symbols.py b/orochi/api/routers/symbols.py index 4fd99e2e..a1af6b06 100644 --- a/orochi/api/routers/symbols.py +++ b/orochi/api/routers/symbols.py @@ -1,17 +1,28 @@ +import concurrent.futures +import json import os import shutil import subprocess from pathlib import Path from typing import List, Optional +from urllib.parse import urlparse import magic +import requests from django.shortcuts import get_object_or_404 +from django.utils.text import slugify from extra_settings.models import Setting from ninja import File, Router from ninja.files import UploadedFile from ninja.security import django_auth -from orochi.api.models import ErrorsOut, SuccessResponse, SymbolsBannerIn, SymbolsIn +from orochi.api.models import ( + ErrorsOut, + ISFIn, + SuccessResponse, + SymbolsBannerIn, + UploadFileIn, +) from orochi.utils.download_symbols import Downloader from orochi.utils.volatility_dask_elk import check_runnable, refresh_symbols from orochi.website.defaults import DUMP_STATUS_COMPLETED @@ -62,7 +73,9 @@ def banner_symbols(request, payload: SymbolsBannerIn): response={200: SuccessResponse, 400: ErrorsOut}, ) def upload_symbols( - request, payload: SymbolsIn, symbols: Optional[List[UploadedFile]] = File(None) + request, + payload: Optional[UploadFileIn], + symbols: Optional[List[UploadedFile]] = File(None), ): """ Uploads a list of symbol files to a specified directory and extracts them if they are in a compressed format. This function handles file writing and type checking to ensure proper processing of the uploaded symbols. @@ -83,10 +96,8 @@ def upload_symbols( if payload.info: for item in payload.info: start = item.local_folder - original_name = item.original_name start = start.replace("/upload/upload", "/media/uploads") - filename = original_name - filepath = f"{path}/{filename}" + filepath = f"{path}/{ item.original_name}" shutil.move(start, filepath) filetype = magic.from_file(filepath, mime=True) if filetype in [ @@ -149,3 +160,115 @@ def delete_symbol(request, path): return 200, {"message": "Symbols deleted."} except Exception as excp: return 400, {"errors": str(excp)} + + +@router.post( + "/isf_download", + url_name="isf_download", + auth=django_auth, + response={200: SuccessResponse, 400: ErrorsOut}, +) +def isf_download(request, payload: ISFIn): + """Download and save symbol files from a given URL. + + This function downloads symbol files for different operating systems from a specified path and saves them locally. It supports concurrent downloading of multiple symbol files. + + Args: + request: The incoming HTTP request. + payload: An ISFIn object containing the download path. + + Returns: + A tuple with status code and response message indicating success or failure. + + Raises: + Exception: If there are issues parsing symbols or downloading files. + + Examples: + POST /isf_download with a payload containing a valid symbol file URL. + (e.g. https://raw.githubusercontent.com/Abyss-W4tcher/volatility3-symbols/master/banners/banners_plain.json) + """ + + try: + path = payload.path + domain = slugify(urlparse(path).netloc) + media_path = Path(f"{Setting.get('VOLATILITY_SYMBOL_PATH')}/{domain}") + media_path.mkdir(exist_ok=True, parents=True) + try: + data = json.loads(requests.get(path).content) + except Exception: + return 400, {"errors": "Error parsing symbols"} + + def download_file(url, path): + try: + response = requests.get(url) + with open(path, "wb") as f: + f.write(response.content) + except Exception as excp: + print(excp) + + with concurrent.futures.ThreadPoolExecutor() as executor: + for key in data: + if key not in ["linux", "mac", "windows"]: + continue + for urls in data[key].values(): + for url in urls: + filename = url.split("/")[-1] + filepath = f"{media_path}/{filename}" + executor.submit(download_file, url, filepath) + + refresh_symbols() + return 200, {"message": "Symbols downloaded successfully"} + except Exception as excp: + return 400, {"errors": str(excp)} + + +@router.post( + "/upload_packages", + url_name="upload_packages", + auth=django_auth, + response={200: SuccessResponse, 400: ErrorsOut}, +) +def upload_packages( + request, + payload: Optional[UploadFileIn], + packages: Optional[List[UploadedFile]] = File(None), +): + """Upload and process symbol packages for analysis. + + This function handles symbol package uploads through either predefined file information or direct file uploads. It processes the uploaded files using a Downloader and refreshes symbol information. + + Args: + request: The incoming HTTP request. + payload: An UploadFileIn object containing file information. + packages: Optional list of uploaded files to process. + + Returns: + A tuple with status code and response message indicating upload success or failure. + + Raises: + Exception: If there are issues processing uploaded files. + + Examples: + POST /upload_packages with file information or direct file uploads. + """ + + try: + file_list = [] + if payload.info: + for item in payload.info: + start = item.local_folder + start = start.replace("/upload/upload", "/media/uploads") + file_list.append((start, item.original_name)) + elif packages: + for package in packages: + filepath = f"/media/{Path(package.name).name}" + with open(filepath, "wb") as f: + f.write(package.read()) + file_list.append((filepath, Path(package.name).name)) + d = Downloader(file_list=file_list) + d.process_list() + os.unlink(filepath) + refresh_symbols() + return 200, {"message": "Symbols uploaded."} + except Exception as excp: + return 400, {"errors": str(excp)} diff --git a/orochi/templates/website/index.html b/orochi/templates/website/index.html index 40bdb58a..36f8cb01 100644 --- a/orochi/templates/website/index.html +++ b/orochi/templates/website/index.html @@ -179,7 +179,7 @@
History Log
}); $("#modal-update").modal('hide'); }, - error: function () { + error: function (data) { $.toast({ title: 'Bookmark status!', content: data.responseJSON.errors, diff --git a/orochi/templates/website/list_symbols.html b/orochi/templates/website/list_symbols.html index da416dab..e737cd98 100644 --- a/orochi/templates/website/list_symbols.html +++ b/orochi/templates/website/list_symbols.html @@ -193,25 +193,46 @@ $("#btn_isf_download").html('Loading...'); $("#btn_isf_download").addClass('disabled'); var form = $(this); + var formData = new FormData(this); + let jsonData = form.serializeArray(); + let obj = { 'info': [] }; + jsonData.forEach(item => { + if (['csrfmiddlewaretoken', 'packages-uploads', 'upload_url', 'form_id', 'packages-metadata'].indexOf(item.name) == -1) { + obj[item.name] = item.value; + } else if (item.name == 'packages-uploads') { + obj['info'].push({ + 'local_folder': '/media/tmp/' + JSON.parse(item.value)[0]['id'], + 'original_name': JSON.parse(item.value)[0]['name'], + }); + } + }); + formData.append("payload", JSON.stringify(obj)); + $.ajaxSetup({ + headers: { 'X-CSRFToken': $('input[name="csrfmiddlewaretoken"]').val() } + }); $.ajax({ url: form.attr("action"), - data: form.serialize(), + data: formData, + contentType: "application/json", type: form.attr("method"), + processData: false, + contentType: false, dataType: 'json', success: function (data) { $("#modal-update").modal('hide'); table.ajax.reload(); $.toast({ title: 'Symbols added!', - content: 'Symbols added.', + content: data.message, type: 'success', delay: 5000 }); }, - error: function () { + error: function (data) { + $("#modal-update").modal('hide'); $.toast({ title: 'Error!', - content: 'Error during symbols upload.', + content: data.responseJSON.errors, type: 'error', delay: 5000 }); diff --git a/orochi/templates/website/partial_isf_download.html b/orochi/templates/website/partial_isf_download.html index 7096d0ff..2feeb62c 100644 --- a/orochi/templates/website/partial_isf_download.html +++ b/orochi/templates/website/partial_isf_download.html @@ -1,6 +1,6 @@ {% load widget_tweaks %} -
+ {{ form.media }} {% csrf_token %}
diff --git a/orochi/templates/website/partial_packages_upload.html b/orochi/templates/website/partial_packages_upload.html index 5e9edc10..0d11bd2f 100644 --- a/orochi/templates/website/partial_packages_upload.html +++ b/orochi/templates/website/partial_packages_upload.html @@ -1,6 +1,6 @@ {% load widget_tweaks %} - + {{ form.media }} {% csrf_token %}
diff --git a/orochi/website/views.py b/orochi/website/views.py index daffadb7..48d77ec4 100644 --- a/orochi/website/views.py +++ b/orochi/website/views.py @@ -1,14 +1,11 @@ -import concurrent.futures import json import mmap import os import re import shlex from pathlib import Path -from urllib.parse import urlparse from urllib.request import pathname2url -import requests from dask.distributed import Client, fire_and_forget from django.conf import settings from django.contrib import messages @@ -23,7 +20,6 @@ from django.shortcuts import get_object_or_404, redirect, render from django.template.loader import render_to_string from django.template.response import TemplateResponse -from django.utils.text import slugify from django.views.decorators.http import require_http_methods from extra_settings.models import Setting from guardian.shortcuts import get_objects_for_user, get_perms @@ -31,14 +27,8 @@ from pymisp.tools import FileObject from volatility3.framework import automagic, contexts -from orochi.utils.download_symbols import Downloader from orochi.utils.timeliner import clean_bodywork -from orochi.utils.volatility_dask_elk import ( - get_parameters, - refresh_symbols, - run_plugin, - unzip_then_run, -) +from orochi.utils.volatility_dask_elk import get_parameters, run_plugin, unzip_then_run from orochi.website.defaults import ( RESULT_STATUS_DISABLED, RESULT_STATUS_EMPTY, @@ -1168,6 +1158,7 @@ def iterate_symbols(request): @login_required @user_passes_test(is_not_readonly) +@require_http_methods(["GET"]) def upload_symbols(request): """Upload symbols""" return JsonResponse( @@ -1183,84 +1174,31 @@ def upload_symbols(request): @login_required @user_passes_test(is_not_readonly) +@require_http_methods(["GET"]) def download_isf(request): """Download all symbols from provided isf server path""" - data = {} - if request.method == "POST": - form = SymbolISFForm(data=request.POST) - if form.is_valid(): - path = form.cleaned_data["path"] - domain = slugify(urlparse(path).netloc) - media_path = Path(f"{Setting.get('VOLATILITY_SYMBOL_PATH')}/{domain}") - media_path.mkdir(exist_ok=True, parents=True) - try: - data = json.loads(requests.get(path).content) - except Exception: - return JsonResponse( - {"status_code": 404, "error": "Error parsing symbols"} - ) - - def download_file(url, path): - try: - response = requests.get(url) - with open(path, "wb") as f: - f.write(response.content) - except Exception as excp: - print(excp) - - with concurrent.futures.ThreadPoolExecutor() as executor: - for key in data: - if key not in ["linux", "mac", "windows"]: - continue - for urls in data[key].values(): - for url in urls: - filename = url.split("/")[-1] - filepath = f"{media_path}/{filename}" - executor.submit(download_file, url, filepath) - - refresh_symbols() - data["form_is_valid"] = True - else: - data["form_is_valid"] = False - else: - form = SymbolISFForm() - - context = {"form": form} - data["html_form"] = render_to_string( - "website/partial_isf_download.html", - context, - request=request, + return JsonResponse( + { + "html_form": render_to_string( + "website/partial_isf_download.html", + {"form": SymbolISFForm()}, + request=request, + ) + } ) - return JsonResponse(data) @login_required @user_passes_test(is_not_readonly) +@require_http_methods(["GET"]) def upload_packages(request): """Generate symbols from uploaded file""" - data = {} - if request.method == "POST": - form = SymbolPackageForm(data=request.POST) - if form.is_valid(): - d = Downloader( - file_list=[ - (package.file.path, package.name) - for package in form.cleaned_data["packages"] - ] + return JsonResponse( + { + "html_form": render_to_string( + "website/partial_packages_upload.html", + {"form": SymbolPackageForm()}, + request=request, ) - d.process_list() - form.delete_temporary_files() - refresh_symbols() - data["form_is_valid"] = True - else: - data["form_is_valid"] = False - else: - form = SymbolPackageForm() - - context = {"form": form} - data["html_form"] = render_to_string( - "website/partial_packages_upload.html", - context, - request=request, + } ) - return JsonResponse(data)