Merge pull request #173 from BenediktMKuehne/tracker-update

Tracker update
e-m-b-a · Dec 6, 2023 · ae8aa60 · ae8aa60
2 parents c68dd39 + 15751b3
commit ae8aa60
Show file tree

Hide file tree

Showing 29 changed files with 611 additions and 520 deletions.
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/dev-tools/check_project.sh b/dev-tools/check_project.sh
@@ -2,7 +2,7 @@
 
 # EMBArk - The firmware security scanning environment
 #
-# Copyright 2020-2021 Siemens Energy AG
+# Copyright 2020-2023 Siemens Energy AG
 # Copyright 2020-2021 Siemens AG
 #
 # EMBArk comes with ABSOLUTELY NO WARRANTY.
@@ -271,19 +271,93 @@ yamlchecker(){
   done
 }
 
+list_linter_exceptions(){
+  # lists all linter exceptions for a given toolname inside a directory 
+  # $1 tool name
+  # $2 directory
+  # $3 excluded dir for find
+  local TOOL_NAME_="${1:-}"
+  local DIR_="${2:-}"
+  local EXCLUDE_="${3:-}"
+  local SEARCH_PAR_=""
+  local SEARCH_TYPE_=""
+  echo -e "\\n""${GREEN}""Checking for ${TOOL_NAME_} exceptions inside ${DIR_}:""${NC}""\\n"
+  case "${TOOL_NAME_}" in
+    jshint)
+      SEARCH_PAR_="jshint ignore"
+      SEARCH_TYPE_="js"
+      ;;
+    shellcheck)
+      SEARCH_PAR_="shellcheck disable"
+      SEARCH_TYPE_="sh"
+      ;;
+    bandit)
+      SEARCH_PAR_="nosec"
+      SEARCH_TYPE_="py"
+      ;;
+    pylint)
+      SEARCH_PAR_="pylint"
+      SEARCH_TYPE_="py"
+      ;;
+    djlint)
+      SEARCH_PAR_="djlint"
+      SEARCH_TYPE_="html"
+      ;;
+  esac
+  mapfile -t EXCEPTION_SCRIPTS < <(find "${DIR_}" -type d -path "${EXCLUDE_}" -prune -false -o -iname "*.${SEARCH_TYPE_}" -exec grep -H "${SEARCH_PAR_}" {} \;)
+  if [[ "${#EXCEPTION_SCRIPTS[@]}" -gt 0 ]]; then
+    for EXCEPTION_ in "${EXCEPTION_SCRIPTS[@]}"; do
+      echo -e "\\n""${GREEN}""Found exception in ${EXCEPTION_%%:*}:""${ORANGE}""${EXCEPTION_##*:}""${NC}""\\n"
+      EXCEPTIONS_TO_CHECK_ARR+=( "${EXCEPTION_%%:*}" )
+    done
+  else
+    echo -e "\\n""${GREEN}""=> Found no exceptions for ${TOOL_NAME_}""${NC}""\\n"
+  fi
+}
+
+copy_right_check(){
+  # checks all Copyright occurences for supplied end-year 
+  # $1 end-year
+  # $2 dir to look in
+  # $3 excluded dir for find
+  local YEAR_="${1:-}"
+  local DIR_="${2:-}"
+  local EXCLUDE_="${3:-}"
+  echo -e "\\n""${ORANGE}""${BOLD}""EMBArk Copyright check""${NC}""\\n""${BOLD}""=================================================================""${NC}"
+  mapfile -t COPYRIGHT_LINE_ < <(find "${DIR_}" -type d -path "${EXCLUDE_}" -prune -false -o -type f -path "${0}" -prune -false -o -iname "*.sh" -exec grep -H "Copyright" {} \;)
+  if [[ "${#COPYRIGHT_LINE_[@]}" -gt 0 ]]; then
+    for LINE_ in "${COPYRIGHT_LINE_[@]}"; do
+      if ! grep -q "${YEAR_}.*Siemens Energy AG" "${LINE_%%:*}" && ! grep -q "Siemens AG" "${LINE_%%:*}"; then
+        ((MODULES_TO_CHECK=MODULES_TO_CHECK+1))
+        MODULES_TO_CHECK_ARR+=( "${LINE_%%:*}" )  
+        echo -e "Found problem with Copyright in ${LINE_%%:*}: ${ORANGE}${LINE_##*:}""${NC}""\\n"
+        echo -e "\\n""${ORANGE}${BOLD}==> FIX ERRORS""${NC}""\\n"
+      fi
+    done
+  else
+    echo -e "\\n""${GREEN}""==> Found no problems with copyrights""${NC}""\\n"
+  fi
+}
+
 #main
 check_tools
 MODULES_TO_CHECK=0
 MODULES_TO_CHECK_ARR=()
+EXCEPTIONS_TO_CHECK_ARR=()
 shellchecker
+list_linter_exceptions "shellcheck" "$PWD"
 dockerchecker
 jscheck
+list_linter_exceptions "jshint" "$PWD"
 templatechecker
+list_linter_exceptions "djlint" "$PWD"
 pycodestyle_check
 banditer
+list_linter_exceptions "bandit" "$PWD" "${PWD}/.venv"
 pylinter
 check_django
 yamlchecker
+copy_right_check 2023 "${PWD}" "${PWD}/emba_logs"
 
 if [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]]; then
   echo -e "\\n\\n""${GREEN}${BOLD}""SUMMARY:${NC}\\n"
@@ -293,5 +367,8 @@ if [[ "${#MODULES_TO_CHECK_ARR[@]}" -gt 0 ]]; then
   done
   exit 1
 fi
+if [[ "${#EXCEPTIONS_TO_CHECK_ARR[@]}" -gt 0 ]]; then
+  echo -e "${ORANGE}${BOLD}==> Please take a look at those Exceptions!""${NC}"
+fi
 echo -e "${GREEN}${BOLD}===> ALL CHECKS SUCCESSFUL""${NC}"
 exit 0
diff --git a/dev-tools/clean-setup.sh b/dev-tools/clean-setup.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # EMBArk - The firmware security scanning environment
 #
-# Copyright 2020-2022 Siemens Energy AG
+# Copyright 2020-2023 Siemens Energy AG
 #
 # EMBArk comes with ABSOLUTELY NO WARRANTY.
 #

diff --git a/dev-tools/debug-server-start.sh b/dev-tools/debug-server-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # EMBArk - The firmware security scanning environment
 #
-# Copyright 2020-2022 Siemens Energy AG
+# Copyright 2020-2023 Siemens Energy AG
 #
 # EMBArk comes with ABSOLUTELY NO WARRANTY.
 #

diff --git a/dev-tools/test.sh b/dev-tools/test.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # EMBArk - The firmware security scanning environment
 #
-# Copyright 2020-2022 Siemens Energy AG
+# Copyright 2020-2023 Siemens Energy AG
 #
 # EMBArk comes with ABSOLUTELY NO WARRANTY.
 #

diff --git a/emba b/emba
diff --git a/embark/dashboard/views.py b/embark/dashboard/views.py
@@ -81,7 +81,7 @@ def service_dashboard(request):
     :return httpresp: html servicedashboard
     """
     form = StopAnalysisForm()
-    form.fields['analysis'].queryset = FirmwareAnalysis.objects.filter(user=request.user).filter(finished=False)
+    form.fields['analysis'].queryset = FirmwareAnalysis.objects.filter(user=request.user).filter(finished=False).exclude(status__work=True)
     return render(request, 'dashboard/serviceDashboard.html', {'username': request.user.username, 'form': form, 'success_message': False})
 
 

diff --git a/embark/embark/__init__.py b/embark/embark/__init__.py
@@ -0,0 +1,25 @@
+from subprocess import Popen, PIPE
+import logging
+
+from django.utils import timezone
+
+logger = logging.getLogger(__name__)
+
+
+def check_tz() -> bool:
+    cmd = "date +%Z"
+    with Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) as process:   # nosec
+        system_tz, _error = process.communicate()
+        ret_code = process.returncode
+    if ret_code != 0:
+        logger.error("check_tz.error: %s", _error)
+        return False
+    system_tz = system_tz.decode("utf-8").rstrip()
+    if system_tz != timezone.get_current_timezone_name():
+        logger.error("SystemTZ=%s and EMBArkTZ=%s are not the same!", system_tz, timezone.get_current_timezone_name())
+        return False
+    logger.debug("SystemTZ=%s and EMBArkTZ=%s are the same!", system_tz, timezone.get_current_timezone_name())
+    return True
+
+
+check_tz()
diff --git a/embark/porter/importer.py b/embark/porter/importer.py
@@ -107,8 +107,8 @@ def f50_csv(file_path, analysis_id):
         res.canary_per = int(res_dict.get("canary_per", 0))
         res.relro = int(res_dict.get("relro", 0))
         res.relro_per = int(res_dict.get("relro_per", 0))
-        res.no_exec = int(res_dict.get("no_exec", 0))
-        res.no_exec_per = int(res_dict.get("no_exec_per", 0))
+        res.no_exec = int(res_dict.get("nx", 0))
+        res.no_exec_per = int(res_dict.get("nx_per", 0))
         res.pie = int(res_dict.get("pie", 0))
         res.pie_per = int(res_dict.get("pie_per", 0))
         res.stripped = int(res_dict.get("stripped", 0))

diff --git a/embark/porter/views.py b/embark/porter/views.py
@@ -173,13 +173,16 @@ def make_zip(request, analysis_id):
     """
     req_logger.info("Zipping Req by user: %s for analysis %s", request.user, analysis_id)
     try:
-        _ = FirmwareAnalysis.objects.get(id=analysis_id)
-        if BoundedExecutor.submit_zip(uuid=analysis_id) is not None:
-            # success
-            logger.info("Successfully submitted zip request %s", str(analysis_id))
-            messages.info(request, 'Zipping ' + str(analysis_id))
-            return redirect('embark-ReportDashboard')
-        messages.error(request, 'zipping failed, queue full?')
+        analysis = FirmwareAnalysis.objects.get(id=analysis_id)
+        # check that the user is authorized
+        if request.user == analysis.user or request.user.is_superuser:
+            if BoundedExecutor.submit_zip(uuid=analysis_id) is not None:
+                # success
+                logger.info("Successfully submitted zip request %s", str(analysis_id))
+                messages.info(request, 'Zipping ' + str(analysis_id))
+                return redirect('embark-dashboard-service')
+            messages.error(request, 'zipping failed, queue full?')
+        messages.error(request, 'Not authorized')
     except FirmwareAnalysis.DoesNotExist:
         messages.error(request, 'No analysis with that id found')
     return redirect('embark-ReportDashboard')
diff --git a/embark/reporter/views.py b/embark/reporter/views.py
@@ -7,12 +7,14 @@
 
 from operator import itemgetter
 from http import HTTPStatus
+from shutil import move
+import codecs
 from uuid import UUID
 
 from django.conf import settings
 from django.forms import model_to_dict
 from django.http.response import Http404
-from django.shortcuts import redirect
+from django.shortcuts import redirect, render
 from django.contrib import messages
 from django.template.loader import get_template
 from django.http import HttpResponse, JsonResponse
@@ -25,6 +27,10 @@
 from uploader.models import FirmwareAnalysis, ResourceTimestamp
 from dashboard.models import Result
 
+
+BLOCKSIZE = 1048576     # for codec change
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -53,13 +59,29 @@ def html_report(request, analysis_id, html_file):
 @require_http_methods(["GET"])
 @login_required(login_url='/' + settings.LOGIN_URL)
 def html_report_path(request, analysis_id, html_path, html_file):
-    report_path = Path(f'{settings.EMBA_LOG_ROOT}{request.path[10:]}')
     if FirmwareAnalysis.objects.filter(id=analysis_id).exists():
         analysis = FirmwareAnalysis.objects.get(id=analysis_id)
         if analysis.hidden is False or analysis.user == request.user or request.user.is_superuser:
-            html_body = get_template(report_path)
+            report_path = f'{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/html-report/{html_path}/{html_file}'
             logger.debug("html_report - analysis_id: %s path: %s html_file: %s", analysis_id, html_path, html_file)
-            return HttpResponse(html_body.render({'embarkBackUrl': reverse('embark-ReportDashboard')}))
+            try:
+                return render(request, report_path, {'embarkBackUrl': reverse('embark-ReportDashboard')}, content_type='text/html')
+            except UnicodeDecodeError as decode_error:
+                logger.error("{%s} with error: %s", report_path, decode_error)
+                # removes all non utf8 chars from html USING: https://stackoverflow.com/questions/191359/how-to-convert-a-file-to-utf-8-in-python
+                # CodeQL issue is not relevant
+                with codecs.open(report_path, "r", encoding='latin1') as source_file:
+                    with codecs.open(f'{report_path}.new', "w", "utf-8") as target_file:
+                        while True:
+                            contents = source_file.read(BLOCKSIZE)
+                            if not contents:
+                                break
+                            target_file.write(contents)
+                # exchange files
+                move(report_path, f'{report_path}.old')
+                move(f'{report_path}.new', report_path)
+                logger.debug("Removed problematic char from %s", report_path)
+                return render(request, report_path, {'embarkBackUrl': reverse('embark-ReportDashboard')}, content_type='text/html')
         messages.error(request, "User not authorized")
     logger.error("could  not get path - %s", request)
     return redirect("..")

diff --git a/embark/static/content/css/globalStyle.css b/embark/static/content/css/globalStyle.css
@@ -11,6 +11,7 @@ body {
     background: #1a1a1a;
     background-image: url('/static/content/images/bg.png');
     background-repeat: repeat;
+    margin-bottom: 10px;
 }
 
 .container-fluid,

diff --git a/embark/static/scripts/individualReportDashboard.js b/embark/static/scripts/individualReportDashboard.js
@@ -191,7 +191,7 @@ get_individual_report().then(function (returnData) {
         "High CVE": returnData.cve_high,
         "Medium CVE": returnData.cve_medium,
         "Low CVE": returnData.cve_low,
-        "NX disabled binaries": returnData.nx,
+        "NX disabled binaries": returnData.no_exec,
         "RELRO disabled binaries": returnData.relro,
         "PIE disabled binaries": returnData.pie,
         "Stack canaries disabled binaries": returnData.canary,

diff --git a/embark/static/scripts/serviceDashboard.js b/embark/static/scripts/serviceDashboard.js
@@ -167,6 +167,28 @@ function add_container_to_finished(status_dict) {
 }
 
 
+function set_container_to_work(status_dict) {
+    "use strict";
+    var $Container = $("#Container_" + status_dict.analysis);
+    $Container.html(`
+        <div class="mainText">
+            <small>`+ status_dict.analysis + `</small>
+            <br>
+            <span>`+ status_dict.firmware_name.split(".")[0] + `</span>
+            <br>
+            <h1> Working </h1>
+        </div>
+        <div class="log tile lastphase">
+            <span>` + status_dict.last_phase +  `</span>
+        </div>
+        <div class="d-flex justify-content-center">
+            <div class="spinner-border text-primary" role="status">
+                <span class="visually-hidden">Loading...</span>
+            </div>
+        </div>`);
+}
+
+
 /**
  * This method is called whenever a message from the backend arrives
  * */
@@ -190,7 +212,7 @@ socket.onmessage = function (event) {
                     // set percentage and other metadata
                     makeProgress(data[analysis_].percentage, data[analysis_].analysis);
                 }
-            } else if (data[analysis_].finished == true){
+            } else if (data[analysis_].finished == true ){
                 newContainer.remove();
                 add_container_to_finished(data[analysis_]);
             } else {
@@ -200,6 +222,9 @@ socket.onmessage = function (event) {
                 // set percentage and other metadata
                 makeProgress(data[analysis_].percentage, data[analysis_].analysis);
             }
+            if (data[analysis_].work == true){
+                set_container_to_work(data[analysis_]);
+            }
         }
     }
     catch(error){

diff --git a/embark/templates/base.html b/embark/templates/base.html
@@ -63,22 +63,41 @@
             </div>
         </div>
         {% block inlinejs %}{% endblock inlinejs %}
+        <footer class="fixed-bottom text-center text-white justify-content-center opacity-75">
+            <div class="d-flex flex-row justify-content-center">
+                <div class="p-1">
+                    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 273.5 222.3" role="img" aria-labelledby="dpefe9ffof6q8i9zrabcwkbbhphn1y5" class="octicon flex-shrink-0" height="16" width="16">
+                        <title id="dpefe9ffof6q8i9zrabcwkbbhphn1y5">Twitter</title>
+                        <path fill="currentColor" d="M273.5 26.3a109.77 109.77 0 0 1-32.2 8.8 56.07 56.07 0 0 0 24.7-31 113.39 113.39 0 0 1-35.7 13.6 56.1 56.1 0 0 0-97 38.4 54 54 0 0 0 1.5 12.8A159.68 159.68 0 0 1 19.1 10.3a56.12 56.12 0 0 0 17.4 74.9 56.06 56.06 0 0 1-25.4-7v.7a56.11 56.11 0 0 0 45 55 55.65 55.65 0 0 1-14.8 2 62.39 62.39 0 0 1-10.6-1 56.24 56.24 0 0 0 52.4 39 112.87 112.87 0 0 1-69.7 24 119 119 0 0 1-13.4-.8 158.83 158.83 0 0 0 86 25.2c103.2 0 159.6-85.5 159.6-159.6 0-2.4-.1-4.9-.2-7.3a114.25 114.25 0 0 0 28.1-29.1"/>
+                    </svg>
+                    <a rel="nofollow me" class="Link--primary" href="https://twitter.com/securefirmware">@securefirmware</a>
+                </div>
+                <div class="p-1">
+                    <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link flex-shrink-0">
+                        <path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"/>
+                    </svg>
+                    <a rel="nofollow" itemprop="url" class="color-fg-default" title="https://www.securefirmware.de" href="https://www.securefirmware.de">https://www.securefirmware.de</a>     
+                </div>
+            </div>
+        </footer>
+        <!--
         <div class="fixed-bottom  justify-content-center">
             <div class="d-flex flex-row justify-content-center">
-                <div class="p-2">
+                <div class="p-1">
                     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 273.5 222.3" role="img" aria-labelledby="dpefe9ffof6q8i9zrabcwkbbhphn1y5" class="octicon flex-shrink-0" height="16" width="16">
                         <title id="dpefe9ffof6q8i9zrabcwkbbhphn1y5">Twitter</title>
                         <path fill="currentColor" d="M273.5 26.3a109.77 109.77 0 0 1-32.2 8.8 56.07 56.07 0 0 0 24.7-31 113.39 113.39 0 0 1-35.7 13.6 56.1 56.1 0 0 0-97 38.4 54 54 0 0 0 1.5 12.8A159.68 159.68 0 0 1 19.1 10.3a56.12 56.12 0 0 0 17.4 74.9 56.06 56.06 0 0 1-25.4-7v.7a56.11 56.11 0 0 0 45 55 55.65 55.65 0 0 1-14.8 2 62.39 62.39 0 0 1-10.6-1 56.24 56.24 0 0 0 52.4 39 112.87 112.87 0 0 1-69.7 24 119 119 0 0 1-13.4-.8 158.83 158.83 0 0 0 86 25.2c103.2 0 159.6-85.5 159.6-159.6 0-2.4-.1-4.9-.2-7.3a114.25 114.25 0 0 0 28.1-29.1"/>
                     </svg>
                     <a rel="nofollow me" class="Link--primary" href="https://twitter.com/securefirmware">@securefirmware</a>
                 </div>
-                <div class="p-2">
+                <div class="p-1">
                     <svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-link flex-shrink-0">
                         <path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"/>
                     </svg>
                     <a rel="nofollow" itemprop="url" class="color-fg-default" title="https://www.securefirmware.de" href="https://www.securefirmware.de">https://www.securefirmware.de</a>     
                 </div>
             </div>
         </div>
+        -->
     </body>
     </html>