Merge pull request #3564 from DefectDojo/release/1.11.0

Release: Merge release into master from: release/1.11.0
DefectDojo · Dec 29, 2020 · b08723d · b08723d
2 parents 0c02ef9 + 0b61a1c
commit b08723d
Show file tree

Hide file tree

Showing 238 changed files with 178,381 additions and 5,442 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -2,4 +2,3 @@
 .gitignore
 *.md
 .env*
-**/local_settings.py
diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md
@@ -15,8 +15,9 @@ Please update any documentation when needed (in the .MD files in this repo, as w
 
 This checklist is for your information.
 
-- [ ] Features/Changes/Bugfixes should be submitted against the `dev` branch by default
-- [ ] Hotfixes should be submitted against master (urgent bugfixes requiring a hotfix release)
+- [ ] Features/Changes/Bugfixes should be submitted against the `dev` branch by default.
+- [ ] Make sure to rebase your PR against the very latest `dev`.
+- [ ] Hotfixes should be submitted against master (urgent bugfixes requiring a hotfix release).
 - [ ] Give a meaningful name to your PR, as it may end up being used in the release notes.
 - [ ] Your code is flake8 compliant.
 - [ ] Your code is python 3.6 compliant (specific python >3.6 syntax is currently not accepted).
@@ -29,7 +30,6 @@ This checklist is for your information.
 
 Please clear everything below when submitting your pull request, it's here purely for your information.
 
-
 Moderators: Labels currently accepted for PRs:
 - Import Scans (for new scanners/importers)
 - enhancement

diff --git a/.github/workflows/k8s-testing.yml b/.github/workflows/k8s-testing.yml
@@ -101,7 +101,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Setup Minikube
-        uses: manusa/actions-setup-minikube@v2.0.1
+        uses: manusa/actions-setup-minikube@v2.3.0
         with:
           minikube version: 'v1.14.2'
           kubernetes version: 'v1.19.2'

diff --git a/.github/workflows/new-release-master-into-dev.yml b/.github/workflows/new-release-master-into-dev.yml
@@ -54,7 +54,7 @@ jobs:
           grep appVersion helm/defectdojo/Chart.yaml
           grep version components/package.json
       - name: Push version changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v4.8.0
         with:
           commit_user_name: "${{ env.GIT_USERNAME }}"
           commit_user_email: "${{ env.GIT_EMAIL }}"

diff --git a/.github/workflows/new-release-pr.yml b/.github/workflows/new-release-pr.yml
@@ -60,7 +60,7 @@ jobs:
           grep appVersion helm/defectdojo/Chart.yaml
           grep version components/package.json
       - name: Push version changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v4.8.0
         with:
           commit_user_name: "${{ env.GIT_USERNAME }}"
           commit_user_email: "${{ env.GIT_EMAIL }}"

diff --git a/.github/workflows/new-release-tag-docker.yml b/.github/workflows/new-release-tag-docker.yml
@@ -46,20 +46,13 @@ jobs:
             Fill in with release drafter information manually for now, then publish.
           draft: true
           prerelease: false
-  job-build-matrix:
-    needs: tag-and-release
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - id: set-matrix
-        # ref should be the tag name
-        run: echo "::set-output name=matrix::{\"docker-image\":[\"django\",\"nginx\"],\"docker-tag\":[\"latest\",\"${{ github.event.inputs.release_number }}\"]}"
+
   job-build-and-push:
-    needs: job-build-matrix
+    needs: tag-and-release
     runs-on: ubuntu-latest
     strategy:
-      matrix: ${{fromJson(needs.job-build-matrix.outputs.matrix)}}
+      matrix: 
+          docker-image: [django, nginx]
     steps:
       - name: Login to DockerHub
         uses: docker/login-action@v1
@@ -82,7 +75,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          tags: ${{ env.REPO_ORG }}/defectdojo-${{ matrix.docker-image}}:${{ matrix.docker-tag }}
+          tags: ${{ env.REPO_ORG }}/defectdojo-${{ matrix.docker-image}}:${{ github.event.inputs.release_number }}, ${{ env.REPO_ORG }}/defectdojo-${{ matrix.docker-image}}:latest
           file: ./Dockerfile.${{ matrix.docker-image }}
           context: .
       - name: Image digest

diff --git a/.github/workflows/plantuml.yml b/.github/workflows/plantuml.yml
@@ -30,7 +30,7 @@ jobs:
         with:
           args: -v -tpng ${{ steps.getfile.outputs.files }}
       - name: Push Local Changes
-        uses: stefanzweifel/git-auto-commit-action@v4.7.2
+        uses: stefanzweifel/git-auto-commit-action@v4.8.0
         with:
           commit_user_name: "PlantUML_bot"
           commit_user_email: "[email protected]"

diff --git a/.travis.yml b/.travis.yml
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -21,29 +21,6 @@ For changes that require additional settings, you can now use local_settings.py
 ## Python3 version
 For compatibility reasons, the code in dev branch should be python3.6 compliant.
 
-## Logging
-Logging is configured in `settings.dist.py` and can be tuned using a `local_settings.py`, see [template for local_settings.py](dojo/settings/template-local_settings)
-Specific logger can be added. For example to activate logs related to the deduplication, change the level from DEBUG to INFO in `local_settings.py`:
-
-
-```
-LOGGING['loggers']['dojo.specific-loggers.deduplication']['level'] = 'DEBUG'
-```
-
-Or you can modify `settings.dist.py` directly, but this adds the risk of having conflicts when `settings.dist.py` gets updated upstream. 
-
-```
-          'dojo.specific-loggers.deduplication': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
-        }
-```
-
-## Debug Toolbar
-In the `dojo/settings/template-local_settings.py` you'll find instructions on how to enable the [Django Debug Toolbar](https://github.com/jazzband/django-debug-toolbar).
-This toolbar allows you to debug SQL queries, and shows some other interesting information.
-
 ## Submitting Pull Requests
 
 The following are things to consider before submitting a pull request to
@@ -70,5 +47,3 @@ DefectDojo.
 [setup_bash]: /setup.bash "Bash setup script"
 [pep8]: https://www.python.org/dev/peps/pep-0008/ "PEP8"
 [flake8 built-in commit hooks]: https://flake8.pycqa.org/en/latest/user/using-hooks.html#built-in-hook-integration
-
-
diff --git a/DOCKER.md b/DOCKER.md
@@ -152,7 +152,7 @@ docker-compose logs initializer | grep "Admin password:"
 
 Make sure you write down the first password generated as you'll need it when re-starting the application.
 
-# Option to change the password 
+## Option to change the password 
 * If you dont have admin password use the below command to change the password. 
 * After starting the container and open another tab in the same folder.  
 * django-defectdojo_uwsgi_1 -- name obtained from running containers using ```zsh docker ps ``` command
@@ -161,6 +161,29 @@ Make sure you write down the first password generated as you'll need it when re-
 docker exec -it django-defectdojo_uwsgi_1 ./manage.py changepassword admin
 ```
 
+# Logging
+For docker-compose release mode the log level is INFO. In the other modes the log level is DEBUG. Logging is configured in `settings.dist.py` and can be tuned using a `local_settings.py`, see [template for local_settings.py](dojo/settings/template-local_settings). For example the deduplication logger can be set to DEBUG in a local_settings.py file:
+
+
+```
+LOGGING['loggers']['dojo.specific-loggers.deduplication']['level'] = 'DEBUG'
+```
+
+Or you can modify `settings.dist.py` directly, but this adds the risk of having conflicts when `settings.dist.py` gets updated upstream. 
+
+```
+          'dojo.specific-loggers.deduplication': {
+            'handlers': ['console'],
+            'level': 'DEBUG',
+            'propagate': False,
+        }
+```
+
+## Debug Toolbar
+In the `dojo/settings/template-local_settings.py` you'll find instructions on how to enable the [Django Debug Toolbar](https://github.com/jazzband/django-debug-toolbar).
+This toolbar allows you to debug SQL queries, and shows some other interesting information.
+
+
 # Exploitation, versioning
 ## Disable the database initialization
 The initializer container can be disabled by exporting: `export DD_INITIALIZE=false`. 

diff --git a/Dockerfile.busybox b/Dockerfile.busybox
@@ -1,2 +1,2 @@
-FROM busybox:1.31.1-musl
+FROM busybox:1.32.0-musl
 ENTRYPOINT ["/bin/echo", "hello world"]
diff --git a/Dockerfile.django b/Dockerfile.django
@@ -1,7 +1,7 @@
 
 # code: language=Dockerfile
 
-# The code for the build image should be idendical with the code in
+# The code for the build image should be identical with the code in
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
@@ -23,8 +23,7 @@ RUN \
   rm -rf /var/lib/apt/lists && \
   true
 COPY requirements.txt ./
-RUN pip3 install --no-cache-dir --upgrade pip
-RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt --use-deprecated=legacy-resolver
+RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
 FROM python:3.6.12-slim-buster@sha256:e5259113df5a7c4dae16ad37c2ca53b1cf722e051cfd5f624e7b76aa72389e0c
 WORKDIR /app
@@ -53,15 +52,14 @@ RUN \
   apt-get clean && \
   rm -rf /var/lib/apt/lists && \
   true
-RUN pip3 install --no-cache-dir --upgrade pip
 COPY --from=build /tmp/wheels /tmp/wheels
 COPY requirements.txt ./
 RUN pip3 install \
 	--no-cache-dir \
 	--no-index \
   --find-links=/tmp/wheels \
-  -r ./requirements.txt \
-  --use-deprecated=legacy-resolver
+  -r ./requirements.txt
+
 COPY \
   docker/entrypoint-celery-beat.sh \
   docker/entrypoint-celery-worker.sh \
@@ -74,7 +72,7 @@ COPY \
   docker/wait-for-it.sh \
   docker/certs/* \
   /
-COPY wsgi.py manage.py tests/unit-tests.sh ./
+COPY wsgi.py manage.py docker/unit-tests.sh ./
 COPY dojo/ ./dojo/
 
 # Add extra fixtures to docker image which are loaded by the initializer
@@ -98,9 +96,12 @@ RUN \
     chown -R ${appuser} /app && \
     chmod 0700 /app && \
     chmod 0750 -R /app/* && \
+	chmod g=u /app && \
+	chmod -R g=u /app/* && \
     mkdir /var/run/${appuser} && \
-    chown ${appuser} /var/run/${appuser}
-USER ${appuser}
+    chown ${appuser} /var/run/${appuser} && \
+	chmod g=u /var/run/${appuser}
+USER ${uid}
 ENV \
   DD_ADMIN_USER=admin \
   [email protected] \

diff --git a/Dockerfile.nginx b/Dockerfile.nginx
@@ -1,6 +1,6 @@
 # code: language=Dockerfile
 
-# The code for the build image should be idendical with the code in
+# The code for the build image should be identical with the code in
 # Dockerfile.django to use the caching mechanism of Docker.
 
 FROM python:3.6.12-slim-buster@sha256:e5259113df5a7c4dae16ad37c2ca53b1cf722e051cfd5f624e7b76aa72389e0c as build
@@ -15,13 +15,13 @@ RUN \
     postgresql-client \
     xmlsec1 \
     git \
+    uuid-runtime \
     && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists && \
   true
 COPY requirements.txt ./
-RUN pip3 install --no-cache-dir --upgrade pip
-RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt --use-deprecated=legacy-resolver
+RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
 FROM build AS collectstatic
 
@@ -47,13 +47,11 @@ RUN \
   rm -rf /var/lib/apt/lists && \
   true
 
-RUN pip3 install --no-cache-dir --upgrade pip
 RUN pip3 install \
 	--no-cache-dir \
 	--no-index \
   --find-links=/tmp/wheels \
-	-r ./requirements.txt \
-    --use-deprecated=legacy-resolver
+	-r ./requirements.txt
 
 COPY components/ ./components/
 COPY manage.py ./
@@ -66,7 +64,7 @@ RUN \
   env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && \
   true
 
-FROM nginx:1.19.4-alpine@sha256:f9ddfb3fd9590a3b6ba095939b7a5aee110a6fb397922e2684d6e189e78329c9
+FROM nginx:1.19.6-alpine@sha256:01747306a7247dbe928db991eab42e4002118bf636dd85b4ffea05dd907e5b66
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/

diff --git a/KUBERNETES.md b/KUBERNETES.md
@@ -165,6 +165,16 @@ docker build --build-arg http_proxy=http://myproxy.com:8080 --build-arg https_pr
 docker build --build-arg http_proxy=http://myproxy.com:8080 --build-arg https_proxy=http://myproxy.com:8080 -t defectdojo/defectdojo-nginx -f Dockerfile.nginx .
 ```
 
+### Debug uWSGI with ptvsd
+
+You can set breakpoints in code that is handled by uWSGI. The feature is meant to be used when you run locally on minikube, and mimics [what can be done with docker-compose](DOCKER.md#run-with-docker-compose-in-development-mode-with-ptvsd-remote-debug).
+
+The port is currently hard-coded to 3000.
+
+* In `values.yaml`, ensure the value for `enable_ptvsd` is set to `true` (the default is `false`). Make sure the change is taken into account in your deployment.
+* Have `DD_DEBUG` set to `True`.
+* Port forward port 3000 to the pod, such as `kubectl port-forward defectdojo-django-7886f49466-7cwm7 3000`.
+
 ### Upgrade the chart
 If you want to change kubernetes configuration of use an updated docker image (evolution of defectDojo code), upgrade the application:
 ```
@@ -308,6 +318,10 @@ However, that doesn't work and I haven't found out why. In a production
 environment, a redundant PostgreSQL cluster is the better option. As it uses
 statefulsets that are kept by default, the problem doesn't exist there.
 
+### Prometheus metrics
+
+It's possible to enable Nginx prometheus exporter by setting `--set monitoring.enabled=true` and `--set monitoring.prometheus.enabled=true`. This adds the Nginx exporter sidecar and the standard Prometheus pod annotations to django deployment.
+
 ## Useful stuff
 
 ```zsh

diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![OWASP Flagship](https://img.shields.io/badge/owasp-flagship%20project-orange.svg)](https://www.owasp.org/index.php/OWASP_DefectDojo_Project) [![GitHub release](https://img.shields.io/github/release/DefectDojo/django-DefectDojo.svg)](https://github.com/DefectDojo/django-DefectDojo) [![YouTube Subscribe](https://img.shields.io/badge/youtube-subscribe-%23c4302b.svg)](https://www.youtube.com/channel/UCWw9qzqptiIvTqSqhOFuCuQ) ![Twitter Follow](https://img.shields.io/twitter/follow/defectdojo.svg?style=social&label=Follow)
 
-[![Build Status](https://travis-ci.org/DefectDojo/django-DefectDojo.svg?branch=master)](https://travis-ci.org/DefectDojo/django-DefectDojo) [![Documentation Status](https://readthedocs.org/projects/defectdojo/badge/?version=latest)](https://defectdojo.readthedocs.io/en/latest/?badge=latest) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2098/badge)](https://bestpractices.coreinfrastructure.org/projects/2098)
+[![Build Status](https://github.com/DefectDojo/django-DefectDojo/actions)](https://github.com/DefectDojo/django-DefectDojo/actions) [![Documentation Status](https://readthedocs.org/projects/defectdojo/badge/?version=latest)](https://defectdojo.readthedocs.io/en/latest/?badge=latest) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2098/badge)](https://bestpractices.coreinfrastructure.org/projects/2098)
 
 ![Screenshot of DefectDojo](https://raw.githubusercontent.com/DefectDojo/Documentation/master/doc/img/screenshot1.png)
 
@@ -22,7 +22,8 @@ cd django-DefectDojo
 docker-compose build
 # running
 docker-compose up
-# obtain admin credentials
+# obtain admin credentials. the initializer can take up to 3 minutes to run
+# use docker-compose logs -f initializer to track progress
 docker-compose logs initializer | grep "Admin password:"
 ```
 

diff --git a/components/package.json b/components/package.json
@@ -1,6 +1,6 @@
 {
   "name": "DefectDojo",
-  "version": "1.10.4",
+  "version": "1.11.0",
   "dependencies": {
     "JUMFlot": "jumjum123/JUMFlot#*",
     "bootstrap": "^3.4.0",
@@ -12,11 +12,11 @@
     "clipboard": "^2.0.6",
     "components-jqueryui": "^1.0.0",
     "datatables.net": "^1.10.22",
-    "datatables.net-bs": "^1.10.22",
+    "datatables.net-bs": "^1.10.23",
     "datatables.net-buttons-bs": "^1.6.5",
     "datatables.net-buttons-dt": "^1.6.5",
-    "datatables.net-colreorder": "^1.5.2",
-    "datatables.net-dt": "^1.10.22",
+    "datatables.net-colreorder": "^1.5.3",
+    "datatables.net-dt": "^1.10.23",
     "drmonty-datatables-plugins": "^1.0.0",
     "drmonty-datatables-responsive": "^1.0.0",
     "easymde": "^2.13.0",
@@ -35,7 +35,7 @@
     "metismenu": "~3.0.6",
     "moment": "^2.29.1",
     "morris.js": "morrisjs/morris.js",
-    "pdfmake": "^0.1.68",
+    "pdfmake": "^0.1.69",
     "startbootstrap-sb-admin-2": "1.0.7"
   },
   "engines": {
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,4 +2,3 @@ @@
     .gitignore
     *.md
     .env*
-    **/local_settings.py