feat: vulnz Docker v2 (#114)

* feat: vulnz Docker

* fix: stop gzipping in vulnz Docker after cache cve

* fix: update documentation according to gzip feature

* Add docs

- Add option to adjust the memory
- Add volume definition, build docs
- Add bash to the docker image

* Add CI integration

* Move version to gradle properties

* Add delay management, fix docs, fix image location

Co-authored-by: Jeremy Long <[email protected]>

* Adjust workflow and add docker hub

---------

Co-authored-by: j.berlin <[email protected]>
Co-authored-by: Jeremy Long <[email protected]>

.github/workflows/vulnz-docker-pr.yml

@@ -0,0 +1,38 @@
+name: docker pr
+
+on:
+  push:
+    branches-ignore:
+      - 'main'
+
+env:
+  IMAGE_FQDN: ghcr.io/jeremylong/open-vulnerability-data-mirror
+  VERSION: 0.0.0-SNAPSHOT
+
+jobs:
+  docker-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Run build
+        run: ./gradlew -x test -Pversion=${{ env.VERSION }} vulnz:build
+      - name: Build docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: vulnz/
+          platforms: linux/amd64
+          push: false
+          tags: |
+            ${{ env.IMAGE_FQDN }}:${{ env.VERSION }}
+          build-args: |
+            BUILD_VERSION=${{ env.VERSION }}

.github/workflows/vulnz-docker-release.yml

@@ -0,0 +1,53 @@
+name: docker pr
+
+on:
+  push:
+    tags:
+      - v*
+
+env:
+  GHCR_IMAGE_FQDN: ghcr.io/jeremylong/open-vulnerability-data-mirror
+  HUB_IMAGE_FQDN: jeremylong/open-vulnerability-data-mirror
+  VERSION: ${{ github.ref_name }}
+
+jobs:
+  docker-release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Run build
+        run: ./gradlew -x test -Pversion=${{ env.VERSION }} vulnz:build
+      - name: Build docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: vulnz/
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ${{ env.GHCR_IMAGE_FQDN }}:${{ env.VERSION }}
+            ${{ env.GHCR_IMAGE_FQDN }}:latest
+            ${{ env.HUB_IMAGE_FQDN }}:${{ env.VERSION }}
+            ${{ env.HUB_IMAGE_FQDN }}:latest
+          build-args: |
+            BUILD_VERSION=${{ env.VERSION }}

buildSrc/src/gradle.properties

@@ -0,0 +1 @@
+version = '5.1.1'

buildSrc/src/main/groovy/vuln.tools.java-common-conventions.gradle

@@ -12,7 +12,6 @@ plugins {
 }
 
 group 'io.github.jeremylong'
-version = '5.1.1'
 
 repositories {
     mavenCentral()

vulnz/Dockerfile

@@ -0,0 +1,44 @@
+FROM httpd:alpine
+
+ARG BUILD_DATE
+ARG BUILD_VERSION
+
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+LABEL authors="derhecht,stevespringett,jeremylong"
+LABEL maintainer="[email protected]"
+LABEL name="jeremylong/vulnz"
+LABEL version=$BUILD_VERSION
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date=$BUILD_DATE
+LABEL org.label-schema.name="jeremylong/vulnz"
+LABEL org.label-schema.description="Persist the data using the open-vulnerability-store."
+LABEL org.label-schema.url="https://github.com/jeremylong/Open-Vulnerability-Project"
+LABEL org.label-schema.vcs-url="https://github.com/jeremylong/Open-Vulnerability-Project"
+LABEL org.label-schema.vendor="jeremylong"
+LABEL org.label-schema.version=$BUILD_VERSION
+LABEL org.label-schema.docker.cmd="docker run -it --rm --name mirror -e NVD_API_KEY=YOUR_API_KEY_HERE -p 80:80 jeremylong/vulnz"
+
+ENV user=mirror
+ENV BUILD_VERSION=$BUILD_VERSION
+ENV JAVA_OPT=-Xmx2g
+
+RUN apk update                                                && \
+    apk add --no-cache bash openjdk11 dcron nss supervisor    && \
+    addgroup -S "$user"                                       && \
+    adduser -S "$user" -G "$user"                             && \
+    chown -R "$user":"$user" /usr/local/apache2/htdocs        && \
+    rm -v /usr/local/apache2/htdocs/index.html
+
+COPY ["/src/docker/conf/supervisord.conf", "/etc/supervisor/conf.d/supervisord.conf"]
+COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"]
+COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"]
+COPY ["/src/docker/conf/mirror.conf", "/usr/local/apache2/conf"]
+COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"]
+
+VOLUME /usr/local/apache2/htdocs
+EXPOSE 80/tcp
+
+CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/conf.d/supervisord.conf", "-l", "/var/log/supervisord.log", "-j", "/var/run/supervisord.pid"]

vulnz/README.md

@@ -10,10 +10,10 @@ for more details. We may add a brew formula in the future.
 After running `install` you may need to restart your shell for the completion to work.
 
 ```bash
-$ ./gradlew build
-$ cd vulnz/build/libs
-$ ./vulnz-1.0.0.jar install
-$ vulnz cve --cveId CVE-2021-44228 --prettyPrint
+./gradlew vulnz:build
+cd vulnz/build/libs
+./vulnz-5.1.1.jar install
+vulnz cve --cveId CVE-2021-44228 --prettyPrint
 ```
 
 Example of using the CLI with an API key stored in [1password](https://1password.com/) using
@@ -36,11 +36,15 @@ is created that will hold the CVEs that have been modified in the last 7 days. A
 the below command you will end up with a directory with:
 
 - `cache.properties`
-- `nvdcve-modified.json`
-- `nvdcve-2002.json`
-- `nvdcve-2003.json`
+- `nvdcve-modified.json.gz`
+- `nvdcve-modified.meta`
+- `nvdcve-2002.json.gz`
+- `nvdcve-2002.meta`
+- `nvdcve-2003.json.gz`
+- `nvdcve-2003.meta`
 - ...
-- `nvdcve-2023.json`
+- `nvdcve-2024.json.gz`
+- `nvdcve-2024.meta`
 
 ### API Key is used and a 403 or 404 error occurs
 
@@ -81,9 +85,7 @@ To create a local cache of the NVD CVE Data you can execute the following comman
 via a daily schedule to keep the cached data current:
 
 ```bash
-vulnz cve --cache --directory ./cache
-cd cache
-for file in *.json; do gzip -k "${file}"; done 
+vulnz cve --cache --directory ./cache 
 ```
 
 Alternatively, without using the above install command:
@@ -98,3 +100,54 @@ When creating the cache all other arguments to the vulnz cli
 will still work except the `--lastModEndDate` and `--lastModStartDate`.
 As such, you can create `--prettyPrint` the cache or create a cache
 of only "application" CVE using the `--virtualMatchString=cpe:2.3:a`.
+
+## Docker image
+
+### Configuration
+
+There are a couple of ENV vars
+
+- `NVD_API_KEY`: define your API key
+- `DELAY`: override the delay. If you do not set an API KEY, the delay will be `10000`
+
+### Run
+
+```bash
+# replace the NVD_API_KEY with your NVD api key
+docker run --name vulnz -e NVD_API_KEY=myapikey ghcr.io/jeremylong/vulnz:5.1.1 
+
+# if you like use a volume 
+docker run --name vulnz -e NVD_API_KEY=myapikey -v cache:/usr/local/apache2/htdocs ghcr.io/jeremylong/vulnz:5.1.1
+
+# adjust the memory usage
+docker run --name vulnz -e JAVA_OPT=-Xmx2g ghcr.io/jeremylong/vulnz:5.1.1
+
+# you can also adjust the delay 
+docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=1000 ghcr.io/jeremylong/vulnz:5.1.1 
+
+```
+
+If you like, run this to pre-populate the database right away
+
+```bash
+docker exec -u mirror vulnz /mirror.sh
+```
+
+### Build
+
+Assuming the current version is `5.1.1`
+
+```bash
+export TARGET_VERSION=5.1.1
+./gradlew vunlz:build -Pversion=$TARGET_VERSION
+docker build vunlz/ -t ghcr.io/jeremylong/vulnz:$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION
+```
+
+### Release
+
+```bash
+# checkout the repo
+git tag vulnz/5.1.1
+git push --tags
+# this will build vulnz 5.1.1 on publish the docker image tagged 5.1.1 
+```

vulnz/src/docker/conf/mirror.conf

@@ -0,0 +1,8 @@
+#
+# Configuration for the httpd mirror
+#
+ServerName localhost
+<Directory /usr/local/apache2/htdocs>
+  Options +Indexes +MultiViews
+  IndexOptions FancyIndexing -SuppressLastModified
+</Directory>

vulnz/src/docker/conf/supervisord.conf

@@ -0,0 +1,23 @@
+[supervisord]
+nodaemon=true
+user=root
+
+[program:crond]
+command=crond -s /var/spool/cron/crontabs -f
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:httpd]
+command=/usr/local/bin/httpd-foreground
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:initialize_htdocs]
+command=/mirror.sh
+autorestart=false
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+user=mirror

vulnz/src/docker/crontab/mirror

@@ -0,0 +1 @@
+0 0 * * * /mirror.sh

vulnz/src/docker/scripts/mirror.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+echo "Updating..."
+
+DELAY_ARG=""
+if [ -z $NVD_API_KEY ]; then
+  DELAY_ARG="--delay=10000"
+fi
+
+if [ -z $DELAY ]; then
+  DELAY_ARG="--delay=$DELAY"
+fi
+
+java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG --cache --directory /usr/local/apache2/htdocs