From e4ffbd716ed8271ce658b95768d983c9ddf4b0bd Mon Sep 17 00:00:00 2001 From: Eugen Mayer <136934+EugenMayer@users.noreply.github.com> Date: Sun, 7 Jan 2024 12:40:05 +0100 Subject: [PATCH] feat: vulnz Docker v2 (#114) * feat: vulnz Docker * fix: stop gzipping in vulnz Docker after cache cve * fix: update documentation according to gzip feature * Add docs - Add option to adjust the memory - Add volume definition, build docs - Add bash to the docker image * Add CI integration * Move version to gradle properties * Add delay management, fix docs, fix image location Co-authored-by: Jeremy Long * Adjust workflow and add docker hub --------- Co-authored-by: j.berlin Co-authored-by: Jeremy Long --- .github/workflows/vulnz-docker-pr.yml | 38 ++++++++++ .github/workflows/vulnz-docker-release.yml | 53 +++++++++++++ buildSrc/src/gradle.properties | 1 + .../vuln.tools.java-common-conventions.gradle | 1 - vulnz/Dockerfile | 44 +++++++++++ vulnz/README.md | 75 ++++++++++++++++--- vulnz/src/docker/conf/mirror.conf | 8 ++ vulnz/src/docker/conf/supervisord.conf | 23 ++++++ vulnz/src/docker/crontab/mirror | 1 + vulnz/src/docker/scripts/mirror.sh | 14 ++++ 10 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/vulnz-docker-pr.yml create mode 100644 .github/workflows/vulnz-docker-release.yml create mode 100644 buildSrc/src/gradle.properties create mode 100644 vulnz/Dockerfile create mode 100644 vulnz/src/docker/conf/mirror.conf create mode 100644 vulnz/src/docker/conf/supervisord.conf create mode 100755 vulnz/src/docker/crontab/mirror create mode 100755 vulnz/src/docker/scripts/mirror.sh diff --git a/.github/workflows/vulnz-docker-pr.yml b/.github/workflows/vulnz-docker-pr.yml new file mode 100644 index 00000000..93b95dac --- /dev/null +++ b/.github/workflows/vulnz-docker-pr.yml @@ -0,0 +1,38 @@ +name: docker pr + +on: + push: + branches-ignore: + - 'main' + +env: + IMAGE_FQDN: ghcr.io/jeremylong/open-vulnerability-data-mirror + VERSION: 0.0.0-SNAPSHOT + +jobs: + docker-pr: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + - name: Run build + run: ./gradlew -x test -Pversion=${{ env.VERSION }} vulnz:build + - name: Build docker image + uses: docker/build-push-action@v3 + with: + context: vulnz/ + platforms: linux/amd64 + push: false + tags: | + ${{ env.IMAGE_FQDN }}:${{ env.VERSION }} + build-args: | + BUILD_VERSION=${{ env.VERSION }} diff --git a/.github/workflows/vulnz-docker-release.yml b/.github/workflows/vulnz-docker-release.yml new file mode 100644 index 00000000..560114e9 --- /dev/null +++ b/.github/workflows/vulnz-docker-release.yml @@ -0,0 +1,53 @@ +name: docker pr + +on: + push: + tags: + - v* + +env: + GHCR_IMAGE_FQDN: ghcr.io/jeremylong/open-vulnerability-data-mirror + HUB_IMAGE_FQDN: jeremylong/open-vulnerability-data-mirror + VERSION: ${{ github.ref_name }} + +jobs: + docker-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + - name: Run build + run: ./gradlew -x test -Pversion=${{ env.VERSION }} vulnz:build + - name: Build docker image + uses: docker/build-push-action@v3 + with: + context: vulnz/ + platforms: linux/amd64 + push: true + tags: | + ${{ env.GHCR_IMAGE_FQDN }}:${{ env.VERSION }} + ${{ env.GHCR_IMAGE_FQDN }}:latest + ${{ env.HUB_IMAGE_FQDN }}:${{ env.VERSION }} + ${{ env.HUB_IMAGE_FQDN }}:latest + build-args: | + BUILD_VERSION=${{ env.VERSION }} diff --git a/buildSrc/src/gradle.properties b/buildSrc/src/gradle.properties new file mode 100644 index 00000000..438c1199 --- /dev/null +++ b/buildSrc/src/gradle.properties @@ -0,0 +1 @@ +version = '5.1.1' \ No newline at end of file diff --git a/buildSrc/src/main/groovy/vuln.tools.java-common-conventions.gradle b/buildSrc/src/main/groovy/vuln.tools.java-common-conventions.gradle index 106ee09a..92b87229 100644 --- a/buildSrc/src/main/groovy/vuln.tools.java-common-conventions.gradle +++ b/buildSrc/src/main/groovy/vuln.tools.java-common-conventions.gradle @@ -12,7 +12,6 @@ plugins { } group 'io.github.jeremylong' -version = '5.1.1' repositories { mavenCentral() diff --git a/vulnz/Dockerfile b/vulnz/Dockerfile new file mode 100644 index 00000000..11740c64 --- /dev/null +++ b/vulnz/Dockerfile @@ -0,0 +1,44 @@ +FROM httpd:alpine + +ARG BUILD_DATE +ARG BUILD_VERSION + +ARG http_proxy +ARG https_proxy +ARG no_proxy + +LABEL authors="derhecht,stevespringett,jeremylong" +LABEL maintainer="jeremy.long@gmail.com" +LABEL name="jeremylong/vulnz" +LABEL version=$BUILD_VERSION +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE +LABEL org.label-schema.name="jeremylong/vulnz" +LABEL org.label-schema.description="Persist the data using the open-vulnerability-store." +LABEL org.label-schema.url="https://github.com/jeremylong/Open-Vulnerability-Project" +LABEL org.label-schema.vcs-url="https://github.com/jeremylong/Open-Vulnerability-Project" +LABEL org.label-schema.vendor="jeremylong" +LABEL org.label-schema.version=$BUILD_VERSION +LABEL org.label-schema.docker.cmd="docker run -it --rm --name mirror -e NVD_API_KEY=YOUR_API_KEY_HERE -p 80:80 jeremylong/vulnz" + +ENV user=mirror +ENV BUILD_VERSION=$BUILD_VERSION +ENV JAVA_OPT=-Xmx2g + +RUN apk update && \ + apk add --no-cache bash openjdk11 dcron nss supervisor && \ + addgroup -S "$user" && \ + adduser -S "$user" -G "$user" && \ + chown -R "$user":"$user" /usr/local/apache2/htdocs && \ + rm -v /usr/local/apache2/htdocs/index.html + +COPY ["/src/docker/conf/supervisord.conf", "/etc/supervisor/conf.d/supervisord.conf"] +COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"] +COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"] +COPY ["/src/docker/conf/mirror.conf", "/usr/local/apache2/conf"] +COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"] + +VOLUME /usr/local/apache2/htdocs +EXPOSE 80/tcp + +CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/conf.d/supervisord.conf", "-l", "/var/log/supervisord.log", "-j", "/var/run/supervisord.pid"] diff --git a/vulnz/README.md b/vulnz/README.md index 2ac33017..277861d1 100644 --- a/vulnz/README.md +++ b/vulnz/README.md @@ -10,10 +10,10 @@ for more details. We may add a brew formula in the future. After running `install` you may need to restart your shell for the completion to work. ```bash -$ ./gradlew build -$ cd vulnz/build/libs -$ ./vulnz-1.0.0.jar install -$ vulnz cve --cveId CVE-2021-44228 --prettyPrint +./gradlew vulnz:build +cd vulnz/build/libs +./vulnz-5.1.1.jar install +vulnz cve --cveId CVE-2021-44228 --prettyPrint ``` Example of using the CLI with an API key stored in [1password](https://1password.com/) using @@ -36,11 +36,15 @@ is created that will hold the CVEs that have been modified in the last 7 days. A the below command you will end up with a directory with: - `cache.properties` -- `nvdcve-modified.json` -- `nvdcve-2002.json` -- `nvdcve-2003.json` +- `nvdcve-modified.json.gz` +- `nvdcve-modified.meta` +- `nvdcve-2002.json.gz` +- `nvdcve-2002.meta` +- `nvdcve-2003.json.gz` +- `nvdcve-2003.meta` - ... -- `nvdcve-2023.json` +- `nvdcve-2024.json.gz` +- `nvdcve-2024.meta` ### API Key is used and a 403 or 404 error occurs @@ -81,9 +85,7 @@ To create a local cache of the NVD CVE Data you can execute the following comman via a daily schedule to keep the cached data current: ```bash -vulnz cve --cache --directory ./cache -cd cache -for file in *.json; do gzip -k "${file}"; done +vulnz cve --cache --directory ./cache ``` Alternatively, without using the above install command: @@ -98,3 +100,54 @@ When creating the cache all other arguments to the vulnz cli will still work except the `--lastModEndDate` and `--lastModStartDate`. As such, you can create `--prettyPrint` the cache or create a cache of only "application" CVE using the `--virtualMatchString=cpe:2.3:a`. + +## Docker image + +### Configuration + +There are a couple of ENV vars + +- `NVD_API_KEY`: define your API key +- `DELAY`: override the delay. If you do not set an API KEY, the delay will be `10000` + +### Run + +```bash +# replace the NVD_API_KEY with your NVD api key +docker run --name vulnz -e NVD_API_KEY=myapikey ghcr.io/jeremylong/vulnz:5.1.1 + +# if you like use a volume +docker run --name vulnz -e NVD_API_KEY=myapikey -v cache:/usr/local/apache2/htdocs ghcr.io/jeremylong/vulnz:5.1.1 + +# adjust the memory usage +docker run --name vulnz -e JAVA_OPT=-Xmx2g ghcr.io/jeremylong/vulnz:5.1.1 + +# you can also adjust the delay +docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=1000 ghcr.io/jeremylong/vulnz:5.1.1 + +``` + +If you like, run this to pre-populate the database right away + +```bash +docker exec -u mirror vulnz /mirror.sh +``` + +### Build + +Assuming the current version is `5.1.1` + +```bash +export TARGET_VERSION=5.1.1 +./gradlew vunlz:build -Pversion=$TARGET_VERSION +docker build vunlz/ -t ghcr.io/jeremylong/vulnz:$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION +``` + +### Release + +```bash +# checkout the repo +git tag vulnz/5.1.1 +git push --tags +# this will build vulnz 5.1.1 on publish the docker image tagged 5.1.1 +``` diff --git a/vulnz/src/docker/conf/mirror.conf b/vulnz/src/docker/conf/mirror.conf new file mode 100644 index 00000000..0fe94c57 --- /dev/null +++ b/vulnz/src/docker/conf/mirror.conf @@ -0,0 +1,8 @@ +# +# Configuration for the httpd mirror +# +ServerName localhost + + Options +Indexes +MultiViews + IndexOptions FancyIndexing -SuppressLastModified + \ No newline at end of file diff --git a/vulnz/src/docker/conf/supervisord.conf b/vulnz/src/docker/conf/supervisord.conf new file mode 100644 index 00000000..ca6fb1b7 --- /dev/null +++ b/vulnz/src/docker/conf/supervisord.conf @@ -0,0 +1,23 @@ +[supervisord] +nodaemon=true +user=root + +[program:crond] +command=crond -s /var/spool/cron/crontabs -f +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:httpd] +command=/usr/local/bin/httpd-foreground +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:initialize_htdocs] +command=/mirror.sh +autorestart=false +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true +user=mirror \ No newline at end of file diff --git a/vulnz/src/docker/crontab/mirror b/vulnz/src/docker/crontab/mirror new file mode 100755 index 00000000..cfad381f --- /dev/null +++ b/vulnz/src/docker/crontab/mirror @@ -0,0 +1 @@ +0 0 * * * /mirror.sh \ No newline at end of file diff --git a/vulnz/src/docker/scripts/mirror.sh b/vulnz/src/docker/scripts/mirror.sh new file mode 100755 index 00000000..b13f250f --- /dev/null +++ b/vulnz/src/docker/scripts/mirror.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +echo "Updating..." + +DELAY_ARG="" +if [ -z $NVD_API_KEY ]; then + DELAY_ARG="--delay=10000" +fi + +if [ -z $DELAY ]; then + DELAY_ARG="--delay=$DELAY" +fi + +java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG --cache --directory /usr/local/apache2/htdocs