Merge branch 'master' into development

.dockerignore

@@ -1 +1,3 @@
 tmp/
+ics-gsa/gsa
+ics-gsa/gsad

.gitignore

@@ -8,6 +8,10 @@ testing/bind
 testing/archive
 *.swp
 gvm-tools
+ics-gsa
 base.sql.xz
 var-lib.tar.xz
 timing
+gsa-final
+certs.tar.xz
+ver.current

Dockerfile

@@ -7,6 +7,7 @@ FROM immauss/ovasbase:latest AS builder
 # Ensure apt doesn't ask any questions 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV LANG=C.UTF-8
+ARG TAG
 ENV VER="$TAG"
 
 # Build/install gvm (by default, everything installs in /usr/local)
@@ -25,8 +26,7 @@ COPY build.d/gvmd.sh /build.d/
 RUN bash /build.d/gvmd.sh
 COPY build.d/openvas-scanner.sh /build.d/
 RUN bash /build.d/openvas-scanner.sh
-COPY build.d/gsa.sh /build.d/
-RUN bash /build.d/gsa.sh
+
 COPY build.d/ospd-openvas.sh /build.d/
 RUN bash /build.d/ospd-openvas.sh
 COPY build.d/gvm-tool.sh /build.d/
@@ -37,11 +37,17 @@ COPY build.d/pg-gvm.sh /build.d/
 RUN bash /build.d/pg-gvm.sh
 COPY build.d/gb-feed-sync.sh /build.d/
 RUN bash /build.d/gb-feed-sync.sh
+
+#COPY build.d/gsa.sh /build.d/
+COPY ics-gsa /ics-gsa
+#RUN bash /build.d/gsa.sh
+COPY build.d/gsad.sh /build.d
+RUN bash /build.d/gsad.sh
+
 COPY build.d/links.sh /build.d/
 RUN bash /build.d/links.sh
 RUN mkdir /branding
-COPY branding/* /branding/
-RUN bash /branding/branding.sh
+
 # Stage 1: Start again with the ovasbase. Dependancies already installed
 # This target is for the image with no database
 # Makes rebuilds for data refresh and scripting changes faster. 
@@ -63,15 +69,18 @@ COPY --from=0 usr/local/sbin /usr/local/sbin
 COPY --from=0 usr/local/share /usr/local/share
 COPY --from=0 usr/share/postgresql /usr/share/postgresql
 COPY --from=0 usr/lib/postgresql /usr/lib/postgresql
+
 COPY confs/gvmd_log.conf /usr/local/etc/gvm/
 COPY confs/openvas_log.conf /usr/local/etc/openvas/
 COPY build.d/links.sh /
 RUN bash /links.sh 
 COPY build.d/gpg-keys.sh /
 RUN bash /gpg-keys.sh
-# Split these off in a new layer makes refresh builds faster.
+# Copy in the prebuilt gsa react code.
+COPY gsa-final/ /usr/local/share/gvm/gsad/web/
 COPY build.rc /gvm-versions
-
+COPY branding/* /branding/
+RUN bash /branding/branding.sh
 COPY scripts/* /scripts/
 # Healthcheck needs be an on image script that will know what service is running and check it. 
 # Current image function stored in /usr/local/etc/running-as
@@ -96,6 +105,6 @@ RUN curl -L --url https://www.immauss.com/openvas/latest.base.sql.xz -o /usr/lib
 COPY scripts/* /scripts/
 # Healthcheck needs be an on image script that will know what service is running and check it. 
 # Current image function stored in /usr/local/etc/running-as
-HEALTHCHECK --interval=60s --start-period=300s --timeout=10s \
+HEALTHCHECK --interval=300s --start-period=300s --timeout=120s \
   CMD /scripts/healthcheck.sh || exit 1
 ENTRYPOINT [ "/scripts/start.sh" ]

Dockerfile.refresh

@@ -1,22 +1,20 @@
 # Environment variables for all
 FROM immauss/openvas:latest-slim AS final
 ENV LANG=C.UTF-8
+ARG TAG
 ENV VER="$TAG"
 LABEL maintainer="[email protected]" \
       version="$VER-full" \
       url="https://hub.docker.com/r/immauss/openvas" \
       source="https://github.com/immauss/openvas"
-# Pull and then Make sure we didn't just pull zero length files 
-RUN curl -L --url https://www.immauss.com/openvas/latest.base.sql.xz -o /usr/lib/base.sql.xz && \
-    curl -L --url https://www.immauss.com/openvas/latest.var-lib.tar.xz -o /usr/lib/var-lib.tar.xz && \
-    bash -c " if [ $(ls -l /usr/lib/base.sql.xz | awk '{print $5}') -lt 1200 ]; then exit 1; fi " && \
-    bash -c " if [ $(ls -l /usr/lib/var-lib.tar.xz | awk '{print $5}') -lt 1200 ]; then exit 1; fi "
-
+# Add the archives.
+COPY base.sql.xz /usr/lib/base.sql.xz
+COPY var-lib.tar.xz /usr/lib/var-lib.tar.xz
 # packages to add to ovasbase
 #RUN apt-get update && apt-get -y install libpaho-mqtt-dev python3-paho-mqtt gir1.2-json-1.0 libjson-glib-1.0-0 libjson-glib-1.0-common
 COPY scripts/* /scripts/
 # Healthcheck needs be an on image script that will know what service is running and check it. 
 # Current image function stored in /usr/local/etc/running-as
-HEALTHCHECK --interval=60s --start-period=300s --timeout=10s \
+HEALTHCHECK --interval=300s --start-period=300s --timeout=120s \
   CMD /scripts/healthcheck.sh || exit 1
 ENTRYPOINT [ "/scripts/start.sh" ]

Readme.md

@@ -11,6 +11,11 @@
 [![Immauss Cybersecurity](https://github.com/immauss/openvas/raw/master/images/ics-hz.png)](https://immauss.com "Immauss Cybersecurity")
 
 [Sponsor immauss](https://github.com/sponsors/immauss)
+OR
+[Sponsor by PayPal](https:/www.immauss.com/container_subscriptions)
+
+## Current Silver Sponsors ##
+[![NOS Informatica](https://raw.githubusercontent.com/immauss/openvas/master/images/NOSinformatica.png)](https://nosinformatica.com/ "NOS Informatica")
 - - - -
 ## Documentation ##
 The current container docs are maintained on github [here](https://immauss.github.io/openvas/)
@@ -21,7 +26,7 @@ For docs on the web interface and scanning, use Greenbone's docs [here](https://
 # Docker Tags  #
 tag              | Description
 ----------------|-------------------------------------------------------------------
-22.4.31 | This is the latest based on GVMd 22.9 available on x86_64, arm64, and armv7.
+22.4.36 | This is the latest based on GVMd 23.0 available on x86_64, arm64, and armv7.
 21.04.09 | This is the last 21.4 build.  
 20.08.04.6 | The last 20.08 image
 pre-20.08   | This is the last image from before the 20.08 update. 
@@ -30,14 +35,12 @@ v1.0             | old out of date image for posterity. (Dont` use this one. . .
 # Greenbone Versions in Latest image: #
 Component | Version | | Component | Version
 ----------|----------|-|----------|---------
-| gvmd | v22.9.0 | | gvm_libs | v22.7.1 |
-| openvas | v22.7.5 | | openvas_scanner | v22.7.5 |
-| openvas_smb | v22.5.3 | | notus_scanner | v22.6.0 |
-| gsa | v22.7.0 | | gsad | v22.6.0 |
-| ospd | v21.4.4 | | ospd_openvas | v22.6.0 |
-| pg_gvm | v22.6.1 | | python_gvm | v23.5.1 |
-| gvm_tools | v23.9.0 | | greenbone_feed_sync | v23.8.0 |
-
+| gvmd | v23.1.0 | | gvm_libs | v22.7.3 |
+| openvas | v22.7.6 | | openvas_smb | v22.5.4 |
+| notus_scanner | v22.6.0 | | gsa | v22.9.0 |
+| gsad | v22.8.0 | | ospd | v21.4.4 |
+| ospd_openvas | v22.6.1 | | pg_gvm | v22.6.1 |
+| python_gvm | v23.10.1 | | gvm_tools | v23.10.0 |
 - - - -
 # 25 August 2023 #
 ## Discussions!!! ##

bin/base-rebuild.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 #
+#Get current gvm versions
+. build.rc
+# Setup some variables
 BUILDHOME=$(pwd)
 STARTTIME=$(date +%s)
 NOBASE="false"
@@ -10,12 +13,13 @@ PRUNESTART=true
 BASESTART=true
 PUBLISH=" "
 RUNOPTIONS=" "
+GSABUILD="false"
 OS=$(uname)
 echo "OS is $OS"
 if [ "$OS" == "Darwin" ]; then
 	STAT="-f %a"
 else
-	STAT="-c %s"
+	STAT="-c %Y"
 fi
 echo "STAT is $STAT"
 TimeMath() {
@@ -27,20 +31,24 @@ TimeMath() {
     printf "%02d:%02d:%02d\n" "$hours" "$minutes" "$seconds"
 }
 PullArchives() {
-	curl -L --url https://www.immauss.com/openvas/latest.base.sql.xz -o base.sql.xz && \
-    curl -L --url https://www.immauss.com/openvas/latest.var-lib.tar.xz -o var-lib.tar.xz && \
-    if [ $(ls -l /usr/lib/base.sql.xz | awk '{print $5}') -lt 1200 ]; then 
+
+	cp /var/lib/openvas/*.xz .
+    if [ $(ls -l base.sql.xz | awk '{print $5}') -lt 1200 ]; then 
 		echo "base.sql.xz size is invalid."
 		exit 1
 	fi 
-    if [ $(ls -l /usr/lib/var-lib.tar.xz | awk '{print $5}') -lt 1200 ]; then 
+    if [ $(ls -l var-lib.tar.xz | awk '{print $5}') -lt 1200 ]; then 
 		echo "var-lib.tar.xz size is invalid."
 		exit 1
 	fi
 }
 
 while ! [ -z "$1" ]; do
   case $1 in
+    -g)
+	shift
+	GSABUILD=true
+	;;
     --push)
 	shift
 	PUBLISH="--push"
@@ -92,8 +100,17 @@ if [ "$tag" == "beta" ]; then
 	NOBASE=true
 elif [ -z $arch ]; then
 	arch="linux/amd64,linux/arm64,linux/arm/v7"
+	#arch="linux/amd64,linux/arm64"
 	ARM="true"
 fi
+# Make the version # in the image meta data consistent
+# This will leave the 
+if [ "$tag" != "latest" ]; then
+	echo $tag > ver.current
+fi
+VER=$(cat ver.current)
+#
+
 # Check to see if we need to pull the latest DB. 
 # yes if it doesn't already exists
 # Yes if the existing is < 7 days old.
@@ -108,41 +125,49 @@ if [ $DBAGE -gt 604800 ]; then
 	PullArchives
 fi
 echo "Building with $tag and $arch"
+
 set -Eeuo pipefail
 if  [ "$NOBASE" == "false" ]; then
+	echo "Building new ovasbase image"
 	cd $BUILDHOME/ovasbase
 	BASESTART=$(date +%s)
 	# Always build all archs for ovasbase.
 	docker buildx build --push  --platform  linux/amd64,linux/arm64,linux/arm/v7 -f Dockerfile -t immauss/ovasbase  .
 	BASEFIN=$(date +%s)
 	cd ..
 fi
+# First we build GSA using a single ovasbase x86_64 container. 
+# this SIGNIFICANTLY speeds the builds.
+# first check to see if the current version has been built already
+if ! [ -f tmp/build/$gsa.tar.gz ] || [ "x$GSABUILD" == "xtrue" ] ; then 
+	echo "Starting container to build GSA" 
+		docker run -it --rm \
+			-v $(pwd)/ics-gsa:/ics-gsa \
+			-v $(pwd)/tmp/build:/build \
+			-v $(pwd):/build.d \
+			-v $(pwd)/gsa-final:/final \
+			immauss/ovasbase -c "cd /build.d; bash build.d/gsa-main.sh "
+else
+	echo "Looks like we have already built gsa $gsa"
+fi
 cd $BUILDHOME
 # Use this to set the version in the Dockerfile.
-# This hould have worked with cmd line args, but does not .... :(
+# This should have worked with cmd line args, but does not .... :(
 	DOCKERFILE=$(mktemp)
-	sed "s/\$VER/$tag/" Dockerfile > $DOCKERFILE
-# Because the arm64 build seems to always fail when building a the same time as the other archs ....
-# We'll build it first to have it cached for the final build. But we only need the slim
-#
-if [ "$ARM" == "true" ]; then
-	ARM64START=$(date +%s)
-	docker buildx build --build-arg TAG=${tag}  \
-	   --platform linux/arm64 -f Dockerfile --target slim -t immauss/openvas:${tag}-slim \
-	   -f $DOCKERFILE .
-	ARM64FIN=$(date +%s)
-fi
+	sed "s/\$VER/$VER/" Dockerfile > $DOCKERFILE
+#DOCKERFILE="Dockerfile"
+
 # Now build everything together. At this point, this will normally only be the arm7 build as the amd64 was likely built and cached as beta.
 SLIMSTART=$(date +%s)
-docker buildx build --build-arg TAG=${tag} $PUBLISH \
+docker buildx build $PUBLISH \
    --platform $arch -f Dockerfile --target slim -t immauss/openvas:${tag}-slim \
    -f $DOCKERFILE .
 SLIMFIN=$(date +%s)
 
 
 
 FINALSTART=$(date +%s)
-docker buildx build --build-arg TAG=${tag} $PUBLISH --platform $arch -f Dockerfile \
+docker buildx build $PUBLISH --platform $arch -f Dockerfile \
    --target final -t immauss/openvas:${tag} \
    -f $DOCKERFILE .
 FINALFIN=$(date +%s)

bin/check-gvm-release.sh

@@ -6,7 +6,7 @@ RC=$(mktemp)
 # Source the api token
 . .token
 # 
-for repo in gvmd gvm-libs openvas openvas-scanner openvas-smb notus-scanner gsa gsad ospd ospd-openvas pg-gvm; do
+for repo in gvmd gvm-libs openvas openvas-smb notus-scanner gsa gsad ospd ospd-openvas pg-gvm; do
 	VERSION=$(curl -s -H "Authorization: token $Oauth" -L https://api.github.com/repos/greenbone/$repo/releases/latest |  jq -r ".tag_name" )
 	#echo "$repo current version is $VERSION"
 	VAR=$( echo $repo | tr - _ )

bin/get-gvm-releases.sh

@@ -10,7 +10,7 @@ echo "# Greenbone Versions in Latest image: #
 Component | Version | | Component | Version
 ----------|----------|-|----------|---------" > versions.md
 
-for repo in gvmd gvm-libs openvas openvas-scanner openvas-smb notus-scanner gsa gsad ospd ospd-openvas pg-gvm; do
+for repo in gvmd gvm-libs openvas openvas-smb notus-scanner gsa gsad ospd ospd-openvas pg-gvm; do
 	VERSION=$(curl -s -H "Authorization: token $Oauth" -L https://api.github.com/repos/greenbone/$repo/releases/latest |  jq -r ".tag_name") 
 	echo "$repo current version is $VERSION"
 	VAR=$( echo $repo | tr - _ )

bin/push2immauss.sh

@@ -0,0 +1,10 @@
+#!/bin/bash 
+TAG=latest
+VER=$(cat ver.current)
+DOCKERFILE=$(mktemp)
+sed "s/\$VER/$VER/" Dockerfile.refresh > $DOCKERFILE
+docker buildx build -f $DOCKERFILE \
+    --target final \
+    -t gitlab.immauss.com:5050/immauss/openvas:latest \
+    --platform linux/arm64,linux/amd64,linux/arm/v7 \
+    --push .

bin/refresh.sh

@@ -11,10 +11,12 @@ WorkDir=$(pwd)
 # Tag to work with. Normally latest but might be using new tag during upgrades.
 TAG="latest"
 SQLBU="${TAG}.base.sql"
-TAR="${TAG}.var-lib.tar.xz"
+VER=$(cat ver.current)
+DOCKERFILE=$(mktemp)
+sed "s/\$VER/$VER/" Dockerfile.refresh > $DOCKERFILE
 # Temp working directory ... needs enough space to pull the entire feed and then compress it. ~2G
 TWD="/var/lib/openvas/" # Must have a trailing "/"
-STIME="30m" # time between resync and archiving.
+STIME="10m" # time between resync and archiving.
 # First, clean TWD and  make sure there's enough storage available before doing anything.
 if [ -d $TWD ]; then # Make sure the TWD exists and is a directory so we don't accidently destroy the system.
 	echo " Cleaning $TWD "
@@ -88,17 +90,21 @@ if [ $SQL_SIZE -le 2000 ] || [ $FEED_SIZE -le 2000 ]; then
 	logger -t db-refresh "SQL_SIZE = $SQL_SIZE : FEED_SIZE = $FEED_SIZE: Failing out"
 	exit
 fi
-echo " Push updates to www"
-scp *.xz [email protected]:/var/www/html/drupal/openvas/
-if [ $? -ne 0 ]; then
-	echo "SCP of new db failed $?"
-	logger -t db-refresh "SCP of new db failed $?"
-	exit
-fi
-# Now rebuild the image
+cp latest.base.sql.xz /home/scott/Projects/openvas/base.sql.xz
+cp latest.var-lib.tar.xz /home/scott/Projects/openvas/var-lib.tar.xz
+
+# echo " Push updates to www"
+# scp *.xz [email protected]:/var/www/html/drupal/openvas/
+# if [ $? -ne 0 ]; then
+# 	echo "SCP of new db failed $?"
+# 	logger -t db-refresh "SCP of new db failed $?"
+# 	exit
+# fi
+echo "Now rebuild the image"
 cd $WorkDir
+echo "$(pwd) Is current working directory."
 date > update.ts
-docker buildx build -f Dockerfile.refresh --build-arg TAG=${TAG} --target final -t immauss/openvas:$TAG --platform linux/arm64,linux/amd64,linux/arm/v7 --push .
+docker buildx build -f $DOCKERFILE --target final -t immauss/openvas:$TAG --platform linux/arm64,linux/amd64,linux/arm/v7 --push .
 if [ $? -ne 0 ]; then
 	echo "Build failed."
 	exit
@@ -108,5 +114,3 @@ echo "Cleaning up"
 cd $TWD
 rm -rf *
 echo "All done"
-
-

branding/branding.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 
 # Replace a few images in the gsa build with Immauss branded images.
+mkdir -p /usr/local/share/gvm/gsad/web/img
 cp /branding/* /usr/local/share/gvm/gsad/web/img/
+