diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile
index 212ecde50..531cbe555 100644
--- a/.docker/prod.Dockerfile
+++ b/.docker/prod.Dockerfile
@@ -1,8 +1,9 @@
ARG VERSION=edge
ARG GVM_LIBS_VERSION=oldstable
ARG DEBIAN_FRONTEND=noninteractive
+ARG IMAGE_REGISTRY=ghcr.io
-FROM greenbone/gvmd-build:${VERSION} as builder
+FROM ${IMAGE_REGISTRY}/greenbone/gvmd-build:${VERSION} as builder
COPY . /source
WORKDIR /source
@@ -99,7 +100,6 @@ RUN apt-get update && \
texlive-fonts-recommended \
texlive-latex-extra \
wget \
- xml-twig-tools \
xmlstarlet \
xsltproc \
zip && \
diff --git a/.docker/start-gvmd.sh b/.docker/start-gvmd.sh
index c4ced08d3..7083b8f81 100644
--- a/.docker/start-gvmd.sh
+++ b/.docker/start-gvmd.sh
@@ -16,28 +16,26 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-#!/bin/sh
-
[ -z "$USER" ] && USER="admin"
[ -z "$PASSWORD" ] && PASSWORD="admin"
[ -z "$GVMD_ARGS" ] && GVMD_ARGS="--listen-mode=666"
[ -z "$GVMD_USER" ] && GVMD_USER="gvmd"
-[ -z "$PGRES_DATA"] && PGRES_DATA="/var/lib/postgresql"
+[ -z "$PGRES_DATA" ] && PGRES_DATA="/var/lib/postgresql"
if [ -n "$GVM_CERTS" ] && [ "$GVM_CERTS" = true ]; then
- echo "Generating certs"
- gvm-manage-certs -a
+ echo "Generating certs"
+ gvm-manage-certs -a
fi
# check for psql connection
FILE=$PGRES_DATA/started
until test -f "$FILE"; do
- echo "waiting 1 second for ready postgres container"
+ echo "waiting 1 second for ready postgres container"
sleep 1
done
until psql -U "$GVMD_USER" -d gvmd -c "SELECT 'connected' as connection"; do
- echo "waiting 1 second to retry psql connection"
- sleep 1
+ echo "waiting 1 second to retry psql connection"
+ sleep 1
done
# migrate db if necessary
@@ -46,11 +44,11 @@ gvmd --migrate || true
gvmd --create-user=$USER --password=$PASSWORD || true
# set the feed import owner
-uid=$(gvmd --get-users --verbose | grep $USER | awk '{print $2}')
+uid=$(gvmd --get-users --verbose | grep "^$USER " | awk '{print $2}')
gvmd --modify-setting 78eceaec-3385-11ea-b237-28d24461215b --value "$uid"
echo "starting gvmd"
gvmd $GVMD_ARGS ||
- (cat /var/log/gvm/gvmd.log && exit 1)
+ (cat /var/log/gvm/gvmd.log && exit 1)
tail -f /var/log/gvm/gvmd.log
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 2277decec..85dac033f 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,6 +1,6 @@
# default reviewers
-* @greenbone/gvmd-maintainers @mattmundell
+* @greenbone/gvmd-maintainers
# dev ops
-.github/ @greenbone/devops @greenbone/gvmd-maintainers @mattmundell
-.docker/ @greenbone/devops @greenbone/gvmd-maintainers @mattmundell
+.github/ @greenbone/devops @greenbone/gvmd-maintainers
+.docker/ @greenbone/devops @greenbone/gvmd-maintainers
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index ce79fbfe8..175d4a263 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -22,7 +22,7 @@ jobs:
scan-build:
name: scan-build (clang static analyzer)
runs-on: ubuntu-latest
- container: greenbone/gvmd-build:stable
+ container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable
steps:
- name: Check out gvmd
uses: actions/checkout@v4
@@ -46,7 +46,7 @@ jobs:
test-units:
name: Unit Tests
runs-on: ubuntu-latest
- container: greenbone/gvmd-build:stable
+ container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable
steps:
- name: Check out gvmd
uses: actions/checkout@v4
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
index d326f8c14..b927fd34d 100644
--- a/.github/workflows/build-container.yml
+++ b/.github/workflows/build-container.yml
@@ -1,19 +1,6 @@
name: Build Container Image Builds
on:
- push:
- branches:
- - main
- tags: ["v*"]
- paths:
- - .github/workflows/build-container.yml
- - .docker/build.Dockerfile
- pull_request:
- branches:
- - main
- paths:
- - .github/workflows/build-container.yml
- - .docker/build.Dockerfile
workflow_dispatch:
repository_dispatch:
schedule:
@@ -22,51 +9,43 @@ on:
jobs:
build-images:
+ strategy:
+ matrix:
+ build:
+ - stable
+ - edge
name: "Build Images"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- - uses: greenbone/actions/is-latest-tag@v3
- id: latest
- name: Setup container meta information
id: meta
uses: docker/metadata-action@v5
with:
- images: ${{ github.repository }}-build
+ images: ${{ vars.IMAGE_REGISTRY }}/${{ github.repository }}-build
labels: |
org.opencontainers.image.vendor=Greenbone
org.opencontainers.image.base.name=greenbone/gvm-libs
flavor: latest=false # no latest container tag for git tags
tags: |
- # use version, major.minor and major for tags
- type=semver,pattern={{version}}
- type=semver,pattern={{major}}.{{minor}}
- type=semver,pattern={{major}}
-
- # use edge for default branch
- type=edge
-
- # set label for non-published pull request builds
- type=ref,event=pr
-
- # when a new git tag is created set stable and a latest tags
- type=raw,value=latest,enable=${{ steps.latest.outputs.is-latest-tag == 'true' }}
- type=raw,value=stable,enable=${{ steps.latest.outputs.is-latest-tag == 'true' }}
+ type=raw,value=latest,enable=${{ matrix.build == 'stable' }}
+ type=raw,value=stable,enable=${{ matrix.build == 'stable' }}
+ type=raw,value=edge,enable=${{ matrix.build == 'edge' }}
- name: Set container build options
id: container-opts
run: |
- if [[ "${{ github.ref_type }}" = 'tag' ]]; then
+ if [[ "${{ matrix.build }}" = 'stable' ]]; then
echo "gvm-libs-version=oldstable" >> $GITHUB_OUTPUT
else
echo "gvm-libs-version=oldstable-edge" >> $GITHUB_OUTPUT
fi
- - name: Login to DockerHub
- if: github.event_name != 'pull_request'
+ - name: Login to GitHub Docker registry
uses: docker/login-action@v3
with:
- username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
+ registry: ghcr.io
+ username: ${{ secrets.GREENBONE_BOT }}
+ password: ${{ secrets.GREENBONE_BOT_PACKAGES_WRITE_TOKEN }}
- run: echo "Build and push ${{ steps.meta.outputs.tags }}"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
@@ -76,17 +55,10 @@ jobs:
uses: docker/build-push-action@v5
with:
context: .
- push: ${{ github.event_name != 'pull_request' }}
+ push: true
build-args: |
GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }}
file: .docker/build.Dockerfile
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
- - name: Trigger libtheia container build
- if: github.event_name != 'pull_request'
- run: |
- curl -X POST https://api.github.com/repos/greenbone/libtheia/actions/workflows/container.yml/dispatches \
- -H "Accept: application/vnd.github.v3+json" \
- -u greenbonebot:${{ secrets.GREENBONE_BOT_TOKEN }} \
- -d '{"ref":"main"}'
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
index f5901f219..6557c2ab6 100644
--- a/.github/workflows/build-docs.yml
+++ b/.github/workflows/build-docs.yml
@@ -16,7 +16,7 @@ jobs:
build-gmp-doc:
name: Build GMP documentation
runs-on: ubuntu-latest
- container: greenbone/gvmd-build:stable
+ container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable
steps:
- name: Check out gvmd
uses: actions/checkout@v4
diff --git a/.github/workflows/codeql-analysis-c.yml b/.github/workflows/codeql-analysis-c.yml
index 42f53118a..add9937b5 100644
--- a/.github/workflows/codeql-analysis-c.yml
+++ b/.github/workflows/codeql-analysis-c.yml
@@ -19,7 +19,7 @@ jobs:
actions: read
contents: read
security-events: write
- container: greenbone/gvmd-build:stable
+ container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable
strategy:
fail-fast: false
diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml
index 78a688f8d..7b534424d 100644
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -71,6 +71,7 @@ jobs:
build-args: |
VERSION=${{ steps.container-opts.outputs.version }}
GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }}
+ IMAGE_REGISTRY=${{ vars.IMAGE_REGISTRY }}
file: .docker/prod.Dockerfile
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a915ed55a..6f5f64a61 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0)
message ("-- Configuring Greenbone Vulnerability Manager...")
project (gvm
- VERSION 23.4.1
+ VERSION 23.5.1
LANGUAGES C)
if (POLICY CMP0005)
diff --git a/INSTALL.md b/INSTALL.md
index 5b743f248..ac327bc57 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -705,9 +705,6 @@ Prerequisites for S/MIME support (e.g. email encryption):
Prerequisites for certificate generation:
* GnuTLS certtool (Debian package: gnutls-bin)
-Prerequisites (recommended) to lower sync RAM usage
-* xml_split (Debian package: xml-twig-tools)
-
## Static code analysis with the Clang Static Analyzer
If you want to use the Clang Static Analyzer (https://clang-analyzer.llvm.org/)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 4a81d9286..c2f3aebb0 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -26,10 +26,10 @@ find_package (Threads)
## list and throw an error, otherwise long install-cmake-install-cmake cycles
## might occur.
-pkg_check_modules (LIBGVM_BASE REQUIRED libgvm_base>=22.8)
-pkg_check_modules (LIBGVM_UTIL REQUIRED libgvm_util>=22.8)
-pkg_check_modules (LIBGVM_OSP REQUIRED libgvm_osp>=22.8)
-pkg_check_modules (LIBGVM_GMP REQUIRED libgvm_gmp>=22.8)
+pkg_check_modules (LIBGVM_BASE REQUIRED libgvm_base>=22.9)
+pkg_check_modules (LIBGVM_UTIL REQUIRED libgvm_util>=22.9)
+pkg_check_modules (LIBGVM_OSP REQUIRED libgvm_osp>=22.9)
+pkg_check_modules (LIBGVM_GMP REQUIRED libgvm_gmp>=22.9)
pkg_check_modules (GNUTLS REQUIRED gnutls>=3.2.15)
pkg_check_modules (GLIB REQUIRED glib-2.0>=2.42)
pkg_check_modules (LIBBSD REQUIRED libbsd)
@@ -55,14 +55,6 @@ elseif ((CMAKE_MATCH_1 EQUAL 9 AND CMAKE_MATCH_2 LESS 6)
message (STATUS "PostgreSQL version ${CMAKE_MATCH_1}.${CMAKE_MATCH_2}${CMAKE_MATCH_3}")
endif (NOT CMAKE_MATCH_1)
-message (STATUS "Looking for xml_split...")
-find_program (XML_SPLIT_EXECUTABLE xml_split DOC "xml_split")
-if (NOT XML_SPLIT_EXECUTABLE)
- message (WARNING "xml_split is recommended to reduce SCAP sync memory usage (Debian package xml-twig-tools).")
-else (NOT XML_SPLIT_EXECUTABLE)
- message (STATUS "Looking for xml_split... ${XML_SPLIT_EXECUTABLE}")
-endif (NOT XML_SPLIT_EXECUTABLE)
-
message (STATUS "Looking for xsltproc...")
find_program (XSLTPROC_EXECUTABLE xsltproc DOC "xsltproc")
if (NOT XSLTPROC_EXECUTABLE)
diff --git a/src/alert_methods/TippingPoint/alert b/src/alert_methods/TippingPoint/alert
index fcde932d2..518cb9a32 100644
--- a/src/alert_methods/TippingPoint/alert
+++ b/src/alert_methods/TippingPoint/alert
@@ -25,12 +25,6 @@ CONVERT_SCRIPT=$4
AUTH_PATH=$5
REPORT_PATH=$6
-# Function to encode for URL
-urlencode () {
- RET=$(python -c "import urllib, sys; print urllib.quote(sys.argv[1])" "$1")
- echo "$RET"
-}
-
# Create temp file for converted report
REPORT_DATE=$(xmlstarlet sel -t -v "report/timestamp" < $REPORT_PATH)
EXIT_CODE=$?
@@ -76,9 +70,9 @@ END_TIME=$(TZ=UTC date -d "$END_TIME" +%Y-%m-%dT%H:%M:%S.000Z)
RUNTIME="$START_TIME/$END_TIME"
# Upload the report
-VENDOR=$(urlencode "Greenbone")
-PRODUCT=$(urlencode "Greenbone Vulnerability Manager")
-FORMAT_VERSION=$(urlencode "1.0.0")
+VENDOR="Greenbone"
+PRODUCT="Greenbone%20Vulnerability%20Manager"
+FORMAT_VERSION="1.0.0"
CN_REPLACEMENT="Tippingpoint"
if [ "1" = $CERT_WORKAROUND ]
diff --git a/src/alert_methods/TippingPoint/report-convert.py b/src/alert_methods/TippingPoint/report-convert.py
index cf4a49322..c2768ff9c 100755
--- a/src/alert_methods/TippingPoint/report-convert.py
+++ b/src/alert_methods/TippingPoint/report-convert.py
@@ -137,6 +137,10 @@ def convert (xml_tree, out_file):
nvt_cve = '';
nvt_elem = result_elem.find ('nvt')
nvt_refs = nvt_elem.find ('refs');
+
+ if (nvt_refs is None):
+ continue
+
for ref in nvt_refs.findall('ref'):
if (ref.attrib['type'] == 'cve'):
if (nvt_cve == ''):
diff --git a/src/gmp.c b/src/gmp.c
index 4c31124ea..492c39920 100644
--- a/src/gmp.c
+++ b/src/gmp.c
@@ -8162,31 +8162,41 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details,
if (include_notes_details == 0)
{
- const char *text = note_iterator_text (notes);
- gchar *excerpt = utf8_substring (text, 0,
- setting_excerpt_size_int ());
+ gchar *excerpt;
+ const char *text;
+
+ text = note_iterator_text (notes);
+ excerpt = utf8_substring (text, 0, setting_excerpt_size_int ());
+
/* This must match send_get_common. */
+
buffer_xml_append_printf (buffer,
"%s"
""
"%s"
"%s"
- ""
- "%s"
- "%s"
- "1"
- "0"
- "%i"
- "%s"
- "%i",
+ "",
get_iterator_owner_name (notes)
? get_iterator_owner_name (notes)
: "",
note_iterator_nvt_oid (notes),
note_iterator_nvt_name (notes),
- note_iterator_nvt_type (notes),
- get_iterator_creation_time (notes),
- get_iterator_modification_time (notes),
+ note_iterator_nvt_type (notes));
+
+ buffer_xml_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_creation_time (notes)));
+
+ buffer_xml_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_modification_time (notes)));
+
+ buffer_xml_append_printf (buffer,
+ "1"
+ "0"
+ "%i"
+ "%s"
+ "%i",
note_iterator_active (notes),
strlen (excerpt) < strlen (text),
excerpt,
@@ -8229,15 +8239,33 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details,
end_time = note_iterator_end_time (notes);
/* This must match send_get_common. */
+
buffer_xml_append_printf
(buffer,
"%s"
""
"%s"
"%s"
- ""
- "%s"
- "%s"
+ "",
+ get_iterator_owner_name (notes)
+ ? get_iterator_owner_name (notes)
+ : "",
+ note_iterator_nvt_oid (notes),
+ note_iterator_nvt_name (notes),
+ note_iterator_nvt_type (notes));
+
+ buffer_xml_append_printf
+ (buffer,
+ "%s",
+ iso_if_time (get_iterator_creation_time (notes)));
+
+ buffer_xml_append_printf
+ (buffer,
+ "%s",
+ iso_if_time (get_iterator_modification_time (notes)));
+
+ buffer_xml_append_printf
+ (buffer,
"1"
"0"
"%i"
@@ -8248,14 +8276,6 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details,
"%s"
"%s%i"
"%i",
- get_iterator_owner_name (notes)
- ? get_iterator_owner_name (notes)
- : "",
- note_iterator_nvt_oid (notes),
- note_iterator_nvt_name (notes),
- note_iterator_nvt_type (notes),
- get_iterator_creation_time (notes),
- get_iterator_modification_time (notes),
note_iterator_active (notes),
end_time > 1 ? iso_time (&end_time) : "",
note_iterator_text (notes),
@@ -8426,18 +8446,36 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides,
if (include_overrides_details == 0)
{
- const char *text = override_iterator_text (overrides);
- gchar *excerpt = utf8_substring (text, 0,
- setting_excerpt_size_int ());
+ gchar *excerpt;
+ const char *text;
+
+ text = override_iterator_text (overrides);
+ excerpt = utf8_substring (text, 0, setting_excerpt_size_int ());
+
/* This must match send_get_common. */
+
buffer_xml_append_printf (buffer,
"%s"
""
"%s"
"%s"
- ""
- "%s"
- "%s"
+ "",
+ get_iterator_owner_name (overrides)
+ ? get_iterator_owner_name (overrides)
+ : "",
+ override_iterator_nvt_oid (overrides),
+ override_iterator_nvt_name (overrides),
+ override_iterator_nvt_type (overrides));
+
+ buffer_xml_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_creation_time (overrides)));
+
+ buffer_xml_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_modification_time (overrides)));
+
+ buffer_xml_append_printf (buffer,
"1"
"0"
"%i"
@@ -8447,14 +8485,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides,
"%s"
"%s"
"%i",
- get_iterator_owner_name (overrides)
- ? get_iterator_owner_name (overrides)
- : "",
- override_iterator_nvt_oid (overrides),
- override_iterator_nvt_name (overrides),
- override_iterator_nvt_type (overrides),
- get_iterator_creation_time (overrides),
- get_iterator_modification_time (overrides),
override_iterator_active (overrides),
strlen (excerpt) < strlen (text),
excerpt,
@@ -8505,15 +8535,33 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides,
end_time = override_iterator_end_time (overrides);
/* This must match send_get_common. */
+
buffer_xml_append_printf
(buffer,
"%s"
""
"%s"
"%s"
- ""
- "%s"
- "%s"
+ "",
+ get_iterator_owner_name (overrides)
+ ? get_iterator_owner_name (overrides)
+ : "",
+ override_iterator_nvt_oid (overrides),
+ override_iterator_nvt_name (overrides),
+ override_iterator_nvt_type (overrides));
+
+ buffer_xml_append_printf
+ (buffer,
+ "%s",
+ iso_if_time (get_iterator_creation_time (overrides)));
+
+ buffer_xml_append_printf
+ (buffer,
+ "%s",
+ iso_if_time (get_iterator_modification_time (overrides)));
+
+ buffer_xml_append_printf
+ (buffer,
"1"
"0"
"%i"
@@ -8527,14 +8575,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides,
"%s"
"%s%i"
"%i",
- get_iterator_owner_name (overrides)
- ? get_iterator_owner_name (overrides)
- : "",
- override_iterator_nvt_oid (overrides),
- override_iterator_nvt_name (overrides),
- override_iterator_nvt_type (overrides),
- get_iterator_creation_time (overrides),
- get_iterator_modification_time (overrides),
override_iterator_active (overrides),
end_time > 1 ? iso_time (&end_time) : "",
override_iterator_text (overrides),
@@ -9334,7 +9374,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task,
int changed, int cert_loaded, int lean, int use_delta_fields)
{
- const char *descr, *name, *comment, *creation_time;
+ const char *descr, *name, *comment;
const char *severity, *original_severity, *original_level;
const char *host, *hostname, *result_id, *port, *path, *asset_id, *qod, *qod_type;
char *detect_oid, *detect_ref, *detect_cpe, *detect_loc, *detect_name;
@@ -9343,6 +9383,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task,
result_t result;
report_t report;
task_t selected_task;
+ time_t creation_time;
comment = get_iterator_comment (results);
name = get_iterator_name (results);
@@ -9414,7 +9455,8 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task,
if (lean == 0)
{
- const char *owner_name, *modification_time;
+ const char *owner_name;
+ time_t modification_time;
if (use_delta_fields)
{
@@ -9435,7 +9477,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task,
if (modification_time)
buffer_xml_append_printf (buffer,
"%s",
- modification_time);
+ iso_time (&modification_time) ?: "");
}
if (comment
@@ -9447,7 +9489,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task,
if (creation_time)
buffer_xml_append_printf (buffer,
"%s",
- creation_time);
+ iso_time (&creation_time) ?: "");
if (include_details)
{
@@ -11671,20 +11713,26 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error)
xml_string_append (result,
""
"%s"
- "%s"
- "%s"
- "%s"
+ "%s",
+ get_iterator_uuid (&identifiers),
+ get_iterator_name (&identifiers),
+ host_identifier_iterator_value (&identifiers));
+
+ xml_string_append (result,
+ "%s",
+ iso_if_time (get_iterator_creation_time (&identifiers)));
+
+ xml_string_append (result,
+ "%s",
+ iso_if_time (get_iterator_modification_time (&identifiers)));
+
+ xml_string_append (result,
"",
- get_iterator_uuid (&identifiers),
- get_iterator_name (&identifiers),
- host_identifier_iterator_value (&identifiers),
- get_iterator_creation_time (&identifiers),
- get_iterator_modification_time (&identifiers),
host_identifier_iterator_source_id
(&identifiers),
source_type,
@@ -14717,17 +14765,12 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error)
task_t task;
/* Send the standard elements. Should match send_get_common. */
+
buffer_xml_append_printf
(prefix,
"%s"
"%s"
- "%s"
- "%s"
- ""
- "%s"
- ""
- "0"
- "0",
+ "%s",
get_iterator_owner_name (&reports)
? get_iterator_owner_name (&reports)
: "",
@@ -14736,13 +14779,20 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error)
: "",
get_iterator_comment (&reports)
? get_iterator_comment (&reports)
- : "",
- get_iterator_creation_time (&reports)
- ? get_iterator_creation_time (&reports)
- : "",
- get_iterator_modification_time (&reports)
- ? get_iterator_modification_time (&reports)
: "");
+
+ buffer_xml_append_printf
+ (prefix,
+ "%s",
+ iso_if_time (get_iterator_creation_time (&reports)));
+
+ buffer_xml_append_printf
+ (prefix,
+ "%s"
+ "0"
+ "0",
+ iso_if_time (get_iterator_modification_time (&reports)));
+
/* Send short task and report format info */
report_task (report, &task);
if (task)
@@ -18627,16 +18677,19 @@ handle_get_vulns (gmp_parser_t *gmp_parser, GError **error)
count ++;
SENDF_TO_CLIENT_OR_FAIL (""
"%s"
- "%s"
- "%s"
- "%s"
- "%1.1f"
- "%d",
+ "%s",
get_iterator_uuid (&vulns),
get_iterator_name (&vulns),
- vuln_iterator_type (&vulns),
- get_iterator_creation_time (&vulns),
- get_iterator_modification_time (&vulns),
+ vuln_iterator_type (&vulns));
+
+ SENDF_TO_CLIENT_OR_FAIL ("%s",
+ iso_if_time (get_iterator_creation_time (&vulns)));
+
+ SENDF_TO_CLIENT_OR_FAIL ("%s",
+ iso_if_time (get_iterator_modification_time (&vulns)));
+
+ SENDF_TO_CLIENT_OR_FAIL ("%1.1f"
+ "%d",
vuln_iterator_severity (&vulns),
vuln_iterator_qod (&vulns));
diff --git a/src/gmp_get.c b/src/gmp_get.c
index b13e73192..3793a49d6 100644
--- a/src/gmp_get.c
+++ b/src/gmp_get.c
@@ -323,12 +323,7 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator,
"<%s id=\"%s\">"
"%s"
"%s"
- "%s"
- "%s"
- "%s"
- "%i"
- "%i"
- "",
+ "%s",
type,
get_iterator_uuid (iterator)
? get_iterator_uuid (iterator)
@@ -341,13 +336,18 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator,
: "",
get_iterator_comment (iterator)
? get_iterator_comment (iterator)
- : "",
- get_iterator_creation_time (iterator)
- ? get_iterator_creation_time (iterator)
- : "",
- get_iterator_modification_time (iterator)
- ? get_iterator_modification_time (iterator)
- : "",
+ : "");
+
+ buffer_xml_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_creation_time (iterator)));
+
+ buffer_xml_append_printf (buffer,
+ "%s"
+ "%i"
+ "%i"
+ "",
+ iso_if_time (get_iterator_modification_time (iterator)),
writable,
in_use);
diff --git a/src/manage.c b/src/manage.c
index 249ab1522..b239b0649 100644
--- a/src/manage.c
+++ b/src/manage.c
@@ -3167,7 +3167,7 @@ static int
fork_cve_scan_handler (task_t task, target_t target)
{
int pid;
- char *report_id, *hosts;
+ char *report_id, *hosts, *exclude_hosts;
gvm_hosts_t *gvm_hosts;
gvm_host_t *gvm_host;
@@ -3234,6 +3234,8 @@ fork_cve_scan_handler (task_t task, target_t target)
exit (1);
}
+ exclude_hosts = target_exclude_hosts (target);
+
reset_task (task);
set_task_start_time_epoch (task, time (NULL));
set_scan_start_time_epoch (global_current_report, time (NULL));
@@ -3242,6 +3244,20 @@ fork_cve_scan_handler (task_t task, target_t target)
gvm_hosts = gvm_hosts_new (hosts);
free (hosts);
+
+ if (gvm_hosts_exclude (gvm_hosts, exclude_hosts ?: "") < 0)
+ {
+ set_task_interrupted (task,
+ "Failed to exclude hosts."
+ " Interrupting scan.");
+ set_report_scan_run_status (global_current_report, TASK_STATUS_INTERRUPTED);
+ gvm_hosts_free (gvm_hosts);
+ free (exclude_hosts);
+ gvm_close_sentry ();
+ exit(1);
+ }
+ free (exclude_hosts);
+
while ((gvm_host = gvm_hosts_next (gvm_hosts)))
if (cve_scan_host (task, global_current_report, gvm_host))
{
@@ -5830,11 +5846,18 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count,
nvt_iterator_detection (nvts));
}
+ g_string_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_creation_time (nvts)));
+
+ g_string_append_printf (buffer,
+ "%s",
+ iso_if_time (get_iterator_modification_time (nvts)));
+
default_timeout = nvt_default_timeout (oid);
+
g_string_append_printf (buffer,
"%s"
- "%s"
- "%s"
"%d"
"%s"
""
@@ -5844,18 +5867,13 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count,
"%s"
"%s",
default_timeout ? default_timeout : "",
- get_iterator_creation_time (nvts)
- ? get_iterator_creation_time (nvts)
- : "",
- get_iterator_modification_time (nvts)
- ? get_iterator_modification_time (nvts)
- : "",
nvt_iterator_category (nvts),
family_text,
nvt_iterator_qod (nvts),
nvt_iterator_qod_type (nvts),
refs_str->str,
nvt_tags->str);
+
free (default_timeout);
g_string_free (nvt_tags, 1);
diff --git a/src/manage.h b/src/manage.h
index ed5d12519..30e0642bb 100644
--- a/src/manage.h
+++ b/src/manage.h
@@ -1564,10 +1564,10 @@ result_iterator_delta_uuid (iterator_t*);
const char *
result_iterator_delta_qod_type (iterator_t*);
-const char *
+time_t
result_iterator_delta_creation_time (iterator_t*);
-const char *
+time_t
result_iterator_delta_modification_time (iterator_t*);
task_t
@@ -3449,6 +3449,9 @@ setting_iterator_comment (iterator_t*);
const char*
setting_iterator_value (iterator_t*);
+int
+setting_value_int (const char *, int *);
+
int
modify_setting (const gchar *, const gchar *, const gchar *, gchar **);
diff --git a/src/manage_get.c b/src/manage_get.c
index 2b81ac2f8..26b32a12c 100644
--- a/src/manage_get.c
+++ b/src/manage_get.c
@@ -154,18 +154,28 @@ get_iterator_comment (iterator_t* iterator)
*
* @param[in] iterator Iterator.
*
- * @return Creation time of the resource or NULL if iteration is complete.
+ * @return Creation time, or 0 if iteration is complete.
*/
-DEF_ACCESS (get_iterator_creation_time, 4);
+time_t
+get_iterator_creation_time (iterator_t* iterator)
+{
+ if (iterator->done) return 0;
+ return iterator_int64 (iterator, 4);
+}
/**
* @brief Get the modification time of the resource from a GET iterator.
*
* @param[in] iterator Iterator.
*
- * @return Modification time of the resource or NULL if iteration is complete.
+ * @return Modification time, or 0 if iteration is complete.
*/
-DEF_ACCESS (get_iterator_modification_time, 5);
+time_t
+get_iterator_modification_time (iterator_t* iterator)
+{
+ if (iterator->done) return 0;
+ return iterator_int64 (iterator, 5);
+}
/**
* @brief Get the owner name of the resource from a GET iterator.
diff --git a/src/manage_get.h b/src/manage_get.h
index 5c44e667c..2b7bfce24 100644
--- a/src/manage_get.h
+++ b/src/manage_get.h
@@ -66,10 +66,10 @@ get_iterator_name (iterator_t*);
const char*
get_iterator_comment (iterator_t*);
-const char*
+time_t
get_iterator_creation_time (iterator_t*);
-const char*
+time_t
get_iterator_modification_time (iterator_t*);
const char*
diff --git a/src/manage_sql.c b/src/manage_sql.c
index c39503e5f..567d8ade6 100644
--- a/src/manage_sql.c
+++ b/src/manage_sql.c
@@ -328,9 +328,6 @@ static void
set_credential_snmp_secret (credential_t, const char *, const char *,
const char *);
-static int
-setting_value_int (const char *, int *);
-
static int
setting_auto_cache_rebuild_int ();
@@ -12142,25 +12139,21 @@ generate_alert_filter_get (alert_t alert, const get_data_t *base_get_data,
if (filter_return)
*filter_return = filter;
+ (*alert_filter_get) = g_malloc0 (sizeof (get_data_t));
+ (*alert_filter_get)->details = base_get_data->details;
+ (*alert_filter_get)->ignore_pagination = base_get_data->ignore_pagination;
+ (*alert_filter_get)->ignore_max_rows_per_page
+ = base_get_data->ignore_max_rows_per_page;
+
if (filter)
{
- (*alert_filter_get) = g_malloc0 (sizeof (get_data_t));
- (*alert_filter_get)->details = base_get_data->details;
- (*alert_filter_get)->ignore_pagination = base_get_data->ignore_pagination;
- (*alert_filter_get)->ignore_max_rows_per_page
- = base_get_data->ignore_max_rows_per_page;
(*alert_filter_get)->filt_id = g_strdup (filt_id);
(*alert_filter_get)->filter = filter_term (filt_id);
}
else
- (*alert_filter_get) = NULL;
-
- ignore_pagination = alert_data (alert, "method",
- "composer_ignore_pagination");
- if (ignore_pagination)
{
- (*alert_filter_get)->ignore_pagination = atoi (ignore_pagination);
- g_free (ignore_pagination);
+ (*alert_filter_get)->filt_id = NULL;
+ (*alert_filter_get)->filter = g_strdup("");
}
/* Adjust filter for report composer.
@@ -12171,39 +12164,44 @@ generate_alert_filter_get (alert_t alert, const get_data_t *base_get_data,
* We simply use these fields to adjust the filter. In the future we'll
* remove the filter terms and extend the way we get the report. */
- if (filter)
+ gchar *include_notes, *include_overrides;
+
+ ignore_pagination = alert_data (alert, "method",
+ "composer_ignore_pagination");
+ if (ignore_pagination)
{
- gchar *include_notes, *include_overrides;
+ (*alert_filter_get)->ignore_pagination = atoi (ignore_pagination);
+ g_free (ignore_pagination);
+ }
- include_notes = alert_data (alert, "method",
- "composer_include_notes");
- if (include_notes)
- {
- gchar *new_filter;
+ include_notes = alert_data (alert, "method",
+ "composer_include_notes");
+ if (include_notes)
+ {
+ gchar *new_filter;
- new_filter = g_strdup_printf ("notes=%i %s",
- atoi (include_notes),
- (*alert_filter_get)->filter);
- g_free ((*alert_filter_get)->filter);
- (*alert_filter_get)->filter = new_filter;
- (*alert_filter_get)->filt_id = NULL;
- g_free (include_notes);
- }
+ new_filter = g_strdup_printf ("notes=%i %s",
+ atoi (include_notes),
+ (*alert_filter_get)->filter);
+ g_free ((*alert_filter_get)->filter);
+ (*alert_filter_get)->filter = new_filter;
+ (*alert_filter_get)->filt_id = NULL;
+ g_free (include_notes);
+ }
- include_overrides = alert_data (alert, "method",
- "composer_include_overrides");
- if (include_overrides)
- {
- gchar *new_filter;
+ include_overrides = alert_data (alert, "method",
+ "composer_include_overrides");
+ if (include_overrides)
+ {
+ gchar *new_filter;
- new_filter = g_strdup_printf ("overrides=%i %s",
- atoi (include_overrides),
- (*alert_filter_get)->filter);
- g_free ((*alert_filter_get)->filter);
- (*alert_filter_get)->filter = new_filter;
- (*alert_filter_get)->filt_id = NULL;
- g_free (include_overrides);
- }
+ new_filter = g_strdup_printf ("overrides=%i %s",
+ atoi (include_overrides),
+ (*alert_filter_get)->filter);
+ g_free ((*alert_filter_get)->filter);
+ (*alert_filter_get)->filter = new_filter;
+ (*alert_filter_get)->filt_id = NULL;
+ g_free (include_overrides);
}
return 0;
@@ -16005,6 +16003,19 @@ check_db_settings ()
" 'Delta Reports Version',"
" 'Version of the generation of the Delta Reports.',"
" '2' );");
+
+ if (sql_int ("SELECT count(*) FROM settings"
+ " WHERE uuid = '" SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "'"
+ " AND " ACL_IS_GLOBAL () ";")
+ == 0)
+ sql ("INSERT into settings (uuid, owner, name, comment, value)"
+ " VALUES"
+ " ('" SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "', NULL,"
+ " 'SecInfo SQL Buffer Threshold',"
+ " 'Buffer size threshold in MiB for running buffered SQL statements'"
+ " || ' in SecInfo updates before the end of the file'"
+ " || ' being processed.',"
+ " '100' );");
}
/**
@@ -21679,8 +21690,8 @@ report_add_results_array (report_t report, GArray *results)
{ "uuid", NULL, KEYWORD_TYPE_STRING }, \
{ "iso_time (creation_time)", "name", KEYWORD_TYPE_STRING }, \
{ "''", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ "(SELECT name FROM users WHERE users.id = reports.owner)", \
@@ -22152,12 +22163,12 @@ where_qod (int min_qod)
"name", \
KEYWORD_TYPE_STRING }, \
{ "''", "comment", KEYWORD_TYPE_STRING }, \
- { " iso_time (date, opts.user_zone)", \
+ { "date", \
"creation_time", \
- KEYWORD_TYPE_STRING }, \
- { " iso_time (date, opts.user_zone)", \
+ KEYWORD_TYPE_INTEGER }, \
+ { "date", \
"modification_time", \
- KEYWORD_TYPE_STRING }, \
+ KEYWORD_TYPE_INTEGER }, \
{ "date", "created", KEYWORD_TYPE_INTEGER }, \
{ "date", "modified", KEYWORD_TYPE_INTEGER }, \
{ "(SELECT name FROM users WHERE users.id = results.owner)", \
@@ -22305,12 +22316,12 @@ where_qod (int min_qod)
"name", \
KEYWORD_TYPE_STRING }, \
{ "''", "comment", KEYWORD_TYPE_STRING }, \
- { " iso_time (date, opts.user_zone)", \
+ { "date", \
"creation_time", \
- KEYWORD_TYPE_STRING }, \
- { " iso_time (date, opts.user_zone)", \
+ KEYWORD_TYPE_INTEGER }, \
+ { "date", \
"modification_time", \
- KEYWORD_TYPE_STRING }, \
+ KEYWORD_TYPE_INTEGER }, \
{ "date", "created", KEYWORD_TYPE_INTEGER }, \
{ "date", "modified", KEYWORD_TYPE_INTEGER }, \
{ "(SELECT name FROM users WHERE users.id = results.owner)", \
@@ -22467,12 +22478,12 @@ where_qod (int min_qod)
{ "comparison.delta_qod", NULL, KEYWORD_TYPE_INTEGER }, \
{ "comparison.delta_uuid", NULL, KEYWORD_TYPE_STRING }, \
{ "delta_qod_type", NULL, KEYWORD_TYPE_STRING }, \
- { " iso_time (delta_date, opts.user_zone)", \
+ { "delta_date", \
"delta_creation_time", \
- KEYWORD_TYPE_STRING }, \
- { " iso_time (delta_date, opts.user_zone)", \
+ KEYWORD_TYPE_INTEGER }, \
+ { "delta_date", \
"delta_modification_time", \
- KEYWORD_TYPE_STRING }, \
+ KEYWORD_TYPE_INTEGER }, \
{ "delta_task", NULL, KEYWORD_TYPE_INTEGER }, \
{ "delta_report", NULL, KEYWORD_TYPE_INTEGER }, \
{ "(SELECT name FROM users WHERE users.id = results.owner)", \
@@ -23906,13 +23917,13 @@ result_iterator_delta_qod_type (iterator_t* iterator)
*
* @param[in] iterator Iterator.
*
- * @return delta creation time if any, else NULL.
+ * @return Time, or 0 if iteration is complete.
*/
-const char *
+time_t
result_iterator_delta_creation_time (iterator_t* iterator)
{
if (iterator->done) return 0;
- return iterator_string (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6);
+ return iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6);
}
/**
@@ -23920,13 +23931,13 @@ result_iterator_delta_creation_time (iterator_t* iterator)
*
* @param[in] iterator Iterator.
*
- * @return delta modification time if any, else NULL.
+ * @return Time, or 0 if iteration is complete.
*/
-const char *
+time_t
result_iterator_delta_modification_time (iterator_t* iterator)
{
if (iterator->done) return 0;
- return iterator_string (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7);
+ return iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7);
}
/**
@@ -27919,23 +27930,41 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta,
"nvts_cols");
extra_with = g_strdup_printf(" comparison AS ("
- " WITH r1 as (SELECT results.id, description, host, report, port,"
+ " WITH r1a as (SELECT results.id, description, host, report, port,"
" severity, nvt, results.qod, results.uuid, hostname,"
" path, r1_lateral.new_severity as new_severity "
" FROM results "
- " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid from nvts)"
+ " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid FROM nvts)"
" AS nvts_cols"
" ON nvts_cols.nvts_oid = results.nvt"
" %s, LATERAL %s AS r1_lateral"
" WHERE report = %llu),"
- " r2 as (SELECT results.*, r2_lateral.new_severity AS new_severity"
+ " r2a as (SELECT results.*, r2_lateral.new_severity AS new_severity"
" FROM results"
- " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid from nvts)"
+ " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid FROM nvts)"
" AS nvts_cols"
" ON nvts_cols.nvts_oid = results.nvt"
" %s, LATERAL %s AS r2_lateral"
- " WHERE report = %llu)"
- " SELECT r1.id AS result1_id,"
+ " WHERE report = %llu),"
+ " r1 as (SELECT DISTINCT ON (r1a.id) r1a.*, r2a.id as r2id, row_number() over w1 as r1_rank"
+ " FROM r1a LEFT JOIN r2a ON r1a.host = r2a.host"
+ " AND normalize_port(r1a.port) = normalize_port(r2a.port)"
+ " AND r1a.nvt = r2a.nvt "
+ " AND (r1a.new_severity = 0) = (r2a.new_severity = 0)"
+ " AND (r1a.description = r2a.description)"
+ " WINDOW w1 AS (PARTITION BY r1a.host, normalize_port(r1a.port),"
+ " r1a.nvt, r1a.new_severity = 0, r2a.id is null ORDER BY r2a.id)"
+ " ORDER BY r1a.id),"
+ " r2 as (SELECT DISTINCT ON (r2a.id) r2a.*, r1a.id as r1id, row_number() over w2 as r2_rank"
+ " FROM r2a LEFT JOIN r1a ON r2a.host = r1a.host"
+ " AND normalize_port(r2a.port) = normalize_port(r1a.port)"
+ " AND r2a.nvt = r1a.nvt "
+ " AND (r2a.new_severity = 0) = (r1a.new_severity = 0)"
+ " AND (r2a.description = r1a.description)"
+ " WINDOW w2 AS (PARTITION BY r2a.host, normalize_port(r2a.port),"
+ " r2a.nvt, r2a.new_severity = 0, r1a.id is null ORDER BY r1a.id)"
+ " ORDER BY r2a.id)"
+ " (SELECT r1.id AS result1_id,"
" r2.id AS result2_id,"
" compare_results("
" r1.description,"
@@ -27961,7 +27990,7 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta,
" r2.path AS delta_path,"
" r2.host AS delta_host,"
RESULT_HOSTNAME_SQL("r2.hostname", "r2.host", "r2.report")
- " AS delta_hostname,"
+ " AS delta_hostname,"
" r2.nvt_version AS delta_nvt_version"
" FROM r1"
" FULL OUTER JOIN r2"
@@ -27969,20 +27998,10 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta,
" AND normalize_port(r1.port) = normalize_port(r2.port)"
" AND r1.nvt = r2.nvt "
" AND (r1.new_severity = 0) = (r2.new_severity = 0)"
- " AND (r1.description = r2.description"
- " OR NOT EXISTS (SELECT * FROM r2"
- " WHERE r1.description = r2.description"
- " AND r1.host = r2.host"
- " AND normalize_port(r1.port) = normalize_port(r2.port)"
- " AND r1.nvt = r2.nvt"
- " AND (r1.new_severity = 0) = (r2.new_severity = 0))"
- " OR NOT EXISTS (SELECT * FROM r1"
- " WHERE r1.description = r2.description"
- " AND r1.host = r2.host"
- " AND normalize_port(r1.port) = normalize_port(r2.port)"
- " AND r1.nvt = r2.nvt"
- " AND (r1.new_severity = 0) = (r2.new_severity = 0)))"
- " )",
+ " AND ((r1id IS NULL AND r2id IS NULL) OR"
+ " r2id = r2.id OR r1id = r1.id)"
+ " AND r1_rank = r2_rank"
+ " ) ) ",
opts_tables,
with_lateral,
report,
@@ -38530,8 +38549,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt,
KEYWORD_TYPE_STRING \
}, \
{ "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (notes.creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (notes.modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "notes.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "notes.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "notes.creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "notes.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ "(SELECT name FROM users WHERE users.id = notes.owner)", \
@@ -38585,8 +38604,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt,
{ "notes_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \
{ "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \
{ "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (notes_trash.creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (notes_trash.modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "notes_trash.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "notes_trash.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "notes_trash.creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "notes_trash.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ "(SELECT name FROM users WHERE users.id = notes_trash.owner)", \
@@ -39767,8 +39786,8 @@ modify_override (const gchar *override_id, const char *active, const char *nvt,
KEYWORD_TYPE_STRING \
}, \
{ "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (overrides.creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (overrides.modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "overrides.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "overrides.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "overrides.creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "overrides.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ \
@@ -39835,12 +39854,12 @@ modify_override (const gchar *override_id, const char *active, const char *nvt,
{ "overrides_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \
{ "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \
{ "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (overrides_trash.creation_time)", \
+ { "overrides_trash.creation_time", \
NULL, \
- KEYWORD_TYPE_STRING }, \
- { "iso_time (overrides_trash.modification_time)", \
+ KEYWORD_TYPE_INTEGER }, \
+ { "overrides_trash.modification_time", \
NULL, \
- KEYWORD_TYPE_STRING }, \
+ KEYWORD_TYPE_INTEGER }, \
{ "overrides_trash.creation_time", \
"created", \
KEYWORD_TYPE_INTEGER }, \
@@ -50155,9 +50174,9 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host,
if (host)
init_iterator (iterator,
- "SELECT id, uuid, name, comment, iso_time (creation_time),"
- " iso_time (modification_time), creation_time,"
- " modification_time, owner, owner, value,"
+ "SELECT id, uuid, name, comment, creation_time,"
+ " modification_time, creation_time AS created,"
+ " modification_time AS modified, owner, owner, value,"
" source_type, source_id, source_data,"
" (CASE WHEN source_type LIKE 'Report%%'"
" THEN NOT EXISTS (SELECT * FROM reports"
@@ -50168,9 +50187,9 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host,
" FROM host_identifiers"
" WHERE host = %llu"
" UNION"
- " SELECT id, uuid, name, comment, iso_time (creation_time),"
- " iso_time (modification_time), creation_time,"
- " modification_time, owner, owner,"
+ " SELECT id, uuid, name, comment, creation_time,"
+ " modification_time, creation_time AS created,"
+ " modification_time AS modified, owner, owner,"
" (SELECT name FROM oss WHERE id = os),"
" source_type, source_id, source_data,"
" (CASE WHEN source_type LIKE 'Report%%'"
@@ -50189,9 +50208,9 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host,
ascending ? "ASC" : "DESC");
else
init_iterator (iterator,
- "SELECT id, uuid, name, comment, iso_time (creation_time),"
- " iso_time (modification_time), creation_time,"
- " modification_time, owner, owner, value,"
+ "SELECT id, uuid, name, comment, creation_time,"
+ " modification_time, creation_time AS created,"
+ " modification_time AS modified, owner, owner, value,"
" source_type, source_id, source_data, 0, '', ''"
" FROM host_identifiers"
" ORDER BY %s %s;",
@@ -50921,8 +50940,8 @@ init_os_host_iterator (iterator_t* iterator, resource_t os)
{
assert (os);
init_iterator (iterator,
- "SELECT id, uuid, name, comment, iso_time (creation_time),"
- " iso_time (modification_time), creation_time,"
+ "SELECT id, uuid, name, comment, creation_time,"
+ " modification_time, creation_time,"
" modification_time, owner, owner,"
" (SELECT round (CAST (severity AS numeric), 1)"
" FROM host_max_severities"
@@ -51978,7 +51997,7 @@ setting_value (const char *uuid, char **value)
*
* @return 0 success, -1 error.
*/
-static int
+int
setting_value_int (const char *uuid, int *value)
{
gchar *quoted_uuid;
@@ -52683,6 +52702,8 @@ setting_name (const gchar *uuid)
return "Feed Import Roles";
if (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0)
return "Delta Reports Version";
+ if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0)
+ return "SecInfo SQL Buffer Threshold";
return NULL;
}
@@ -52722,12 +52743,15 @@ setting_description (const gchar *uuid)
return "Roles given access to new resources from feed.";
if (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0)
return "Version of the generation of the Delta Reports.";
+ if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0)
+ return "Buffer size threshold in MiB for running buffered SQL statements"
+ " in SecInfo updates before the end of the file being processed.";
return NULL;
}
/**
- * @brief Get the name of a setting.
+ * @brief Verify the value of a setting.
*
* @param[in] uuid UUID of setting.
* @param[in] value Value of setting, to verify.
@@ -52815,6 +52839,14 @@ setting_verify (const gchar *uuid, const gchar *value, const gchar *user)
return 1;
}
+ if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD))
+ {
+ int threshold;
+ threshold = atoi (value);
+ if (threshold < 0 || threshold > (INT_MAX / 1048576))
+ return 1;
+ }
+
return 0;
}
@@ -52870,6 +52902,15 @@ setting_normalise (const gchar *uuid, const gchar *value)
return g_string_free (normalised, FALSE);
}
+ if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0)
+ {
+ int threshold;
+ threshold = atoi (value);
+ if (threshold < 0)
+ return NULL;
+ return g_strdup_printf ("%i", threshold);
+ }
+
return g_strdup (value);
}
@@ -52900,7 +52941,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database,
&& strcmp (uuid, SETTING_UUID_LSC_DEB_MAINTAINER)
&& strcmp (uuid, SETTING_UUID_FEED_IMPORT_OWNER)
&& strcmp (uuid, SETTING_UUID_FEED_IMPORT_ROLES)
- && strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION))
+ && strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION)
+ && strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD))
{
fprintf (stderr, "Error in setting UUID.\n");
return 3;
@@ -52927,7 +52969,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database,
if ((strcmp (uuid, SETTING_UUID_DEFAULT_CA_CERT) == 0)
|| (strcmp (uuid, SETTING_UUID_FEED_IMPORT_OWNER) == 0)
|| (strcmp (uuid, SETTING_UUID_FEED_IMPORT_ROLES) == 0)
- || (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0))
+ || (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0)
+ || (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0))
{
sql_rollback ();
fprintf (stderr,
@@ -55289,8 +55332,8 @@ user_resources_in_use (user_t user,
{ "uuid", "uuid", KEYWORD_TYPE_STRING }, \
{ "name", "name", KEYWORD_TYPE_STRING }, \
{ "''", "comment", KEYWORD_TYPE_STRING }, \
- { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ "cast (null AS text)", "_owner", KEYWORD_TYPE_INTEGER }, \
@@ -58297,23 +58340,36 @@ manage_optimize (GSList *log_config, const db_conn_info_t *database,
}
else if (strcasecmp (name, "cleanup-config-prefs") == 0)
{
- int removed, fixed_values;
sql ("DELETE FROM config_preferences WHERE id NOT IN"
" (SELECT min(id) FROM config_preferences"
" GROUP BY config, type, name, value);");
- removed = sql_changes();
sql ("UPDATE config_preferences"
" SET value = (SELECT value FROM nvt_preferences"
" WHERE name='scanner_plugins_timeout')"
" WHERE name = 'scanner_plugins_timeout'"
" AND value = 'SCANNER_NVT_TIMEOUT';");
- fixed_values = sql_changes();
- success_text = g_strdup_printf ("Optimized: cleanup-config-prefs."
- " Duplicate config preferences removed:"
- " %d. Corrected preference values: %d",
- removed, fixed_values);
+ sql ("UPDATE config_preferences"
+ " SET pref_nvt = NULL,"
+ " pref_id = NULL,"
+ " pref_type = NULL,"
+ " pref_name = NULL"
+ " WHERE type = 'SERVER_PREFS' AND pref_nvt IS NOT NULL;");
+
+ sql ("UPDATE config_preferences"
+ " SET pref_nvt = substring (name, '^([^:]*)'),"
+ " pref_id = CAST(substring (name, '^[^:]*:([0-9]+)') AS integer),"
+ " pref_type = substring (name, '^[^:]*:[0-9]+:([^:]*):'),"
+ " pref_name = substring (name, '^[^:]*:[0-9]+:[^:]*:(.*)')"
+ " WHERE type = 'PLUGINS_PREFS'"
+ " AND (pref_nvt = '(null)' OR pref_nvt IS NULL"
+ " OR pref_type = '(null)' OR pref_type IS NULL"
+ " OR pref_name = '(null)' OR pref_name IS NULL)"
+ " AND name ~ '^[^:]*:[0-9]+:[^:]*:.*'"
+ " AND type = 'PLUGINS_PREFS';");
+
+ success_text = g_strdup_printf ("Optimized: cleanup-config-prefs.");
}
else if (strcasecmp (name, "cleanup-feed-permissions") == 0)
{
diff --git a/src/manage_sql.h b/src/manage_sql.h
index 7d98f1994..9f15f1269 100644
--- a/src/manage_sql.h
+++ b/src/manage_sql.h
@@ -142,6 +142,10 @@
*/
#define SETTING_UUID_DELTA_REPORTS_VERSION "985a0c05-2140-4e66-9989-ce9a0906a5a9"
+/**
+ * @brief UUID of 'SecInfo SQL Buffer Threshold' setting.
+ */
+#define SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "316275a9-3629-49ad-9cea-5b3ab155b93f"
/**
* @brief Trust constant for error.
@@ -248,9 +252,9 @@ typedef struct
*
* @param[in] prefix Column prefix.
*/
-#define GET_ITERATOR_COLUMNS_STRING \
- "id, uuid, name, comment, iso_time (creation_time)," \
- " iso_time (modification_time), creation_time AS created," \
+#define GET_ITERATOR_COLUMNS_STRING \
+ "id, uuid, name, comment, creation_time," \
+ " modification_time, creation_time AS created," \
" modification_time AS modified"
/**
@@ -263,8 +267,8 @@ typedef struct
{ prefix "uuid", NULL, KEYWORD_TYPE_STRING }, \
{ prefix "name", NULL, KEYWORD_TYPE_STRING }, \
{ prefix "comment", NULL, KEYWORD_TYPE_STRING }, \
- { " iso_time (" prefix "creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { " iso_time (" prefix "modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { prefix "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { prefix "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ prefix "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ prefix "modification_time", "modified", KEYWORD_TYPE_INTEGER }
diff --git a/src/manage_sql_configs.c b/src/manage_sql_configs.c
index 4fe5b0814..ae4b9454f 100644
--- a/src/manage_sql_configs.c
+++ b/src/manage_sql_configs.c
@@ -3627,6 +3627,13 @@ modify_config_preference (config_t config, const char* nvt,
g_free (quoted_name);
quoted_name = sql_quote (splits[3]);
}
+ else
+ {
+ quoted_pref_nvt = sql_quote (splits[0]);
+ pref_id = atoi (splits[1]);
+ quoted_pref_type = sql_quote (splits[2]);
+ quoted_pref_name = sql_quote (splits[3]);
+ }
}
g_strfreev (splits);
@@ -3639,12 +3646,25 @@ modify_config_preference (config_t config, const char* nvt,
config,
nvt ? "= 'PLUGINS_PREFS'" : "= 'SERVER_PREFS'",
quoted_name);
- sql ("INSERT INTO config_preferences"
- " (config, type, name, value, pref_nvt, pref_id, pref_type, pref_name)"
- " VALUES (%llu, %s, '%s', '%s', '%s', %i, '%s', '%s');",
- config, nvt ? "'PLUGINS_PREFS'" : "'SERVER_PREFS'", quoted_name,
- quoted_value, quoted_pref_nvt, pref_id, quoted_pref_type,
- quoted_pref_name);
+ if (nvt)
+ {
+ sql ("INSERT INTO config_preferences"
+ " (config, type, name, value,"
+ " pref_nvt, pref_id, pref_type, pref_name)"
+ " VALUES (%llu, 'PLUGINS_PREFS', '%s', '%s',"
+ " '%s', %i, '%s', '%s');",
+ config, quoted_name, quoted_value,
+ quoted_pref_nvt, pref_id, quoted_pref_type, quoted_pref_name);
+ }
+ else
+ {
+ sql ("INSERT INTO config_preferences"
+ " (config, type, name, value,"
+ " pref_nvt, pref_id, pref_type, pref_name)"
+ " VALUES (%llu, 'SERVER_PREFS', '%s', '%s',"
+ " NULL, NULL, NULL, NULL);",
+ config, quoted_name, quoted_value);
+ }
g_free (quoted_value);
g_free (quoted_name);
diff --git a/src/manage_sql_report_configs.c b/src/manage_sql_report_configs.c
index ed158db2f..6d4b44c15 100644
--- a/src/manage_sql_report_configs.c
+++ b/src/manage_sql_report_configs.c
@@ -661,8 +661,8 @@ restore_report_config (const char *report_config_id)
{ "uuid", NULL, KEYWORD_TYPE_STRING }, \
{ "name", NULL, KEYWORD_TYPE_STRING }, \
{ "comment", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ \
@@ -696,8 +696,8 @@ restore_report_config (const char *report_config_id)
{ "uuid", NULL, KEYWORD_TYPE_STRING }, \
{ "name", NULL, KEYWORD_TYPE_STRING }, \
{ "comment", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ \
diff --git a/src/manage_sql_report_formats.c b/src/manage_sql_report_formats.c
index dd0f3da50..3f6df153f 100644
--- a/src/manage_sql_report_formats.c
+++ b/src/manage_sql_report_formats.c
@@ -2653,8 +2653,8 @@ report_format_trust (report_format_t report_format)
{ "uuid", NULL, KEYWORD_TYPE_STRING }, \
{ "name", NULL, KEYWORD_TYPE_STRING }, \
{ "''", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ \
@@ -2690,8 +2690,8 @@ report_format_trust (report_format_t report_format)
{ "uuid", NULL, KEYWORD_TYPE_STRING }, \
{ "name", NULL, KEYWORD_TYPE_STRING }, \
{ "''", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \
- { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \
+ { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \
+ { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \
{ "creation_time", "created", KEYWORD_TYPE_INTEGER }, \
{ "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \
{ \
diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c
index 4911b73d8..0a2d90cc7 100644
--- a/src/manage_sql_secinfo.c
+++ b/src/manage_sql_secinfo.c
@@ -51,6 +51,7 @@
#include
#include
#include
+#include
#undef G_LOG_DOMAIN
/**
@@ -62,7 +63,7 @@
/* Static variables. */
/**
- * @brief Maximum number of rows in an INSERT.
+ * @brief Maximum number of rows in a CPEs INSERT.
*/
#define CPE_MAX_CHUNK_SIZE 10000
@@ -185,130 +186,21 @@ increment_transaction_size (int* current_size)
}
}
+/* Helper: buffer structure for INSERTs. */
+
/**
- * @brief Split a file.
- *
- * @param[in] path Path to file.
- * @param[in] size Approx size of split files. In same format that
- * xml_split accepts, eg "200Kb".
- * @param[in] tail Text to replace last line of split files.
- *
- * @return Temp dir holding split files.
+ * @brief Get the SQL buffer size threshold converted from MiB to bytes.
*/
-static const gchar *
-split_xml_file (gchar *path, const gchar *size, const gchar *tail)
+int
+setting_secinfo_sql_buffer_threshold_bytes ()
{
- int ret;
- static gchar dir[] = "/tmp/gvmd-split-xml-file-XXXXXX";
- gchar *previous_dir, *command;
-
- if (mkdtemp (dir) == NULL)
- {
- g_warning ("%s: Failed to make temp dir: %s",
- __func__,
- strerror (errno));
- return NULL;
- }
-
- previous_dir = getcwd (NULL, 0);
- if (previous_dir == NULL)
- {
- g_warning ("%s: Failed to getcwd: %s",
- __func__,
- strerror (errno));
- return NULL;
- }
-
- if (chdir (dir))
- {
- g_warning ("%s: Failed to chdir: %s",
- __func__,
- strerror (errno));
- g_free (previous_dir);
- return NULL;
- }
-
- if (gvm_file_copy (path, "split.xml") == FALSE)
- {
- g_free (previous_dir);
- return NULL;
- }
-
- /* xml_split will chop split.xml into files that are roughly 'size' big.
- *
- * The generated files are always put in the directory that holds
- * split.xml, as follows:
- *
- * split.xml Source XML.
- * split-00.xml Master generated XML. No content, just includes other
- * files. The include statements are wrapped in the
- * root element from split.xml.
- * split-01.xml Generated XML content. Wrapped in an
- * element.
- * split-02.xml Second generated content file.
- * ...
- * split-112.xml Last content, for example.
- *
- * Parsing the generated files independently will only work if the files
- * contain the original root element (for example, because the parser
- * requires the namespace definitions to be present).
- *
- * So the command below needs to mess around a little bit to replace the
- * wrapper XML element in split-01.xml, split-02.xml, etc with the root
- * element from split-00.xml.
- *
- * Using tail and head is not super robust, but it's simple and it will
- * work as long as xml_split keeps the opening of the wrapper element
- * in split-00.xml on a dedicated line. (It doesn't do this for the
- * closing element, so we use the tail argument instead.)
- */
-
- command = g_strdup_printf
- ("xml_split -s%s split.xml"
- " && head -n 2 split-00.xml > head.xml"
- " && echo '%s' > tail.xml"
- " && for F in split-*.xml; do"
- /* Remove the first two lines and last line. */
- " awk 'NR>3 {print last} {last=$0}' $F > body.xml"
- /* Combine with new start and end. */
- " && cat head.xml body.xml tail.xml > $F;"
- " done",
- size,
- tail);
-
- g_debug ("%s: command: %s", __func__, command);
- ret = system (command);
- if ((ret == -1) || WIFEXITED(ret) == 0 || WEXITSTATUS (ret))
- {
- g_warning ("%s: system failed with ret %i, %i (%i), %s",
- __func__,
- ret,
- WIFEXITED (ret),
- WIFEXITED (ret) ? WEXITSTATUS (ret) : 0,
- command);
- g_free (command);
-
- if (chdir (previous_dir))
- g_warning ("%s: and failed to chdir back", __func__);
- g_free (previous_dir);
+ int threshold;
- return NULL;
- }
-
- g_free (command);
-
- if (chdir (previous_dir))
- g_warning ("%s: Failed to chdir back (will continue anyway)",
- __func__);
+ setting_value_int (SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD, &threshold);
- g_free (previous_dir);
-
- return dir;
+ return threshold * 1048576;
}
-
-/* Helper: buffer structure for INSERTs. */
-
/**
* @brief Buffer for INSERT statements.
*/
@@ -316,26 +208,34 @@ typedef struct
{
array_t *statements; ///< Buffered statements.
GString *statement; ///< Current statement.
+ int statements_size; ///< Sum of lengths of all statements buffered.
+ int max_statements_size; ///< Auto-run at this statement_size, 0 for never.
int current_chunk_size; ///< Number of rows in current statement.
int max_chunk_size; ///< Max number of rows per INSERT.
gchar *open_sql; ///< SQL to open each statement.
gchar *close_sql; ///< SQL to close each statement.
} inserts_t;
+static void
+inserts_run (inserts_t *, gboolean);
+
/**
* @brief Check size of current statement.
*
* @param[in] inserts Insert buffer.
- * @param[in] max_chunk_size Max chunk size.
+ * @param[in] max_chunk_size Max chunk size per statement.
+ * @param[in] max_statements_size Automatically run at this statements size.
* @param[in] open_sql SQL to to start each statement.
* @param[in] close_sql SQL to append to the end of each statement.
*/
static void
-inserts_init (inserts_t *inserts, int max_chunk_size, const gchar *open_sql,
- const gchar *close_sql)
+inserts_init (inserts_t *inserts, int max_chunk_size, int max_statements_size,
+ const gchar *open_sql, const gchar *close_sql)
{
inserts->statements = make_array ();
inserts->statement = NULL;
+ inserts->statements_size = 0;
+ inserts->max_statements_size = max_statements_size;
inserts->current_chunk_size = 0;
inserts->max_chunk_size = max_chunk_size;
inserts->open_sql = open_sql ? g_strdup (open_sql) : NULL;
@@ -377,8 +277,15 @@ inserts_check_size (inserts_t *inserts)
{
inserts_statement_close (inserts);
array_add (inserts->statements, inserts->statement);
+ inserts->statements_size += inserts->statement->len;
inserts->statement = NULL;
inserts->current_chunk_size = 0;
+
+ if (inserts->max_statements_size
+ && inserts-> statements_size >= inserts->max_statements_size)
+ {
+ inserts_run (inserts, FALSE);
+ }
}
if (inserts->statement == NULL)
@@ -392,7 +299,26 @@ inserts_check_size (inserts_t *inserts)
}
/**
- * @brief Free everything.
+ * @brief Free only the statements in an inserts buffer so it can be reused.
+ *
+ * @param[in] inserts Insert buffer.
+ */
+static void
+inserts_free_statements (inserts_t *inserts)
+{
+ int index;
+
+ for (index = 0; index < inserts->statements->len; index++)
+ {
+ g_string_free (g_ptr_array_index (inserts->statements, index), TRUE);
+ inserts->statements->pdata[index] = NULL;
+ }
+ g_ptr_array_set_size (inserts->statements, 0);
+ inserts->statements_size = 0;
+}
+
+/**
+ * @brief Free all fields in an inserts buffer.
*
* @param[in] inserts Insert buffer.
*/
@@ -413,9 +339,10 @@ inserts_free (inserts_t *inserts)
* @brief Run the INSERT SQL, freeing the buffers.
*
* @param[in] inserts Insert buffer.
+ * @param[in] finalize Whether to free the whole inserts buffer afterwards.
*/
static void
-inserts_run (inserts_t *inserts)
+inserts_run (inserts_t *inserts, gboolean finalize)
{
guint index;
@@ -435,7 +362,10 @@ inserts_run (inserts_t *inserts)
sql ("%s", statement->str);
}
- inserts_free (inserts);
+ if (finalize)
+ inserts_free (inserts);
+ else
+ inserts_free_statements (inserts);
}
@@ -2119,45 +2049,40 @@ insert_scap_cpe_details (inserts_t *inserts, element_t cpe_item)
static int
update_scap_cpes_from_file (const gchar *path)
{
- GError *error;
- element_t element, cpe_list, cpe_item;
- gchar *xml;
- gsize xml_len;
- inserts_t inserts, details_inserts;
-
- g_debug ("%s: parsing %s", __func__, path);
-
- error = NULL;
- g_file_get_contents (path, &xml, &xml_len, &error);
- if (error)
- {
- g_warning ("%s: Failed to get contents: %s",
- __func__,
- error->message);
- g_error_free (error);
- return -1;
- }
-
- if (parse_element (xml, &element))
- {
- g_free (xml);
- g_warning ("%s: Failed to parse element", __func__);
- return -1;
- }
- g_free (xml);
+ int ret;
+ element_t cpe_item;
+ inserts_t inserts;
+ xml_file_iterator_t file_iterator;
+ gchar *error_message = NULL;
- cpe_list = element;
- if (strcmp (element_name (cpe_list), "cpe-list"))
+ file_iterator = xml_file_iterator_new ();
+ ret = xml_file_iterator_init_from_file_path (file_iterator, path, 1);
+ switch (ret)
{
- element_free (element);
- g_warning ("%s: CPE dictionary missing CPE-LIST", __func__);
- return -1;
+ case 0:
+ break;
+ case 2:
+ g_warning ("%s: Could not open file '%s' for XML file iterator: %s",
+ __func__, path, strerror(errno));
+ xml_file_iterator_free (file_iterator);
+ return -1;
+ case 3:
+ g_warning ("%s: Could not create parser context for XML file iterator",
+ __func__);
+ xml_file_iterator_free (file_iterator);
+ return -1;
+ default:
+ g_warning ("%s: Could not initialize XML file iterator",
+ __func__);
+ xml_file_iterator_free (file_iterator);
+ return -1;
}
sql_begin_immediate ();
inserts_init (&inserts,
CPE_MAX_CHUNK_SIZE,
+ setting_secinfo_sql_buffer_threshold_bytes (),
"INSERT INTO scap2.cpes"
" (uuid, name, title, creation_time,"
" modification_time, status, deprecated_by_id,"
@@ -2171,7 +2096,15 @@ update_scap_cpes_from_file (const gchar *path)
" status = EXCLUDED.status,"
" deprecated_by_id = EXCLUDED.deprecated_by_id,"
" nvd_id = EXCLUDED.nvd_id");
- cpe_item = element_first_child (cpe_list);
+
+ cpe_item = xml_file_iterator_next (file_iterator, &error_message);
+ if (error_message)
+ {
+ g_warning ("%s: could not get first CPE XML element: %s",
+ __func__, error_message);
+ g_free (error_message);
+ goto fail;
+ }
while (cpe_item)
{
gchar *modification_date;
@@ -2180,7 +2113,15 @@ update_scap_cpes_from_file (const gchar *path)
if (strcmp (element_name (cpe_item), "cpe-item"))
{
- cpe_item = element_next (cpe_item);
+ element_free (cpe_item);
+ cpe_item = xml_file_iterator_next (file_iterator, &error_message);
+ if (error_message)
+ {
+ g_warning ("%s: could not get next CPE XML element: %s",
+ __func__, error_message);
+ g_free (error_message);
+ goto fail;
+ }
continue;
}
@@ -2205,47 +2146,90 @@ update_scap_cpes_from_file (const gchar *path)
if (insert_scap_cpe (&inserts, cpe_item, item_metadata,
modification_time))
goto fail;
- cpe_item = element_next (cpe_item);
+
+ element_free (cpe_item);
+ cpe_item = xml_file_iterator_next (file_iterator, &error_message);
+ if (error_message)
+ {
+ g_warning ("%s: could not get next CPE XML element: %s",
+ __func__, error_message);
+ g_free (error_message);
+ error_message = NULL;
+ goto fail;
+ }
}
- inserts_run (&inserts);
+ inserts_run (&inserts, TRUE);
sql_commit ();
+ sql_begin_immediate();
+
+ if (xml_file_iterator_rewind (file_iterator))
+ {
+ g_warning ("%s: Could not create parser context for XML file iterator"
+ " for details.",
+ __func__);
+ goto fail;
+ }
// Extract and save details XML.
- inserts_init (&details_inserts,
+ inserts_init (&inserts,
CPE_MAX_CHUNK_SIZE,
+ setting_secinfo_sql_buffer_threshold_bytes (),
"INSERT INTO scap2.cpe_details"
" (cpe_id, details_xml)"
" VALUES",
" ON CONFLICT (cpe_id) DO UPDATE"
" SET details_xml = EXCLUDED.details_xml");
- cpe_item = element_first_child (cpe_list);
+ cpe_item = xml_file_iterator_next (file_iterator, &error_message);
+ if (error_message)
+ {
+ g_warning ("%s: could not get first CPE XML element for details: %s",
+ __func__, error_message);
+ g_free (error_message);
+ error_message = NULL;
+ goto fail;
+ }
while (cpe_item)
{
if (strcmp (element_name (cpe_item), "cpe-item"))
{
- cpe_item = element_next (cpe_item);
+ element_free (cpe_item);
+ cpe_item = xml_file_iterator_next (file_iterator, &error_message);
+ if (error_message)
+ {
+ g_warning ("%s: could not get next CPE XML element"
+ " for details: %s",
+ __func__, error_message);
+ g_free (error_message);
+ goto fail;
+ }
continue;
}
- if (insert_scap_cpe_details (&details_inserts, cpe_item))
+ if (insert_scap_cpe_details (&inserts, cpe_item))
goto fail;
- cpe_item = element_next (cpe_item);
+ element_free (cpe_item);
+ cpe_item = xml_file_iterator_next (file_iterator, &error_message);
+ if (error_message)
+ {
+ g_warning ("%s: could not get next CPE XML element for details: %s",
+ __func__, error_message);
+ g_free (error_message);
+ goto fail;
+ }
}
- element_free (element);
-
- sql_begin_immediate();
- inserts_run (&details_inserts);
+ inserts_run (&inserts, TRUE);
sql_commit();
+ xml_file_iterator_free (file_iterator);
return 0;
fail:
inserts_free (&inserts);
- element_free (element);
g_warning ("Update of CPEs failed");
sql_commit ();
+ xml_file_iterator_free (file_iterator);
return -1;
}
@@ -2258,9 +2242,8 @@ static int
update_scap_cpes ()
{
gchar *full_path;
- const gchar *split_dir;
GStatBuf state;
- int index;
+ int ret;
full_path = g_build_filename (GVM_SCAP_DATA_DIR,
"official-cpe-dictionary_v2.2.xml",
@@ -2277,44 +2260,9 @@ update_scap_cpes ()
g_info ("Updating CPEs");
- split_dir = split_xml_file (full_path, "40Mb", "");
- if (split_dir == NULL)
- {
- int ret;
-
- g_warning ("%s: Failed to split CPEs, attempting with full file",
- __func__);
- ret = update_scap_cpes_from_file (full_path);
- g_free (full_path);
- return ret;
- }
- g_free (full_path);
-
- for (index = 1; 1; index++)
- {
- int ret;
- gchar *path, *name;
-
- name = g_strdup_printf ("split-%02i.xml", index);
- path = g_build_filename (split_dir, name, NULL);
- g_free (name);
-
- if (g_stat (path, &state))
- {
- g_free (path);
- break;
- }
-
- ret = update_scap_cpes_from_file (path);
- g_free (path);
- if (ret < 0)
- {
- gvm_file_remove_recurse (split_dir);
- return -1;
- }
- }
-
- gvm_file_remove_recurse (split_dir);
+ ret = update_scap_cpes_from_file (full_path);
+ if (ret)
+ return -1;
return 0;
}
@@ -2422,8 +2370,7 @@ insert_cve_products (element_t list, resource_t cve,
gchar *quoted_product, *product_decoded;
gchar *product_tilde;
- product_decoded = g_uri_unescape_string
- (element_text (product), NULL);
+ product_decoded = g_uri_unescape_string (product_text, NULL);
product_tilde = string_replace (product_decoded,
"~", "%7E", "%7e",
NULL);
@@ -2693,12 +2640,13 @@ insert_cve_from_entry (element_t entry, element_t last_modified,
static int
update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes)
{
- GError *error;
- element_t element, entry;
- gchar *xml, *full_path;
- gsize xml_len;
+ gchar *error_message = NULL;
+ xml_file_iterator_t iterator;
+ element_t entry;
+ gchar *full_path;
GStatBuf state;
int transaction_size = 0;
+ int ret;
full_path = g_build_filename (GVM_SCAP_DATA_DIR, xml_path, NULL);
@@ -2712,29 +2660,31 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes)
g_info ("Updating %s", full_path);
- error = NULL;
- g_file_get_contents (full_path, &xml, &xml_len, &error);
- if (error)
- {
- g_warning ("%s: Failed to get contents: %s",
- __func__,
- error->message);
- g_error_free (error);
- g_free (full_path);
- return -1;
- }
-
- if (parse_element (xml, &element))
+ iterator = xml_file_iterator_new ();
+ ret = xml_file_iterator_init_from_file_path (iterator, full_path, 1);
+ switch (ret)
{
- g_free (xml);
- g_warning ("%s: Failed to parse element", __func__);
- g_free (full_path);
- return -1;
+ case 0:
+ break;
+ case 2:
+ g_warning ("%s: Could not open file '%s' for XML file iterator: %s",
+ __func__, full_path, strerror(errno));
+ xml_file_iterator_free (iterator);
+ return -1;
+ case 3:
+ g_warning ("%s: Could not create parser context for XML file iterator",
+ __func__);
+ xml_file_iterator_free (iterator);
+ return -1;
+ default:
+ g_warning ("%s: Could not initialize XML file iterator",
+ __func__);
+ xml_file_iterator_free (iterator);
+ return -1;
}
- g_free (xml);
sql_begin_immediate ();
- entry = element_first_child (element);
+ entry = xml_file_iterator_next (iterator, &error_message);
while (entry)
{
if (strcmp (element_name (entry), "entry") == 0)
@@ -2744,28 +2694,40 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes)
last_modified = element_child (entry, "vuln:last-modified-datetime");
if (last_modified == NULL)
{
- g_warning ("%s: vuln:last-modified-datetime missing",
- __func__);
+ error_message = g_strdup ("vuln:last-modified-datetime missing");
goto fail;
}
if (insert_cve_from_entry (entry, last_modified, hashed_cpes,
&transaction_size))
- goto fail;
+ {
+ error_message = g_strdup ("Insert of CVE into database failed");
+ goto fail;
+ }
}
- entry = element_next (entry);
+
+ element_free (entry);
+ entry = xml_file_iterator_next (iterator, &error_message);
}
- element_free (element);
+ if (error_message)
+ goto fail;
+
+ xml_file_iterator_free (iterator);
g_free (full_path);
sql_commit ();
return 0;
fail:
- element_free (element);
- g_warning ("Update of CVEs failed at file '%s'",
- full_path);
+ xml_file_iterator_free (iterator);
+ if (error_message)
+ g_warning ("Update of CVEs failed at file '%s': %s",
+ full_path, error_message);
+ else
+ g_warning ("Update of CVEs failed at file '%s'",
+ full_path);
g_free (full_path);
+ g_free (error_message);
sql_commit ();
return -1;
}
@@ -3427,38 +3389,15 @@ update_scap_placeholders ()
" WHERE cpe=cpes.id))"
" WHERE cpes.title IS NULL;");
}
-
+
/**
- * @brief Finish scap update.
- *
- * @return 0 success, -1 error.
+ * @brief Update CERT data that depends on SCAP.
*/
-static int
-update_scap_end ()
+static void
+update_cert_data ()
{
int cert_db_version;
- g_debug ("%s: update timestamp", __func__);
-
- update_scap_timestamp ();
-
- /* Replace the real scap schema with the new one. */
-
- if (sql_int ("SELECT EXISTS (SELECT schema_name FROM"
- " information_schema.schemata"
- " WHERE schema_name = 'scap');"))
- {
- sql ("ALTER SCHEMA scap RENAME TO scap3;");
- sql ("ALTER SCHEMA scap2 RENAME TO scap;");
- sql ("DROP SCHEMA scap3 CASCADE;");
- /* View 'vulns' contains references into the SCAP schema, so it is
- * removed by the CASCADE. */
- create_view_vulns ();
- }
- else
- sql ("ALTER SCHEMA scap2 RENAME TO scap;");
-
- /* Update CERT data that depends on SCAP. */
cert_db_version = manage_cert_db_version();
if (cert_db_version == -1)
@@ -3490,6 +3429,37 @@ update_scap_end ()
update_cvss_dfn_cert (1, last_cert_update, last_scap_update);
update_cvss_cert_bund (1, last_cert_update, last_scap_update);
}
+}
+
+/**
+ * @brief Finish scap update.
+ */
+static void
+update_scap_end ()
+{
+ g_debug ("%s: update timestamp", __func__);
+
+ update_scap_timestamp ();
+
+ /* Replace the real scap schema with the new one. */
+
+ if (sql_int ("SELECT EXISTS (SELECT schema_name FROM"
+ " information_schema.schemata"
+ " WHERE schema_name = 'scap');"))
+ {
+ sql ("ALTER SCHEMA scap RENAME TO scap3;");
+ sql ("ALTER SCHEMA scap2 RENAME TO scap;");
+ sql ("DROP SCHEMA scap3 CASCADE;");
+ /* View 'vulns' contains references into the SCAP schema, so it is
+ * removed by the CASCADE. */
+ create_view_vulns ();
+ }
+ else
+ sql ("ALTER SCHEMA scap2 RENAME TO scap;");
+
+ /* Update CERT data that depends on SCAP. */
+
+ update_cert_data ();
/* Analyze. */
@@ -3499,8 +3469,60 @@ update_scap_end ()
g_info ("%s: Updating SCAP info succeeded", __func__);
setproctitle ("Syncing SCAP: done");
+}
- return 0;
+/**
+ * @brief Abort scap update.
+ */
+static void
+abort_scap_update ()
+{
+ g_debug ("%s: update timestamp", __func__);
+
+ if (sql_int ("SELECT EXISTS (SELECT schema_name FROM"
+ " information_schema.schemata"
+ " WHERE schema_name = 'scap');"))
+ {
+ update_scap_timestamp ();
+ sql ("UPDATE scap.meta SET value = "
+ " (SELECT value from scap2.meta WHERE name = 'last_update')"
+ " WHERE name = 'last_update';");
+ sql ("DROP SCHEMA scap2 CASCADE;");
+ /* View 'vulns' contains references into the SCAP schema, so it is
+ * removed by the CASCADE. */
+ create_view_vulns ();
+ /* Update CERT data that depends on SCAP. */
+ update_cert_data ();
+ }
+ else
+ {
+ /* reset scap2 schema */
+ if (manage_db_init ("scap"))
+ {
+ g_warning ("%s: could not reset scap2 schema, db init failed", __func__);
+ }
+ else if (manage_db_init_indexes ("scap"))
+ {
+ g_warning ("%s: could not reset scap2 schema, init indexes failed", __func__);
+ }
+ else if (manage_db_add_constraints ("scap"))
+ {
+ g_warning ("%s: could not reset scap2 schema, add constrains failed", __func__);
+ }
+
+ if (sql_int ("SELECT EXISTS (SELECT schema_name FROM"
+ " information_schema.schemata"
+ " WHERE schema_name = 'scap2');"))
+ {
+ update_scap_timestamp ();
+ sql ("ALTER SCHEMA scap2 RENAME TO scap;");
+ /* Update CERT data that depends on SCAP. */
+ update_cert_data ();
+ }
+ }
+
+ g_info ("%s: Updating SCAP data aborted", __func__);
+ setproctitle ("Syncing SCAP: aborted");
}
/**
@@ -3562,7 +3584,8 @@ try_load_csv ()
return -1;
}
- return update_scap_end ();
+ update_scap_end ();
+ return 0;
}
return 1;
}
@@ -3661,7 +3684,7 @@ update_scap (gboolean reset_scap_db)
if (update_scap_cpes () == -1)
{
- update_scap_timestamp ();
+ abort_scap_update ();
return -1;
}
@@ -3670,7 +3693,7 @@ update_scap (gboolean reset_scap_db)
if (update_scap_cves () == -1)
{
- update_scap_timestamp ();
+ abort_scap_update ();
return -1;
}
@@ -3689,7 +3712,8 @@ update_scap (gboolean reset_scap_db)
update_scap_placeholders ();
- return update_scap_end ();
+ update_scap_end ();
+ return 0;
}
/**
diff --git a/src/utils.c b/src/utils.c
index 83c55f7fd..efe108b23 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -483,7 +483,7 @@ iso_time_internal (time_t *epoch_time, const char **abbrev)
/**
* @brief Create an ISO time from seconds since epoch.
*
- * @param[in] epoch_time Time in seconds from epoch.
+ * @param[in] epoch_time Pointer to time in seconds from epoch.
*
* @return Pointer to ISO time in static memory, or NULL on error.
*/
@@ -496,7 +496,7 @@ iso_time (time_t *epoch_time)
/**
* @brief Create an ISO time from seconds since epoch, given a timezone.
*
- * @param[in] epoch_time Time in seconds from epoch.
+ * @param[in] epoch_time Pointer to time in seconds from epoch.
* @param[in] zone Timezone.
* @param[out] abbrev Timezone abbreviation.
*
@@ -543,6 +543,28 @@ iso_time_tz (time_t *epoch_time, const char *zone, const char **abbrev)
return ret;
}
+/**
+ * @brief Create an ISO time from seconds since epoch, with a 0 check.
+ *
+ * @param[in] epoch_time Time in seconds from epoch.
+ *
+ * @return ISO time string in static memory. If epoch_time is 0 then string is empty.
+ */
+char *
+iso_if_time (time_t epoch_time)
+{
+ static char *empty = "";
+ if (epoch_time)
+ {
+ char *ret;
+
+ ret = iso_time (&epoch_time);
+ if (ret)
+ return ret;
+ }
+ return empty;
+}
+
/* Locks. */
diff --git a/src/utils.h b/src/utils.h
index 50333d71e..dfeed44b6 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -55,6 +55,9 @@ iso_time (time_t *);
char *
iso_time_tz (time_t *, const char *, const char **);
+char *
+iso_if_time (time_t epoch_time);
+
/**
* @brief Lockfile.
*/