diff --git a/.docker/prod.Dockerfile b/.docker/prod.Dockerfile index 212ecde50..531cbe555 100644 --- a/.docker/prod.Dockerfile +++ b/.docker/prod.Dockerfile @@ -1,8 +1,9 @@ ARG VERSION=edge ARG GVM_LIBS_VERSION=oldstable ARG DEBIAN_FRONTEND=noninteractive +ARG IMAGE_REGISTRY=ghcr.io -FROM greenbone/gvmd-build:${VERSION} as builder +FROM ${IMAGE_REGISTRY}/greenbone/gvmd-build:${VERSION} as builder COPY . /source WORKDIR /source @@ -99,7 +100,6 @@ RUN apt-get update && \ texlive-fonts-recommended \ texlive-latex-extra \ wget \ - xml-twig-tools \ xmlstarlet \ xsltproc \ zip && \ diff --git a/.docker/start-gvmd.sh b/.docker/start-gvmd.sh index c4ced08d3..7083b8f81 100644 --- a/.docker/start-gvmd.sh +++ b/.docker/start-gvmd.sh @@ -16,28 +16,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -#!/bin/sh - [ -z "$USER" ] && USER="admin" [ -z "$PASSWORD" ] && PASSWORD="admin" [ -z "$GVMD_ARGS" ] && GVMD_ARGS="--listen-mode=666" [ -z "$GVMD_USER" ] && GVMD_USER="gvmd" -[ -z "$PGRES_DATA"] && PGRES_DATA="/var/lib/postgresql" +[ -z "$PGRES_DATA" ] && PGRES_DATA="/var/lib/postgresql" if [ -n "$GVM_CERTS" ] && [ "$GVM_CERTS" = true ]; then - echo "Generating certs" - gvm-manage-certs -a + echo "Generating certs" + gvm-manage-certs -a fi # check for psql connection FILE=$PGRES_DATA/started until test -f "$FILE"; do - echo "waiting 1 second for ready postgres container" + echo "waiting 1 second for ready postgres container" sleep 1 done until psql -U "$GVMD_USER" -d gvmd -c "SELECT 'connected' as connection"; do - echo "waiting 1 second to retry psql connection" - sleep 1 + echo "waiting 1 second to retry psql connection" + sleep 1 done # migrate db if necessary @@ -46,11 +44,11 @@ gvmd --migrate || true gvmd --create-user=$USER --password=$PASSWORD || true # set the feed import owner -uid=$(gvmd --get-users --verbose | grep $USER | awk '{print $2}') +uid=$(gvmd --get-users --verbose | grep "^$USER " | awk '{print $2}') gvmd --modify-setting 78eceaec-3385-11ea-b237-28d24461215b --value "$uid" echo "starting gvmd" gvmd $GVMD_ARGS || - (cat /var/log/gvm/gvmd.log && exit 1) + (cat /var/log/gvm/gvmd.log && exit 1) tail -f /var/log/gvm/gvmd.log diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2277decec..85dac033f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,6 @@ # default reviewers -* @greenbone/gvmd-maintainers @mattmundell +* @greenbone/gvmd-maintainers # dev ops -.github/ @greenbone/devops @greenbone/gvmd-maintainers @mattmundell -.docker/ @greenbone/devops @greenbone/gvmd-maintainers @mattmundell +.github/ @greenbone/devops @greenbone/gvmd-maintainers +.docker/ @greenbone/devops @greenbone/gvmd-maintainers diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ce79fbfe8..175d4a263 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -22,7 +22,7 @@ jobs: scan-build: name: scan-build (clang static analyzer) runs-on: ubuntu-latest - container: greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 @@ -46,7 +46,7 @@ jobs: test-units: name: Unit Tests runs-on: ubuntu-latest - container: greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index d326f8c14..b927fd34d 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -1,19 +1,6 @@ name: Build Container Image Builds on: - push: - branches: - - main - tags: ["v*"] - paths: - - .github/workflows/build-container.yml - - .docker/build.Dockerfile - pull_request: - branches: - - main - paths: - - .github/workflows/build-container.yml - - .docker/build.Dockerfile workflow_dispatch: repository_dispatch: schedule: @@ -22,51 +9,43 @@ on: jobs: build-images: + strategy: + matrix: + build: + - stable + - edge name: "Build Images" runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - uses: greenbone/actions/is-latest-tag@v3 - id: latest - name: Setup container meta information id: meta uses: docker/metadata-action@v5 with: - images: ${{ github.repository }}-build + images: ${{ vars.IMAGE_REGISTRY }}/${{ github.repository }}-build labels: | org.opencontainers.image.vendor=Greenbone org.opencontainers.image.base.name=greenbone/gvm-libs flavor: latest=false # no latest container tag for git tags tags: | - # use version, major.minor and major for tags - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - # use edge for default branch - type=edge - - # set label for non-published pull request builds - type=ref,event=pr - - # when a new git tag is created set stable and a latest tags - type=raw,value=latest,enable=${{ steps.latest.outputs.is-latest-tag == 'true' }} - type=raw,value=stable,enable=${{ steps.latest.outputs.is-latest-tag == 'true' }} + type=raw,value=latest,enable=${{ matrix.build == 'stable' }} + type=raw,value=stable,enable=${{ matrix.build == 'stable' }} + type=raw,value=edge,enable=${{ matrix.build == 'edge' }} - name: Set container build options id: container-opts run: | - if [[ "${{ github.ref_type }}" = 'tag' ]]; then + if [[ "${{ matrix.build }}" = 'stable' ]]; then echo "gvm-libs-version=oldstable" >> $GITHUB_OUTPUT else echo "gvm-libs-version=oldstable-edge" >> $GITHUB_OUTPUT fi - - name: Login to DockerHub - if: github.event_name != 'pull_request' + - name: Login to GitHub Docker registry uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ secrets.GREENBONE_BOT }} + password: ${{ secrets.GREENBONE_BOT_PACKAGES_WRITE_TOKEN }} - run: echo "Build and push ${{ steps.meta.outputs.tags }}" - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -76,17 +55,10 @@ jobs: uses: docker/build-push-action@v5 with: context: . - push: ${{ github.event_name != 'pull_request' }} + push: true build-args: | GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }} file: .docker/build.Dockerfile platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: Trigger libtheia container build - if: github.event_name != 'pull_request' - run: | - curl -X POST https://api.github.com/repos/greenbone/libtheia/actions/workflows/container.yml/dispatches \ - -H "Accept: application/vnd.github.v3+json" \ - -u greenbonebot:${{ secrets.GREENBONE_BOT_TOKEN }} \ - -d '{"ref":"main"}' diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index f5901f219..6557c2ab6 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -16,7 +16,7 @@ jobs: build-gmp-doc: name: Build GMP documentation runs-on: ubuntu-latest - container: greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable steps: - name: Check out gvmd uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis-c.yml b/.github/workflows/codeql-analysis-c.yml index 42f53118a..add9937b5 100644 --- a/.github/workflows/codeql-analysis-c.yml +++ b/.github/workflows/codeql-analysis-c.yml @@ -19,7 +19,7 @@ jobs: actions: read contents: read security-events: write - container: greenbone/gvmd-build:stable + container: ${{ vars.IMAGE_REGISTRY }}/greenbone/gvmd-build:stable strategy: fail-fast: false diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 78a688f8d..7b534424d 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -71,6 +71,7 @@ jobs: build-args: | VERSION=${{ steps.container-opts.outputs.version }} GVM_LIBS_VERSION=${{ steps.container-opts.outputs.gvm-libs-version }} + IMAGE_REGISTRY=${{ vars.IMAGE_REGISTRY }} file: .docker/prod.Dockerfile platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} diff --git a/CMakeLists.txt b/CMakeLists.txt index a915ed55a..6f5f64a61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0) message ("-- Configuring Greenbone Vulnerability Manager...") project (gvm - VERSION 23.4.1 + VERSION 23.5.1 LANGUAGES C) if (POLICY CMP0005) diff --git a/INSTALL.md b/INSTALL.md index 5b743f248..ac327bc57 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -705,9 +705,6 @@ Prerequisites for S/MIME support (e.g. email encryption): Prerequisites for certificate generation: * GnuTLS certtool (Debian package: gnutls-bin) -Prerequisites (recommended) to lower sync RAM usage -* xml_split (Debian package: xml-twig-tools) - ## Static code analysis with the Clang Static Analyzer If you want to use the Clang Static Analyzer (https://clang-analyzer.llvm.org/) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a81d9286..c2f3aebb0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,10 +26,10 @@ find_package (Threads) ## list and throw an error, otherwise long install-cmake-install-cmake cycles ## might occur. -pkg_check_modules (LIBGVM_BASE REQUIRED libgvm_base>=22.8) -pkg_check_modules (LIBGVM_UTIL REQUIRED libgvm_util>=22.8) -pkg_check_modules (LIBGVM_OSP REQUIRED libgvm_osp>=22.8) -pkg_check_modules (LIBGVM_GMP REQUIRED libgvm_gmp>=22.8) +pkg_check_modules (LIBGVM_BASE REQUIRED libgvm_base>=22.9) +pkg_check_modules (LIBGVM_UTIL REQUIRED libgvm_util>=22.9) +pkg_check_modules (LIBGVM_OSP REQUIRED libgvm_osp>=22.9) +pkg_check_modules (LIBGVM_GMP REQUIRED libgvm_gmp>=22.9) pkg_check_modules (GNUTLS REQUIRED gnutls>=3.2.15) pkg_check_modules (GLIB REQUIRED glib-2.0>=2.42) pkg_check_modules (LIBBSD REQUIRED libbsd) @@ -55,14 +55,6 @@ elseif ((CMAKE_MATCH_1 EQUAL 9 AND CMAKE_MATCH_2 LESS 6) message (STATUS "PostgreSQL version ${CMAKE_MATCH_1}.${CMAKE_MATCH_2}${CMAKE_MATCH_3}") endif (NOT CMAKE_MATCH_1) -message (STATUS "Looking for xml_split...") -find_program (XML_SPLIT_EXECUTABLE xml_split DOC "xml_split") -if (NOT XML_SPLIT_EXECUTABLE) - message (WARNING "xml_split is recommended to reduce SCAP sync memory usage (Debian package xml-twig-tools).") -else (NOT XML_SPLIT_EXECUTABLE) - message (STATUS "Looking for xml_split... ${XML_SPLIT_EXECUTABLE}") -endif (NOT XML_SPLIT_EXECUTABLE) - message (STATUS "Looking for xsltproc...") find_program (XSLTPROC_EXECUTABLE xsltproc DOC "xsltproc") if (NOT XSLTPROC_EXECUTABLE) diff --git a/src/alert_methods/TippingPoint/alert b/src/alert_methods/TippingPoint/alert index fcde932d2..518cb9a32 100644 --- a/src/alert_methods/TippingPoint/alert +++ b/src/alert_methods/TippingPoint/alert @@ -25,12 +25,6 @@ CONVERT_SCRIPT=$4 AUTH_PATH=$5 REPORT_PATH=$6 -# Function to encode for URL -urlencode () { - RET=$(python -c "import urllib, sys; print urllib.quote(sys.argv[1])" "$1") - echo "$RET" -} - # Create temp file for converted report REPORT_DATE=$(xmlstarlet sel -t -v "report/timestamp" < $REPORT_PATH) EXIT_CODE=$? @@ -76,9 +70,9 @@ END_TIME=$(TZ=UTC date -d "$END_TIME" +%Y-%m-%dT%H:%M:%S.000Z) RUNTIME="$START_TIME/$END_TIME" # Upload the report -VENDOR=$(urlencode "Greenbone") -PRODUCT=$(urlencode "Greenbone Vulnerability Manager") -FORMAT_VERSION=$(urlencode "1.0.0") +VENDOR="Greenbone" +PRODUCT="Greenbone%20Vulnerability%20Manager" +FORMAT_VERSION="1.0.0" CN_REPLACEMENT="Tippingpoint" if [ "1" = $CERT_WORKAROUND ] diff --git a/src/alert_methods/TippingPoint/report-convert.py b/src/alert_methods/TippingPoint/report-convert.py index cf4a49322..c2768ff9c 100755 --- a/src/alert_methods/TippingPoint/report-convert.py +++ b/src/alert_methods/TippingPoint/report-convert.py @@ -137,6 +137,10 @@ def convert (xml_tree, out_file): nvt_cve = ''; nvt_elem = result_elem.find ('nvt') nvt_refs = nvt_elem.find ('refs'); + + if (nvt_refs is None): + continue + for ref in nvt_refs.findall('ref'): if (ref.attrib['type'] == 'cve'): if (nvt_cve == ''): diff --git a/src/gmp.c b/src/gmp.c index 4c31124ea..492c39920 100644 --- a/src/gmp.c +++ b/src/gmp.c @@ -8162,31 +8162,41 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, if (include_notes_details == 0) { - const char *text = note_iterator_text (notes); - gchar *excerpt = utf8_substring (text, 0, - setting_excerpt_size_int ()); + gchar *excerpt; + const char *text; + + text = note_iterator_text (notes); + excerpt = utf8_substring (text, 0, setting_excerpt_size_int ()); + /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" - "1" - "0" - "%i" - "%s" - "%i", + "", get_iterator_owner_name (notes) ? get_iterator_owner_name (notes) : "", note_iterator_nvt_oid (notes), note_iterator_nvt_name (notes), - note_iterator_nvt_type (notes), - get_iterator_creation_time (notes), - get_iterator_modification_time (notes), + note_iterator_nvt_type (notes)); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (notes))); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_modification_time (notes))); + + buffer_xml_append_printf (buffer, + "1" + "0" + "%i" + "%s" + "%i", note_iterator_active (notes), strlen (excerpt) < strlen (text), excerpt, @@ -8229,15 +8239,33 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, end_time = note_iterator_end_time (notes); /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" + "", + get_iterator_owner_name (notes) + ? get_iterator_owner_name (notes) + : "", + note_iterator_nvt_oid (notes), + note_iterator_nvt_name (notes), + note_iterator_nvt_type (notes)); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_creation_time (notes))); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_modification_time (notes))); + + buffer_xml_append_printf + (buffer, "1" "0" "%i" @@ -8248,14 +8276,6 @@ buffer_notes_xml (GString *buffer, iterator_t *notes, int include_notes_details, "%s" "%s%i" "%i", - get_iterator_owner_name (notes) - ? get_iterator_owner_name (notes) - : "", - note_iterator_nvt_oid (notes), - note_iterator_nvt_name (notes), - note_iterator_nvt_type (notes), - get_iterator_creation_time (notes), - get_iterator_modification_time (notes), note_iterator_active (notes), end_time > 1 ? iso_time (&end_time) : "", note_iterator_text (notes), @@ -8426,18 +8446,36 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, if (include_overrides_details == 0) { - const char *text = override_iterator_text (overrides); - gchar *excerpt = utf8_substring (text, 0, - setting_excerpt_size_int ()); + gchar *excerpt; + const char *text; + + text = override_iterator_text (overrides); + excerpt = utf8_substring (text, 0, setting_excerpt_size_int ()); + /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" + "", + get_iterator_owner_name (overrides) + ? get_iterator_owner_name (overrides) + : "", + override_iterator_nvt_oid (overrides), + override_iterator_nvt_name (overrides), + override_iterator_nvt_type (overrides)); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (overrides))); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_modification_time (overrides))); + + buffer_xml_append_printf (buffer, "1" "0" "%i" @@ -8447,14 +8485,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, "%s" "%s" "%i", - get_iterator_owner_name (overrides) - ? get_iterator_owner_name (overrides) - : "", - override_iterator_nvt_oid (overrides), - override_iterator_nvt_name (overrides), - override_iterator_nvt_type (overrides), - get_iterator_creation_time (overrides), - get_iterator_modification_time (overrides), override_iterator_active (overrides), strlen (excerpt) < strlen (text), excerpt, @@ -8505,15 +8535,33 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, end_time = override_iterator_end_time (overrides); /* This must match send_get_common. */ + buffer_xml_append_printf (buffer, "%s" "" "%s" "%s" - "" - "%s" - "%s" + "", + get_iterator_owner_name (overrides) + ? get_iterator_owner_name (overrides) + : "", + override_iterator_nvt_oid (overrides), + override_iterator_nvt_name (overrides), + override_iterator_nvt_type (overrides)); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_creation_time (overrides))); + + buffer_xml_append_printf + (buffer, + "%s", + iso_if_time (get_iterator_modification_time (overrides))); + + buffer_xml_append_printf + (buffer, "1" "0" "%i" @@ -8527,14 +8575,6 @@ buffer_overrides_xml (GString *buffer, iterator_t *overrides, "%s" "%s%i" "%i", - get_iterator_owner_name (overrides) - ? get_iterator_owner_name (overrides) - : "", - override_iterator_nvt_oid (overrides), - override_iterator_nvt_name (overrides), - override_iterator_nvt_type (overrides), - get_iterator_creation_time (overrides), - get_iterator_modification_time (overrides), override_iterator_active (overrides), end_time > 1 ? iso_time (&end_time) : "", override_iterator_text (overrides), @@ -9334,7 +9374,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, int changed, int cert_loaded, int lean, int use_delta_fields) { - const char *descr, *name, *comment, *creation_time; + const char *descr, *name, *comment; const char *severity, *original_severity, *original_level; const char *host, *hostname, *result_id, *port, *path, *asset_id, *qod, *qod_type; char *detect_oid, *detect_ref, *detect_cpe, *detect_loc, *detect_name; @@ -9343,6 +9383,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, result_t result; report_t report; task_t selected_task; + time_t creation_time; comment = get_iterator_comment (results); name = get_iterator_name (results); @@ -9414,7 +9455,8 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, if (lean == 0) { - const char *owner_name, *modification_time; + const char *owner_name; + time_t modification_time; if (use_delta_fields) { @@ -9435,7 +9477,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, if (modification_time) buffer_xml_append_printf (buffer, "%s", - modification_time); + iso_time (&modification_time) ?: ""); } if (comment @@ -9447,7 +9489,7 @@ buffer_results_xml (GString *buffer, iterator_t *results, task_t task, if (creation_time) buffer_xml_append_printf (buffer, "%s", - creation_time); + iso_time (&creation_time) ?: ""); if (include_details) { @@ -11671,20 +11713,26 @@ handle_get_assets (gmp_parser_t *gmp_parser, GError **error) xml_string_append (result, "" "%s" - "%s" - "%s" - "%s" + "%s", + get_iterator_uuid (&identifiers), + get_iterator_name (&identifiers), + host_identifier_iterator_value (&identifiers)); + + xml_string_append (result, + "%s", + iso_if_time (get_iterator_creation_time (&identifiers))); + + xml_string_append (result, + "%s", + iso_if_time (get_iterator_modification_time (&identifiers))); + + xml_string_append (result, "" "%s" "%s" "%i" "%s" "", - get_iterator_uuid (&identifiers), - get_iterator_name (&identifiers), - host_identifier_iterator_value (&identifiers), - get_iterator_creation_time (&identifiers), - get_iterator_modification_time (&identifiers), host_identifier_iterator_source_id (&identifiers), source_type, @@ -14717,17 +14765,12 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) task_t task; /* Send the standard elements. Should match send_get_common. */ + buffer_xml_append_printf (prefix, "%s" "%s" - "%s" - "%s" - "" - "%s" - "" - "0" - "0", + "%s", get_iterator_owner_name (&reports) ? get_iterator_owner_name (&reports) : "", @@ -14736,13 +14779,20 @@ handle_get_reports (gmp_parser_t *gmp_parser, GError **error) : "", get_iterator_comment (&reports) ? get_iterator_comment (&reports) - : "", - get_iterator_creation_time (&reports) - ? get_iterator_creation_time (&reports) - : "", - get_iterator_modification_time (&reports) - ? get_iterator_modification_time (&reports) : ""); + + buffer_xml_append_printf + (prefix, + "%s", + iso_if_time (get_iterator_creation_time (&reports))); + + buffer_xml_append_printf + (prefix, + "%s" + "0" + "0", + iso_if_time (get_iterator_modification_time (&reports))); + /* Send short task and report format info */ report_task (report, &task); if (task) @@ -18627,16 +18677,19 @@ handle_get_vulns (gmp_parser_t *gmp_parser, GError **error) count ++; SENDF_TO_CLIENT_OR_FAIL ("" "%s" - "%s" - "%s" - "%s" - "%1.1f" - "%d", + "%s", get_iterator_uuid (&vulns), get_iterator_name (&vulns), - vuln_iterator_type (&vulns), - get_iterator_creation_time (&vulns), - get_iterator_modification_time (&vulns), + vuln_iterator_type (&vulns)); + + SENDF_TO_CLIENT_OR_FAIL ("%s", + iso_if_time (get_iterator_creation_time (&vulns))); + + SENDF_TO_CLIENT_OR_FAIL ("%s", + iso_if_time (get_iterator_modification_time (&vulns))); + + SENDF_TO_CLIENT_OR_FAIL ("%1.1f" + "%d", vuln_iterator_severity (&vulns), vuln_iterator_qod (&vulns)); diff --git a/src/gmp_get.c b/src/gmp_get.c index b13e73192..3793a49d6 100644 --- a/src/gmp_get.c +++ b/src/gmp_get.c @@ -323,12 +323,7 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator, "<%s id=\"%s\">" "%s" "%s" - "%s" - "%s" - "%s" - "%i" - "%i" - "", + "%s", type, get_iterator_uuid (iterator) ? get_iterator_uuid (iterator) @@ -341,13 +336,18 @@ send_get_common (const char *type, get_data_t *get, iterator_t *iterator, : "", get_iterator_comment (iterator) ? get_iterator_comment (iterator) - : "", - get_iterator_creation_time (iterator) - ? get_iterator_creation_time (iterator) - : "", - get_iterator_modification_time (iterator) - ? get_iterator_modification_time (iterator) - : "", + : ""); + + buffer_xml_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (iterator))); + + buffer_xml_append_printf (buffer, + "%s" + "%i" + "%i" + "", + iso_if_time (get_iterator_modification_time (iterator)), writable, in_use); diff --git a/src/manage.c b/src/manage.c index 249ab1522..b239b0649 100644 --- a/src/manage.c +++ b/src/manage.c @@ -3167,7 +3167,7 @@ static int fork_cve_scan_handler (task_t task, target_t target) { int pid; - char *report_id, *hosts; + char *report_id, *hosts, *exclude_hosts; gvm_hosts_t *gvm_hosts; gvm_host_t *gvm_host; @@ -3234,6 +3234,8 @@ fork_cve_scan_handler (task_t task, target_t target) exit (1); } + exclude_hosts = target_exclude_hosts (target); + reset_task (task); set_task_start_time_epoch (task, time (NULL)); set_scan_start_time_epoch (global_current_report, time (NULL)); @@ -3242,6 +3244,20 @@ fork_cve_scan_handler (task_t task, target_t target) gvm_hosts = gvm_hosts_new (hosts); free (hosts); + + if (gvm_hosts_exclude (gvm_hosts, exclude_hosts ?: "") < 0) + { + set_task_interrupted (task, + "Failed to exclude hosts." + " Interrupting scan."); + set_report_scan_run_status (global_current_report, TASK_STATUS_INTERRUPTED); + gvm_hosts_free (gvm_hosts); + free (exclude_hosts); + gvm_close_sentry (); + exit(1); + } + free (exclude_hosts); + while ((gvm_host = gvm_hosts_next (gvm_hosts))) if (cve_scan_host (task, global_current_report, gvm_host)) { @@ -5830,11 +5846,18 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, nvt_iterator_detection (nvts)); } + g_string_append_printf (buffer, + "%s", + iso_if_time (get_iterator_creation_time (nvts))); + + g_string_append_printf (buffer, + "%s", + iso_if_time (get_iterator_modification_time (nvts))); + default_timeout = nvt_default_timeout (oid); + g_string_append_printf (buffer, "%s" - "%s" - "%s" "%d" "%s" "" @@ -5844,18 +5867,13 @@ get_nvt_xml (iterator_t *nvts, int details, int pref_count, "%s" "%s", default_timeout ? default_timeout : "", - get_iterator_creation_time (nvts) - ? get_iterator_creation_time (nvts) - : "", - get_iterator_modification_time (nvts) - ? get_iterator_modification_time (nvts) - : "", nvt_iterator_category (nvts), family_text, nvt_iterator_qod (nvts), nvt_iterator_qod_type (nvts), refs_str->str, nvt_tags->str); + free (default_timeout); g_string_free (nvt_tags, 1); diff --git a/src/manage.h b/src/manage.h index ed5d12519..30e0642bb 100644 --- a/src/manage.h +++ b/src/manage.h @@ -1564,10 +1564,10 @@ result_iterator_delta_uuid (iterator_t*); const char * result_iterator_delta_qod_type (iterator_t*); -const char * +time_t result_iterator_delta_creation_time (iterator_t*); -const char * +time_t result_iterator_delta_modification_time (iterator_t*); task_t @@ -3449,6 +3449,9 @@ setting_iterator_comment (iterator_t*); const char* setting_iterator_value (iterator_t*); +int +setting_value_int (const char *, int *); + int modify_setting (const gchar *, const gchar *, const gchar *, gchar **); diff --git a/src/manage_get.c b/src/manage_get.c index 2b81ac2f8..26b32a12c 100644 --- a/src/manage_get.c +++ b/src/manage_get.c @@ -154,18 +154,28 @@ get_iterator_comment (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return Creation time of the resource or NULL if iteration is complete. + * @return Creation time, or 0 if iteration is complete. */ -DEF_ACCESS (get_iterator_creation_time, 4); +time_t +get_iterator_creation_time (iterator_t* iterator) +{ + if (iterator->done) return 0; + return iterator_int64 (iterator, 4); +} /** * @brief Get the modification time of the resource from a GET iterator. * * @param[in] iterator Iterator. * - * @return Modification time of the resource or NULL if iteration is complete. + * @return Modification time, or 0 if iteration is complete. */ -DEF_ACCESS (get_iterator_modification_time, 5); +time_t +get_iterator_modification_time (iterator_t* iterator) +{ + if (iterator->done) return 0; + return iterator_int64 (iterator, 5); +} /** * @brief Get the owner name of the resource from a GET iterator. diff --git a/src/manage_get.h b/src/manage_get.h index 5c44e667c..2b7bfce24 100644 --- a/src/manage_get.h +++ b/src/manage_get.h @@ -66,10 +66,10 @@ get_iterator_name (iterator_t*); const char* get_iterator_comment (iterator_t*); -const char* +time_t get_iterator_creation_time (iterator_t*); -const char* +time_t get_iterator_modification_time (iterator_t*); const char* diff --git a/src/manage_sql.c b/src/manage_sql.c index c39503e5f..567d8ade6 100644 --- a/src/manage_sql.c +++ b/src/manage_sql.c @@ -328,9 +328,6 @@ static void set_credential_snmp_secret (credential_t, const char *, const char *, const char *); -static int -setting_value_int (const char *, int *); - static int setting_auto_cache_rebuild_int (); @@ -12142,25 +12139,21 @@ generate_alert_filter_get (alert_t alert, const get_data_t *base_get_data, if (filter_return) *filter_return = filter; + (*alert_filter_get) = g_malloc0 (sizeof (get_data_t)); + (*alert_filter_get)->details = base_get_data->details; + (*alert_filter_get)->ignore_pagination = base_get_data->ignore_pagination; + (*alert_filter_get)->ignore_max_rows_per_page + = base_get_data->ignore_max_rows_per_page; + if (filter) { - (*alert_filter_get) = g_malloc0 (sizeof (get_data_t)); - (*alert_filter_get)->details = base_get_data->details; - (*alert_filter_get)->ignore_pagination = base_get_data->ignore_pagination; - (*alert_filter_get)->ignore_max_rows_per_page - = base_get_data->ignore_max_rows_per_page; (*alert_filter_get)->filt_id = g_strdup (filt_id); (*alert_filter_get)->filter = filter_term (filt_id); } else - (*alert_filter_get) = NULL; - - ignore_pagination = alert_data (alert, "method", - "composer_ignore_pagination"); - if (ignore_pagination) { - (*alert_filter_get)->ignore_pagination = atoi (ignore_pagination); - g_free (ignore_pagination); + (*alert_filter_get)->filt_id = NULL; + (*alert_filter_get)->filter = g_strdup(""); } /* Adjust filter for report composer. @@ -12171,39 +12164,44 @@ generate_alert_filter_get (alert_t alert, const get_data_t *base_get_data, * We simply use these fields to adjust the filter. In the future we'll * remove the filter terms and extend the way we get the report. */ - if (filter) + gchar *include_notes, *include_overrides; + + ignore_pagination = alert_data (alert, "method", + "composer_ignore_pagination"); + if (ignore_pagination) { - gchar *include_notes, *include_overrides; + (*alert_filter_get)->ignore_pagination = atoi (ignore_pagination); + g_free (ignore_pagination); + } - include_notes = alert_data (alert, "method", - "composer_include_notes"); - if (include_notes) - { - gchar *new_filter; + include_notes = alert_data (alert, "method", + "composer_include_notes"); + if (include_notes) + { + gchar *new_filter; - new_filter = g_strdup_printf ("notes=%i %s", - atoi (include_notes), - (*alert_filter_get)->filter); - g_free ((*alert_filter_get)->filter); - (*alert_filter_get)->filter = new_filter; - (*alert_filter_get)->filt_id = NULL; - g_free (include_notes); - } + new_filter = g_strdup_printf ("notes=%i %s", + atoi (include_notes), + (*alert_filter_get)->filter); + g_free ((*alert_filter_get)->filter); + (*alert_filter_get)->filter = new_filter; + (*alert_filter_get)->filt_id = NULL; + g_free (include_notes); + } - include_overrides = alert_data (alert, "method", - "composer_include_overrides"); - if (include_overrides) - { - gchar *new_filter; + include_overrides = alert_data (alert, "method", + "composer_include_overrides"); + if (include_overrides) + { + gchar *new_filter; - new_filter = g_strdup_printf ("overrides=%i %s", - atoi (include_overrides), - (*alert_filter_get)->filter); - g_free ((*alert_filter_get)->filter); - (*alert_filter_get)->filter = new_filter; - (*alert_filter_get)->filt_id = NULL; - g_free (include_overrides); - } + new_filter = g_strdup_printf ("overrides=%i %s", + atoi (include_overrides), + (*alert_filter_get)->filter); + g_free ((*alert_filter_get)->filter); + (*alert_filter_get)->filter = new_filter; + (*alert_filter_get)->filt_id = NULL; + g_free (include_overrides); } return 0; @@ -16005,6 +16003,19 @@ check_db_settings () " 'Delta Reports Version'," " 'Version of the generation of the Delta Reports.'," " '2' );"); + + if (sql_int ("SELECT count(*) FROM settings" + " WHERE uuid = '" SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "'" + " AND " ACL_IS_GLOBAL () ";") + == 0) + sql ("INSERT into settings (uuid, owner, name, comment, value)" + " VALUES" + " ('" SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "', NULL," + " 'SecInfo SQL Buffer Threshold'," + " 'Buffer size threshold in MiB for running buffered SQL statements'" + " || ' in SecInfo updates before the end of the file'" + " || ' being processed.'," + " '100' );"); } /** @@ -21679,8 +21690,8 @@ report_add_results_array (report_t report, GArray *results) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "iso_time (creation_time)", "name", KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = reports.owner)", \ @@ -22152,12 +22163,12 @@ where_qod (int min_qod) "name", \ KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + { "date", \ "creation_time", \ - KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + KEYWORD_TYPE_INTEGER }, \ + { "date", \ "modification_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "date", "created", KEYWORD_TYPE_INTEGER }, \ { "date", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = results.owner)", \ @@ -22305,12 +22316,12 @@ where_qod (int min_qod) "name", \ KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + { "date", \ "creation_time", \ - KEYWORD_TYPE_STRING }, \ - { " iso_time (date, opts.user_zone)", \ + KEYWORD_TYPE_INTEGER }, \ + { "date", \ "modification_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "date", "created", KEYWORD_TYPE_INTEGER }, \ { "date", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = results.owner)", \ @@ -22467,12 +22478,12 @@ where_qod (int min_qod) { "comparison.delta_qod", NULL, KEYWORD_TYPE_INTEGER }, \ { "comparison.delta_uuid", NULL, KEYWORD_TYPE_STRING }, \ { "delta_qod_type", NULL, KEYWORD_TYPE_STRING }, \ - { " iso_time (delta_date, opts.user_zone)", \ + { "delta_date", \ "delta_creation_time", \ - KEYWORD_TYPE_STRING }, \ - { " iso_time (delta_date, opts.user_zone)", \ + KEYWORD_TYPE_INTEGER }, \ + { "delta_date", \ "delta_modification_time", \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "delta_task", NULL, KEYWORD_TYPE_INTEGER }, \ { "delta_report", NULL, KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = results.owner)", \ @@ -23906,13 +23917,13 @@ result_iterator_delta_qod_type (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return delta creation time if any, else NULL. + * @return Time, or 0 if iteration is complete. */ -const char * +time_t result_iterator_delta_creation_time (iterator_t* iterator) { if (iterator->done) return 0; - return iterator_string (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6); + return iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 6); } /** @@ -23920,13 +23931,13 @@ result_iterator_delta_creation_time (iterator_t* iterator) * * @param[in] iterator Iterator. * - * @return delta modification time if any, else NULL. + * @return Time, or 0 if iteration is complete. */ -const char * +time_t result_iterator_delta_modification_time (iterator_t* iterator) { if (iterator->done) return 0; - return iterator_string (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7); + return iterator_int64 (iterator, RESULT_ITERATOR_DELTA_COLUMN_OFFSET + 7); } /** @@ -27919,23 +27930,41 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta, "nvts_cols"); extra_with = g_strdup_printf(" comparison AS (" - " WITH r1 as (SELECT results.id, description, host, report, port," + " WITH r1a as (SELECT results.id, description, host, report, port," " severity, nvt, results.qod, results.uuid, hostname," " path, r1_lateral.new_severity as new_severity " " FROM results " - " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid from nvts)" + " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid FROM nvts)" " AS nvts_cols" " ON nvts_cols.nvts_oid = results.nvt" " %s, LATERAL %s AS r1_lateral" " WHERE report = %llu)," - " r2 as (SELECT results.*, r2_lateral.new_severity AS new_severity" + " r2a as (SELECT results.*, r2_lateral.new_severity AS new_severity" " FROM results" - " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid from nvts)" + " LEFT JOIN (SELECT cvss_base, oid AS nvts_oid FROM nvts)" " AS nvts_cols" " ON nvts_cols.nvts_oid = results.nvt" " %s, LATERAL %s AS r2_lateral" - " WHERE report = %llu)" - " SELECT r1.id AS result1_id," + " WHERE report = %llu)," + " r1 as (SELECT DISTINCT ON (r1a.id) r1a.*, r2a.id as r2id, row_number() over w1 as r1_rank" + " FROM r1a LEFT JOIN r2a ON r1a.host = r2a.host" + " AND normalize_port(r1a.port) = normalize_port(r2a.port)" + " AND r1a.nvt = r2a.nvt " + " AND (r1a.new_severity = 0) = (r2a.new_severity = 0)" + " AND (r1a.description = r2a.description)" + " WINDOW w1 AS (PARTITION BY r1a.host, normalize_port(r1a.port)," + " r1a.nvt, r1a.new_severity = 0, r2a.id is null ORDER BY r2a.id)" + " ORDER BY r1a.id)," + " r2 as (SELECT DISTINCT ON (r2a.id) r2a.*, r1a.id as r1id, row_number() over w2 as r2_rank" + " FROM r2a LEFT JOIN r1a ON r2a.host = r1a.host" + " AND normalize_port(r2a.port) = normalize_port(r1a.port)" + " AND r2a.nvt = r1a.nvt " + " AND (r2a.new_severity = 0) = (r1a.new_severity = 0)" + " AND (r2a.description = r1a.description)" + " WINDOW w2 AS (PARTITION BY r2a.host, normalize_port(r2a.port)," + " r2a.nvt, r2a.new_severity = 0, r1a.id is null ORDER BY r1a.id)" + " ORDER BY r2a.id)" + " (SELECT r1.id AS result1_id," " r2.id AS result2_id," " compare_results(" " r1.description," @@ -27961,7 +27990,7 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta, " r2.path AS delta_path," " r2.host AS delta_host," RESULT_HOSTNAME_SQL("r2.hostname", "r2.host", "r2.report") - " AS delta_hostname," + " AS delta_hostname," " r2.nvt_version AS delta_nvt_version" " FROM r1" " FULL OUTER JOIN r2" @@ -27969,20 +27998,10 @@ init_v2_delta_iterator (report_t report, iterator_t *results, report_t delta, " AND normalize_port(r1.port) = normalize_port(r2.port)" " AND r1.nvt = r2.nvt " " AND (r1.new_severity = 0) = (r2.new_severity = 0)" - " AND (r1.description = r2.description" - " OR NOT EXISTS (SELECT * FROM r2" - " WHERE r1.description = r2.description" - " AND r1.host = r2.host" - " AND normalize_port(r1.port) = normalize_port(r2.port)" - " AND r1.nvt = r2.nvt" - " AND (r1.new_severity = 0) = (r2.new_severity = 0))" - " OR NOT EXISTS (SELECT * FROM r1" - " WHERE r1.description = r2.description" - " AND r1.host = r2.host" - " AND normalize_port(r1.port) = normalize_port(r2.port)" - " AND r1.nvt = r2.nvt" - " AND (r1.new_severity = 0) = (r2.new_severity = 0)))" - " )", + " AND ((r1id IS NULL AND r2id IS NULL) OR" + " r2id = r2.id OR r1id = r1.id)" + " AND r1_rank = r2_rank" + " ) ) ", opts_tables, with_lateral, report, @@ -38530,8 +38549,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt, KEYWORD_TYPE_STRING \ }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes.creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes.modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "notes.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "notes.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "notes.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "notes.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = notes.owner)", \ @@ -38585,8 +38604,8 @@ modify_note (const gchar *note_id, const char *active, const char *nvt, { "notes_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes_trash.creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (notes_trash.modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "notes_trash.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "notes_trash.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "notes_trash.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "notes_trash.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "(SELECT name FROM users WHERE users.id = notes_trash.owner)", \ @@ -39767,8 +39786,8 @@ modify_override (const gchar *override_id, const char *active, const char *nvt, KEYWORD_TYPE_STRING \ }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides.creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides.modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "overrides.creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "overrides.modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "overrides.creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "overrides.modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -39835,12 +39854,12 @@ modify_override (const gchar *override_id, const char *active, const char *nvt, { "overrides_trash.uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ { "CAST ('' AS TEXT)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides_trash.creation_time)", \ + { "overrides_trash.creation_time", \ NULL, \ - KEYWORD_TYPE_STRING }, \ - { "iso_time (overrides_trash.modification_time)", \ + KEYWORD_TYPE_INTEGER }, \ + { "overrides_trash.modification_time", \ NULL, \ - KEYWORD_TYPE_STRING }, \ + KEYWORD_TYPE_INTEGER }, \ { "overrides_trash.creation_time", \ "created", \ KEYWORD_TYPE_INTEGER }, \ @@ -50155,9 +50174,9 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, if (host) init_iterator (iterator, - "SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," - " modification_time, owner, owner, value," + "SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time AS created," + " modification_time AS modified, owner, owner, value," " source_type, source_id, source_data," " (CASE WHEN source_type LIKE 'Report%%'" " THEN NOT EXISTS (SELECT * FROM reports" @@ -50168,9 +50187,9 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, " FROM host_identifiers" " WHERE host = %llu" " UNION" - " SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," - " modification_time, owner, owner," + " SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time AS created," + " modification_time AS modified, owner, owner," " (SELECT name FROM oss WHERE id = os)," " source_type, source_id, source_data," " (CASE WHEN source_type LIKE 'Report%%'" @@ -50189,9 +50208,9 @@ init_host_identifier_iterator (iterator_t* iterator, host_t host, ascending ? "ASC" : "DESC"); else init_iterator (iterator, - "SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," - " modification_time, owner, owner, value," + "SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time AS created," + " modification_time AS modified, owner, owner, value," " source_type, source_id, source_data, 0, '', ''" " FROM host_identifiers" " ORDER BY %s %s;", @@ -50921,8 +50940,8 @@ init_os_host_iterator (iterator_t* iterator, resource_t os) { assert (os); init_iterator (iterator, - "SELECT id, uuid, name, comment, iso_time (creation_time)," - " iso_time (modification_time), creation_time," + "SELECT id, uuid, name, comment, creation_time," + " modification_time, creation_time," " modification_time, owner, owner," " (SELECT round (CAST (severity AS numeric), 1)" " FROM host_max_severities" @@ -51978,7 +51997,7 @@ setting_value (const char *uuid, char **value) * * @return 0 success, -1 error. */ -static int +int setting_value_int (const char *uuid, int *value) { gchar *quoted_uuid; @@ -52683,6 +52702,8 @@ setting_name (const gchar *uuid) return "Feed Import Roles"; if (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0) return "Delta Reports Version"; + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0) + return "SecInfo SQL Buffer Threshold"; return NULL; } @@ -52722,12 +52743,15 @@ setting_description (const gchar *uuid) return "Roles given access to new resources from feed."; if (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0) return "Version of the generation of the Delta Reports."; + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0) + return "Buffer size threshold in MiB for running buffered SQL statements" + " in SecInfo updates before the end of the file being processed."; return NULL; } /** - * @brief Get the name of a setting. + * @brief Verify the value of a setting. * * @param[in] uuid UUID of setting. * @param[in] value Value of setting, to verify. @@ -52815,6 +52839,14 @@ setting_verify (const gchar *uuid, const gchar *value, const gchar *user) return 1; } + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD)) + { + int threshold; + threshold = atoi (value); + if (threshold < 0 || threshold > (INT_MAX / 1048576)) + return 1; + } + return 0; } @@ -52870,6 +52902,15 @@ setting_normalise (const gchar *uuid, const gchar *value) return g_string_free (normalised, FALSE); } + if (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0) + { + int threshold; + threshold = atoi (value); + if (threshold < 0) + return NULL; + return g_strdup_printf ("%i", threshold); + } + return g_strdup (value); } @@ -52900,7 +52941,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database, && strcmp (uuid, SETTING_UUID_LSC_DEB_MAINTAINER) && strcmp (uuid, SETTING_UUID_FEED_IMPORT_OWNER) && strcmp (uuid, SETTING_UUID_FEED_IMPORT_ROLES) - && strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION)) + && strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) + && strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD)) { fprintf (stderr, "Error in setting UUID.\n"); return 3; @@ -52927,7 +52969,8 @@ manage_modify_setting (GSList *log_config, const db_conn_info_t *database, if ((strcmp (uuid, SETTING_UUID_DEFAULT_CA_CERT) == 0) || (strcmp (uuid, SETTING_UUID_FEED_IMPORT_OWNER) == 0) || (strcmp (uuid, SETTING_UUID_FEED_IMPORT_ROLES) == 0) - || (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0)) + || (strcmp (uuid, SETTING_UUID_DELTA_REPORTS_VERSION) == 0) + || (strcmp (uuid, SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD) == 0)) { sql_rollback (); fprintf (stderr, @@ -55289,8 +55332,8 @@ user_resources_in_use (user_t user, { "uuid", "uuid", KEYWORD_TYPE_STRING }, \ { "name", "name", KEYWORD_TYPE_STRING }, \ { "''", "comment", KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { "cast (null AS text)", "_owner", KEYWORD_TYPE_INTEGER }, \ @@ -58297,23 +58340,36 @@ manage_optimize (GSList *log_config, const db_conn_info_t *database, } else if (strcasecmp (name, "cleanup-config-prefs") == 0) { - int removed, fixed_values; sql ("DELETE FROM config_preferences WHERE id NOT IN" " (SELECT min(id) FROM config_preferences" " GROUP BY config, type, name, value);"); - removed = sql_changes(); sql ("UPDATE config_preferences" " SET value = (SELECT value FROM nvt_preferences" " WHERE name='scanner_plugins_timeout')" " WHERE name = 'scanner_plugins_timeout'" " AND value = 'SCANNER_NVT_TIMEOUT';"); - fixed_values = sql_changes(); - success_text = g_strdup_printf ("Optimized: cleanup-config-prefs." - " Duplicate config preferences removed:" - " %d. Corrected preference values: %d", - removed, fixed_values); + sql ("UPDATE config_preferences" + " SET pref_nvt = NULL," + " pref_id = NULL," + " pref_type = NULL," + " pref_name = NULL" + " WHERE type = 'SERVER_PREFS' AND pref_nvt IS NOT NULL;"); + + sql ("UPDATE config_preferences" + " SET pref_nvt = substring (name, '^([^:]*)')," + " pref_id = CAST(substring (name, '^[^:]*:([0-9]+)') AS integer)," + " pref_type = substring (name, '^[^:]*:[0-9]+:([^:]*):')," + " pref_name = substring (name, '^[^:]*:[0-9]+:[^:]*:(.*)')" + " WHERE type = 'PLUGINS_PREFS'" + " AND (pref_nvt = '(null)' OR pref_nvt IS NULL" + " OR pref_type = '(null)' OR pref_type IS NULL" + " OR pref_name = '(null)' OR pref_name IS NULL)" + " AND name ~ '^[^:]*:[0-9]+:[^:]*:.*'" + " AND type = 'PLUGINS_PREFS';"); + + success_text = g_strdup_printf ("Optimized: cleanup-config-prefs."); } else if (strcasecmp (name, "cleanup-feed-permissions") == 0) { diff --git a/src/manage_sql.h b/src/manage_sql.h index 7d98f1994..9f15f1269 100644 --- a/src/manage_sql.h +++ b/src/manage_sql.h @@ -142,6 +142,10 @@ */ #define SETTING_UUID_DELTA_REPORTS_VERSION "985a0c05-2140-4e66-9989-ce9a0906a5a9" +/** + * @brief UUID of 'SecInfo SQL Buffer Threshold' setting. + */ +#define SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD "316275a9-3629-49ad-9cea-5b3ab155b93f" /** * @brief Trust constant for error. @@ -248,9 +252,9 @@ typedef struct * * @param[in] prefix Column prefix. */ -#define GET_ITERATOR_COLUMNS_STRING \ - "id, uuid, name, comment, iso_time (creation_time)," \ - " iso_time (modification_time), creation_time AS created," \ +#define GET_ITERATOR_COLUMNS_STRING \ + "id, uuid, name, comment, creation_time," \ + " modification_time, creation_time AS created," \ " modification_time AS modified" /** @@ -263,8 +267,8 @@ typedef struct { prefix "uuid", NULL, KEYWORD_TYPE_STRING }, \ { prefix "name", NULL, KEYWORD_TYPE_STRING }, \ { prefix "comment", NULL, KEYWORD_TYPE_STRING }, \ - { " iso_time (" prefix "creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { " iso_time (" prefix "modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { prefix "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { prefix "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { prefix "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { prefix "modification_time", "modified", KEYWORD_TYPE_INTEGER } diff --git a/src/manage_sql_configs.c b/src/manage_sql_configs.c index 4fe5b0814..ae4b9454f 100644 --- a/src/manage_sql_configs.c +++ b/src/manage_sql_configs.c @@ -3627,6 +3627,13 @@ modify_config_preference (config_t config, const char* nvt, g_free (quoted_name); quoted_name = sql_quote (splits[3]); } + else + { + quoted_pref_nvt = sql_quote (splits[0]); + pref_id = atoi (splits[1]); + quoted_pref_type = sql_quote (splits[2]); + quoted_pref_name = sql_quote (splits[3]); + } } g_strfreev (splits); @@ -3639,12 +3646,25 @@ modify_config_preference (config_t config, const char* nvt, config, nvt ? "= 'PLUGINS_PREFS'" : "= 'SERVER_PREFS'", quoted_name); - sql ("INSERT INTO config_preferences" - " (config, type, name, value, pref_nvt, pref_id, pref_type, pref_name)" - " VALUES (%llu, %s, '%s', '%s', '%s', %i, '%s', '%s');", - config, nvt ? "'PLUGINS_PREFS'" : "'SERVER_PREFS'", quoted_name, - quoted_value, quoted_pref_nvt, pref_id, quoted_pref_type, - quoted_pref_name); + if (nvt) + { + sql ("INSERT INTO config_preferences" + " (config, type, name, value," + " pref_nvt, pref_id, pref_type, pref_name)" + " VALUES (%llu, 'PLUGINS_PREFS', '%s', '%s'," + " '%s', %i, '%s', '%s');", + config, quoted_name, quoted_value, + quoted_pref_nvt, pref_id, quoted_pref_type, quoted_pref_name); + } + else + { + sql ("INSERT INTO config_preferences" + " (config, type, name, value," + " pref_nvt, pref_id, pref_type, pref_name)" + " VALUES (%llu, 'SERVER_PREFS', '%s', '%s'," + " NULL, NULL, NULL, NULL);", + config, quoted_name, quoted_value); + } g_free (quoted_value); g_free (quoted_name); diff --git a/src/manage_sql_report_configs.c b/src/manage_sql_report_configs.c index ed158db2f..6d4b44c15 100644 --- a/src/manage_sql_report_configs.c +++ b/src/manage_sql_report_configs.c @@ -661,8 +661,8 @@ restore_report_config (const char *report_config_id) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "comment", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -696,8 +696,8 @@ restore_report_config (const char *report_config_id) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "comment", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ diff --git a/src/manage_sql_report_formats.c b/src/manage_sql_report_formats.c index dd0f3da50..3f6df153f 100644 --- a/src/manage_sql_report_formats.c +++ b/src/manage_sql_report_formats.c @@ -2653,8 +2653,8 @@ report_format_trust (report_format_t report_format) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ @@ -2690,8 +2690,8 @@ report_format_trust (report_format_t report_format) { "uuid", NULL, KEYWORD_TYPE_STRING }, \ { "name", NULL, KEYWORD_TYPE_STRING }, \ { "''", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (creation_time)", NULL, KEYWORD_TYPE_STRING }, \ - { "iso_time (modification_time)", NULL, KEYWORD_TYPE_STRING }, \ + { "creation_time", NULL, KEYWORD_TYPE_INTEGER }, \ + { "modification_time", NULL, KEYWORD_TYPE_INTEGER }, \ { "creation_time", "created", KEYWORD_TYPE_INTEGER }, \ { "modification_time", "modified", KEYWORD_TYPE_INTEGER }, \ { \ diff --git a/src/manage_sql_secinfo.c b/src/manage_sql_secinfo.c index 4911b73d8..0a2d90cc7 100644 --- a/src/manage_sql_secinfo.c +++ b/src/manage_sql_secinfo.c @@ -51,6 +51,7 @@ #include #include #include +#include #undef G_LOG_DOMAIN /** @@ -62,7 +63,7 @@ /* Static variables. */ /** - * @brief Maximum number of rows in an INSERT. + * @brief Maximum number of rows in a CPEs INSERT. */ #define CPE_MAX_CHUNK_SIZE 10000 @@ -185,130 +186,21 @@ increment_transaction_size (int* current_size) } } +/* Helper: buffer structure for INSERTs. */ + /** - * @brief Split a file. - * - * @param[in] path Path to file. - * @param[in] size Approx size of split files. In same format that - * xml_split accepts, eg "200Kb". - * @param[in] tail Text to replace last line of split files. - * - * @return Temp dir holding split files. + * @brief Get the SQL buffer size threshold converted from MiB to bytes. */ -static const gchar * -split_xml_file (gchar *path, const gchar *size, const gchar *tail) +int +setting_secinfo_sql_buffer_threshold_bytes () { - int ret; - static gchar dir[] = "/tmp/gvmd-split-xml-file-XXXXXX"; - gchar *previous_dir, *command; - - if (mkdtemp (dir) == NULL) - { - g_warning ("%s: Failed to make temp dir: %s", - __func__, - strerror (errno)); - return NULL; - } - - previous_dir = getcwd (NULL, 0); - if (previous_dir == NULL) - { - g_warning ("%s: Failed to getcwd: %s", - __func__, - strerror (errno)); - return NULL; - } - - if (chdir (dir)) - { - g_warning ("%s: Failed to chdir: %s", - __func__, - strerror (errno)); - g_free (previous_dir); - return NULL; - } - - if (gvm_file_copy (path, "split.xml") == FALSE) - { - g_free (previous_dir); - return NULL; - } - - /* xml_split will chop split.xml into files that are roughly 'size' big. - * - * The generated files are always put in the directory that holds - * split.xml, as follows: - * - * split.xml Source XML. - * split-00.xml Master generated XML. No content, just includes other - * files. The include statements are wrapped in the - * root element from split.xml. - * split-01.xml Generated XML content. Wrapped in an - * element. - * split-02.xml Second generated content file. - * ... - * split-112.xml Last content, for example. - * - * Parsing the generated files independently will only work if the files - * contain the original root element (for example, because the parser - * requires the namespace definitions to be present). - * - * So the command below needs to mess around a little bit to replace the - * wrapper XML element in split-01.xml, split-02.xml, etc with the root - * element from split-00.xml. - * - * Using tail and head is not super robust, but it's simple and it will - * work as long as xml_split keeps the opening of the wrapper element - * in split-00.xml on a dedicated line. (It doesn't do this for the - * closing element, so we use the tail argument instead.) - */ - - command = g_strdup_printf - ("xml_split -s%s split.xml" - " && head -n 2 split-00.xml > head.xml" - " && echo '%s' > tail.xml" - " && for F in split-*.xml; do" - /* Remove the first two lines and last line. */ - " awk 'NR>3 {print last} {last=$0}' $F > body.xml" - /* Combine with new start and end. */ - " && cat head.xml body.xml tail.xml > $F;" - " done", - size, - tail); - - g_debug ("%s: command: %s", __func__, command); - ret = system (command); - if ((ret == -1) || WIFEXITED(ret) == 0 || WEXITSTATUS (ret)) - { - g_warning ("%s: system failed with ret %i, %i (%i), %s", - __func__, - ret, - WIFEXITED (ret), - WIFEXITED (ret) ? WEXITSTATUS (ret) : 0, - command); - g_free (command); - - if (chdir (previous_dir)) - g_warning ("%s: and failed to chdir back", __func__); - g_free (previous_dir); + int threshold; - return NULL; - } - - g_free (command); - - if (chdir (previous_dir)) - g_warning ("%s: Failed to chdir back (will continue anyway)", - __func__); + setting_value_int (SETTING_UUID_SECINFO_SQL_BUFFER_THRESHOLD, &threshold); - g_free (previous_dir); - - return dir; + return threshold * 1048576; } - -/* Helper: buffer structure for INSERTs. */ - /** * @brief Buffer for INSERT statements. */ @@ -316,26 +208,34 @@ typedef struct { array_t *statements; ///< Buffered statements. GString *statement; ///< Current statement. + int statements_size; ///< Sum of lengths of all statements buffered. + int max_statements_size; ///< Auto-run at this statement_size, 0 for never. int current_chunk_size; ///< Number of rows in current statement. int max_chunk_size; ///< Max number of rows per INSERT. gchar *open_sql; ///< SQL to open each statement. gchar *close_sql; ///< SQL to close each statement. } inserts_t; +static void +inserts_run (inserts_t *, gboolean); + /** * @brief Check size of current statement. * * @param[in] inserts Insert buffer. - * @param[in] max_chunk_size Max chunk size. + * @param[in] max_chunk_size Max chunk size per statement. + * @param[in] max_statements_size Automatically run at this statements size. * @param[in] open_sql SQL to to start each statement. * @param[in] close_sql SQL to append to the end of each statement. */ static void -inserts_init (inserts_t *inserts, int max_chunk_size, const gchar *open_sql, - const gchar *close_sql) +inserts_init (inserts_t *inserts, int max_chunk_size, int max_statements_size, + const gchar *open_sql, const gchar *close_sql) { inserts->statements = make_array (); inserts->statement = NULL; + inserts->statements_size = 0; + inserts->max_statements_size = max_statements_size; inserts->current_chunk_size = 0; inserts->max_chunk_size = max_chunk_size; inserts->open_sql = open_sql ? g_strdup (open_sql) : NULL; @@ -377,8 +277,15 @@ inserts_check_size (inserts_t *inserts) { inserts_statement_close (inserts); array_add (inserts->statements, inserts->statement); + inserts->statements_size += inserts->statement->len; inserts->statement = NULL; inserts->current_chunk_size = 0; + + if (inserts->max_statements_size + && inserts-> statements_size >= inserts->max_statements_size) + { + inserts_run (inserts, FALSE); + } } if (inserts->statement == NULL) @@ -392,7 +299,26 @@ inserts_check_size (inserts_t *inserts) } /** - * @brief Free everything. + * @brief Free only the statements in an inserts buffer so it can be reused. + * + * @param[in] inserts Insert buffer. + */ +static void +inserts_free_statements (inserts_t *inserts) +{ + int index; + + for (index = 0; index < inserts->statements->len; index++) + { + g_string_free (g_ptr_array_index (inserts->statements, index), TRUE); + inserts->statements->pdata[index] = NULL; + } + g_ptr_array_set_size (inserts->statements, 0); + inserts->statements_size = 0; +} + +/** + * @brief Free all fields in an inserts buffer. * * @param[in] inserts Insert buffer. */ @@ -413,9 +339,10 @@ inserts_free (inserts_t *inserts) * @brief Run the INSERT SQL, freeing the buffers. * * @param[in] inserts Insert buffer. + * @param[in] finalize Whether to free the whole inserts buffer afterwards. */ static void -inserts_run (inserts_t *inserts) +inserts_run (inserts_t *inserts, gboolean finalize) { guint index; @@ -435,7 +362,10 @@ inserts_run (inserts_t *inserts) sql ("%s", statement->str); } - inserts_free (inserts); + if (finalize) + inserts_free (inserts); + else + inserts_free_statements (inserts); } @@ -2119,45 +2049,40 @@ insert_scap_cpe_details (inserts_t *inserts, element_t cpe_item) static int update_scap_cpes_from_file (const gchar *path) { - GError *error; - element_t element, cpe_list, cpe_item; - gchar *xml; - gsize xml_len; - inserts_t inserts, details_inserts; - - g_debug ("%s: parsing %s", __func__, path); - - error = NULL; - g_file_get_contents (path, &xml, &xml_len, &error); - if (error) - { - g_warning ("%s: Failed to get contents: %s", - __func__, - error->message); - g_error_free (error); - return -1; - } - - if (parse_element (xml, &element)) - { - g_free (xml); - g_warning ("%s: Failed to parse element", __func__); - return -1; - } - g_free (xml); + int ret; + element_t cpe_item; + inserts_t inserts; + xml_file_iterator_t file_iterator; + gchar *error_message = NULL; - cpe_list = element; - if (strcmp (element_name (cpe_list), "cpe-list")) + file_iterator = xml_file_iterator_new (); + ret = xml_file_iterator_init_from_file_path (file_iterator, path, 1); + switch (ret) { - element_free (element); - g_warning ("%s: CPE dictionary missing CPE-LIST", __func__); - return -1; + case 0: + break; + case 2: + g_warning ("%s: Could not open file '%s' for XML file iterator: %s", + __func__, path, strerror(errno)); + xml_file_iterator_free (file_iterator); + return -1; + case 3: + g_warning ("%s: Could not create parser context for XML file iterator", + __func__); + xml_file_iterator_free (file_iterator); + return -1; + default: + g_warning ("%s: Could not initialize XML file iterator", + __func__); + xml_file_iterator_free (file_iterator); + return -1; } sql_begin_immediate (); inserts_init (&inserts, CPE_MAX_CHUNK_SIZE, + setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpes" " (uuid, name, title, creation_time," " modification_time, status, deprecated_by_id," @@ -2171,7 +2096,15 @@ update_scap_cpes_from_file (const gchar *path) " status = EXCLUDED.status," " deprecated_by_id = EXCLUDED.deprecated_by_id," " nvd_id = EXCLUDED.nvd_id"); - cpe_item = element_first_child (cpe_list); + + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get first CPE XML element: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } while (cpe_item) { gchar *modification_date; @@ -2180,7 +2113,15 @@ update_scap_cpes_from_file (const gchar *path) if (strcmp (element_name (cpe_item), "cpe-item")) { - cpe_item = element_next (cpe_item); + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } continue; } @@ -2205,47 +2146,90 @@ update_scap_cpes_from_file (const gchar *path) if (insert_scap_cpe (&inserts, cpe_item, item_metadata, modification_time)) goto fail; - cpe_item = element_next (cpe_item); + + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element: %s", + __func__, error_message); + g_free (error_message); + error_message = NULL; + goto fail; + } } - inserts_run (&inserts); + inserts_run (&inserts, TRUE); sql_commit (); + sql_begin_immediate(); + + if (xml_file_iterator_rewind (file_iterator)) + { + g_warning ("%s: Could not create parser context for XML file iterator" + " for details.", + __func__); + goto fail; + } // Extract and save details XML. - inserts_init (&details_inserts, + inserts_init (&inserts, CPE_MAX_CHUNK_SIZE, + setting_secinfo_sql_buffer_threshold_bytes (), "INSERT INTO scap2.cpe_details" " (cpe_id, details_xml)" " VALUES", " ON CONFLICT (cpe_id) DO UPDATE" " SET details_xml = EXCLUDED.details_xml"); - cpe_item = element_first_child (cpe_list); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get first CPE XML element for details: %s", + __func__, error_message); + g_free (error_message); + error_message = NULL; + goto fail; + } while (cpe_item) { if (strcmp (element_name (cpe_item), "cpe-item")) { - cpe_item = element_next (cpe_item); + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element" + " for details: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } continue; } - if (insert_scap_cpe_details (&details_inserts, cpe_item)) + if (insert_scap_cpe_details (&inserts, cpe_item)) goto fail; - cpe_item = element_next (cpe_item); + element_free (cpe_item); + cpe_item = xml_file_iterator_next (file_iterator, &error_message); + if (error_message) + { + g_warning ("%s: could not get next CPE XML element for details: %s", + __func__, error_message); + g_free (error_message); + goto fail; + } } - element_free (element); - - sql_begin_immediate(); - inserts_run (&details_inserts); + inserts_run (&inserts, TRUE); sql_commit(); + xml_file_iterator_free (file_iterator); return 0; fail: inserts_free (&inserts); - element_free (element); g_warning ("Update of CPEs failed"); sql_commit (); + xml_file_iterator_free (file_iterator); return -1; } @@ -2258,9 +2242,8 @@ static int update_scap_cpes () { gchar *full_path; - const gchar *split_dir; GStatBuf state; - int index; + int ret; full_path = g_build_filename (GVM_SCAP_DATA_DIR, "official-cpe-dictionary_v2.2.xml", @@ -2277,44 +2260,9 @@ update_scap_cpes () g_info ("Updating CPEs"); - split_dir = split_xml_file (full_path, "40Mb", ""); - if (split_dir == NULL) - { - int ret; - - g_warning ("%s: Failed to split CPEs, attempting with full file", - __func__); - ret = update_scap_cpes_from_file (full_path); - g_free (full_path); - return ret; - } - g_free (full_path); - - for (index = 1; 1; index++) - { - int ret; - gchar *path, *name; - - name = g_strdup_printf ("split-%02i.xml", index); - path = g_build_filename (split_dir, name, NULL); - g_free (name); - - if (g_stat (path, &state)) - { - g_free (path); - break; - } - - ret = update_scap_cpes_from_file (path); - g_free (path); - if (ret < 0) - { - gvm_file_remove_recurse (split_dir); - return -1; - } - } - - gvm_file_remove_recurse (split_dir); + ret = update_scap_cpes_from_file (full_path); + if (ret) + return -1; return 0; } @@ -2422,8 +2370,7 @@ insert_cve_products (element_t list, resource_t cve, gchar *quoted_product, *product_decoded; gchar *product_tilde; - product_decoded = g_uri_unescape_string - (element_text (product), NULL); + product_decoded = g_uri_unescape_string (product_text, NULL); product_tilde = string_replace (product_decoded, "~", "%7E", "%7e", NULL); @@ -2693,12 +2640,13 @@ insert_cve_from_entry (element_t entry, element_t last_modified, static int update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) { - GError *error; - element_t element, entry; - gchar *xml, *full_path; - gsize xml_len; + gchar *error_message = NULL; + xml_file_iterator_t iterator; + element_t entry; + gchar *full_path; GStatBuf state; int transaction_size = 0; + int ret; full_path = g_build_filename (GVM_SCAP_DATA_DIR, xml_path, NULL); @@ -2712,29 +2660,31 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) g_info ("Updating %s", full_path); - error = NULL; - g_file_get_contents (full_path, &xml, &xml_len, &error); - if (error) - { - g_warning ("%s: Failed to get contents: %s", - __func__, - error->message); - g_error_free (error); - g_free (full_path); - return -1; - } - - if (parse_element (xml, &element)) + iterator = xml_file_iterator_new (); + ret = xml_file_iterator_init_from_file_path (iterator, full_path, 1); + switch (ret) { - g_free (xml); - g_warning ("%s: Failed to parse element", __func__); - g_free (full_path); - return -1; + case 0: + break; + case 2: + g_warning ("%s: Could not open file '%s' for XML file iterator: %s", + __func__, full_path, strerror(errno)); + xml_file_iterator_free (iterator); + return -1; + case 3: + g_warning ("%s: Could not create parser context for XML file iterator", + __func__); + xml_file_iterator_free (iterator); + return -1; + default: + g_warning ("%s: Could not initialize XML file iterator", + __func__); + xml_file_iterator_free (iterator); + return -1; } - g_free (xml); sql_begin_immediate (); - entry = element_first_child (element); + entry = xml_file_iterator_next (iterator, &error_message); while (entry) { if (strcmp (element_name (entry), "entry") == 0) @@ -2744,28 +2694,40 @@ update_cve_xml (const gchar *xml_path, GHashTable *hashed_cpes) last_modified = element_child (entry, "vuln:last-modified-datetime"); if (last_modified == NULL) { - g_warning ("%s: vuln:last-modified-datetime missing", - __func__); + error_message = g_strdup ("vuln:last-modified-datetime missing"); goto fail; } if (insert_cve_from_entry (entry, last_modified, hashed_cpes, &transaction_size)) - goto fail; + { + error_message = g_strdup ("Insert of CVE into database failed"); + goto fail; + } } - entry = element_next (entry); + + element_free (entry); + entry = xml_file_iterator_next (iterator, &error_message); } - element_free (element); + if (error_message) + goto fail; + + xml_file_iterator_free (iterator); g_free (full_path); sql_commit (); return 0; fail: - element_free (element); - g_warning ("Update of CVEs failed at file '%s'", - full_path); + xml_file_iterator_free (iterator); + if (error_message) + g_warning ("Update of CVEs failed at file '%s': %s", + full_path, error_message); + else + g_warning ("Update of CVEs failed at file '%s'", + full_path); g_free (full_path); + g_free (error_message); sql_commit (); return -1; } @@ -3427,38 +3389,15 @@ update_scap_placeholders () " WHERE cpe=cpes.id))" " WHERE cpes.title IS NULL;"); } - + /** - * @brief Finish scap update. - * - * @return 0 success, -1 error. + * @brief Update CERT data that depends on SCAP. */ -static int -update_scap_end () +static void +update_cert_data () { int cert_db_version; - g_debug ("%s: update timestamp", __func__); - - update_scap_timestamp (); - - /* Replace the real scap schema with the new one. */ - - if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" - " information_schema.schemata" - " WHERE schema_name = 'scap');")) - { - sql ("ALTER SCHEMA scap RENAME TO scap3;"); - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - sql ("DROP SCHEMA scap3 CASCADE;"); - /* View 'vulns' contains references into the SCAP schema, so it is - * removed by the CASCADE. */ - create_view_vulns (); - } - else - sql ("ALTER SCHEMA scap2 RENAME TO scap;"); - - /* Update CERT data that depends on SCAP. */ cert_db_version = manage_cert_db_version(); if (cert_db_version == -1) @@ -3490,6 +3429,37 @@ update_scap_end () update_cvss_dfn_cert (1, last_cert_update, last_scap_update); update_cvss_cert_bund (1, last_cert_update, last_scap_update); } +} + +/** + * @brief Finish scap update. + */ +static void +update_scap_end () +{ + g_debug ("%s: update timestamp", __func__); + + update_scap_timestamp (); + + /* Replace the real scap schema with the new one. */ + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + sql ("ALTER SCHEMA scap RENAME TO scap3;"); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + sql ("DROP SCHEMA scap3 CASCADE;"); + /* View 'vulns' contains references into the SCAP schema, so it is + * removed by the CASCADE. */ + create_view_vulns (); + } + else + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + + /* Update CERT data that depends on SCAP. */ + + update_cert_data (); /* Analyze. */ @@ -3499,8 +3469,60 @@ update_scap_end () g_info ("%s: Updating SCAP info succeeded", __func__); setproctitle ("Syncing SCAP: done"); +} - return 0; +/** + * @brief Abort scap update. + */ +static void +abort_scap_update () +{ + g_debug ("%s: update timestamp", __func__); + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap');")) + { + update_scap_timestamp (); + sql ("UPDATE scap.meta SET value = " + " (SELECT value from scap2.meta WHERE name = 'last_update')" + " WHERE name = 'last_update';"); + sql ("DROP SCHEMA scap2 CASCADE;"); + /* View 'vulns' contains references into the SCAP schema, so it is + * removed by the CASCADE. */ + create_view_vulns (); + /* Update CERT data that depends on SCAP. */ + update_cert_data (); + } + else + { + /* reset scap2 schema */ + if (manage_db_init ("scap")) + { + g_warning ("%s: could not reset scap2 schema, db init failed", __func__); + } + else if (manage_db_init_indexes ("scap")) + { + g_warning ("%s: could not reset scap2 schema, init indexes failed", __func__); + } + else if (manage_db_add_constraints ("scap")) + { + g_warning ("%s: could not reset scap2 schema, add constrains failed", __func__); + } + + if (sql_int ("SELECT EXISTS (SELECT schema_name FROM" + " information_schema.schemata" + " WHERE schema_name = 'scap2');")) + { + update_scap_timestamp (); + sql ("ALTER SCHEMA scap2 RENAME TO scap;"); + /* Update CERT data that depends on SCAP. */ + update_cert_data (); + } + } + + g_info ("%s: Updating SCAP data aborted", __func__); + setproctitle ("Syncing SCAP: aborted"); } /** @@ -3562,7 +3584,8 @@ try_load_csv () return -1; } - return update_scap_end (); + update_scap_end (); + return 0; } return 1; } @@ -3661,7 +3684,7 @@ update_scap (gboolean reset_scap_db) if (update_scap_cpes () == -1) { - update_scap_timestamp (); + abort_scap_update (); return -1; } @@ -3670,7 +3693,7 @@ update_scap (gboolean reset_scap_db) if (update_scap_cves () == -1) { - update_scap_timestamp (); + abort_scap_update (); return -1; } @@ -3689,7 +3712,8 @@ update_scap (gboolean reset_scap_db) update_scap_placeholders (); - return update_scap_end (); + update_scap_end (); + return 0; } /** diff --git a/src/utils.c b/src/utils.c index 83c55f7fd..efe108b23 100644 --- a/src/utils.c +++ b/src/utils.c @@ -483,7 +483,7 @@ iso_time_internal (time_t *epoch_time, const char **abbrev) /** * @brief Create an ISO time from seconds since epoch. * - * @param[in] epoch_time Time in seconds from epoch. + * @param[in] epoch_time Pointer to time in seconds from epoch. * * @return Pointer to ISO time in static memory, or NULL on error. */ @@ -496,7 +496,7 @@ iso_time (time_t *epoch_time) /** * @brief Create an ISO time from seconds since epoch, given a timezone. * - * @param[in] epoch_time Time in seconds from epoch. + * @param[in] epoch_time Pointer to time in seconds from epoch. * @param[in] zone Timezone. * @param[out] abbrev Timezone abbreviation. * @@ -543,6 +543,28 @@ iso_time_tz (time_t *epoch_time, const char *zone, const char **abbrev) return ret; } +/** + * @brief Create an ISO time from seconds since epoch, with a 0 check. + * + * @param[in] epoch_time Time in seconds from epoch. + * + * @return ISO time string in static memory. If epoch_time is 0 then string is empty. + */ +char * +iso_if_time (time_t epoch_time) +{ + static char *empty = ""; + if (epoch_time) + { + char *ret; + + ret = iso_time (&epoch_time); + if (ret) + return ret; + } + return empty; +} + /* Locks. */ diff --git a/src/utils.h b/src/utils.h index 50333d71e..dfeed44b6 100644 --- a/src/utils.h +++ b/src/utils.h @@ -55,6 +55,9 @@ iso_time (time_t *); char * iso_time_tz (time_t *, const char *, const char **); +char * +iso_if_time (time_t epoch_time); + /** * @brief Lockfile. */