Skip to content

Commit

Permalink
Merge branch 'main' into fix-nvt-indexes
Browse files Browse the repository at this point in the history
  • Loading branch information
a-h-abdelsalam authored Aug 21, 2024
2 parents ca06a8c + 5ce67ca commit 0ba2653
Show file tree
Hide file tree
Showing 11 changed files with 250 additions and 4 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ cmake_minimum_required (VERSION 3.0)
message ("-- Configuring Greenbone Vulnerability Manager...")

project (gvm
VERSION 23.7.1
VERSION 23.8.2
LANGUAGES C)

if (POLICY CMP0005)
Expand Down
6 changes: 6 additions & 0 deletions doc/gvmd.8
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,15 @@ Maximum size of user-defined message text in alert emails, in bytes.
\fB--max-ips-per-target=\fINUMBER\fB\f1
Maximum number of IPs per target.
.TP
\fB--mem-wait-retries=\fINUMBER\fB\f1
How often to try waiting for available memory. Default: 30. Each retry will wait for 10 seconds.
.TP
\fB-m, --migrate\f1
Migrate the database and exit.
.TP
\fB--min-mem-feed-update=\fINUMBER\fB\f1
Minimum memory in MiB for feed updates. Default: 0. Feed updates are skipped if less physical memory is available.
.TP
\fB--modify-scanner=\fISCANNER-UUID\fB\f1
Modify scanner SCANNER-UUID and exit.
.TP
Expand Down
18 changes: 18 additions & 0 deletions doc/gvmd.8.xml
Original file line number Diff line number Diff line change
Expand Up @@ -287,12 +287,30 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<p>Maximum number of IPs per target.</p>
</optdesc>
</option>
<option>
<p><opt>--mem-wait-retries=<arg>NUMBER</arg></opt></p>
<optdesc>
<p>
How often to try waiting for available memory. Default: 30.
Each retry will wait for 10 seconds.
</p>
</optdesc>
</option>
<option>
<p><opt>-m, --migrate</opt></p>
<optdesc>
<p>Migrate the database and exit.</p>
</optdesc>
</option>
<option>
<p><opt>--min-mem-feed-update=<arg>NUMBER</arg></opt></p>
<optdesc>
<p>
Minimum memory in MiB for feed updates. Default: 0.
Feed updates are skipped if less physical memory is available.
</p>
</optdesc>
</option>
<option>
<p><opt>--modify-scanner=<arg>SCANNER-UUID</arg></opt></p>
<optdesc>
Expand Down
18 changes: 18 additions & 0 deletions doc/gvmd.html
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,30 @@ <h2>Options</h2>



<p><b>--mem-wait-retries=<em>NUMBER</em></b></p>

<p>
How often to try waiting for available memory. Default: 30.
Each retry will wait for 10 seconds.
</p>



<p><b>-m, --migrate</b></p>

<p>Migrate the database and exit.</p>



<p><b>--min-mem-feed-update=<em>NUMBER</em></b></p>

<p>
Minimum memory in MiB for feed updates. Default: 0.
Feed updates are skipped if less physical memory is available.
</p>



<p><b>--modify-scanner=<em>SCANNER-UUID</em></b></p>

<p>Modify scanner SCANNER-UUID and exit.</p>
Expand Down
18 changes: 18 additions & 0 deletions src/gvmd.c
Original file line number Diff line number Diff line change
Expand Up @@ -1895,6 +1895,8 @@ gvmd (int argc, char** argv, char *env[])
static gchar *broker_address = NULL;
static gchar *feed_lock_path = NULL;
static int feed_lock_timeout = 0;
static int mem_wait_retries = 30;
static int min_mem_feed_update = 0;
static int vt_ref_insert_size = VT_REF_INSERT_SIZE_DEFAULT;
static int vt_sev_insert_size = VT_SEV_INSERT_SIZE_DEFAULT;
static gchar *vt_verification_collation = NULL;
Expand Down Expand Up @@ -2088,10 +2090,20 @@ gvmd (int argc, char** argv, char *env[])
&max_ips_per_target,
"Maximum number of IPs per target.",
"<number>" },
{ "mem-wait-retries", '\0', 0, G_OPTION_ARG_INT,
&mem_wait_retries,
"How often to try waiting for available memory. Default: 30."
" Each retry will wait for 10 seconds.",
"<number>" },
{ "migrate", 'm', 0, G_OPTION_ARG_NONE,
&migrate_database,
"Migrate the database and exit.",
NULL },
{ "min-mem-feed-update", '\0', 0, G_OPTION_ARG_INT,
&min_mem_feed_update,
"Minimum memory in MiB for feed updates. Default: 0."
" Feed updates are skipped if less physical memory is available.",
"<number>" },
{ "modify-scanner", '\0', 0, G_OPTION_ARG_STRING,
&modify_scanner,
"Modify scanner <scanner-uuid> and exit.",
Expand Down Expand Up @@ -2447,6 +2459,12 @@ gvmd (int argc, char** argv, char *env[])
}
}

/* Set number of retries waiting for memory */
set_mem_wait_retries (mem_wait_retries);

/* Set minimum memory for feed updates */
set_min_mem_feed_update (min_mem_feed_update);

/* Set relay mapper */
if (relay_mapper)
{
Expand Down
143 changes: 141 additions & 2 deletions src/manage.c
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,16 @@ static gchar *feed_lock_path = NULL;
*/
static int feed_lock_timeout = 0;

/**
* @brief Retries for waiting for memory to be available.
*/
static int mem_wait_retries = 0;

/**
* @brief Minimum available memory in MiB for running a feed update.
*/
static int min_mem_feed_update = 0;

/**
* @brief Path to the relay mapper executable, NULL to disable relays.
*/
Expand Down Expand Up @@ -5050,7 +5060,52 @@ feed_sync_required ()
return FALSE;
}

/**
* @brief Wait for memory
*
* @param[in] check_func Function to check memory, should return 1 if enough.
* @param[in] retries Number of retries.
* @param[in] min_mem Minimum memory in MiB, for logging only
* @param[in] action Short descriptor of action waiting for memory.
*
* @return 0 if enough memory is available, 1 gave up
*/
static int
wait_for_mem (int check_func(),
int retries,
int min_mem,
const char *action)
{
int retry_number = 0;
while (check_func () == 0)
{
if (retry_number == 0)
{
g_info ("%s: not enough memory for %s"
" (%lld / %d) MiB",
__func__,
action,
phys_mem_available () / 1048576llu,
min_mem);
}
else
{
g_debug ("%s: waiting for memory for %s"
" (%lld / %d) MiB",
__func__,
action,
phys_mem_available () / 1048576llu,
min_mem);
}

retry_number ++;
if (retry_number > retries)
return 1;

sleep (SCHEDULE_PERIOD);
}
return 0;
}

/**
* @brief Perform any syncing that is due.
Expand All @@ -5074,7 +5129,11 @@ manage_sync (sigset_t *sigmask_current,

if (feed_sync_required ())
{
if (feed_lockfile_lock (&lockfile) == 0)
if (wait_for_mem (check_min_mem_feed_update,
mem_wait_retries,
min_mem_feed_update,
"SecInfo feed sync") == 0
&& feed_lockfile_lock (&lockfile) == 0)
{
pid_t nvts_pid, scap_pid, cert_pid;
nvts_pid = manage_sync_nvts (fork_update_nvt_cache);
Expand All @@ -5096,7 +5155,11 @@ manage_sync (sigset_t *sigmask_current,
|| should_sync_port_lists ()
|| should_sync_report_formats ()))
{
if (feed_lockfile_lock (&lockfile) == 0)
if (wait_for_mem (check_min_mem_feed_update,
mem_wait_retries,
min_mem_feed_update,
"data objects feed sync") == 0
&& feed_lockfile_lock (&lockfile) == 0)
{
manage_sync_configs ();
manage_sync_port_lists ();
Expand Down Expand Up @@ -6352,6 +6415,82 @@ set_feed_lock_timeout (int new_timeout)
feed_lock_timeout = new_timeout;
}

/**
* @brief Get the number of retries when waiting for memory to be available.
*
* @return The current number of retries.
*/
int
get_mem_wait_retries()
{
return mem_wait_retries;
}

/**
* @brief Set the number of retries when waiting for memory to be available.
*
* @param[in] new_retries The new number of retries.
*/
void
set_mem_wait_retries (int new_retries)
{
if (new_retries < 0)
min_mem_feed_update = 0;
else
min_mem_feed_update = new_retries;
}

/**
* @brief Check if the minimum memory for feed updates is available
*
* @return 1 if minimum memory amount is available, 0 if not
*/
int
check_min_mem_feed_update ()
{
if (min_mem_feed_update)
{
guint64 min_mem_bytes = (guint64)min_mem_feed_update * 1048576llu;
return phys_mem_available () >= min_mem_bytes ? 1 : 0;
}
return 1;
}

/**
* @brief Get the minimum memory for feed updates.
*
* @return The current minimum memory for feed updates in MiB.
*/
int
get_min_mem_feed_update ()
{
return min_mem_feed_update;
}

/**
* @brief Get the minimum memory for feed updates.
*
* @param[in] new_min_mem The new minimum memory for feed updates in MiB.
*/
void
set_min_mem_feed_update (int new_min_mem)
{
guint64 min_mem_bytes = (guint64)new_min_mem * 1048576llu;
if (new_min_mem < 0)
min_mem_feed_update = 0;
else if (min_mem_bytes > phys_mem_total ())
{
g_warning ("%s: requested feed minimum memory limit (%d MiB)"
" exceeds total physical memory (%lld MiB)."
" The setting is ignored.",
__func__,
new_min_mem,
phys_mem_total () / 1048576llu);
}
else
min_mem_feed_update = new_min_mem;
}

/**
* @brief Write start time to sync lock file.
*
Expand Down
15 changes: 15 additions & 0 deletions src/manage.h
Original file line number Diff line number Diff line change
Expand Up @@ -3870,6 +3870,21 @@ get_feed_lock_timeout ();
void
set_feed_lock_timeout (int);

int
get_mem_wait_retries ();

void
set_mem_wait_retries (int);

int
check_min_mem_feed_update ();

int
get_min_mem_feed_update ();

void
set_min_mem_feed_update (int);

void
write_sync_start (int);

Expand Down
4 changes: 3 additions & 1 deletion src/manage_pg.c
Original file line number Diff line number Diff line change
Expand Up @@ -1849,9 +1849,11 @@ create_view_result_vt_epss ()
" epss_score,"
" epss_percentile,"
" epss_cve,"
" epss_severity,"
" max_epss_score,"
" max_epss_percentile,"
" max_epss_cve"
" max_epss_cve,"
" max_epss_severity"
" FROM nvts);");

sql ("SELECT create_index ('result_vt_epss_by_vt_id',"
Expand Down
1 change: 1 addition & 0 deletions src/manage_sql_nvts.c
Original file line number Diff line number Diff line change
Expand Up @@ -1979,6 +1979,7 @@ update_nvts_from_vts (element_t *get_vts_response,
if (rebuild) {
sql ("DROP VIEW IF EXISTS results_autofp;");
sql ("DROP VIEW vulns;");
sql ("DROP MATERIALIZED VIEW IF EXISTS result_vt_epss;");
sql ("DROP TABLE nvts, nvt_preferences, vt_refs, vt_severities;");

sql ("ALTER TABLE vt_refs_rebuild RENAME TO vt_refs;");
Expand Down
23 changes: 23 additions & 0 deletions src/utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -959,3 +959,26 @@ wait_for_pid (pid_t pid, const char *context)
}
}
}

/**
* @brief Get the available physical memory in bytes.
*
* @return The available memory
*/
guint64
phys_mem_available ()
{
return (unsigned long long)(sysconf(_SC_AVPHYS_PAGES))
* sysconf(_SC_PAGESIZE);
}

/**
* @brief Get the total physical memory in bytes.
*
* @return The total memory
*/
guint64
phys_mem_total ()
{
return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
}
6 changes: 6 additions & 0 deletions src/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,10 @@ fork_with_handlers ();
void
wait_for_pid (pid_t, const char *);

guint64
phys_mem_available ();

guint64
phys_mem_total ();

#endif /* not _GVMD_UTILS_H */

0 comments on commit 0ba2653

Please sign in to comment.