From 1a6c7db8bf06627749c2e2f734a09f26dd6e084b Mon Sep 17 00:00:00 2001 From: Michal Sedlak Date: Sun, 1 Sep 2024 16:04:18 +0200 Subject: [PATCH] FDS Output: enable selection of fields saved in output based on configuration --- src/plugins/output/fds/src/Storage.cpp | 212 ++++++++++++++++++++++--- src/plugins/output/fds/src/Storage.hpp | 7 + 2 files changed, 201 insertions(+), 18 deletions(-) diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp index 5867c344..f3e309cb 100644 --- a/src/plugins/output/fds/src/Storage.cpp +++ b/src/plugins/output/fds/src/Storage.cpp @@ -1,6 +1,7 @@ /** * \file src/plugins/output/fds/src/Storage.cpp * \author Lukas Hutak + * \author Michal Sedlak * \brief FDS file storage (source file) * \date June 2019 * @@ -13,13 +14,154 @@ #include #include #include +#include #include #include #include "Storage.hpp" const std::string TMP_SUFFIX = ".tmp"; -Storage::Storage(ipx_ctx_t *ctx, const Config &cfg) : m_ctx(ctx), m_path(cfg.m_path) +static void +buf_write(std::vector &buffer, const uint8_t *data, std::size_t data_len) +{ + std::size_t idx = buffer.size(); + buffer.resize(idx + data_len); + std::memcpy(&buffer[idx], data, data_len); +} + +template +static void +buf_write(std::vector &buffer, T &&value) +{ + buf_write(buffer, reinterpret_cast(&value), sizeof(value)); +} + +static bool +contains_element(const std::vector &elements, const fds_tfield &field) +{ + auto it = std::find_if(elements.begin(), elements.end(), + [&](const Config::element &elem) { return elem.id == field.id && elem.pen == field.en; }); + return it != elements.end(); +} + +void +create_modified_template(const fds_template *tmplt, + const std::vector &selected_elements, + std::vector &out_buffer) +{ + out_buffer.clear(); + + // Collect the fields we want in the resulting template + std::vector fields; + for (uint16_t i = 0; i < tmplt->fields_cnt_total; i++) { + const fds_tfield &field = tmplt->fields[i]; + if (contains_element(selected_elements, field)) { + fields.push_back(&field); + } + } + + + /** + * Build the template definition... + * + * Template: + * [Template Record Header] + * [Field Specifier] + * [Field Specifier] + * ... + * [Field Specifier] + * + * Template Record Header: + * [Template ID : 16 bit] [Field Count : 16 bit] + * + * Example: + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Template ID = 256 | Field Count = N | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |1| Information Element id. 1.1 | Field Length 1.1 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Enterprise Number 1.1 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |0| Information Element id. 1.2 | Field Length 1.2 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ... | ... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |1| Information Element id. 1.N | Field Length 1.N | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Enterprise Number 1.N | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Template ID = 257 | Field Count = M | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |0| Information Element id. 2.1 | Field Length 2.1 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |1| Information Element id. 2.2 | Field Length 2.2 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Enterprise Number 2.2 | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ... | ... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |1| Information Element id. 2.M | Field Length 2.M | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Enterprise Number 2.M | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Padding (opt) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- + */ + + // Write the header + buf_write(out_buffer, htons(tmplt->id)); + buf_write(out_buffer, htons(fields.size())); + + // Write the template fields + for (const fds_tfield *field : fields) { + if (field->en == 0) { + buf_write(out_buffer, htons(field->id)); + buf_write(out_buffer, htons(field->length)); + } else { + // If PEN is specified, MSB of element ID is set to 1 + buf_write(out_buffer, htons(field->id | 0x8000)); + buf_write(out_buffer, htons(field->length)); + buf_write(out_buffer, htonl(field->en)); + } + } +} + +static void +create_modified_data_record(fds_drec &drec, + const std::vector &selected_elements, + std::vector &out_buffer) +{ + out_buffer.clear(); + + /** + * Data record consists of a sequence of field values. + * + * Data Record: + * +--------------------------------------------------+ + * | Field Value | + * +--------------------------------------------------+ + * | Field Value | + * +--------------------------------------------------+ + * ... + * +--------------------------------------------------+ + * | Field Value | + * +--------------------------------------------------+ + */ + + fds_drec_iter iter; + fds_drec_iter_init(&iter, &drec, 0); + while (fds_drec_iter_next(&iter) != FDS_EOC) { + if (contains_element(selected_elements, *iter.field.info)) { + buf_write(out_buffer, iter.field.data, iter.field.size); + } + } +} + +Storage::Storage(ipx_ctx_t *ctx, const Config &cfg) : + m_ctx(ctx), + m_path(cfg.m_path), + m_selection_used(cfg.m_selection_used), + m_selection(cfg.m_selection) { // Check if the directory exists struct stat file_info; @@ -142,6 +284,16 @@ Storage::process_msg(ipx_msg_ipfix_t *msg) uint16_t rec_size = rec_ptr->rec.size; uint16_t tmplt_id = rec_ptr->rec.tmplt->id; + if (m_selection_used) { + create_modified_data_record(rec_ptr->rec, m_selection, m_buffer); + rec_data = m_buffer.data(); + rec_size = m_buffer.size(); + if (rec_size == 0) { + // No record fields were selected -> the record is empty, skip + continue; + } + } + if (fds_file_write_rec(m_file.get(), tmplt_id, rec_data, rec_size) != FDS_OK) { const char *err_msg = fds_file_error(m_file.get()); throw FDS_exception("Failed to add a Data Record: " + std::string(err_msg)); @@ -160,6 +312,12 @@ struct tmplt_update_data { fds_file_t *file; /// Set of processed Templates in the snapshot std::set ids; + /// Whether selection is used + bool selection_used; + /// The selection + std::vector *selection; + /// Buffer for building modified templates + std::vector *buffer; }; /** @@ -175,21 +333,40 @@ struct tmplt_update_data { static bool tmplt_update_cb(const struct fds_template *tmplt, void *data) { - // Template type, raw data and size - enum fds_template_type t_type; - const uint8_t *t_data; - uint16_t t_size; - auto info = reinterpret_cast(data); // No exceptions can be thrown in the C callback! try { - uint16_t t_id = tmplt->id; - info->ids.emplace(t_id); + if (info->selection_used && tmplt->type != FDS_TYPE_TEMPLATE) { + // Skip Option Templates in case field selection is used + return info->is_ok; + } + + // Get definition of the template we want to store + fds_template_type new_t_type = tmplt->type; + const uint8_t *new_t_data = tmplt->raw.data; + uint16_t new_t_size = tmplt->raw.length; + + if (info->selection_used) { + create_modified_template(tmplt, *info->selection, *info->buffer); + new_t_data = info->buffer->data(); + new_t_size = info->buffer->size(); + + if (new_t_size == 0) { + // None of the template fields have been selected -> skip this template + return info->is_ok; + } + } + + // Only now store the template ID as we're now sure that this is an active template + info->ids.emplace(tmplt->id); // Get definition of the Template specified in the file - int res = fds_file_write_tmplt_get(info->file, t_id, &t_type, &t_data, &t_size); + fds_template_type old_t_type; + const uint8_t *old_t_data; + uint16_t old_t_size; + int res = fds_file_write_tmplt_get(info->file, tmplt->id, &old_t_type, &old_t_data, &old_t_size); if (res != FDS_OK && res != FDS_ERR_NOTFOUND) { // Something bad happened const char *err_msg = fds_file_error(info->file); @@ -198,21 +375,17 @@ tmplt_update_cb(const struct fds_template *tmplt, void *data) // Should we add/redefine the definition of the Template if (res == FDS_OK - && tmplt->type == t_type - && tmplt->raw.length == t_size - && memcmp(tmplt->raw.data, t_data, t_size) == 0) { + && old_t_type == new_t_type + && old_t_size == new_t_size + && memcmp(old_t_data, new_t_data, new_t_size) == 0) { // The same -> nothing to do return info->is_ok; } // Add the definition (i.e. templates are different or the template hasn't been defined) - IPX_CTX_DEBUG(info->ctx, "Adding/updating definition of Template ID %" PRIu16, t_id); - - t_type = tmplt->type; - t_data = tmplt->raw.data; - t_size = tmplt->raw.length; + IPX_CTX_DEBUG(info->ctx, "Adding/updating definition of Template ID %" PRIu16, tmplt->id); - if (fds_file_write_tmplt_add(info->file, t_type, t_data, t_size) != FDS_OK) { + if (fds_file_write_tmplt_add(info->file, new_t_type, new_t_data, new_t_size) != FDS_OK) { const char *err_msg = fds_file_error(info->file); throw FDS_exception("fds_file_write_tmplt_add() failed: " + std::string(err_msg)); } @@ -258,6 +431,9 @@ Storage::tmplts_update(struct snap_info &info, const fds_tsnapshot_t *snap) data.ctx = m_ctx; data.file = m_file.get(); data.ids.clear(); + data.selection_used = m_selection_used; + data.selection = &m_selection; + data.buffer = &m_buffer; // Update templates fds_tsnapshot_for(snap, &tmplt_update_cb, &data); diff --git a/src/plugins/output/fds/src/Storage.hpp b/src/plugins/output/fds/src/Storage.hpp index 56bbcc14..3d27339e 100644 --- a/src/plugins/output/fds/src/Storage.hpp +++ b/src/plugins/output/fds/src/Storage.hpp @@ -110,6 +110,13 @@ class Storage { /// Mapping of Transport Sessions to FDS specific parameters std::map m_session2params; + /// Whether field selection is used + bool m_selection_used; + /// The fields to select + std::vector m_selection; + /// Data buffer for preparing modified template/data records if necessary + std::vector m_buffer; + std::string filename_gen(const time_t &ts); static void