From 0ca9227132f42e59983b15a61dcbb889ee81c9d0 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Mon, 8 Jul 2024 18:35:03 +0200 Subject: [PATCH] add a custom command based log_upload workflow Signed-off-by: Reuben Miller --- .../cumulocity/log/workflow/log_upload.sh | 57 ++++++++++++++++++ .../cumulocity/log/workflow/log_upload.toml | 29 +++++++++ .../log/workflow/log_workflow.robot | 59 +++++++++++++++++++ .../log/workflow/tedge-log-plugin.toml | 8 +++ 4 files changed, 153 insertions(+) create mode 100755 tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.sh create mode 100644 tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.toml create mode 100644 tests/RobotFramework/tests/cumulocity/log/workflow/log_workflow.robot create mode 100644 tests/RobotFramework/tests/cumulocity/log/workflow/tedge-log-plugin.toml diff --git a/tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.sh b/tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.sh new file mode 100755 index 00000000000..b5db88102af --- /dev/null +++ b/tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.sh @@ -0,0 +1,57 @@ +#!/bin/sh +set -e + +preprocess_sqlite() { + # + # Create a log file in the location + # that is defined in the tedge-log-plugin.toml + # + LOG_TYPE="$1" + TEDGE_FTS_URL="$2" + DATE_FROM="$3" + DATE_TO="$4" + SEARCH_TEXT="$5" + + TMP_LOG_FILE=/tmp/${LOG_TYPE}.log + + # delete any existing log file (from a previous run) + rm -f "$TMP_LOG_FILE" + + # Run the query + echo "Running sqlite query. dateFrom=$DATE_FROM, dateTo=$DATE_TO, searchText=$SEARCH_TEXT" >&2 + echo "File will be upload to $TEDGE_FTS_URL" >&2 + + cat << EOT > "$TMP_LOG_FILE" +Running some sqlite query... +Parameters: + dateFrom=$DATE_FROM + dateTo=$DATE_TO +EOT +} + +# +# Main +# +if [ $# -lt 1 ]; then + echo "Missing required positional argument" >&2 + exit 2 +fi + +COMMAND="$1" +shift + +case "$COMMAND" in + preprocess) + LOG_TYPE="$1" + case "$LOG_TYPE" in + sqlite) + preprocess_sqlite "$@" + ;; + *) + echo "Log type does not require a pre-processing. type=$LOG_TYPE" >&2 + ;; + esac + ;; +esac + +exit 0 diff --git a/tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.toml b/tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.toml new file mode 100644 index 00000000000..5fa1fbb4753 --- /dev/null +++ b/tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.toml @@ -0,0 +1,29 @@ +#:schema https://gist.githubusercontent.com/reubenmiller/4e28e8403fe0c54b7461ac7d1d6838c2/raw/4ad7e3bc3ce2e3a7542a5b4873a0196cbe91f35d/tedge.workflow.json + +operation = "log_upload" +on_error = "failed" + +[init] + action = "proceed" + on_success = "preprocess" + +[preprocess] + script = "/usr/bin/log_upload.sh ${.payload.status} ${.payload.type} ${.payload.tedgeUrl} ${.payload.dateFrom} ${.payload.dateTo} ${.payload.searchText}" + on_success = "scheduled" + # Proceed on any unexpected errors to ensure the operations status changed are handled, the operation + # will still fail if the sqlite handler (in the script) did not generate a file + on_error = "scheduled" + +[scheduled] + action = "builtin" + on_success = "executing" + +[executing] + action = "builtin" + on_success = "successful" + +[successful] + action = "cleanup" + +[failed] + action = "cleanup" diff --git a/tests/RobotFramework/tests/cumulocity/log/workflow/log_workflow.robot b/tests/RobotFramework/tests/cumulocity/log/workflow/log_workflow.robot new file mode 100644 index 00000000000..aa2de91320c --- /dev/null +++ b/tests/RobotFramework/tests/cumulocity/log/workflow/log_workflow.robot @@ -0,0 +1,59 @@ +*** Settings *** +Resource ../../../../resources/common.resource +Library Cumulocity +Library DateTime +Library ThinEdgeIO +Library String + +Suite Setup Custom Setup +Test Teardown Get Logs + +Test Tags theme:c8y theme:log + +*** Test Cases *** + +Custom log workflow with pre-processor + [Documentation] Use a custom log_upload workflow to support command based logs. It uses a custom preprocessor + ... step to run a custom command (via the log_upload.sh script), which can be used to read logs from any source + ... (sqlite in this example) + Cumulocity.Should Support Log File Types sqlite includes=${True} + + ${start_timestamp}= Get Current Date UTC -24 hours result_format=%Y-%m-%dT%H:%M:%S+0000 + ${end_timestamp}= Get Current Date UTC +60 seconds result_format=%Y-%m-%dT%H:%M:%S+0000 + + ${operation}= Cumulocity.Get Log File type=sqlite date_from=${start_timestamp} date_to=${end_timestamp} maximum_lines=10 + ${operation}= Operation Should Be SUCCESSFUL ${operation} + Log File Contents Should Be Equal operation=${operation} expected_pattern=filename: sqlite.log\\nRunning some sqlite query...\\nParameters:\\n\\s+dateFrom=.+\\n\\s+dateTo=.+\\n + + +*** Keywords *** + +Custom Setup + ${DEVICE_SN}= Setup + Set Suite Variable $DEVICE_SN + Device Should Exist ${DEVICE_SN} + + Setup LogFiles + +Setup LogFiles + ThinEdgeIO.Transfer To Device ${CURDIR}/tedge-log-plugin.toml /etc/tedge/plugins/tedge-log-plugin.toml + + # Custom workflow and handler script + ThinEdgeIO.Transfer To Device ${CURDIR}/log_upload.toml /etc/tedge/operations/ + ThinEdgeIO.Transfer To Device ${CURDIR}/log_upload.sh /usr/bin/log_upload.sh + + ThinEdgeio.Restart Service tedge-agent + ThinEdgeIO.Service Health Status Should Be Up tedge-agent + ThinEdgeIO.Service Health Status Should Be Up tedge-mapper-c8y + + +Log File Contents Should Be Equal + [Arguments] ${operation} ${expected_pattern} ${encoding}=utf-8 ${expected_filename}=^${DEVICE_SN}_[\\w\\W]+-c8y-mapper-\\d+$ ${expected_mime_type}=text/plain + ${event_url_parts}= Split String ${operation["c8y_LogfileRequest"]["file"]} separator=/ + ${event_id}= Set Variable ${event_url_parts}[-2] + ${contents}= Cumulocity.Event Should Have An Attachment + ... ${event_id} + ... expected_pattern=${expected_pattern} + ... encoding=${encoding} + ${event}= Cumulocity.Event Attachment Should Have File Info ${event_id} name=${expected_filename} mime_type=${expected_mime_type} + RETURN ${contents} diff --git a/tests/RobotFramework/tests/cumulocity/log/workflow/tedge-log-plugin.toml b/tests/RobotFramework/tests/cumulocity/log/workflow/tedge-log-plugin.toml new file mode 100644 index 00000000000..7efffcb4973 --- /dev/null +++ b/tests/RobotFramework/tests/cumulocity/log/workflow/tedge-log-plugin.toml @@ -0,0 +1,8 @@ +[[files]] +type = "software-management" +path = "/var/log/tedge/agent/software-*" + +[[files]] +type = "sqlite" +# This path can be any dummy location that the tedge-agent has write access to +path = "/tmp/sqlite.log"