add a custom command based log_upload workflow

Signed-off-by: Reuben Miller <[email protected]>

tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+set -e
+
+preprocess_sqlite() {
+    #
+    # Create a log file in the location
+    # that is defined in the tedge-log-plugin.toml
+    #
+    LOG_TYPE="$1"
+    TEDGE_FTS_URL="$2"
+    DATE_FROM="$3"
+    DATE_TO="$4"
+    SEARCH_TEXT="$5"
+
+    TMP_LOG_FILE=/tmp/${LOG_TYPE}.log
+
+    # delete any existing log file (from a previous run)
+    rm -f "$TMP_LOG_FILE"
+
+    # Run the query
+    echo "Running sqlite query. dateFrom=$DATE_FROM, dateTo=$DATE_TO, searchText=$SEARCH_TEXT" >&2
+    echo "File will be upload to $TEDGE_FTS_URL" >&2
+
+    cat << EOT > "$TMP_LOG_FILE"
+Running some sqlite query...
+Parameters:
+    dateFrom=$DATE_FROM
+    dateTo=$DATE_TO
+EOT
+}
+
+#
+# Main
+#
+if [ $# -lt 1 ]; then
+    echo "Missing required positional argument" >&2
+    exit 2
+fi
+
+COMMAND="$1"
+shift
+
+case "$COMMAND" in
+    preprocess)
+        LOG_TYPE="$1"
+        case "$LOG_TYPE" in
+            sqlite)
+                preprocess_sqlite "$@"
+                ;;
+            *)
+                echo "Log type does not require a pre-processing. type=$LOG_TYPE" >&2
+                ;;
+        esac
+        ;;
+esac
+
+exit 0

tests/RobotFramework/tests/cumulocity/log/workflow/log_upload.toml

@@ -0,0 +1,29 @@
+#:schema https://gist.githubusercontent.com/reubenmiller/4e28e8403fe0c54b7461ac7d1d6838c2/raw/4ad7e3bc3ce2e3a7542a5b4873a0196cbe91f35d/tedge.workflow.json
+
+operation = "log_upload"
+on_error = "failed"
+
+[init]
+  action = "proceed"
+  on_success = "preprocess"
+
+[preprocess]
+  script = "/usr/bin/log_upload.sh ${.payload.status} ${.payload.type} ${.payload.tedgeUrl} ${.payload.dateFrom} ${.payload.dateTo} ${.payload.searchText}"
+  on_success = "scheduled"
+  # Proceed on any unexpected errors to ensure the operations status changed are handled, the operation
+  # will still fail if the sqlite handler (in the script) did not generate a file
+  on_error = "scheduled"
+
+[scheduled]
+  action = "builtin"
+  on_success = "executing"
+
+[executing]
+  action = "builtin"
+  on_success = "successful"
+
+[successful]
+  action = "cleanup"
+
+[failed]
+  action = "cleanup"

tests/RobotFramework/tests/cumulocity/log/workflow/log_workflow.robot

@@ -0,0 +1,59 @@
+*** Settings ***
+Resource            ../../../../resources/common.resource
+Library             Cumulocity
+Library             DateTime
+Library             ThinEdgeIO
+Library             String
+
+Suite Setup         Custom Setup
+Test Teardown       Get Logs
+
+Test Tags           theme:c8y    theme:log
+
+*** Test Cases ***
+
+Custom log workflow with pre-processor
+    [Documentation]    Use a custom log_upload workflow to support command based logs. It uses a custom preprocessor
+    ...   step to run a custom command (via the log_upload.sh script), which can be used to read logs from any source
+    ...   (sqlite in this example)
+    Cumulocity.Should Support Log File Types    sqlite    includes=${True}
+
+    ${start_timestamp}=    Get Current Date    UTC    -24 hours    result_format=%Y-%m-%dT%H:%M:%S+0000
+    ${end_timestamp}=    Get Current Date    UTC    +60 seconds    result_format=%Y-%m-%dT%H:%M:%S+0000
+
+    ${operation}=    Cumulocity.Get Log File    type=sqlite    date_from=${start_timestamp}    date_to=${end_timestamp}    maximum_lines=10
+    ${operation}=    Operation Should Be SUCCESSFUL    ${operation}
+    Log File Contents Should Be Equal    operation=${operation}    expected_pattern=filename: sqlite.log\\nRunning some sqlite query...\\nParameters:\\n\\s+dateFrom=.+\\n\\s+dateTo=.+\\n
+
+
+*** Keywords ***
+
+Custom Setup
+    ${DEVICE_SN}=    Setup
+    Set Suite Variable    $DEVICE_SN
+    Device Should Exist    ${DEVICE_SN}
+
+    Setup LogFiles
+
+Setup LogFiles
+    ThinEdgeIO.Transfer To Device    ${CURDIR}/tedge-log-plugin.toml    /etc/tedge/plugins/tedge-log-plugin.toml
+
+    # Custom workflow and handler script
+    ThinEdgeIO.Transfer To Device    ${CURDIR}/log_upload.toml    /etc/tedge/operations/
+    ThinEdgeIO.Transfer To Device    ${CURDIR}/log_upload.sh    /usr/bin/log_upload.sh
+
+    ThinEdgeio.Restart Service    tedge-agent
+    ThinEdgeIO.Service Health Status Should Be Up    tedge-agent
+    ThinEdgeIO.Service Health Status Should Be Up    tedge-mapper-c8y
+
+
+Log File Contents Should Be Equal
+    [Arguments]    ${operation}    ${expected_pattern}    ${encoding}=utf-8    ${expected_filename}=^${DEVICE_SN}_[\\w\\W]+-c8y-mapper-\\d+$    ${expected_mime_type}=text/plain
+    ${event_url_parts}=    Split String    ${operation["c8y_LogfileRequest"]["file"]}    separator=/
+    ${event_id}=    Set Variable    ${event_url_parts}[-2]
+    ${contents}=    Cumulocity.Event Should Have An Attachment
+    ...    ${event_id}
+    ...    expected_pattern=${expected_pattern}
+    ...    encoding=${encoding}
+    ${event}=    Cumulocity.Event Attachment Should Have File Info    ${event_id}    name=${expected_filename}    mime_type=${expected_mime_type}
+    RETURN    ${contents}

tests/RobotFramework/tests/cumulocity/log/workflow/tedge-log-plugin.toml

@@ -0,0 +1,8 @@
+[[files]]
+type = "software-management"
+path = "/var/log/tedge/agent/software-*"
+
+[[files]]
+type = "sqlite"
+# This path can be any dummy location that the tedge-agent has write access to
+path = "/tmp/sqlite.log"