Merge pull request #17476 from rpastrana/HPCC-29518-ZapFilterESP

HPCC-29518 ZAP log filter

esp/scm/ws_logaccess.ecm

@@ -100,6 +100,17 @@ ESPenum LogAccessFilterOperator : int
     OR(2, "OR")
 };
 
+ESPenum LogEventClass : string
+{
+    All("ALL"),
+    Disaster("DIS"),
+    Error("ERR"),
+    Warning("WRN"),
+    Info("INF"),
+    Progress("PRO"),
+    Metric("MET")
+};
+
 /*
 * Provides mechanism to query log entries
 *

esp/scm/ws_workunits.ecm

@@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp);
 
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.96"), default_client_version("1.96"), cache_group("ESPWsWUs"),
+    version("1.97"), default_client_version("1.97"), cache_group("ESPWsWUs"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/scm/ws_workunits_req_resp.ecm

@@ -20,6 +20,7 @@
 
 EspInclude(common);
 EspInclude(ws_workunits_struct);
+EspInclude(ws_logaccess);
 
 ESPrequest [nil_remove] WUCreateRequest
 {
@@ -927,6 +928,7 @@ ESPrequest [nil_remove] WUCreateZAPInfoRequest
     [min_ver("1.73")] string EmailFrom;
     [min_ver("1.73")] string EmailSubject;
     [min_ver("1.73")] string EmailBody;
+    [min_ver("1.97")] ESPStruct LogAccessFilter LogFilter;
 };
 
 ESPresponse [exceptions_inline] WUCreateZAPInfoResponse

esp/scm/ws_workunits_struct.ecm

@@ -882,3 +882,18 @@ ESPenum WUProtectFilter : string
     Protected("Protected"),
     NotProtected("NotProtected")
 };
+
+ESPStruct LogAccessFilter
+{
+    string WildcardFilter;
+    ESPStruct TimeRange AbsoluteTimeRange;
+    unsigned RelativeTimeRangeBuffer;
+    unsigned LineLimit(10000);
+    int64 LineStartFrom(0);
+    ESPenum LogSelectColumnMode SelectColumnMode(1);
+    ESParray<string> CustomColumns;
+    ESParray<string> ComponentsFilter;
+    LogAccessLogFormat Format("CSV");
+    ESPenum SortDirection sortByTimeDirection(1);
+    ESPenum LogEventClass LogEventType;
+};

esp/services/ws_workunits/ws_workunitsAuditLogs.cpp

@@ -1407,6 +1407,11 @@ void CWsWorkunitsSoapBindingEx::createAndDownloadWUZAPFile(IEspContext& context,
         if (zapInfoReq.esp.isEmpty())
             zapInfoReq.esp.set(espApplicationName.get());
         request->getParameter("ThorProcesses", zapInfoReq.thor);
+
+        if (version >= 1.97)
+        {
+            zapInfoReq.populateLogFilter(request);
+        }
     }
     else
     {

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -207,21 +207,9 @@ WsWUExceptions::WsWUExceptions(IConstWorkUnit& wu): numerr(0), numwrn(0), numinf
     }
 }
 
-void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRecords, const LogAccessReturnColsMode retColsMode,
-                                         const LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs)
+void streamFilteredLogsToFile(const char* outFile, LogAccessConditions & logFetchOptions, const LogAccessLogFormat logFormat)
 {
-    if (!queryRemoteLogAccessor())
-        throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!");
-
-    if (isEmptyString(outFile))
-        throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!");
-
-    LogAccessConditions logFetchOptions;
-    logFetchOptions.setFilter(getJobIDLogAccessFilter(wuid));
-    setLogTimeRange(logFetchOptions, wuLogSearchTimeBuffSecs);
-
-    logFetchOptions.setReturnColsMode(retColsMode);
-    logFetchOptions.setLimit(maxLogRecords);
+    unsigned maxLogRecords = logFetchOptions.getLimit();
 
     Owned<IFileIOStream> outIOS;
     outIOS.setown(createBufferedIOStreamFromFile(outFile, IFOcreate));
@@ -286,6 +274,42 @@ void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRec
     }
 }
 
+void WsWuInfo::readWorkunitComponentLogs(const char* outFile, CWsWuZAPInfoReq& zapLogFilterOptions)
+{
+    if (!queryRemoteLogAccessor())
+        throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!");
+
+    if (isEmptyString(outFile))
+        throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!");
+
+    const LogAccessTimeRange& trange = zapLogFilterOptions.logFilter.logFetchOptions.getTimeRange();
+
+    if (trange.getStartt().isNull())
+        setLogTimeRange(zapLogFilterOptions.logFilter.logFetchOptions, zapLogFilterOptions.logFilter.wuLogSearchTimeBuffSecs);
+
+    streamFilteredLogsToFile(outFile, zapLogFilterOptions.logFilter.logFetchOptions, zapLogFilterOptions.logFilter.logDataFormat);
+}
+
+void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRecords, const LogAccessReturnColsMode retColsMode,
+                                         const LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs)
+{
+    if (!queryRemoteLogAccessor())
+        throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!");
+
+    if (isEmptyString(outFile))
+        throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!");
+
+    LogAccessConditions logFetchOptions;
+
+    logFetchOptions.setFilter(getJobIDLogAccessFilter(wuid));
+    setLogTimeRange(logFetchOptions, wuLogSearchTimeBuffSecs);
+
+    logFetchOptions.setReturnColsMode(retColsMode);
+    logFetchOptions.setLimit(maxLogRecords);
+
+    streamFilteredLogsToFile(outFile, logFetchOptions, logFormat);
+}
+
 void WsWuInfo::setLogTimeRange(LogAccessConditions& logFetchOptions, unsigned wuLogSearchTimeBuffSecs)
 {
     struct LogAccessTimeRange range;
@@ -4126,11 +4150,13 @@ void CWsWuFileHelper::createZAPECLQueryArchiveFiles(IConstWorkUnit* cwu, const c
     }
 }
 
-void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, unsigned maxLogRecords, LogAccessReturnColsMode retColsMode, LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs)
+void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions)
 {
     if (cwu->getWuidVersion() == 0)
         return;
 
+    LogAccessLogFormat logFormat = zapLogFilterOptions.logFilter.logDataFormat;
+
     StringBuffer logfileextension;
     if (logFormat == LOGACCESS_LOGFORMAT_csv)
         logfileextension.set("csv");
@@ -4142,9 +4168,10 @@ void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, cons
         logfileextension.set("log");
 
     VStringBuffer fileName("%s%c%s-log.%s", path, PATHSEPCHAR, cwu->queryWuid(), logfileextension.str());
+
     try
     {
-        winfo.readWorkunitComponentLogs(fileName.str(), maxLogRecords, retColsMode, logFormat, wuLogSearchTimeBuffSecs);
+        winfo.readWorkunitComponentLogs(fileName.str(), zapLogFilterOptions);
     }
     catch(IException* e)
     {
@@ -4402,12 +4429,7 @@ void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu,
     if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
         createThorSlaveLogfile(cwu, winfo, tempDirName);
 #else
-    //These options should ultimately be drawn from req
-    unsigned maxLogRecords = defaultMaxLogRecords;
-    LogAccessReturnColsMode retColsMode = RETURNCOLS_MODE_default;
-    LogAccessLogFormat logFormat = LOGACCESS_LOGFORMAT_csv;
-    unsigned wuLogSearchTimeBuffSecs = defaultWULogSearchTimeBufferSecs;
-    createWULogFile(cwu, winfo, tempDirName, maxLogRecords, retColsMode, LOGACCESS_LOGFORMAT_csv, wuLogSearchTimeBuffSecs);
+    createWULogFile(cwu, winfo, tempDirName, request);
 #endif
 
     //Write out to ZIP file