diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm index ec09859b72f..455561e7f7f 100644 --- a/esp/scm/ws_logaccess.ecm +++ b/esp/scm/ws_logaccess.ecm @@ -100,6 +100,17 @@ ESPenum LogAccessFilterOperator : int OR(2, "OR") }; +ESPenum LogEventClass : string +{ + All("ALL"), + Disaster("DIS"), + Error("ERR"), + Warning("WRN"), + Info("INF"), + Progress("PRO"), + Metric("MET") +}; + /* * Provides mechanism to query log entries * diff --git a/esp/scm/ws_workunits.ecm b/esp/scm/ws_workunits.ecm index 5bd95366b24..03064b548f5 100644 --- a/esp/scm/ws_workunits.ecm +++ b/esp/scm/ws_workunits.ecm @@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp); ESPservice [ auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization - version("1.96"), default_client_version("1.96"), cache_group("ESPWsWUs"), + version("1.97"), default_client_version("1.97"), cache_group("ESPWsWUs"), noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits { ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")] WUQuery(WUQueryRequest, WUQueryResponse); diff --git a/esp/scm/ws_workunits_req_resp.ecm b/esp/scm/ws_workunits_req_resp.ecm index 9d9391d0a97..5047bd31e4e 100644 --- a/esp/scm/ws_workunits_req_resp.ecm +++ b/esp/scm/ws_workunits_req_resp.ecm @@ -20,6 +20,7 @@ EspInclude(common); EspInclude(ws_workunits_struct); +EspInclude(ws_logaccess); ESPrequest [nil_remove] WUCreateRequest { @@ -927,6 +928,7 @@ ESPrequest [nil_remove] WUCreateZAPInfoRequest [min_ver("1.73")] string EmailFrom; [min_ver("1.73")] string EmailSubject; [min_ver("1.73")] string EmailBody; + [min_ver("1.97")] ESPStruct LogAccessFilter LogFilter; }; ESPresponse [exceptions_inline] WUCreateZAPInfoResponse diff --git a/esp/scm/ws_workunits_struct.ecm b/esp/scm/ws_workunits_struct.ecm index 4952df23f58..fbe88028819 100644 --- a/esp/scm/ws_workunits_struct.ecm +++ b/esp/scm/ws_workunits_struct.ecm @@ -882,3 +882,18 @@ ESPenum WUProtectFilter : string Protected("Protected"), NotProtected("NotProtected") }; + +ESPStruct LogAccessFilter +{ + string WildcardFilter; + ESPStruct TimeRange AbsoluteTimeRange; + unsigned RelativeTimeRangeBuffer; + unsigned LineLimit(10000); + int64 LineStartFrom(0); + ESPenum LogSelectColumnMode SelectColumnMode(1); + ESParray CustomColumns; + ESParray ComponentsFilter; + LogAccessLogFormat Format("CSV"); + ESPenum SortDirection sortByTimeDirection(1); + ESPenum LogEventClass LogEventType; +}; \ No newline at end of file diff --git a/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp b/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp index 24ebe355c6f..a0893fa9396 100644 --- a/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp +++ b/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp @@ -1407,6 +1407,11 @@ void CWsWorkunitsSoapBindingEx::createAndDownloadWUZAPFile(IEspContext& context, if (zapInfoReq.esp.isEmpty()) zapInfoReq.esp.set(espApplicationName.get()); request->getParameter("ThorProcesses", zapInfoReq.thor); + + if (version >= 1.97) + { + zapInfoReq.populateLogFilter(request); + } } else { diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index a43d82d562f..6c80b0a0e11 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -207,21 +207,9 @@ WsWUExceptions::WsWUExceptions(IConstWorkUnit& wu): numerr(0), numwrn(0), numinf } } -void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRecords, const LogAccessReturnColsMode retColsMode, - const LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs) +void streamFilteredLogsToFile(const char* outFile, LogAccessConditions & logFetchOptions, const LogAccessLogFormat logFormat) { - if (!queryRemoteLogAccessor()) - throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!"); - - if (isEmptyString(outFile)) - throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!"); - - LogAccessConditions logFetchOptions; - logFetchOptions.setFilter(getJobIDLogAccessFilter(wuid)); - setLogTimeRange(logFetchOptions, wuLogSearchTimeBuffSecs); - - logFetchOptions.setReturnColsMode(retColsMode); - logFetchOptions.setLimit(maxLogRecords); + unsigned maxLogRecords = logFetchOptions.getLimit(); Owned outIOS; outIOS.setown(createBufferedIOStreamFromFile(outFile, IFOcreate)); @@ -286,6 +274,42 @@ void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRec } } +void WsWuInfo::readWorkunitComponentLogs(const char* outFile, CWsWuZAPInfoReq& zapLogFilterOptions) +{ + if (!queryRemoteLogAccessor()) + throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!"); + + if (isEmptyString(outFile)) + throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!"); + + const LogAccessTimeRange& trange = zapLogFilterOptions.logFilter.logFetchOptions.getTimeRange(); + + if (trange.getStartt().isNull()) + setLogTimeRange(zapLogFilterOptions.logFilter.logFetchOptions, zapLogFilterOptions.logFilter.wuLogSearchTimeBuffSecs); + + streamFilteredLogsToFile(outFile, zapLogFilterOptions.logFilter.logFetchOptions, zapLogFilterOptions.logFilter.logDataFormat); +} + +void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRecords, const LogAccessReturnColsMode retColsMode, + const LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs) +{ + if (!queryRemoteLogAccessor()) + throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: Remote Log Access plug-in not available!"); + + if (isEmptyString(outFile)) + throw makeStringException(ECLWATCH_INVALID_FILE_NAME, "WsWuInfo: Target filename not provided!"); + + LogAccessConditions logFetchOptions; + + logFetchOptions.setFilter(getJobIDLogAccessFilter(wuid)); + setLogTimeRange(logFetchOptions, wuLogSearchTimeBuffSecs); + + logFetchOptions.setReturnColsMode(retColsMode); + logFetchOptions.setLimit(maxLogRecords); + + streamFilteredLogsToFile(outFile, logFetchOptions, logFormat); +} + void WsWuInfo::setLogTimeRange(LogAccessConditions& logFetchOptions, unsigned wuLogSearchTimeBuffSecs) { struct LogAccessTimeRange range; @@ -4126,11 +4150,13 @@ void CWsWuFileHelper::createZAPECLQueryArchiveFiles(IConstWorkUnit* cwu, const c } } -void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, unsigned maxLogRecords, LogAccessReturnColsMode retColsMode, LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs) +void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions) { if (cwu->getWuidVersion() == 0) return; + LogAccessLogFormat logFormat = zapLogFilterOptions.logFilter.logDataFormat; + StringBuffer logfileextension; if (logFormat == LOGACCESS_LOGFORMAT_csv) logfileextension.set("csv"); @@ -4142,9 +4168,10 @@ void CWsWuFileHelper::createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, cons logfileextension.set("log"); VStringBuffer fileName("%s%c%s-log.%s", path, PATHSEPCHAR, cwu->queryWuid(), logfileextension.str()); + try { - winfo.readWorkunitComponentLogs(fileName.str(), maxLogRecords, retColsMode, logFormat, wuLogSearchTimeBuffSecs); + winfo.readWorkunitComponentLogs(fileName.str(), zapLogFilterOptions); } catch(IException* e) { @@ -4402,12 +4429,7 @@ void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu, if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on")) createThorSlaveLogfile(cwu, winfo, tempDirName); #else - //These options should ultimately be drawn from req - unsigned maxLogRecords = defaultMaxLogRecords; - LogAccessReturnColsMode retColsMode = RETURNCOLS_MODE_default; - LogAccessLogFormat logFormat = LOGACCESS_LOGFORMAT_csv; - unsigned wuLogSearchTimeBuffSecs = defaultWULogSearchTimeBufferSecs; - createWULogFile(cwu, winfo, tempDirName, maxLogRecords, retColsMode, LOGACCESS_LOGFORMAT_csv, wuLogSearchTimeBuffSecs); + createWULogFile(cwu, winfo, tempDirName, request); #endif //Write out to ZIP file diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.hpp b/esp/services/ws_workunits/ws_workunitsHelpers.hpp index d43f0849601..4b11906939b 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.hpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.hpp @@ -154,7 +154,233 @@ struct WUComponentLogOptions { LogAccessConditions logFetchOptions; LogAccessLogFormat logFormat = LOGACCESS_LOGFORMAT_csv; + LogAccessLogFormat logDataFormat = LOGACCESS_LOGFORMAT_csv; unsigned wuLogSearchTimeBuffSecs = defaultWULogSearchTimeBufferSecs; + + ILogAccessFilter * getOredComponentsLogFilter(StringArray & components, unsigned index = 0) + { + if (index + 1 == components.length()) + return getComponentLogAccessFilter(components.item(index)); + else + return getBinaryLogAccessFilter + ( + getComponentLogAccessFilter(components.item(index)), + getOredComponentsLogFilter(components, index+1), + LOGACCESS_FILTER_or + ); + } + + void populateTimeRange(const char * start, const char * end, unsigned relativeTimeBufferSecs) + { + if (!isEmptyString(start) && !isEmptyString(end)) + { + struct LogAccessTimeRange absoluteTimeRange; + absoluteTimeRange.setStart(start); + absoluteTimeRange.setEnd(end); + logFetchOptions.setTimeRange(absoluteTimeRange); + } + else if (!isEmptyString(start)) + { + if (isEmptyString(end)) + throw makeStringException(ECLWATCH_INVALID_INPUT, "ZapLogFilter: Empty 'Absolute TimeRange End' detected!"); + } + else if (!isEmptyString(end)) + { + if (isEmptyString(end)) + throw makeStringException(ECLWATCH_INVALID_INPUT, "ZapLogFilter: Empty 'Absolute TimeRange Start' detected!"); + } + else + { + if (relativeTimeBufferSecs > 0 ) + wuLogSearchTimeBuffSecs = relativeTimeBufferSecs; + } + } + + void populateLogFilter(const char * wuid, CHttpRequest * zapHttpRequest) + { + ILogAccessFilter * logFetchFilter = getJobIDLogAccessFilter(wuid); + + StringBuffer requestedLogDataFormat; + zapHttpRequest->getParameter("LogFilter_Format", requestedLogDataFormat); + if (!requestedLogDataFormat.isEmpty()) + logDataFormat = logAccessFormatFromName(requestedLogDataFormat.str()); + + StringBuffer start; // Absolute query time range start in YYYY-DD-MMTHH:MM:SS + zapHttpRequest->getParameter("LogFilter_AbsoluteTimeRange_Start", start); + StringBuffer end; // Absolute query time range end in YYYY-DD-MMTHH:MM:SS + zapHttpRequest->getParameter("LogFilter_AbsoluteTimeRange_End", end); + // Query time range based on WU Time +- Buffer in seconds + unsigned bufferSecs = (unsigned)zapHttpRequest->getParameterInt("LogFilter_RelativeTimeRangeBuffer", 0); + + populateTimeRange(start, end, bufferSecs); + + //int 0 ==MIN, 1==DEFAULT, 2==ALL, 3==CUSTOM + int colMode = zapHttpRequest->getParameterInt("LogFilter_ColumnMode", -1); + if (colMode != -1) + { + StringArray customFields; //comma delimited list of available columns, only if ColumnMode==3 + if (colMode == 3) + { + StringBuffer customFieldsList; + zapHttpRequest->getParameter("LogFilter_CustomColumns", customFieldsList); + + if(!customFieldsList.isEmpty()) + customFields.appendList(customFieldsList.str(), ","); + } + + setReturnColumMode(colMode, customFields); + } + + StringBuffer lineLimit; + zapHttpRequest->getParameter("LogFilter_LineLimit", lineLimit); + if (!lineLimit.isEmpty()) + { + int limit = strtoll(lineLimit.str(), nullptr, 10); + if (limit < 0) + throw makeStringException(ECLWATCH_INVALID_INPUT, "Zap LogFilter encountered negative line limit!"); + + logFetchOptions.setLimit((unsigned)limit); + } + + StringBuffer startFrom; + zapHttpRequest->getParameter("LogFilter_LineStartFrom", startFrom); + if (!startFrom.isEmpty()) + { + int start = strtoll(startFrom.str(), nullptr, 10); + if (start < 0) + throw makeStringException(ECLWATCH_INVALID_INPUT, "Zap LogFilter encountered negative startFrom!"); + logFetchOptions.setStartFrom((unsigned)start); + } + + ILogAccessFilter * componentsFilterObj = nullptr; + StringBuffer componentsFilterList; + zapHttpRequest->getParameter("LogFilter_ComponentsFilter", componentsFilterList); + if (!componentsFilterList.isEmpty()) + { + StringArray componentsFilter; + componentsFilter.appendList(componentsFilterList.str(), ","); + componentsFilterObj = getOredComponentsLogFilter(componentsFilter); + } + + if (componentsFilterObj != nullptr) + logFetchFilter = getBinaryLogAccessFilter(logFetchFilter, componentsFilterObj, LOGACCESS_FILTER_and); + + ILogAccessFilter * logEventTypeFilterObj = nullptr; + StringBuffer logType; //"DIS","ERR","WRN","INF","PRO","MET","ALL" + zapHttpRequest->getParameter("LogFilter_LogEventType", logType); + if (!logType.isEmpty() && strcmp(logType.str(), "ALL") != 0) + logEventTypeFilterObj = getClassLogAccessFilter(LogMsgClassFromAbbrev(logType.str())); + + if (logEventTypeFilterObj != nullptr) + logFetchFilter = getBinaryLogAccessFilter(logFetchFilter, logEventTypeFilterObj, LOGACCESS_FILTER_and); + + StringBuffer wildCharFilter; + zapHttpRequest->getParameter("LogFilter_WildcardFilter", wildCharFilter); + if (!wildCharFilter.isEmpty()) + logFetchFilter = getBinaryLogAccessFilter(logFetchFilter, getWildCardLogAccessFilter(wildCharFilter.str()), LOGACCESS_FILTER_or); + + logFetchOptions.setFilter(logFetchFilter); + + //"ASC", "DSC" + StringBuffer sortByTimeDirection; + zapHttpRequest->getParameter("LogFilter_SortByTimeDirection", sortByTimeDirection); + logFetchOptions.addSortByCondition(LOGACCESS_MAPPEDFIELD_timestamp, "", + strcmp(sortByTimeDirection, "ASC") == 0 + ? SORTBY_DIRECTION_ascending : SORTBY_DIRECTION_descending); + + } + + void setReturnColumMode(int columnModeCode, const StringArray & customFields) + { + logFetchOptions.setReturnColsMode(RETURNCOLS_MODE_default); + switch (columnModeCode) + { + case 0: + logFetchOptions.setReturnColsMode(RETURNCOLS_MODE_min); + break; + case 1: + logFetchOptions.setReturnColsMode(RETURNCOLS_MODE_default); + break; + case 2: + logFetchOptions.setReturnColsMode(RETURNCOLS_MODE_all); + break; + case 3: + logFetchOptions.setReturnColsMode(RETURNCOLS_MODE_custom); + if (customFields.length() > 0 ) + logFetchOptions.copyLogFieldNames(customFields); + else + throw makeStringException(ECLWATCH_LOGACCESS_UNAVAILABLE, "WsWuInfo: LogFilter empty custom colums detected!"); + break; + default: + break; + } + } + + void populateLogFilter(const char * wuid, IConstLogAccessFilter & logFilterReq) + { + ILogAccessFilter * logFetchFilter = getJobIDLogAccessFilter(wuid); + + const char * requestedLogFormat = logFilterReq.getFormat(); + if (!isEmptyString(requestedLogFormat)) + logDataFormat = logAccessFormatFromName(requestedLogFormat); + + struct LogAccessTimeRange absoluteTimeRange; + const char * start = logFilterReq.getAbsoluteTimeRange().getStartDate(); + const char * end = logFilterReq.getAbsoluteTimeRange().getEndDate(); + + populateTimeRange(start, end, logFilterReq.getRelativeTimeRangeBuffer()); + + if (!isEmptyString(logFilterReq.getSelectColumnModeAsString())) + setReturnColumMode(logFilterReq.getSelectColumnMode(), logFilterReq.getCustomColumns()); + + logFetchOptions.setLimit(logFilterReq.getLineLimit()); + logFetchOptions.setStartFrom(logFilterReq.getLineStartFrom()); + + ILogAccessFilter * componentsFilterObj = nullptr; + if (logFilterReq.getComponentsFilter().length() > 0) + componentsFilterObj = getOredComponentsLogFilter(logFilterReq.getComponentsFilter()); + + if (componentsFilterObj != nullptr) + logFetchFilter = getBinaryLogAccessFilter(logFetchFilter, componentsFilterObj, LOGACCESS_FILTER_and); + + ILogAccessFilter * logEventTypeFilterObj = nullptr; + const char * logType = logFilterReq.getLogEventTypeAsString(); + if (!isEmptyString(logType) && strcmp(logType,"ALL") != 0) + logEventTypeFilterObj = getClassLogAccessFilter(LogMsgClassFromAbbrev(logType)); + + if (logEventTypeFilterObj != nullptr) + logFetchFilter = getBinaryLogAccessFilter(logFetchFilter, logEventTypeFilterObj, LOGACCESS_FILTER_and); + + const char * wildCharFilter = logFilterReq.getWildcardFilter(); + if (!isEmptyString(wildCharFilter)) + logFetchFilter = getBinaryLogAccessFilter(logFetchFilter, getWildCardLogAccessFilter(wildCharFilter), LOGACCESS_FILTER_or); + + logFetchOptions.setFilter(logFetchFilter); + CSortDirection espSortDirection = logFilterReq.getSortByTimeDirection(); + logFetchOptions.addSortByCondition(LOGACCESS_MAPPEDFIELD_timestamp, "", espSortDirection == CSortDirection_ASC + ? SORTBY_DIRECTION_ascending : SORTBY_DIRECTION_descending); + } +}; + +struct CWsWuZAPInfoReq +{ + StringBuffer wuid, esp, url, thor, problemDesc, whatChanged, whereSlow, includeThorSlaveLog, zapFileName, password; + StringBuffer emailFrom, emailTo, emailServer, emailSubject, emailBody; + + bool sendEmail, attachZAPReportToEmail; + unsigned maxAttachmentSize, port; + + WUComponentLogOptions logFilter; + + void populateLogFilter(IConstLogAccessFilter & logFilterReq) + { + logFilter.populateLogFilter(wuid.str(), logFilterReq); + } + + void populateLogFilter(CHttpRequest * httpRequest) + { + logFilter.populateLogFilter(wuid.str(), httpRequest); + } }; class WsWuInfo @@ -169,6 +395,7 @@ class WsWuInfo void readWorkunitThorLog(const char* processName, const char* logSpec, const char* slaveIPAddress, unsigned slaveNum, MemoryBuffer& buf, const char* outFile); void readWorkunitThorLogOneDay(IFile* ios, unsigned& processID, MemoryBuffer& buf, IFileIOStream* outIOS); #endif + void setLogTimeRange(LogAccessConditions& logFetchOptions, unsigned wuLogSearchTimeBuffSecs); unsigned sendComponentLogContent(IEspContext* context, IRemoteLogAccessStream* logreader, IXmlStreamFlusher* flusher, const WUComponentLogOptions& options); void sendComponentLogCSV(IEspContext* context, IRemoteLogAccessStream* logreader, IXmlStreamFlusher* flusher, const WUComponentLogOptions& options); void sendComponentLogJSON(IEspContext* context, IRemoteLogAccessStream* logreader, IXmlStreamFlusher* flusher, const WUComponentLogOptions& options); @@ -177,7 +404,6 @@ class WsWuInfo const char* sourceAlias, MemoryBuffer &mb, bool forDownload); void copyContentFromRemoteFile(const char* sourceFileName, const char* sourceIPAddress, const char* sourceAlias, const char *outFileName); - void setLogTimeRange(LogAccessConditions& logFetchOptions, unsigned wuLogSearchTimeBuffSecs); void getPostMortemFiles(IFile* file, unsigned& helpersCount, StringArray& postMortemFiles); void addPostMortemFiles(StringArray& postMortemFiles, IArrayOf& helpers); void validatePostMortemFile(IFile* file, const char* fileToBeValidated, bool& validated); @@ -193,6 +419,8 @@ class WsWuInfo */ void readWorkunitComponentLogs(const char* outFile, unsigned maxLogRecords, const LogAccessReturnColsMode retColsMode, const LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs); + + void readWorkunitComponentLogs(const char* outFile, CWsWuZAPInfoReq & zapLogFilterOptions); void sendWorkunitComponentLogs(IEspContext* context, CHttpResponse* response, WUComponentLogOptions& options); void sendImportedWorkunitComponentLog(const char* logFile, CHttpResponse* response); @@ -644,14 +872,6 @@ class NewWsWorkunit : public Owned } }; -struct CWsWuZAPInfoReq -{ - StringBuffer wuid, esp, url, thor, problemDesc, whatChanged, whereSlow, includeThorSlaveLog, zapFileName, password; - StringBuffer emailFrom, emailTo, emailServer, emailSubject, emailBody; - bool sendEmail, attachZAPReportToEmail; - unsigned maxAttachmentSize, port; -}; - class CWsWuFileHelper { IPropertyTree* directories; @@ -680,7 +900,7 @@ class CWsWuFileHelper void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path); #endif LogAccessLogFormat getComponentLogFormatFromLogName(const char *log); - void createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, unsigned maxLogRecords, LogAccessReturnColsMode retColsMode, LogAccessLogFormat logFormat, unsigned wuLogSearchTimeBuffSecs); + void createWULogFile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq & zapLogFilterOptions); void writeZAPWUInfoToIOStream(IFileIOStream *outFile, const char *name, SCMStringBuffer &value); void writeZAPWUInfoToIOStream(IFileIOStream *outFile, const char *name, const char *value); void readWUIDRequest(CHttpRequest *request, StringBuffer &wuid); diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index ab0aea632ae..e3cb94f672e 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -4892,6 +4892,11 @@ bool CWsWorkunitsEx::onWUCreateZAPInfo(IEspContext &context, IEspWUCreateZAPInfo if (zapInfoReq.esp.isEmpty()) zapInfoReq.esp.set(espApplicationName.get()); zapInfoReq.thor = req.getThorProcesses(); + + if (version >= 1.97) + { + zapInfoReq.populateLogFilter(req.getLogFilter()); + } } else { diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index cc6987ba8dc..8e44015fb6d 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -3151,6 +3151,11 @@ ILogAccessFilter * getLogAccessFilterFromPTree(IPropertyTree * xml) throwUnexpectedX("getLogAccessFilterFromPTree : unrecognized LogAccessFilter type"); } +ILogAccessFilter * getWildCardLogAccessFilter(const char * wildcardfilter) +{ + return new FieldLogAccessFilter(wildcardfilter, LOGACCESS_FILTER_wildcard); +} + ILogAccessFilter * getWildCardLogAccessFilter() { return new FieldLogAccessFilter("", LOGACCESS_FILTER_wildcard); diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index b711cfc40f1..cfb7692af02 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1658,6 +1658,7 @@ extern jlib_decl ILogAccessFilter * getClassLogAccessFilter(LogMsgClass logclass extern jlib_decl ILogAccessFilter * getBinaryLogAccessFilter(ILogAccessFilter * arg1, ILogAccessFilter * arg2, LogAccessFilterType type); extern jlib_decl ILogAccessFilter * getBinaryLogAccessFilterOwn(ILogAccessFilter * arg1, ILogAccessFilter * arg2, LogAccessFilterType type); extern jlib_decl ILogAccessFilter * getWildCardLogAccessFilter(); +extern jlib_decl ILogAccessFilter * getWildCardLogAccessFilter(const char * wildcardfilter); extern jlib_decl ILogAccessFilter * getColumnLogAccessFilter(const char * columnName, const char * value); // Helper functions to actuate log access query diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index ad386c18721..40adcac1db2 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -540,6 +540,7 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr StringBuffer queryValue; std::string queryField = m_globalSearchColName.str(); + std::string queryOperator = " =~ "; filter->toString(queryValue); switch (filter->filterType()) @@ -641,7 +642,13 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr break; } case LOGACCESS_FILTER_wildcard: - throw makeStringExceptionV(-1, "%s: Wild Card filter detected within exact term filter!", COMPONENT_NAME); + if (queryValue.isEmpty()) + throw makeStringExceptionV(-1, "%s: Wildcard filter cannot be empty!", COMPONENT_NAME); + + queryOperator = " contains "; + DBGLOG("%s: Searching log entries by wildcard filter: '%s %s %s'...", COMPONENT_NAME, queryField.c_str(), queryOperator.c_str(), queryValue.str()); + + break; case LOGACCESS_FILTER_or: case LOGACCESS_FILTER_and: { @@ -669,9 +676,8 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr //KQL structure: //TableName - //| where Fieldname =~ 'value' - //queryString.append("\n| where ").append(queryField.c_str()).append(" =~ '").append(queryValue.str()).append("'"); - queryString.append(" ").append(queryField.c_str()).append(" =~ '").append(queryValue.str()).append("'"); + //| where Fieldname OPERATOR 'value' + queryString.append(" ").append(queryField.c_str()).append(queryOperator.c_str()).append("'").append(queryValue.str()).append("'"); } void AzureLogAnalyticsCurlClient::declareContainerIndexJoinTable(StringBuffer & queryString, const LogAccessConditions & options) @@ -707,7 +713,7 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr queryString.append(queryIndex); generateHPCCLogColumnstAllColumns(queryString, m_globalSearchColName.str()); - if (options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter + if (options.queryFilter() == nullptr || options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter { //no where clause queryIndex.set(m_globalIndexSearchPattern.str()); diff --git a/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp b/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp index 198b6a8512c..260766acb60 100644 --- a/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp +++ b/system/logaccess/ElasticStack/ElasticStackLogAccess.cpp @@ -711,7 +711,11 @@ void ElasticStackLogAccess::populateESQueryQueryString(std::string & queryString break; } case LOGACCESS_FILTER_wildcard: - throw makeStringExceptionV(-1, "%s: Wild Card filter detected within exact term filter!", COMPONENT_NAME); + if (queryValue.isEmpty()) + throw makeStringExceptionV(-1, "%s: Wildcard filter cannot be empty!", COMPONENT_NAME); + + DBGLOG("%s: Searching log entries by wildcard filter: '%s: %s'...", COMPONENT_NAME, queryField.c_str(), queryValue.str()); + break; case LOGACCESS_FILTER_or: case LOGACCESS_FILTER_and: queryString += " ( "; @@ -746,7 +750,7 @@ void ElasticStackLogAccess::populateQueryStringAndQueryIndex(std::string & query esSearchMetaData(queryString, options.getReturnColsMode(), options.getLogFieldNames(), options.getSortByConditions(), options.getLimit(), options.getStartFrom()); queryString += "\"query\": { \"bool\": { \"filter\": [ "; - if (options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter + if (options.queryFilter() == nullptr || options.queryFilter()->filterType() == LOGACCESS_FILTER_wildcard) // No filter { queryIndex = m_globalIndexSearchPattern.str(); }