From 0fee5c21e8c0f8b231601124e39abe9e752cef38 Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Sun, 30 Jul 2023 02:00:28 -0400 Subject: [PATCH] HPCC-29332 WIP - Removes system/tracing/tracemanager Signed-off-by: Rodrigo Pastrana --- esp/bindings/http/platform/httpservice.cpp | 38 +- esp/bindings/http/platform/httpservice.hpp | 1 + esp/platform/espp.cpp | 18 + system/jlib/CMakeLists.txt | 23 ++ system/jlib/jtrace.cpp | 193 ++++++++- system/jlib/jtrace.hpp | 321 ++++++++++++++- system/tracing/CMakeLists.txt | 53 +-- system/tracing/opentel/CMakeLists.txt | 27 ++ .../opentel/exporters/jlog/CMakeLists.txt | 62 +++ .../exporters/jlog/spanexporterfactory.cpp | 107 +++++ .../exporters/jlog/spanexporterfactory.hpp | 36 ++ .../opentel/instrumented/CMakeLists.txt | 23 +- .../opentel/instrumented/httpclient.cpp | 95 +++-- .../opentel/instrumented/httpserver.cpp | 42 +- system/tracing/tracemanager.hpp | 384 ------------------ 15 files changed, 902 insertions(+), 521 deletions(-) create mode 100644 system/tracing/opentel/CMakeLists.txt create mode 100644 system/tracing/opentel/exporters/jlog/CMakeLists.txt create mode 100644 system/tracing/opentel/exporters/jlog/spanexporterfactory.cpp create mode 100644 system/tracing/opentel/exporters/jlog/spanexporterfactory.hpp delete mode 100644 system/tracing/tracemanager.hpp diff --git a/esp/bindings/http/platform/httpservice.cpp b/esp/bindings/http/platform/httpservice.cpp index ea3cbb1f3e2..d3f33ee69bb 100644 --- a/esp/bindings/http/platform/httpservice.cpp +++ b/esp/bindings/http/platform/httpservice.cpp @@ -184,6 +184,34 @@ void checkSetCORSAllowOrigin(EspHttpBinding *binding, CHttpRequest *req, CHttpRe int CEspHttpServer::processRequest() { + TraceManager traceManager("esp"); //we'd use an appropriate module/lib name here + auto tracer = traceManager.getTracer(); + + //Extract parent(caller) context from http header, likely done earlier in the process + //We'd need a setParentContextFromHeaders version supporting httptransport's StringArray m_headers; + //TraceManager::setParentContextFromHeaders(const_cast &>(request.headers), options); + //or labda function to extract parent context from http header + + //Options used to annotate span representing the processing of http requests + opentelemetry::trace::StartSpanOptions options; + options.kind = opentelemetry::trace::SpanKind::kServer; + + //Declare the span, provide appropriate attributes, and options + //Trace ID generated if no parent context is provided + auto processingRequestSpan = + tracer->StartSpan("ProcessingHTTPRequest", + { //Declare whatever span attributes we have at this point + //More can be attached along the way + //{"stype", stype}, //span attributes + {opentelemetry::trace::SemanticConventions::kNetHostPort, "8010"}, + //{opentelemetry::trace::SemanticConventions::kHttpMethod, methodName.str()}, + //{opentelemetry::trace::SemanticConventions::kRpcService, serviceName.str()}, + {opentelemetry::trace::SemanticConventions::kHttpScheme, "http"}}, + options); //options.parent is set as parent context for current span + + //activate the span + auto scope = tracer->WithActiveSpan(processingRequestSpan); + IEspContext* ctx = m_request->queryContext(); StringBuffer errMessage; m_request->setPersistentEnabled(m_apport->queryProtocol()->persistentEnabled() && !shouldClose); @@ -234,11 +262,14 @@ int CEspHttpServer::processRequest() StringBuffer serviceName; StringBuffer methodName; m_request->getEspPathInfo(stype, &pathEx, &serviceName, &methodName, false); + ESPLOG(LogNormal,"sub service type: %s. parm: %s", getSubServiceDesc(stype), m_request->queryParamStr()); +//all thesee attributes could/should be tracked by opentel trace/spans m_request->updateContext(); ctx->setServiceName(serviceName.str()); ctx->setHTTPMethod(method.str()); + processingRequestSpan->SetAttribute(opentelemetry::trace::SemanticConventions::kHttpMethod, method.str()); ctx->setServiceMethod(methodName.str()); ctx->addTraceSummaryValue(LogMin, "app.protocol", method.str(), TXSUMMARY_GRP_ENTERPRISE); ctx->addTraceSummaryValue(LogMin, "app.service", serviceName.str(), TXSUMMARY_GRP_ENTERPRISE); @@ -257,6 +288,7 @@ int CEspHttpServer::processRequest() } ctx->addTraceSummaryValue(LogMin, "custom_fields.URL", url.str(), TXSUMMARY_GRP_ENTERPRISE); + //TraceManager::injectKeyValue(C & carrier, const char * key, const char * val) m_response->setHeader(HTTP_HEADER_HPCC_GLOBAL_ID, ctx->getGlobalId()); if(strieq(method.str(), OPTIONS_METHOD)) @@ -268,7 +300,8 @@ int CEspHttpServer::processRequest() ESPLOG(LogMin, "%s %s, from %s", method.str(), m_request->getPath(pathStr).str(), m_request->getPeer(peerStr).str()); else //user ID is in HTTP header ESPLOG(LogMin, "%s %s, from %s@%s", method.str(), m_request->getPath(pathStr).str(), userid, m_request->getPeer(peerStr).str()); - +//checkUserAuth could declare nested span +//and/or declare this as an event on the processingRequestSpan authState = checkUserAuth(); if ((authState == authTaskDone) || (authState == authFailed)) return 0; @@ -450,6 +483,9 @@ int CEspHttpServer::processRequest() return 0; } + //need to ensure that the span is ended when out of scope Owend processingRequestSpan? + processingRequestSpan->End(); + return 0; } diff --git a/esp/bindings/http/platform/httpservice.hpp b/esp/bindings/http/platform/httpservice.hpp index 4b44472deb2..fd373b6a539 100644 --- a/esp/bindings/http/platform/httpservice.hpp +++ b/esp/bindings/http/platform/httpservice.hpp @@ -34,6 +34,7 @@ #include "espsession.ipp" #include "jhash.hpp" +#include "jtrace.hpp" typedef enum espAuthState_ { diff --git a/esp/platform/espp.cpp b/esp/platform/espp.cpp index 734b9357016..b0c0e9964b5 100644 --- a/esp/platform/espp.cpp +++ b/esp/platform/espp.cpp @@ -23,6 +23,7 @@ #include "jliball.hpp" #include "jstats.h" #include "jutil.hpp" +#include "jtrace.hpp" //CRT / OS #ifndef _WIN32 @@ -426,6 +427,11 @@ int init_main(int argc, const char* argv[]) //save off generated config to register with container. Legacy can always reference the config file, application based ESP needs generated config saved off Owned appConfig; + TraceManager traceManager("esp"); + auto espTracer = traceManager.getTracer();//All instances of this esp process can share the same tracer same as TraceManager::getTracer("esp") + auto rootEspProcspan = espTracer->StartSpan(__func__); //Dummy top level span for the esp process + auto scope = espTracer->WithActiveSpan(rootEspProcspan); //Set the active span. The span will remain active until the returned Scope object is destroyed. + try { const char* cfgfile = NULL; @@ -483,6 +489,16 @@ int init_main(int argc, const char* argv[]) const char * processName = procpt->queryProp("@name"); setStatisticsComponentName(SCTesp, processName, true); + rootEspProcspan->SetAttribute("procname", processName); //Span can be annotated with attributes + rootEspProcspan->SetAttribute("config", cfgfile);//Span can be annotated with attributes + //rootEspProcspan->SetAttribute("application", application); + + std::string traceId; + traceManager.getCurrentTraceId(traceId); + + std::string spanId; + traceManager.getCurrentSpanID(spanId); + openEspLogFile(envpt.get(), procpt.get()); DBGLOG("Esp starting %s", hpccBuildInfo.buildTag); @@ -511,6 +527,8 @@ int init_main(int argc, const char* argv[]) config.setown(cfg); abortHandler.setConfig(cfg); } + + rootEspProcspan->End(); } catch(IException* e) { diff --git a/system/jlib/CMakeLists.txt b/system/jlib/CMakeLists.txt index e6ed9160474..672556b61f4 100644 --- a/system/jlib/CMakeLists.txt +++ b/system/jlib/CMakeLists.txt @@ -42,6 +42,15 @@ endif(NOT TARGET libbase58) find_package(yaml CONFIG REQUIRED) +#For http exporter +find_package(Protobuf REQUIRED) +#For http exporter +find_package(CURL REQUIRED) + +find_package(opentelemetry-cpp CONFIG REQUIRED) + +#add_subdirectory(${HPCC_SOURCE_DIR}/system/tracing/opentel/exporters/jlog ${CMAKE_BINARY_DIR}/system/tracing/opentel/exporters/jlog) + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") set ( SRCS @@ -184,6 +193,7 @@ set ( INCLUDES ${HPCC_SOURCE_DIR}/system/security/cryptohelper/digisign.cpp ${HPCC_SOURCE_DIR}/system/security/cryptohelper/pke.cpp ${HPCC_SOURCE_DIR}/system/security/cryptohelper/ske.cpp + #${HPCC_SOURCE_DIR}/system/tracing/tracemanager.hpp ) set_source_files_properties(jmd5.cpp jsort.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) @@ -207,6 +217,8 @@ include_directories ( ${HPCC_SOURCE_DIR}/system/security/shared ${HPCC_SOURCE_DIR}/system/security/cryptohelper ${HPCC_SOURCE_DIR}/system/httplib + #${HPCC_SOURCE_DIR}/system/tracing + ${OPENTELEMETRY_CPP_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR} # for generated jelog.h file ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/oss @@ -230,6 +242,17 @@ if ( ${HAVE_LIBCRYPT} ) target_link_libraries ( jlib crypt) endif ( ${HAVE_LIBCRYPT} ) +target_link_libraries ( jlib + opentelemetry-cpp::api + opentelemetry-cpp::ext + opentelemetry-cpp::sdk + opentelemetry-cpp::logs #might not need + opentelemetry-cpp::ostream_span_exporter + opentelemetry-cpp::metrics #might not need + opentelemetry-cpp::otlp_http_exporter #might not need + opentelemetry-cpp::http_client_curl #might not need + ) + IF (USE_OPENSSL) target_link_libraries ( jlib ${OPENSSL_LIBRARIES}) endif (USE_OPENSSL) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 8a486b052e9..91b345bc82d 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -15,7 +15,6 @@ limitations under the License. ############################################################################## */ - #include "platform.h" #include "jmisc.hpp" @@ -44,6 +43,11 @@ LogTrace::LogTrace(const char * globalId) setGlobalId(globalId); } +LogTrace::LogTrace() +{ + assignLocalId(); +} + const char* LogTrace::assignLocalId() { localId.set(createUniqueIdString().c_str()); @@ -69,3 +73,190 @@ const char* LogTrace::queryLocalId() const { return localId.get(); } + +void TraceManager::initTracer() +{ + //Handle HPCC specific tracing configuration here + //Target exporter? exporter connection info? + //Target processor(s)? batch/interactive? prob not + //Target propogator? http? grpc? binary? custom? + //HPCC component tracing switches? + Owned traceConfig; + try + { + traceConfig.setown(getComponentConfigSP()->getPropTree("tracing")); + } + catch (IException * e) + { + EXCLOG(e); + e->Release(); + } + + if (traceConfig) + { + Owned exportConfig = traceConfig->getPropTree("exporter"); + if (exportConfig) + { + if (exportConfig->getPropBool("OS", false)) //To stdout/err + DBGLOG("Tracing to stdout/err"); + else if (exportConfig->getPropBool("OTLP", false)) + DBGLOG("Tracing to OTLP"); + else if (exportConfig->getPropBool("Jaeger", false)) + DBGLOG("Tracing to Jaeger"); + else if (exportConfig->getPropBool("Zipkin", false)) + DBGLOG("Tracing to Zipkin"); + else if (exportConfig->getPropBool("Prometheus", false)) + DBGLOG("Tracing to Prometheus"); + else if (exportConfig->getPropBool("HPCC", false)) + DBGLOG("Tracing to HPCC JLog"); + } + } + else + { + using namespace opentelemetry::trace; + //OStream exporter, useful for development and debugging tasks and simplest to set up. + auto exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); + + //SimpleSpanProcesser sends spans one by one to an exporter. + //We could use a batchspanprocessor, which will group several spans together, before sending them to an exporter. + auto processor = opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); + std::vector> processors; + processors.push_back(std::move(processor)); + + // Default is an always-on sampler. + std::shared_ptr context = + opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); + std::shared_ptr provider = + opentelemetry::sdk::trace::TracerProviderFactory::Create(context); + + // Set the global trace provider + opentelemetry::trace::Provider::SetTracerProvider(provider); + + // set global propagator + // Injects Context into and extracts it from carriers that travel in-band + // across process boundaries. Encoding is expected to conform to the HTTP + // Header Field semantics. + // Values are often encoded as RPC/HTTP request headers. + opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator( + opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::propagation::HttpTraceContext())); + } +} + +void TraceManager::cleanupTracer() +{ + std::shared_ptr none; + opentelemetry::trace::Provider::SetTracerProvider(none); +} + +//convenience non-static method to get the default tracer, uses stored tracer/module name +opentelemetry::nostd::shared_ptr TraceManager::getTracer() +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + return provider->GetTracer(tracerName); // (library_name [,library_version][,schema_url]) +} + +//convenience Static method to get the default tracer, uses provided module name +opentelemetry::nostd::shared_ptr TraceManager::getTracer(std::string moduleName) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + return provider->GetTracer(moduleName); // (library_name [,library_version][,schema_url]) +} + +void TraceManager::getCallerSpanId(context::propagation::TextMapCarrier &carrier, std::string & callerSpanId) +{ + callerSpanId.clear(); + + // Inject current context into http header + auto currentCtx = context::RuntimeContext::GetCurrent(); + auto propagator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propagator->Inject(carrier, currentCtx); + + // Extract parent span context from the TextMapCarrier + auto parentSpanContext = propagator->Extract(carrier,currentCtx); + + // Get the value of the HPCC-Caller-Id header from the TextMapCarrier + auto callerIdHeader = carrier.Get("traceparent"); + + callerSpanId = callerIdHeader.data(); +} + +void TraceManager::getParentSpanId(std::map requestHeaders, std::string & callerSpanId) +{ + const HttpTextMapCarrier> carrier(requestHeaders); + TraceManager::getParentSpanId(carrier, callerSpanId); +} + +void TraceManager::getParentSpanId(const HttpTextMapCarrier> carrier, std::string & callerSpanId) +{ + callerSpanId.clear(); + + auto propagator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto current_ctx = context::RuntimeContext::GetCurrent(); + auto new_context = propagator->Extract(carrier, current_ctx); + auto parentSpan = opentel_trace::GetSpan(new_context)->GetContext(); + + char span_id[16] = {0}; + parentSpan.span_id().ToLowerBase16(span_id); + callerSpanId = std::string(span_id, 16).c_str(); +} + +void TraceManager::getCurrentTraceId(std::string & traceId) const +{ + traceId.clear(); + + nostd::shared_ptr currntSpan = + TraceManager::getTracer(std::string(tracerName))->GetCurrentSpan(); + + if (currntSpan->IsRecording()) + { + auto spanCtx = currntSpan->GetContext(); + + char trace_id[32] = {0}; + spanCtx.trace_id().ToLowerBase16(trace_id); + traceId = std::string(trace_id, 32); + } +} +void TraceManager::getParentContext(std::map& request_headers, opentel_trace::StartSpanOptions & options) +{ + const HttpTextMapCarrier> carrier(request_headers); + auto prop = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto current_ctx = context::RuntimeContext::GetCurrent(); + auto new_context = prop->Extract(carrier, current_ctx); + options.parent = opentelemetry::trace::GetSpan(new_context)->GetContext(); +} + +void TraceManager::getCurrentSpanID(std::string & spanId) const +{ + spanId.clear(); + nostd::shared_ptr currntSpan = + TraceManager::getTracer(std::string(tracerName))->GetCurrentSpan(); + + if (currntSpan->IsRecording()) + { + char span_id[16] = {0}; + currntSpan->GetContext().span_id().ToLowerBase16(span_id); + + spanId = std::string(span_id, 16); + } +} + + + +static Singleton traceManager; +MODULE_INIT(INIT_PRIORITY_STANDARD) +{ + TraceManager::initTracer(); //Initialize the tracer based on HPCC configuration + return true; +} + +MODULE_EXIT() +{ + traceManager.destroy(); +} + +//Name of module/library that is being traced/instrumented +TraceManager * queryTraceManager(const char * moduleName) +{ + return new TraceManager(moduleName); +} diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 6aa184bbdc0..b9bd4c037ad 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -18,6 +18,180 @@ #ifndef JTRACE_HPP #define JTRACE_HPP +#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED +#include "opentelemetry/exporters/ostream/span_exporter_factory.h" +#include "opentelemetry/sdk/trace/exporter.h" +#include "opentelemetry/sdk/trace/processor.h" +#include "opentelemetry/sdk/trace/simple_processor_factory.h" +#include "opentelemetry/sdk/trace/tracer_context.h" +#include "opentelemetry/sdk/trace/tracer_context_factory.h" +#include "opentelemetry/sdk/trace/tracer_provider_factory.h" +#include "opentelemetry/trace/provider.h" + +#include "opentelemetry/context/propagation/global_propagator.h" +#include "opentelemetry/context/propagation/text_map_propagator.h" +#include "opentelemetry/trace/propagation/http_trace_context.h" + +#include "opentelemetry/ext/http/client/http_client_factory.h" +#include "opentelemetry/ext/http/common/url_parser.h" +#include "opentelemetry/trace/semantic_conventions.h" + +#include + +//using namespace opentelemetry::trace; +namespace http_client = opentelemetry::ext::http::client; +namespace context = opentelemetry::context; +namespace nostd = opentelemetry::nostd; +namespace trace_sdk = opentelemetry::sdk::trace; +namespace opentel_trace = opentelemetry::trace; + +//#include "jexcept.hpp" //re-define UNIMPLEMENTED +#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) +#include "jprop.hpp" + +// TextMapCarrier is the storage medium used by TextMapPropagator. +// pure virtual Get(key) returns the value associated with the passed key. +// pure virtual Set(key, value) stores the key-value pair. +// virtual Keys(nostd::function_ref /* callback */) +// list of all the keys in the carrier. +// By default, it returns true without invoking callback */ +template +class HttpTextMapCarrier : public opentelemetry::context::propagation::TextMapCarrier +{ +public: + HttpTextMapCarrier(T &headers) : httpHeaders(headers) {} + HttpTextMapCarrier() = default; + + virtual opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override + { + std::string theKey = key.data(); + + // perform any key mapping needed... + { + //Instrumented http client/server Capitalizes the first letter of the header name + if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState ) + theKey[0] = toupper(theKey[0]); + } + + //now search for the vaule + auto it = httpHeaders.find(theKey); + if (it != httpHeaders.end()) + return it->second; + + return ""; + } + + virtual void Set(opentelemetry::nostd::string_view key, + opentelemetry::nostd::string_view value) noexcept override + { + httpHeaders.insert(std::pair(std::string(key), std::string(value))); + } + + T httpHeaders; +}; + +template +class HPCCHttpTextMapCarrier : public opentelemetry::context::propagation::TextMapCarrier +{ +public: + HPCCHttpTextMapCarrier(R &headers) : httpHeaders(headers) {} + HPCCHttpTextMapCarrier() = default; + + virtual opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override + { + std::string theKey = key.data(); + + // perform any key mapping needed... + { + //Instrumented http client/server Capitalizes the first letter of the header name + if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState ) + theKey[0] = toupper(theKey[0]); + } + + return httpHeaders->queryProp(theKey.c_str()); + } + + virtual void Set(opentelemetry::nostd::string_view key, + opentelemetry::nostd::string_view value) noexcept override + { + httpHeaders->setProp(std::string(key).c_str(), std::string(value).c_str()); + //httpHeaders.insert(std::pair(std::string(key), std::string(value))); + } + + Owned httpHeaders = createProperties(); + //R httpHeaders = createProperties(); +}; + +/* +template +class HPCCStringArrayHttpTextMapCarrier : public opentelemetry::context::propagation::TextMapCarrier +{ +public: + HPCCHttpTextMapCarrier(R &headers) : httpHeaders(headers) {} + HPCCHttpTextMapCarrier() = default; + + virtual opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override + { + std::string theKey = key.data(); + std::string headerval; + + // perform any key mapping needed... + { + //Instrumented http client/server Capitalizes the first letter of the header name + if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState ) + theKey[0] = toupper(theKey[0]); + } + + ForEachItemIn(x, httpHeaders) + { + const char* header = httpHeaders.item(x); + if(header == nullptr) + continue; + + const char* colon = strchr(header, ':'); + if(colon == nullptr) + continue; + + unsigned len = colon - header; + if((strlen(headername) == len) && (strnicmp(headername, header, len) == 0)) + { + headerval.append(colon + 2); + break; + } + } + return headerval; + } + + virtual void Set(opentelemetry::nostd::string_view key, + opentelemetry::nostd::string_view value) noexcept override + { + if(!key || !*key) + return; + + StringBuffer kv; + kv.append(key).append(": ").append(value); + ForEachItemIn(x, m_headers) + { + const char* curst = m_headers.item(x); + if(!curst) + continue; + const char* colon = strchr(curst, ':'); + if(!colon) + continue; + if(!strnicmp(headername, curst, colon - curst)) + { + m_headers.replace(kv.str(), x); + return; + } + } + + m_headers.append(kv.str()); + } + + StringArray httpHeaders; +}; +*/ + class jlib_decl LogTrace { private: @@ -25,6 +199,8 @@ class jlib_decl LogTrace StringAttr callerId; StringAttr localId; + HPCCHttpTextMapCarrier carrier; //Injects/extracts context and other info in/from http headers + StringAttr globalIdHTTPHeaderName = "HPCC-Global-Id"; StringAttr callerIdHTTPHeaderName = "HPCC-Caller-Id"; @@ -32,7 +208,7 @@ class jlib_decl LogTrace public: - LogTrace() {}; + LogTrace(); LogTrace(const char * globalId); const char* queryGlobalId() const; @@ -55,6 +231,145 @@ class jlib_decl LogTrace void setLocalId(const char* id); }; +/** + * @brief This follows open telemetry's span attribute naming conventions + * Known HPCC span Keys could be added here + * Specialized span keys can also be defined within the scope of a span + */ +namespace HPCCSemanticConventions +{ +static constexpr const char *kGLOBALIDHTTPHeader = "HPCC-Global-Id"; +static constexpr const char *kCallerIdHTTPHeader = "HPCC-Caller-Id"; +} + +// github copilot generated comment: +// This class provides a high-level interface for managing tracing and profiling information in the HPCC Systems platform. +// The TraceManager class is responsible for creating and managing instances of the Tracer class, which is used to instrument code for tracing and profiling. The Tracer class provides methods for starting and ending spans, adding attributes to spans, and propagating trace context across different services and systems. +// Overall, the TraceManager class provides a convenient and flexible way to instrument code for tracing and profiling in the HPCC Systems platform. By using the Tracer class and the TraceManager class, developers can gain insight into the performance and behavior of their applications and diagnose issues in distributed systems. +class TraceManager +{ +private: + //Used as the opentel trace name, refered to as name of library being instrumented + const char * tracerName = nullptr; + +public: + TraceManager(const char * moduleName) + { + //InitModuleObjects(); + initTracer(); //Still not sure where this should be done, + //but it needs to be done once per process before any tracers/spans are created + tracerName = moduleName; //1 tracer for each module/library being instrumented + }; + + TraceManager(const std::string & moduleName) + { + TraceManager(moduleName.c_str()); + }; + + ~TraceManager() {}; + + static void initTracer(); + static void cleanupTracer(); + + //convenience non-static method to get the default tracer, uses stored tracer/module name + opentelemetry::nostd::shared_ptr getTracer(); + + //convenience Static method to get the tracer for the provided module name + static opentelemetry::nostd::shared_ptr getTracer(std::string moduleName); + + //Extracts parent contex from the carrier's headers and returns it as callerSpanId + static void getCallerSpanId(context::propagation::TextMapCarrier &carrier, std::string & callerSpanId); + static void getParentSpanId(std::map requestHeaders, std::string & callerSpanId); + static void getParentSpanId(const HttpTextMapCarrier> carrier, std::string & callerSpanId); + static void getParentContext(std::map& request_headers, opentel_trace::StartSpanOptions & options); + + //Get the parentSpan from ANY Carrier implementation + // HPCCHttpTextMapCarrier , HttpTextMapCarrier, etc + // The carrier must implement the TextMapCarrier interface + // and it will determine the header which contains the parent span + template + static void getParentSpanId(const CARRIER carrier, std::string & callerSpanId) + { + // extract caller span id from http header + auto propagator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto ctx = propagator->Extract(carrier); + auto spanContext = ctx->GetSpanContext(); + callerSpanId = spanContext.span_id(); + } + + void getCurrentTraceId(std::string & traceId) const; + void getCurrentSpanID(std::string & spanId) const; + + /* + static void injectCurrentHTTPContext(HPCCHttpTextMapCarrier & hpccHttpHeaders) + { + // inject current context into http header + auto currentCtx = context::RuntimeContext::GetCurrent(); + auto propegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propegator->Inject(carrier, currentCtx); //injects current context as parent + }*/ + + //Injects current opentelemetry context into the carrier + //Typically done by client before sending request + //Context is injected as parent + //Default propegator targets "traceparent" and "tracestate" headers + template + static void injectCurretContext(C & carrier) + { + //HPCCHttpTextMapCarrier | HttpTextMapCarrier + + // inject current context into http header + auto currentCtx = context::RuntimeContext::GetCurrent(); + auto propegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propegator->Inject(carrier, currentCtx); //injects current context as parent + } + + void injectCurretContext(HttpTextMapCarrier & carrier) const + { + // inject current context into http header + auto currentCtx = context::RuntimeContext::GetCurrent(); + //auto propegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + //propegator->Inject(carrier, currentCtx); //injects current context as parent + + //return true; + } + + //Injects custom key/val pairs into provided carrier + //HPCC components can inject + template //HPCCHttpTextMapCarrier | HttpTextMapCarrier + static void injectKeyValue(C & carrier, const char * key, const char * val) + { + // inject current key/val pair into carrier (http headers?) + carrier.Set(key, val); + } + + template //HPCCHttpTextMapCarrier | HttpTextMapCarrier + static void injectHPCCGlobalID(C & carrier, const char * val) + { + carrier.Set(HPCCSemanticConventions::kGLOBALIDHTTPHeader, val); + } + + template //HPCCHttpTextMapCarrier | HttpTextMapCarrier + static void injectHPCCCallerID(C & carrier, const char * val) + { + carrier.Set(HPCCSemanticConventions::kCallerIdHTTPHeader, val); + } + + template //HPCCHttpTextMapCarrier | HttpTextMapCarrier + static void extractHPCCCallerID(C & carrier, const char * val) + { + TraceManager::getParentSpanId(carrier, val); + carrier.Set(HPCCSemanticConventions::kCallerIdHTTPHeader, val); + } + + //const char* queryCallerId(context::propagation::TextMapCarrier &carrier) const; + //static bool setParentContextFromHeaders(std::map& request_headers, opentel_trace::StartSpanOptions & options); + //static bool extractCallerSpanId(std::map request_headers, std::string & callerSpanId); + //static bool extractCallerSpanId(const HttpTextMapCarrier> carrier, std::string & callerSpanId); +}; + +extern jlib_decl TraceManager * queryTraceManager(const char * name); + /* To use feature-level tracing flags, protect the tracing with a test such as: @@ -181,8 +496,6 @@ constexpr std::initializer_list roxieTraceOptions TRACEOPT(traceSmartStepping), }; -interface IPropertyTree; - extern jlib_decl bool doTrace(TraceFlags featureFlag, TraceFlags level=TraceFlags::Standard); // Overwrites current trace flags for active thread (and optionally the global default for new threads) @@ -196,7 +509,7 @@ extern jlib_decl TraceFlags queryDefaultTraceFlags(); // Load trace flags from a property tree - typically the global config // See also the workunit-variant in workunit.hpp - +interface IPropertyTree; extern jlib_decl TraceFlags loadTraceFlags(const IPropertyTree * globals, const std::initializer_list & y, TraceFlags dft); diff --git a/system/tracing/CMakeLists.txt b/system/tracing/CMakeLists.txt index 408fdc90d0f..457207459ee 100644 --- a/system/tracing/CMakeLists.txt +++ b/system/tracing/CMakeLists.txt @@ -14,60 +14,13 @@ # limitations under the License. ################################################################################ -# Component: jlib +# Component: openteltracing ##################################################### # Description: # ------------ -# Cmake Input File for jlib +# Cmake Input File for openteltracing ##################################################### project( tracing ) -#For http exporter -find_package(Protobuf REQUIRED) -#For http exporter -find_package(CURL REQUIRED) - -find_package(opentelemetry-cpp CONFIG REQUIRED) - -if (opentelemetry-cpp_FOUND) - MESSAGE(STATUS "Found Opentelemetry") - MESSAGE("^^^^^^^^^${OPENTELEMETRY_CPP_INCLUDE_DIRS}") - HPCC_ADD_SUBDIRECTORY (opentel/instrumented) - - include_directories ( - ${HPCC_SOURCE_DIR}/system/win32 - ${HPCC_SOURCE_DIR}/system/include - ${HPCC_SOURCE_DIR}/system/jlib - ${OPENTELEMETRY_CPP_INCLUDE_DIRS} - ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/oss - ${HPCC_SOURCE_DIR}/system/tracing - ) - #SET ( SRCS tracemanager.hpp opentel/instrumented/httpserver.cpp opentel/instrumented/httpclient.cpp) - SET ( SRCS tracemanager.hpp) - SET ( INCLUDES tracemanager.hpp ) - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") - HPCC_ADD_LIBRARY( tracing SHARED ${SRCS} ${INCLUDES} ) - - target_link_libraries ( tracing - opentelemetry-cpp::api - opentelemetry-cpp::ext - opentelemetry-cpp::sdk - opentelemetry-cpp::logs - opentelemetry-cpp::ostream_span_exporter - #opentelemetry-cpp::jaeger_trace_exporter - #opentelemetry-cpp::otlp_grpc_exporter - #opentelemetry-cpp::otlp_grpc_log_exporter - opentelemetry-cpp::metrics - opentelemetry-cpp::otlp_http_exporter - opentelemetry-cpp::http_client_curl - jlib - ) -else() - MESSAGE(STATUS "Opentelemetry libs/headers not found") -endif() - - - - +HPCC_ADD_SUBDIRECTORY (opentel) #opentelemetry based tracing projects diff --git a/system/tracing/opentel/CMakeLists.txt b/system/tracing/opentel/CMakeLists.txt new file mode 100644 index 00000000000..d13ff222d4a --- /dev/null +++ b/system/tracing/opentel/CMakeLists.txt @@ -0,0 +1,27 @@ +################################################################################ +# HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Component: openteltracing +##################################################### +# Description: +# ------------ +# Cmake Input File for openteltracing +##################################################### + +project( openteltracing ) + +HPCC_ADD_SUBDIRECTORY (instrumented) #sample instrumented HTTP client/server +HPCC_ADD_SUBDIRECTORY (exporters/jlog) #jlog exporter diff --git a/system/tracing/opentel/exporters/jlog/CMakeLists.txt b/system/tracing/opentel/exporters/jlog/CMakeLists.txt new file mode 100644 index 00000000000..086232c1de5 --- /dev/null +++ b/system/tracing/opentel/exporters/jlog/CMakeLists.txt @@ -0,0 +1,62 @@ +################################################################################ +# HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Component: jlogspanexporter +##################################################### +# Description: +# ------------ +# Cmake Input File for jlogspanexporter +##################################################### + +project( jlogspanexporter ) + +#For http exporter +find_package(Protobuf REQUIRED) +#For http exporter +find_package(CURL REQUIRED) + +find_package(opentelemetry-cpp CONFIG REQUIRED) + +if (opentelemetry-cpp_FOUND) + MESSAGE(STATUS "Found Opentelemetry") + include_directories ( + ${HPCC_SOURCE_DIR}/system/win32 + ${HPCC_SOURCE_DIR}/system/include + ${HPCC_SOURCE_DIR}/system/jlib + ${OPENTELEMETRY_CPP_INCLUDE_DIRS} + ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}/oss + ) + + SET ( SRCS spanexporterfactory.cpp ) + SET ( INCLUDES spanexporterfactory.hpp ) + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") + + HPCC_ADD_LIBRARY( jlogspanexporter SHARED ${SRCS} ${INCLUDES} ) + + target_link_libraries ( jlogspanexporter + opentelemetry-cpp::api + opentelemetry-cpp::ext + opentelemetry-cpp::sdk + opentelemetry-cpp::logs + opentelemetry-cpp::ostream_span_exporter + opentelemetry-cpp::metrics + opentelemetry-cpp::otlp_http_exporter + opentelemetry-cpp::http_client_curl + jlib) +else() + MESSAGE(STATUS "Opentelemetry libs/headers not found") +endif() diff --git a/system/tracing/opentel/exporters/jlog/spanexporterfactory.cpp b/system/tracing/opentel/exporters/jlog/spanexporterfactory.cpp new file mode 100644 index 00000000000..dbbb4d32558 --- /dev/null +++ b/system/tracing/opentel/exporters/jlog/spanexporterfactory.cpp @@ -0,0 +1,107 @@ +#include "spanexporterfactory.hpp" +#include "opentelemetry/sdk_config.h" +#include +//#include "opentelemetry/sdk/trace/span_exporter.h" + +namespace nostd = opentelemetry::nostd; +namespace trace_sdk = opentelemetry::sdk::trace; +namespace trace_api = opentelemetry::trace; +namespace sdkcommon = opentelemetry::sdk::common; + +class JLogSpanExporter : public trace_sdk::SpanExporter +{ +public: + JLogSpanExporter() {} + + virtual ~JLogSpanExporter() {} + + virtual std::unique_ptr MakeRecordable() noexcept override + { + return std::unique_ptr(new trace_sdk::SpanData); + } + + virtual sdkcommon::ExportResult Export(const nostd::span> &spans) noexcept override + //virtual opentelemetry::sdk::common::ExportResult Export(const opentelemetry::sdk::trace::Span>&spans) noexcept override + { + if (isShutdown()) + { + // OTEL_INTERNAL_LOG_ERROR("[Ostream Trace Exporter] Exporting " + // << spans.size() << " span(s) failed, exporter is shutdown"); + + return sdkcommon::ExportResult::kFailure; + } + + for (auto &recordable : spans) + { + auto span = std::unique_ptr( + static_cast(recordable.release()));s +/* + if (span != nullptr) + { + char trace_id[32] = {0}; + char span_id[16] = {0}; + char parent_span_id[16] = {0}; + + span->GetTraceId().ToLowerBase16(trace_id); + span->GetSpanId().ToLowerBase16(span_id); + span->GetParentSpanId().ToLowerBase16(parent_span_id); + + sout_ << "{" + << "\n name : " << span->GetName() + << "\n trace_id : " << std::string(trace_id, 32) + << "\n span_id : " << std::string(span_id, 16) + << "\n tracestate : " << span->GetSpanContext().trace_state()->ToHeader() + << "\n parent_span_id: " << std::string(parent_span_id, 16) + << "\n start : " << span->GetStartTime().time_since_epoch().count() + << "\n duration : " << span->GetDuration().count() + << "\n description : " << span->GetDescription() + << "\n span kind : " << span->GetSpanKind() + << "\n status : " << statusMap[int(span->GetStatus())] + << "\n attributes : "; + printAttributes(span->GetAttributes()); + sout_ << "\n events : "; + printEvents(span->GetEvents()); + sout_ << "\n links : "; + printLinks(span->GetLinks()); + sout_ << "\n resources : "; + printResources(span->GetResource()); + sout_ << "\n instr-lib : "; + printInstrumentationScope(span->GetInstrumentationScope()); + sout_ << "\n}\n"; + } +*/ + } + + return sdkcommon::ExportResult::kSuccess; + } + + bool isShutdown() const noexcept + { + //const std::lock_guard locked(lock_); + //return is_shutdown_; + return false; + } + + virtual bool Shutdown(std::chrono::microseconds timeout = std::chrono::microseconds::max()) noexcept + { + return true; + } +/* + virtual sdkcommon::ExportResult Export(const std::vector> &spans) noexcept override + { + // TODO: Implement this method to export the spans to your backend + return sdkcommon::ExportResult::kSuccess; + } + + virtual std::unique_ptr Clone() const noexcept override + { + // TODO: Implement this method to create a new instance of the exporter + return nullptr; + }*/ +}; + +std::unique_ptr JLogSpanExporterFactory::Create() +{ + std::unique_ptr exporter(new JLogSpanExporter()); + return exporter; +} diff --git a/system/tracing/opentel/exporters/jlog/spanexporterfactory.hpp b/system/tracing/opentel/exporters/jlog/spanexporterfactory.hpp new file mode 100644 index 00000000000..692e729e881 --- /dev/null +++ b/system/tracing/opentel/exporters/jlog/spanexporterfactory.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include "opentelemetry/sdk/version/version.h" +#include "opentelemetry/sdk/trace/exporter.h" + +//OPENTELEMETRY_BEGIN_NAMESPACE +//namespace sdk +//{ +//namespace trace +//{ +//class SpanExporter; +//} // namespace trace +//} // namespace sdk + +//namespace exporter +//{ +//namespace trace +//{ + +class JLogSpanExporterFactory +{ +public: + /** + * Creates a JLogSpanExporterFactory writing to the default location. + */ + static std::unique_ptr Create(); + + //static std::unique_ptr Create(std::ostream &sout); +}; + +//} // namespace trace +//} // namespace exporter +//OPENTELEMETRY_END_NAMESPACE diff --git a/system/tracing/opentel/instrumented/CMakeLists.txt b/system/tracing/opentel/instrumented/CMakeLists.txt index ad3e0b5f3e4..1849fc0219b 100644 --- a/system/tracing/opentel/instrumented/CMakeLists.txt +++ b/system/tracing/opentel/instrumented/CMakeLists.txt @@ -24,11 +24,11 @@ project( instrumentedhttp ) #For http exporter -#find_package(Protobuf REQUIRED) +find_package(Protobuf REQUIRED) #For http exporter -#find_package(CURL REQUIRED) +find_package(CURL REQUIRED) -#find_package(opentelemetry-cpp CONFIG REQUIRED) +find_package(opentelemetry-cpp CONFIG REQUIRED) if (opentelemetry-cpp_FOUND) MESSAGE(STATUS "Found Opentelemetry") @@ -39,15 +39,16 @@ if (opentelemetry-cpp_FOUND) ${OPENTELEMETRY_CPP_INCLUDE_DIRS} ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/oss - ${HPCC_SOURCE_DIR}/system/tracing + ${HPCC_SOURCE_DIR}/system/tracing/opentel/exporters/jlog ) - SET ( SERVER_SRCS httpserver.cpp httpclient.cpp) - SET ( INCLUDES httpserver.hpp ) + SET ( SERVER_SRCS httpserver.cpp) + SET ( SERVER_INCLUDES httpserver.hpp ) + SET ( CLIENT_SRCS httpclient.cpp) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STRICT_CXX_FLAGS}") - HPCC_ADD_EXECUTABLE(instrumented_http_client httpclient.cpp) - HPCC_ADD_EXECUTABLE(instrumented_http_server httpserver.cpp) + HPCC_ADD_EXECUTABLE(instrumented_http_client ${CLIENT_SRCS}) + HPCC_ADD_EXECUTABLE(instrumented_http_server ${SERVER_SRCS}) target_link_libraries(instrumented_http_server ${CMAKE_THREAD_LIBS_INIT} opentelemetry-cpp::api @@ -58,7 +59,8 @@ if (opentelemetry-cpp_FOUND) opentelemetry-cpp::metrics opentelemetry-cpp::otlp_http_exporter opentelemetry-cpp::ostream_span_exporter - jlib) + jlib + ) target_link_libraries ( instrumented_http_client opentelemetry-cpp::api @@ -66,9 +68,6 @@ if (opentelemetry-cpp_FOUND) opentelemetry-cpp::sdk opentelemetry-cpp::logs opentelemetry-cpp::ostream_span_exporter - #opentelemetry-cpp::jaeger_trace_exporter - #opentelemetry-cpp::otlp_grpc_exporter - #opentelemetry-cpp::otlp_grpc_log_exporter opentelemetry-cpp::metrics opentelemetry-cpp::otlp_http_exporter opentelemetry-cpp::http_client_curl diff --git a/system/tracing/opentel/instrumented/httpclient.cpp b/system/tracing/opentel/instrumented/httpclient.cpp index 88e173d4807..da07436b3c5 100644 --- a/system/tracing/opentel/instrumented/httpclient.cpp +++ b/system/tracing/opentel/instrumented/httpclient.cpp @@ -19,7 +19,8 @@ #include "opentelemetry/ext/http/common/url_parser.h" #include "opentelemetry/trace/semantic_conventions.h" -#include "tracemanager.hpp" +#include "jtrace.hpp" +//#include "spanexporterfactory.hpp" namespace { @@ -29,12 +30,33 @@ namespace http_client = opentelemetry::ext::http::client; namespace context = opentelemetry::context; namespace nostd = opentelemetry::nostd; -//std::string MODULE_NAME = "SimulatedESPClient"; -std::string MODULE_NAME = "http_server"; +std::string MODULE_NAME = "SimulatedESPClient"; +//std::string MODULE_NAME = "http_server"; +/* +void subTask(TraceManager * traceman) +{ + StartSpanOptions options; + + auto span = traceman->getTracer()->StartSpan("subTask", options); + auto scope = traceman->getTracer()->WithActiveSpan(span); + + span->End(); +} + +void compoundSubTask(TraceManager * traceman) +{ + StartSpanOptions options; + auto span = traceman->getTracer()->StartSpan("compoundSubTask", options); + auto scope = traceman->getTracer()->WithActiveSpan(span); + subTask(traceman); + span->End(); +}*/ void sendRequest(const std::string &url) { auto http_client = http_client::HttpClientFactory::CreateSync(); + //OStream exporter, useful for development and debugging tasks and simplest to set up. + //auto exporter = JLogSpanExporterFactory::Create(); opentelemetry::ext::http::common::UrlParser espReqURL(url); //url parts used as sample span options/attributes @@ -67,22 +89,24 @@ void sendRequest(const std::string &url) //activate the span Scope scope = tracer->WithActiveSpan(clientReqSpan); - const char * mytraceid = traceManager.queryTraceId(); - fprintf(stdout, "mytraceid %s", mytraceid); + //simulate subtask with its own sub-span + //subTask(&traceManager); - // inject current context into http header - auto currentCtx = context::RuntimeContext::GetCurrent(); - HttpTextMapCarrier carrier1; - HPCCHttpTextMapCarrier carrier; - - auto propegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - propegator->Inject(carrier, currentCtx); //injects current context as parent - auto a = carrier.httpHeaders->queryProp(HPCCSemanticConventions::kGLOBALIDHTTPHeader); - const char * callerId = traceManager.queryCallerId(carrier); - fprintf (stdout, "callerId %s and %s", callerId, a); + //std::string myLocalId; + //traceManager.queryCurrentSpanID(myLocalId); + //DBGLOG("myLocalId %s", myLocalId.c_str()); + + //std::string myTraceId; + //traceManager.queryTraceId(myTraceId); + //DBGLOG("mytraceid %s", myTraceId.c_str()); + + //HPCCHttpTextMapCarrier carrier; + HttpTextMapCarrier carrier; + // inject current context into http or hpcchttp carrier + traceManager.injectCurretContext(carrier); // send http request - http_client::Result result = http_client->GetNoSsl(url, carrier1.httpHeaders); + http_client::Result result = http_client->GetNoSsl(url, carrier.httpHeaders); if (result) { @@ -92,15 +116,15 @@ void sendRequest(const std::string &url) result.GetResponse().ForEachHeader( [&clientReqSpan](nostd::string_view header_name, nostd::string_view header_value) { - clientReqSpan->SetAttribute("http.header." + std::string(header_name.data()), header_value); - return true; + clientReqSpan->SetAttribute("http.header." + std::string(header_name.data()), header_value); + return true; }); if (status_code >= 400) { clientReqSpan->SetStatus(StatusCode::kError); // kUnset(default), - // kOk(Operation completed) - // kError(peration encountered error) + // kOk(Operation completed) + // kError(peration encountered error) } } else @@ -118,37 +142,10 @@ void sendRequest(const std::string &url) } // namespace - -void subTask(TraceManager * traceman) -{ - StartSpanOptions options; - - auto span = traceman->getTracer()->StartSpan("subTask", options); - auto scope = traceman->getTracer()->WithActiveSpan(span); - - span->End(); -} - -void mainTask(TraceManager * traceman) -{ - StartSpanOptions options; - auto span = traceman->getTracer()->StartSpan("mainTask", options); - auto scope = traceman->getTracer()->WithActiveSpan(span); - subTask(traceman); - span->End(); -} - int main(int argc, char *argv[]) { - TraceManager traceManager(MODULE_NAME); - TraceManager::initTracer(); //@ init_module - //sets up default provider and http propegator - mainTask(&traceManager); - const char * mytraceid = traceManager.queryTraceId(); - if (isEmptyString(mytraceid)) - fprintf(stderr, "Span is not active"); - else - fprintf(stderr, "Span id: %s", mytraceid); + TraceManager traceManager(MODULE_NAME); //@ init_module + //sets up default provider and http propegator constexpr char default_host[] = "localhost"; constexpr char default_path[] = "/helloworld"; diff --git a/system/tracing/opentel/instrumented/httpserver.cpp b/system/tracing/opentel/instrumented/httpserver.cpp index 548435d2906..78e25dca971 100644 --- a/system/tracing/opentel/instrumented/httpserver.cpp +++ b/system/tracing/opentel/instrumented/httpserver.cpp @@ -4,11 +4,13 @@ #include "httpserver.hpp" #include "opentelemetry/trace/context.h" #include "opentelemetry/trace/semantic_conventions.h" -#include "tracemanager.hpp" +#include "jtrace.hpp" #include #include +std::string MODULE_NAME = "SimulatedESPServer"; + namespace { @@ -25,11 +27,14 @@ class RequestHandler : public HTTP_SERVER_NS::HttpRequestCallback virtual int onHttpRequest(HTTP_SERVER_NS::HttpRequest const &request, HTTP_SERVER_NS::HttpResponse &response) override { + TraceManager traceManager(MODULE_NAME); //@ init_module, we need a getTraceManager() + auto tracer = traceManager.getTracer(); + StartSpanOptions options; options.kind = SpanKind::kServer; // extract parent(caller) context from http header, set as parent context for current span - TraceManager::setParentContextFromHeaders(const_cast &>(request.headers), options); + TraceManager::getParentContext(const_cast &>(request.headers), options); //Above call replaces the following code: //const HttpTextMapCarrier> carrier(request_headers); //auto prop = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); @@ -37,16 +42,14 @@ class RequestHandler : public HTTP_SERVER_NS::HttpRequestCallback //auto new_context = prop->Extract(carrier, current_ctx); //options.parent = GetSpan(new_context)->GetContext(); - std::string callerSpanID; - TraceManager::extractCallerSpanId(request.headers, callerSpanID); - DBGLOG("Caller Span ID: %s", callerSpanID.c_str()); + //std::string callerSpanID; + //TraceManager::extractCallerSpanId(request.headers, callerSpanID); + //DBGLOG("Caller Span ID: %s", callerSpanID.c_str()); std::string spanName = request.uri; - TraceManager traceManager(COMPONENT_NAME); - auto tracer = traceManager.getTracer(); //Declare current span with parent context extracted from http header - auto span = + auto processingRequestSpan = tracer->StartSpan(spanName, {{SemanticConventions::kNetHostName, server_name}, //span attributes {SemanticConventions::kNetHostPort, server_port}, @@ -57,26 +60,26 @@ class RequestHandler : public HTTP_SERVER_NS::HttpRequestCallback {SemanticConventions::kHttpClientIp, request.client}}, options); //options.parent is set as parent context for current span - auto scope = tracer->WithActiveSpan(span); + auto scope = tracer->WithActiveSpan(processingRequestSpan); for (auto &kv : request.headers) { - span->SetAttribute("http.header." + std::string(kv.first.data()), kv.second); + processingRequestSpan->SetAttribute("http.header." + std::string(kv.first.data()), kv.second); } if (request.uri == "/helloworld") { - span->AddEvent("Processing request"); - span->AddEvent("Setting response headers"); + processingRequestSpan->AddEvent("Processing request"); + processingRequestSpan->AddEvent("Setting response headers"); response.headers[HTTP_SERVER_NS::CONTENT_TYPE] = HTTP_SERVER_NS::CONTENT_TYPE_TEXT; - span->End(); + processingRequestSpan->End(); return 200; } else { - span->AddEvent("Processing Error request"); - DBGLOG("Error request: %s", request.uri.c_str()); - span->End(); + processingRequestSpan->AddEvent("Processing Error request"); + //DBGLOG("Error request: %s", request.uri.c_str()); + processingRequestSpan->End(); return 404; } } @@ -85,8 +88,6 @@ class RequestHandler : public HTTP_SERVER_NS::HttpRequestCallback int main(int argc, char *argv[]) { - TraceManager::initTracer(); - // The port the validation service listens to can be specified via the command line. if (argc > 1) { @@ -101,9 +102,10 @@ int main(int argc, char *argv[]) Scope scope(root_span); //not active span? Just for scope? http_server.Start(); std::cout << "Server is running..Press ctrl-c to exit...\n"; - while (1) + + //while (1) { - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::seconds(60)); } http_server.Stop(); root_span->End(); diff --git a/system/tracing/tracemanager.hpp b/system/tracing/tracemanager.hpp deleted file mode 100644 index 9f7c044dc83..00000000000 --- a/system/tracing/tracemanager.hpp +++ /dev/null @@ -1,384 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -#ifndef TRACER_HPP -#define TRACER_HPP - -#include "opentelemetry/exporters/ostream/span_exporter_factory.h" -#include "opentelemetry/sdk/trace/exporter.h" -#include "opentelemetry/sdk/trace/processor.h" -#include "opentelemetry/sdk/trace/simple_processor_factory.h" -#include "opentelemetry/sdk/trace/tracer_context.h" -#include "opentelemetry/sdk/trace/tracer_context_factory.h" -#include "opentelemetry/sdk/trace/tracer_provider_factory.h" -#include "opentelemetry/trace/provider.h" - -#include "opentelemetry/context/propagation/global_propagator.h" -#include "opentelemetry/context/propagation/text_map_propagator.h" -#include "opentelemetry/trace/propagation/http_trace_context.h" - -#include "opentelemetry/ext/http/client/http_client_factory.h" -#include "opentelemetry/ext/http/common/url_parser.h" -#include "opentelemetry/trace/semantic_conventions.h" - -#include -//#include - - -#include "jlib.hpp" -#include "jliball.hpp" - -using namespace opentelemetry::trace; -namespace http_client = opentelemetry::ext::http::client; -namespace context = opentelemetry::context; -namespace nostd = opentelemetry::nostd; -namespace trace_sdk = opentelemetry::sdk::trace; - -namespace HPCCSemanticConventions -{ -/** - * Known HPCC span Keys could be added here - * Specialized span keys can also be defined within the scope of a span - */ -static constexpr const char *kGLOBALIDHTTPHeader = "HPCC-Global-Id"; -static constexpr const char *kCallerIdHTTPHeader = "HPCC-Caller-Id"; -} - - -template -class HPCCHttpTextMapCarrier : public opentelemetry::context::propagation::TextMapCarrier -{ -public: - HPCCHttpTextMapCarrier(R &headers) : httpHeaders(headers) {} - HPCCHttpTextMapCarrier() = default; - - virtual opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override - { - std::string theKey = key.data(); - - // perform any key mapping needed... - { - //Instrumented http client/server Capitalizes the first letter of the header name - if (key == propagation::kTraceParent || key == propagation::kTraceState ) - theKey[0] = toupper(theKey[0]); - } - - return httpHeaders->queryProp(theKey.c_str()); - } - - virtual void Set(opentelemetry::nostd::string_view key, - opentelemetry::nostd::string_view value) noexcept override - { - httpHeaders->setProp(std::string(key).c_str(), std::string(value).c_str()); - //httpHeaders.insert(std::pair(std::string(key), std::string(value))); - } - - Owned httpHeaders = createProperties(); - //R httpHeaders = createProperties(); -}; - - -// TextMapCarrier is the storage medium used by TextMapPropagator. -// pure virtual Get(key) returns the value associated with the passed key. -// pure virtual Set(key, value) stores the key-value pair. -// virtual Keys(nostd::function_ref /* callback */) -// list of all the keys in the carrier. -// By default, it returns true without invoking callback */ -template -class HttpTextMapCarrier : public opentelemetry::context::propagation::TextMapCarrier -{ -public: - HttpTextMapCarrier(T &headers) : httpHeaders(headers) {} - HttpTextMapCarrier() = default; - - virtual opentelemetry::nostd::string_view Get(opentelemetry::nostd::string_view key) const noexcept override - { - std::string theKey = key.data(); - - // perform any key mapping needed... - { - //Instrumented http client/server Capitalizes the first letter of the header name - if (key == propagation::kTraceParent || key == propagation::kTraceState ) - theKey[0] = toupper(theKey[0]); - } - - //now search for the vaule - auto it = httpHeaders.find(theKey); - if (it != httpHeaders.end()) - return it->second; - - return ""; - } - - virtual void Set(opentelemetry::nostd::string_view key, - opentelemetry::nostd::string_view value) noexcept override - { - httpHeaders.insert(std::pair(std::string(key), std::string(value))); - } - - T httpHeaders; -}; - -class TraceManager -{ -private: - //Used as the opentel trace name, refered to as name of library being instrumented - const char * tracerName = nullptr; - const char * callerId = nullptr; - - //const char * globalIdHTTPHeaderName = "HPCC-Global-Id"; - //const char * callerIdHTTPHeaderName = "HPCC-Caller-Id"; - -public: - TraceManager(const std::string & moduleName) - { - //1 tracer for each module - tracerName = moduleName.c_str(); - }; - - ~TraceManager() {}; - - static void initTracer() - { - //Handle HPCC specific tracing configuration here - //Target exporter? exporter connection info? - //Target processor(s)? batch/interactive? - //Target propogator? http? grpc? binary? custom? - //HPCC component tracing switches? - - /* - Owned traceConfig = getComponentConfigSP()->getPropTree("tracing"); - if (traceConfig) - { - Owned exportConfig = traceConfig->getPropTree("exporter"); - if (exportConfig) - { - if (exportConfig->getPropBool("OS", false)) //To stdout/err - else if (exportConfig->getPropBool("OTLP", false)) - else if (exportConfig->getPropBool("Jaeger", false)) - else if (exportConfig->getPropBool("Zipkin", false)) - else if (exportConfig->getPropBool("Prometheus", false)) - else if (exportConfig->getPropBool("HPCC", false)) - //perhaps a custom exporter for HPCC, - //which will send spans to a HPCC service, or - //allow us to interogate the internal spandata (vs spancontext) which includes - //the span's parent, attributes, events, links, and status. - } - } - */ - - //OStream exporter, useful for development and debugging tasks and simplest to set up. - auto exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); - - //SimpleSpanProcesser sends spans one by one to an exporter. - //We could use a batchspanprocessor, which will group several spans together, before sending them to an exporter. - auto processor = opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); - std::vector> processors; - processors.push_back(std::move(processor)); - - // Default is an always-on sampler. - std::shared_ptr context = - opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); - std::shared_ptr provider = - opentelemetry::sdk::trace::TracerProviderFactory::Create(context); - - // Set the global trace provider - opentelemetry::trace::Provider::SetTracerProvider(provider); - - // set global propagator - // Injects Context into and extracts it from carriers that travel in-band - // across process boundaries. Encoding is expected to conform to the HTTP - // Header Field semantics. - // Values are often encoded as RPC/HTTP request headers. - opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator( - opentelemetry::nostd::shared_ptr( - new opentelemetry::trace::propagation::HttpTraceContext())); - } - - //convenience non-static method to get the default tracer, uses stored tracer/module name - opentelemetry::nostd::shared_ptr getTracer() - { - auto provider = opentelemetry::trace::Provider::GetTracerProvider(); - return provider->GetTracer(tracerName); // (library_name [,library_version][,schema_url]) - } - - //convenience Static method to get the default tracer, uses provided module name - static opentelemetry::nostd::shared_ptr getTracer(std::string moduleName) - { - auto provider = opentelemetry::trace::Provider::GetTracerProvider(); - return provider->GetTracer(moduleName); // (library_name [,library_version][,schema_url]) - } - - static void cleanupTracer() - { - std::shared_ptr none; - opentelemetry::trace::Provider::SetTracerProvider(none); - } - - //SpanContex does not expose parent span. - //If we need parent info, we might need to track them - - //Not convinced we should be tracking parent span context, but the StartSpan is - //here to explore the idea. - - //By intersecting call to tracer->StartSpan(), we can preemtively interogate current span - //for what will ultimately be the parent span of the new span. - //template ::value> * = nullptr> - //nostd::shared_ptr StartSpan(const char * spanName, - // const T &attributes, - // const StartSpanOptions &options = {}) noexcept - //{ - // nostd::shared_ptr currntSpan = - // TraceManager::getTracer(tracerName)->GetCurrentSpan(); - - // opentelemetry::v1::trace::SpanContext spanCtx = currntSpan->GetContext(); - - // if (currntSpan->IsRecording()) - // { - // fprintf(stderr, "Parent span detected...\n"); - //if (spanCtx.IsRemote()) //this span hasn't started, it wouldn't be remote - // { - // char trace_id[32] = {0}; - // spanCtx.trace_id().ToLowerBase16(trace_id); - // fprintf(stderr, "Parent trace id: '%s' ", std::string(trace_id, 32).c_str()); - - // char span_id[16] = {0}; - // spanCtx.span_id().ToLowerBase16(span_id); - // fprintf(stderr, "Parent span id: '%s'", std::string(span_id, 32).c_str()); - // } - // } - - // return TraceManager::getTracer(tracerName)->StartSpan(spanName, attributes, options); - //} - - const char* queryCallerId(context::propagation::TextMapCarrier &carrier) const - { - // Inject current context into http header - auto currentCtx = context::RuntimeContext::GetCurrent(); - auto propagator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - propagator->Inject(carrier, currentCtx); - - // Extract parent span context from the TextMapCarrier - auto parentSpanContext = propagator->Extract(carrier,currentCtx); - - // Get the value of the HPCC-Caller-Id header from the TextMapCarrier - auto callerIdHeader = carrier.Get("traceparent"); - - return callerIdHeader.empty() ? "" : callerIdHeader.data(); - } - - static bool setParentContextFromHeaders(std::map& request_headers, StartSpanOptions & options) - { - const HttpTextMapCarrier> carrier(request_headers); - auto prop = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - auto current_ctx = context::RuntimeContext::GetCurrent(); - auto new_context = prop->Extract(carrier, current_ctx); - options.parent = GetSpan(new_context)->GetContext(); - - return true; - } - - static bool extractCallerSpanId(std::map request_headers, std::string & callerSpanId) - { - const HttpTextMapCarrier> carrier(request_headers); - return TraceManager::extractCallerSpanId(carrier, callerSpanId); - } - - static bool extractCallerSpanId(const HttpTextMapCarrier> carrier, std::string & callerSpanId) - { - auto propagator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - auto current_ctx = context::RuntimeContext::GetCurrent(); - auto new_context = propagator->Extract(carrier, current_ctx); - auto parentSpan = GetSpan(new_context)->GetContext(); - - char span_id[16] = {0}; - parentSpan.span_id().ToLowerBase16(span_id); - callerSpanId = std::string(span_id, 32).c_str(); - return true; - } - - bool setCallerId(std::map request_headers) - { - const HttpTextMapCarrier> carrier(request_headers); - return setCallerId(carrier); - } - - bool setCallerId(const HttpTextMapCarrier> carrier) - { - auto prop = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - auto current_ctx = context::RuntimeContext::GetCurrent(); - auto new_context = prop->Extract(carrier, current_ctx); - auto parentSpan = GetSpan(new_context)->GetContext(); - - char span_id[16] = {0}; - parentSpan.span_id().ToLowerBase16(span_id); - callerId = std::string(span_id, 32).c_str(); - - return (callerId && *callerId) ? true : false; - } - - const char* queryCallerId() const - { - return callerId; - } - - const char* queryTraceId() const - { - nostd::shared_ptr currntSpan = - TraceManager::getTracer(std::string(tracerName))->GetCurrentSpan(); - - if (currntSpan->IsRecording()) - { - auto spanCtx = currntSpan->GetContext(); - - char trace_id[32] = {0}; - spanCtx.trace_id().ToLowerBase16(trace_id); - return std::string(trace_id, 32).c_str(); - } - else - return ""; - } - - const char* queryLocalId() const - { - nostd::shared_ptr currntSpan = - TraceManager::getTracer(std::string(tracerName))->GetCurrentSpan(); - - if (currntSpan->IsRecording()) - { - char span_id[16] = {0}; - currntSpan->GetContext().span_id().ToLowerBase16(span_id); - - return std::string(span_id, 32).c_str(); - } - else - return nullptr; - } - - //const char* queryGlobalIdHTTPHeaderName() const { return globalIdHTTPHeaderName; } - //const char* queryCallerIdHTTPHeaderName() const { return callerIdHTTPHeaderName; } - - //void setHttpIdHeaderNames(const char *global, const char *caller) - //{ - // if (global && *global) - // globalIdHTTPHeaderName = global; - // if (caller && *caller) - // callerIdHTTPHeaderName = caller; - //} -}; - -#endif