diff --git a/esp/bindings/http/platform/httpservice.cpp b/esp/bindings/http/platform/httpservice.cpp index 06bf7539e33..1b712dd661b 100644 --- a/esp/bindings/http/platform/httpservice.cpp +++ b/esp/bindings/http/platform/httpservice.cpp @@ -203,16 +203,19 @@ int CEspHttpServer::processRequest() mockHTTPHeadersSA.append("HPCC-Caller-Id:IncomingCID"); //This span tracks processing of httprequest - //Owned reqProcessSpan = queryTraceManager("esp")->createTransactionSpan("ProcessingHTTPRequest", mockHTTPHeaders); - Owned reqProcessSpan = queryTraceManager("esp")->createTransactionSpan("ProcessingHTTPRequest", mockHTTPHeadersSA); + //Owned reqProcessSpan = queryTraceManager("esp")->createServerSpan("ProcessingHTTPRequest", mockHTTPHeaders); - reqProcessSpan->setAttribute("http.request_port", "8010"); - reqProcessSpan->setAttribute("app.name", "esp"); - reqProcessSpan->setAttribute("app.version", "1.0.0"); - reqProcessSpan->setAttribute("app.instance", "esp1"); - reqProcessSpan->setAttribute("http.method", m_request->queryMethod()); - reqProcessSpan->setAttribute("http.url", m_request->queryPath()); - reqProcessSpan->setAttribute("http.host", m_request->queryHost()); + StringArray mockEmptyHTTPHeaders; + Owned rootSpan = queryTraceManager().createServerSpan("rootProcessingHTTPRequest", mockEmptyHTTPHeaders, nullptr); + Owned reqProcessSpan = queryTraceManager().createServerSpan("childProcessingHTTPRequest", mockHTTPHeadersSA, rootSpan.get()); + + reqProcessSpan->setSpanAttribute("http.request_port", "8010"); + reqProcessSpan->setSpanAttribute("app.name", "esp"); + reqProcessSpan->setSpanAttribute("app.version", "1.0.0"); + reqProcessSpan->setSpanAttribute("app.instance", "esp1"); + reqProcessSpan->setSpanAttribute("http.method", m_request->queryMethod()); + reqProcessSpan->setSpanAttribute("http.url", m_request->queryPath()); + reqProcessSpan->setSpanAttribute("http.host", m_request->queryHost()); IEspContext* ctx = m_request->queryContext(); StringBuffer errMessage; @@ -270,7 +273,7 @@ int CEspHttpServer::processRequest() m_request->updateContext(); ctx->setServiceName(serviceName.str()); ctx->setHTTPMethod(method.str()); - reqProcessSpan->setAttribute("http.method", method.str()); + reqProcessSpan->setSpanAttribute("http.method", method.str()); ctx->setServiceMethod(methodName.str()); ctx->addTraceSummaryValue(LogMin, "app.protocol", method.str(), TXSUMMARY_GRP_ENTERPRISE); ctx->addTraceSummaryValue(LogMin, "app.service", serviceName.str(), TXSUMMARY_GRP_ENTERPRISE); diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 2467cc73920..8771d121e4c 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -9068,10 +9068,7 @@ jlib_decl IPropertyTree * loadConfiguration(IPropertyTree *componentDefault, con if (monitor) configFileUpdater->startMonitoring(); - DBGLOG("Initializing OpenTel based tracing for component: '%s'", componentTag); - //Owned tracer = - queryTraceManager(componentTag); //inits opentel singleton internals - + initTraceManager(componentTag, componentConfiguration.get()); return componentConfiguration.getLink(); } diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 7ed2ef5ca9b..19848b8a55b 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -18,6 +18,10 @@ #include "platform.h" +#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED +#include "opentelemetry/trace/provider.h" //StartSpanOptions +#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) + #include "jmisc.hpp" #include "jtrace.hpp" #include "lnuid.h" @@ -32,6 +36,10 @@ #include "opentelemetry/trace/propagation/http_trace_context.h" //opentel_trace::propagation::kTraceParent +namespace context = opentelemetry::context; +namespace nostd = opentelemetry::nostd; +namespace opentel_trace = opentelemetry::trace; + using namespace ln_uid; /* @@ -85,25 +93,6 @@ const char* LogTrace::queryLocalId() const return localId.get(); } -CSpan::CSpan() : span(nullptr) {}; -CSpan::~CSpan() -{ - if (span != nullptr) - span->End(); -} - -void CSpan::setAttribute(const char * key, const char * val) -{ - if (span) - span->SetAttribute(key, val); -} - -void CSpan::addEvent(const char * eventName) -{ - if (span && !isEmptyString(eventName)) - span->AddEvent(eventName); -} - class CHPCCHttpTextMapCarrier : public opentelemetry::context::propagation::TextMapCarrier { public: @@ -209,321 +198,543 @@ class HPCCStringArrayHttpTextMapCarrier : public opentelemetry::context::propaga }; */ -void CTraceManager::initTracer() +class CNoOpSpan : public CInterfaceOf { - Owned traceConfig; - try + void setSpanAttribute(const char * key, const char * val) override {}; + void setSpanAttributes(const IProperties * attributes) override {}; + void addSpanEvent(const char * eventName) override {}; + void setActive() override {}; + const char * queryTraceID() override { return ""; }; + const char * queryTraceFlags() override { return ""; }; + const char * queryTraceName() override { return ""; }; + const char * querySpanID() override { return ""; }; + const char * querySpanName() override { return ""; }; + const char * queryHPCCGlobalID() override { return ""; }; + const char * queryHPCCCallerID() override { return ""; }; +}; + +class CSpan : public CInterfaceOf +{ +public: + CSpan() : span(nullptr) {}; + ~CSpan() { - traceConfig.setown(getComponentConfigSP()->getPropTree("tracing")); + if (span != nullptr) + span->End(); + } -#ifdef TRACECONFIGDEBUG - if (!traceConfig) + + void setActive() override + { + if (span != nullptr) { - const char * simulatedGlobalYaml = R"!!(global: - tracing: - enabled: true - exporter: - OS: true - processor: - batchSpan: true - simpleSpan: false - )!!"; - Owned testTree = createPTreeFromYAMLString(simulatedGlobalYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr); - traceConfig.setown(testTree->getPropTree("global/tracing")); + if (!isEmptyString(tracerName.get())) + { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer(tracerName.get()); + /* Set the active span. The span will remain active until the returned Scope + * object is destroyed. + * @param span the span that should be set as the new active span. + * @return a Scope that controls how long the span will be active. + */ + activeSpanScope = tracer->WithActiveSpan(span); + } } -#endif - if (traceConfig && traceConfig->getPropBool("enable", true)) + } + + void setSpanAttributes(const IProperties * attributes) override + { + Owned iter = attributes->getIterator(); + ForEach(*iter) { - DBGLOG("OpenTel tracing enabled"); - using namespace opentelemetry::trace; - std::unique_ptr exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); - - Owned exportConfig = traceConfig->getPropTree("exporter"); - StringBuffer xml; - toXML(exportConfig, xml); - DBGLOG("exportConfig tree: %s", xml.str()); - if (exportConfig) - { - if (exportConfig->getPropBool("@OS", false)) //To stdout/err - DBGLOG("Tracing to stdout/err"); - else if (exportConfig->getPropBool("@OTLP", false)) - { - //namespace otlp = opentelemetry::exporter::otlp; + const char * key = iter->getPropKey(); + const char * val = attributes->queryProp(key); + setSpanAttribute(key, val); + } + } - //otlp::OtlpGrpcExporterOptions opts; - //opts.endpoint = "localhost:4317"; - //opts.use_ssl_credentials = true; - //opts.ssl_credentials_cacert_as_string = "ssl-certificate"; + void setSpanAttribute(const char * key, const char * val) override + { + if (span && !isEmptyString(key) && !isEmptyString(val)) + span->SetAttribute(key, val); + } - //exporter = otlp::OtlpGrpcExporterFactory::Create(opts); - //DBGLOG("Tracing to OTLP"); - } - else if (exportConfig->getPropBool("@Prometheus", false)) - DBGLOG("Tracing to Prometheus"); - else if (exportConfig->getPropBool("@HPCC", false)) - DBGLOG("Tracing to HPCC JLog"); - } + void addSpanEvent(const char * eventName) override + { + if (span && !isEmptyString(eventName)) + span->AddEvent(eventName); + } - std::unique_ptr processor; - if (exportConfig->getPropBool("processor/batchSpan", true)) - { - //Groups several spans together, before sending them to an exporter. - opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; - //The time interval between two consecutive exports - //std::chrono::milliseconds(5000); - //The maximum batch size of every export. It must be smaller or - //equal to max_queue_size. - //size_t max_export_batch_size = 512 - processor = opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), options); - DBGLOG("OpenTel tracing using batch Span Processor"); - } - else - { - //SimpleSpanProcesser sends spans one by one to an exporter. - processor = opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); - DBGLOG("OpenTel tracing using Simple Span Processor"); - } + const char * queryTraceID() override { return traceID.get(); }; + const char * queryTraceName() override { return tracerName.get();} + const char * querySpanID() override { return spanID.get(); }; + const char * queryTraceFlags() override { return traceFlags.get(); } + const char * querySpanName() override { return name.get(); } + const char * queryHPCCGlobalID() override { return hpccGlobalId.get(); } + const char * queryHPCCCallerID() override { return hpccCallerId.get(); } + +protected: + CSpan(const char * spanName, const char * tracerName_) + { + name.set(spanName); + tracerName.set(tracerName_); + } + + void init(ISpan * parentSpan) + { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer(tracerName.get()); + + if (parentSpan != nullptr) + injectParentSpan(parentSpan); - std::vector> processors; - processors.push_back(std::move(processor)); - - // Default is an always-on sampler. - std::shared_ptr context = - opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); - std::shared_ptr provider = - opentelemetry::sdk::trace::TracerProviderFactory::Create(context); - - // Set the global trace provider - opentelemetry::trace::Provider::SetTracerProvider(provider); - - // set global propagator - // Injects Context into and extracts it from carriers that travel in-band - // across process boundaries. Encoding is expected to conform to the HTTP - // Header Field semantics. - // Values are often encoded as RPC/HTTP request headers. - opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator( - opentelemetry::nostd::shared_ptr( - new opentelemetry::trace::propagation::HttpTraceContext())); + span = tracer->StartSpan(name.get(), {}, opts); + + if (span != nullptr) + storeSpanContext(); + } + + void storeSpanContext() + { + if (span != nullptr) + { + storeTraceID(); + storeSpanID(); + storeTraceFlags(); } } - catch (IException * e) + + void injectParentSpan(ISpan * parentSpan) { - EXCLOG(e); - e->Release(); + if (parentSpan != nullptr) + { + nostd::string_view parentTraceID = parentSpan->queryTraceID(); + opentelemetry::v1::trace::TraceId tid = opentelemetry::trace::propagation::HttpTraceContext::TraceIdFromHex(parentTraceID); + + nostd::string_view parentSpanID = parentSpan->querySpanID(); + opentelemetry::v1::trace::SpanId sid = opentelemetry::trace::propagation::HttpTraceContext::SpanIdFromHex(parentSpanID); + + nostd::string_view parentTraceFlags = parentSpan->queryTraceFlags(); + opentelemetry::v1::trace::TraceFlags tf = opentelemetry::trace::propagation::HttpTraceContext::TraceFlagsFromHex(parentTraceFlags); + + opts.parent = opentel_trace::SpanContext(tid, sid, tf, true); + } } -} -void CTraceManager::cleanupTracer() -{ - std::shared_ptr none; - opentelemetry::trace::Provider::SetTracerProvider(none); -} + void storeTraceID() + { + traceID.clear(); -void CSpan::setAttributes(const IProperties * attributes) -{ - Owned iter = attributes->getIterator(); - ForEach(*iter) + if (!span) + return; + + auto spanCtx = span->GetContext(); + if (!spanCtx.IsValid()) + return; + + if (!spanCtx.trace_id().IsValid()) + return; + + char trace_id[32] = {0}; + + spanCtx.trace_id().ToLowerBase16(trace_id); + traceID.set(trace_id, 32); + } + + void storeSpanID() { - const char * key = iter->getPropKey(); - const char * val = attributes->queryProp(key); - span->SetAttribute(key, val); + spanID.clear(); + + if (!span) + return; + + char span_id[16] = {0}; + span->GetContext().span_id().ToLowerBase16(span_id); + + spanID.set(span_id, 16); } -} -void CTransactionSpan::setAttributesFromHTTPHeaders(StringArray & httpHeaders, opentelemetry::trace::StartSpanOptions * options) -{ - Owned httpHeaderProps = createProperties(); - ForEachItemIn(currentHeaderIndex, httpHeaders) + void storeTraceFlags() { - const char* httHeader = httpHeaders.item(currentHeaderIndex); - if(!httHeader) - continue; + traceFlags.clear(); - const char* colon = strchr(httHeader, ':'); - if(colon == nullptr) - continue; + if (!span) + return; - StringBuffer key, value; - key.append(colon - httHeader, httHeader); - value.set(colon + 1); + char trace_flags[2] = {0}; + span->GetContext().trace_flags().ToLowerBase16(trace_flags); - httpHeaderProps->setProp(key, value); + traceFlags.set(trace_flags, 2); } - setAttributesFromHTTPHeaders(httpHeaderProps, options); + StringAttr name; + StringAttr tracerName; + StringAttr traceFlags; + StringAttr traceID; + StringAttr spanID; + StringAttr hpccGlobalId; + StringAttr hpccCallerId; + StringAttr opentelTraceParent; + StringAttr opentelTraceState; + + opentelemetry::trace::StartSpanOptions opts; + nostd::shared_ptr span; + //Not sure if we should attempt to support declaring the Active span + opentelemetry::v1::trace::Scope activeSpanScope = + opentelemetry::v1::trace::Scope(opentelemetry::nostd::shared_ptr(nullptr)); +}; -/* - if(stricmp(httHeader, HPCCSemanticConventions::kGLOBALIDHTTPHeader) == 0) +class CTransactionSpan : public CSpan +{ +private: + opentelemetry::v1::trace::SpanContext parentContext = opentelemetry::trace::SpanContext::GetInvalid(); + + void setSpanContext(StringArray & httpHeaders, const char kvDelineator = ':') + { + Owned contextProps = createProperties(); + ForEachItemIn(currentHeaderIndex, httpHeaders) { - hpccGlobalId.set(httpHeaders.item(currentHeaderIndex+1)); - continue; + const char* httpHeader = httpHeaders.item(currentHeaderIndex); + if(!httpHeader) + continue; + + const char* delineator = strchr(httpHeader, kvDelineator); + if(delineator == nullptr) + continue; + + StringBuffer key, value; + key.append(delineator - httpHeader, httpHeader); + value.set(delineator + 1); + + contextProps->setProp(key, value); } - if(stricmp(httHeader, HPCCSemanticConventions::kCallerIdHTTPHeader) == 0) + setSpanContext(contextProps); + + //Instead of copying StringArray to IProperties, we could operate directly on the array: + // if(stricmp(httHeader, HPCCSemanticConventions::kGLOBALIDHTTPHeader) == 0) + // { + // hpccGlobalId.set(httpHeaders.item(currentHeaderIndex+1)); + // continue; + // } + + // if(stricmp(httHeader, HPCCSemanticConventions::kCallerIdHTTPHeader) == 0) + // { + // hpccCallerId.set(httpHeaders.item(currentHeaderIndex+1)); + // continue; + // } + // } + + // const HPCCStringArrayHttpTextMapCarrier carrier(httpHeaders); + // auto globalPropegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + // auto currentContext = context::RuntimeContext::GetCurrent(); + // auto newContext = globalPropegator->Extract(carrier, currentContext); + // parentContext = opentelemetry::trace::GetSpan(newContext)->GetContext(); + // options.parent = parentContext; + } + + void setSpanContext(const IProperties * httpHeaders) + { + if (httpHeaders) { - hpccCallerId.set(httpHeaders.item(currentHeaderIndex+1)); - continue; + // perform any key mapping needed... + //Instrumented http client/server Capitalizes the first letter of the header name + //if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState ) + // theKey[0] = toupper(theKey[0]); + + hpccGlobalId.set(httpHeaders->queryProp(HPCCSemanticConventions::kGLOBALIDHTTPHeader)); + hpccCallerId.set(httpHeaders->queryProp(HPCCSemanticConventions::kCallerIdHTTPHeader)); + + const CHPCCHttpTextMapCarrier carrier(httpHeaders); + auto globalPropegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto currentContext = context::RuntimeContext::GetCurrent(); + auto newContext = globalPropegator->Extract(carrier, currentContext); + parentContext = opentelemetry::trace::GetSpan(newContext)->GetContext(); + opts.parent = parentContext; } } - const HPCCStringArrayHttpTextMapCarrier carrier(httpHeaders); - auto globalPropegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - auto currentContext = context::RuntimeContext::GetCurrent(); - auto newContext = globalPropegator->Extract(carrier, currentContext); - parentContext = opentelemetry::trace::GetSpan(newContext)->GetContext(); - options.parent = parentContext; -*/ -} +public: + CTransactionSpan(const char * spanName, const char * tracerName_, StringArray & httpHeaders, ISpan * parentSpan) + : CSpan(spanName, tracerName_) + { + opts.kind = opentelemetry::trace::SpanKind::kServer; + setSpanContext(httpHeaders); + init(parentSpan); + } -void CTransactionSpan::setAttributesFromHTTPHeaders(const IProperties * httpHeaders, opentelemetry::trace::StartSpanOptions * options) -{ - if (httpHeaders) + CTransactionSpan(const char * spanName, const char * tracerName_, const IProperties * httpHeaders, ISpan * parentSpan) + : CSpan(spanName, tracerName_) { - // perform any key mapping needed... - //Instrumented http client/server Capitalizes the first letter of the header name - //if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState ) - // theKey[0] = toupper(theKey[0]); + opts.kind = opentelemetry::trace::SpanKind::kServer; + setSpanContext(httpHeaders); + init(parentSpan); + } - hpccGlobalId.set(httpHeaders->queryProp(HPCCSemanticConventions::kGLOBALIDHTTPHeader)); - hpccCallerId.set(httpHeaders->queryProp(HPCCSemanticConventions::kCallerIdHTTPHeader)); + bool queryOTParentSpanID(StringAttr & parentSpanId) //override + { + if (!parentContext.IsValid()) + return false; + + if (!parentContext.trace_id().IsValid()) + return false; + + char span_id[16] = {0}; + parentContext.span_id().ToLowerBase16(span_id); + parentSpanId.set(span_id, 16); - const CHPCCHttpTextMapCarrier carrier(httpHeaders); - auto globalPropegator = context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - auto currentContext = context::RuntimeContext::GetCurrent(); - auto newContext = globalPropegator->Extract(carrier, currentContext); - parentContext = opentelemetry::trace::GetSpan(newContext)->GetContext(); - options->parent = parentContext; + return true; } -} +}; -CClientSpan::CClientSpan(const char * spanName, const char * tracerName_) -//: CSpan(opentelemetry::trace::SpanKind::kClient, spanName, tracerName_) +class CInternalSpan : public CSpan { - opentelemetry::trace::StartSpanOptions options; - options.kind = opentelemetry::trace::SpanKind::kClient; - CSpan(&options, spanName, tracerName_); -} +public: + CInternalSpan(const char * spanName, const char * tracerName_, ISpan * parentSpan) + : CSpan(spanName, tracerName_) + { + init(parentSpan); + } +}; -CTransactionSpan::CTransactionSpan(const char * spanName, const char * tracerName_, StringArray & httpHeaders) +class CClientSpan : public CSpan { - opentelemetry::trace::StartSpanOptions options; - options.kind = opentelemetry::trace::SpanKind::kServer; - setAttributesFromHTTPHeaders(httpHeaders, &options); - CSpan(&options, spanName, tracerName_); -} +public: + CClientSpan(const char * spanName, const char * tracerName_, ISpan * parentSpan) + : CSpan(spanName, tracerName_) + { + opts.kind = opentelemetry::trace::SpanKind::kClient; + init(parentSpan); + } -CTransactionSpan::CTransactionSpan(const char * spanName, const char * tracerName_, const IProperties * httpHeaders) -{ - opentelemetry::trace::StartSpanOptions options; - options.kind = opentelemetry::trace::SpanKind::kServer; - setAttributesFromHTTPHeaders(httpHeaders, &options); - CSpan(&options, spanName, tracerName_); -} + bool injectClientContext(CHPCCHttpTextMapCarrier * carrier) + { + if (!carrier) + return false; -void CSpan::setOTTraceID() -{ - openTelTraceID.clear(); + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); - if (!span) - return; + //get current context... + auto currentCtx = opentelemetry::context::RuntimeContext::GetCurrent(); + //and have the propagator inject the ctx into carrier + propagator->Inject(*carrier, currentCtx); - auto spanCtx = span->GetContext(); - if (!spanCtx.IsValid()) - return; + if (!isEmptyString(hpccGlobalId.get())) + carrier->Set(HPCCSemanticConventions::kGLOBALIDHTTPHeader, hpccGlobalId.get()); - if (!spanCtx.trace_id().IsValid()) - return; + if (!isEmptyString(hpccCallerId.get())) + carrier->Set(HPCCSemanticConventions::kCallerIdHTTPHeader, hpccCallerId.get()); - char trace_id[32] = {0}; - - spanCtx.trace_id().ToLowerBase16(trace_id); - openTelTraceID.set(trace_id, 32); -} + return true; + } -void CSpan::setOTSpanID() -{ - openTelSpanID.clear(); + // inject current opentel context http headers container directly + // using homebrewed mechanisms + bool injectClientContext(IProperties * httpHeaders) + { + if (!httpHeaders) + return false; - if (!span) - return; + if (isEmptyString(traceID.get())) + return false; - char span_id[16] = {0}; - span->GetContext().span_id().ToLowerBase16(span_id); + if (isEmptyString(spanID.get())) + return false; - openTelSpanID.set(span_id, 16); -} + if (isEmptyString(traceFlags.get())) + return false; -CSpan::CSpan(opentelemetry::trace::StartSpanOptions * options, const char * spanName, const char * tracerName_) -{ - name.set(spanName); - tracerName.set(tracerName_); + StringBuffer contextHTTPHeader; + //The traceparent header uses the version-trace_id-parent_id-trace_flags format where: + //version is always 00. trace_id is a hex-encoded trace id. span_id is a hex-encoded span id. trace_flags is a hex-encoded 8-bit field that contains tracing flags such as sampling, trace level, etc. + //Example: "traceparent", "00-beca49ca8f3138a2842e5cf21402bfff-4b960b3e4647da3f-01" + contextHTTPHeader.append("00-").append(traceID.get()).append("-").append(spanID.get()).append(traceFlags.get()); + //opentelemetry::trace::propagation::kTraceParent + httpHeaders->setProp("traceparent", contextHTTPHeader.str()); - auto provider = opentelemetry::trace::Provider::GetTracerProvider(); - auto tracer = provider->GetTracer(tracerName.get()); - span = tracer->StartSpan(spanName, {}, *options); + StringBuffer traceStateHTTPHeader; + traceStateHTTPHeader.append("hpcc=").append(spanID.get()); + //opentelemetry::trace::propagation::kTraceState + httpHeaders->setProp("tracestate", contextHTTPHeader.str()); - StringAttr openTelTraceID; - StringAttr openTelSpanID; + if (!isEmptyString(hpccGlobalId.get())) + httpHeaders->setProp(HPCCSemanticConventions::kGLOBALIDHTTPHeader, hpccGlobalId.get()); - auto scope = tracer->WithActiveSpan(span); -} + if (!isEmptyString(hpccCallerId.get())) + httpHeaders->setProp(HPCCSemanticConventions::kCallerIdHTTPHeader, hpccCallerId.get()); + } +}; -// inject current opentel context into carrier via propagator -bool CClientSpan::injectClientContext(CHPCCHttpTextMapCarrier * carrier) +class CTraceManager : implements ITraceManager, public CInterface +//class CTraceManager : CInterfaceOf { - if (!carrier) - return false; +private: + bool enabled = true; + StringAttr moduleName; + + void initTracer(IPropertyTree * traceConfig) + { + //Owned traceConfig; + try + { + //traceConfig.setown(getComponentConfigSP()->getPropTree("tracing")); - auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); +//#ifdef TRACECONFIGDEBUG + if (!traceConfig || !traceConfig->hasProp("tracing")) + { + const char * simulatedGlobalYaml = R"!!(global: + tracing: + enable: + true + exporter: + OS: true + processor: + batchSpan: true + simpleSpan: false + )!!"; + Owned testTree = createPTreeFromYAMLString(simulatedGlobalYaml, ipt_none, ptr_ignoreWhiteSpace, nullptr); + traceConfig = testTree->getPropTree("global/tracing"); + } +//#endif + StringBuffer xml; + toXML(traceConfig, xml); + DBGLOG("traceConfig tree: %s", xml.str()); - //get current context... - auto currentCtx = opentelemetry::context::RuntimeContext::GetCurrent(); - //and have the propagator inject the ctx into carrier - propagator->Inject(*carrier, currentCtx); + if (traceConfig && traceConfig->getPropBool("@enable", false)) + { + DBGLOG("OpenTel tracing enabled"); + using namespace opentelemetry::trace; + std::unique_ptr exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); - if (!isEmptyString(hpccGlobalId.get())) - carrier->Set(HPCCSemanticConventions::kGLOBALIDHTTPHeader, hpccGlobalId.get()); + Owned exportConfig = traceConfig->getPropTree("exporter"); + if (exportConfig) + { + if (exportConfig->getPropBool("@OS", false)) //To stdout/err + DBGLOG("Tracing to stdout/err"); + else if (exportConfig->getPropBool("@OTLP", false)) + { + //namespace otlp = opentelemetry::exporter::otlp; + + //otlp::OtlpGrpcExporterOptions opts; + //opts.endpoint = "localhost:4317"; + //opts.use_ssl_credentials = true; + //opts.ssl_credentials_cacert_as_string = "ssl-certificate"; + + //exporter = otlp::OtlpGrpcExporterFactory::Create(opts); + DBGLOG("Tracing to OTLP currently not supported"); + } + else if (exportConfig->getPropBool("@Prometheus", false)) + DBGLOG("Tracing to Prometheus currently not supported"); + else if (exportConfig->getPropBool("@HPCC", false)) + DBGLOG("Tracing to HPCC JLog currently not supported"); + } - if (!isEmptyString(hpccCallerId.get())) - carrier->Set(HPCCSemanticConventions::kCallerIdHTTPHeader, hpccCallerId.get()); + Owned processorConfig = traceConfig->getPropTree("processor"); + std::unique_ptr processor; + if (exportConfig && exportConfig->getPropBool("@batchSpan", false)) + { + //Groups several spans together, before sending them to an exporter. + opentelemetry::v1::sdk::trace::BatchSpanProcessorOptions options; //size_t max_queue_size = 2048; + //The time interval between two consecutive exports + //std::chrono::milliseconds(5000); + //The maximum batch size of every export. It must be smaller or + //equal to max_queue_size. + //size_t max_export_batch_size = 512 + processor = opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), options); + DBGLOG("OpenTel tracing using batch Span Processor"); + } + else + { + //SimpleSpanProcesser sends spans one by one to an exporter. + processor = opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); + DBGLOG("OpenTel tracing using Simple Span Processor"); + } - return true; -} + std::vector> processors; + processors.push_back(std::move(processor)); + + // Default is an always-on sampler. + std::shared_ptr context = + opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); + std::shared_ptr provider = + opentelemetry::sdk::trace::TracerProviderFactory::Create(context); + + // Set the global trace provider + opentelemetry::trace::Provider::SetTracerProvider(provider); + + // set global propagator + // Injects Context into and extracts it from carriers that travel in-band + // across process boundaries. Encoding is expected to conform to the HTTP + // Header Field semantics. + // Values are often encoded as RPC/HTTP request headers. + opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator( + opentelemetry::nostd::shared_ptr( + new opentelemetry::trace::propagation::HttpTraceContext())); + } + } + catch (IException * e) + { + EXCLOG(e); + e->Release(); + } + } -// inject current opentel context http headers container directly -// using homebrewed mechanisms -bool CClientSpan::injectClientContext(IProperties * httpHeaders) -{ - if (!httpHeaders) - return false; + void cleanupTracer() + { + std::shared_ptr none; + opentelemetry::trace::Provider::SetTracerProvider(none); + } + + nostd::shared_ptr tracer; + +public: + IMPLEMENT_IINTERFACE; + CTraceManager(const char * componentName, IPropertyTree * traceConfig) + { + moduleName.set(componentName); + initTracer(traceConfig); - if (isEmptyString(openTelTraceID.get())) - return false; + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + tracer = provider->GetTracer(moduleName.get()); + } - if (isEmptyString(openTelSpanID.get())) - return false; + CTraceManager() {} - StringBuffer contextHTTPHeader; - //The traceparent header uses the version-trace_id-parent_id-trace_flags format where: - //version is always 00. trace_id is a hex-encoded trace id. span_id is a hex-encoded span id. trace_flags is a hex-encoded 8-bit field that contains tracing flags such as sampling, trace level, etc. - //Example: "traceparent", "00-beca49ca8f3138a2842e5cf21402bfff-4b960b3e4647da3f-01" - contextHTTPHeader.append("00-").append(openTelTraceID.get()).append("-").append(openTelSpanID.get()).append("00"); - //opentelemetry::trace::propagation::kTraceParent - httpHeaders->setProp("traceparent", contextHTTPHeader.str()); + ISpan * createServerSpan(const char * name, StringArray & httpHeaders, ISpan * parentSpan) override + { + if (!enabled) + return new CNoOpSpan(); - StringBuffer traceStateHTTPHeader; - traceStateHTTPHeader.append("hpcc=").append(openTelSpanID.get()); - //opentelemetry::trace::propagation::kTraceState - httpHeaders->setProp("tracestate", contextHTTPHeader.str()); + return new CTransactionSpan(name, moduleName.get(), httpHeaders, parentSpan); + } - if (!isEmptyString(hpccGlobalId.get())) - httpHeaders->setProp(HPCCSemanticConventions::kGLOBALIDHTTPHeader, hpccGlobalId.get()); + ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, ISpan * parentSpan) override + { + if (!enabled) + return new CNoOpSpan(); - if (!isEmptyString(hpccCallerId.get())) - httpHeaders->setProp(HPCCSemanticConventions::kCallerIdHTTPHeader, hpccCallerId.get()); -} + return new CTransactionSpan(name, moduleName.get(), httpHeaders, parentSpan); + } + + ISpan * createClientSpan(const char * name, ISpan * parentSpan) override + { + if (!enabled) + return new CNoOpSpan(); + + return new CClientSpan(name, moduleName.get(), parentSpan); + } + + ISpan * createInternalSpan(const char * name, ISpan * parentSpan) override + { + if (!enabled) + return new CNoOpSpan(); + + return new CInternalSpan(name, moduleName.get(), parentSpan); + } +}; static Singleton theTraceManager; + MODULE_INIT(INIT_PRIORITY_STANDARD) { return true; @@ -534,7 +745,13 @@ MODULE_EXIT() theTraceManager.destroy(); } -CTraceManager * queryTraceManager(const char * componentName) +void initTraceManager(const char * componentName, IPropertyTree * config) +{ + theTraceManager.query([=] () { return new CTraceManager(componentName, config); }); +} + +ITraceManager & queryTraceManager() { - return theTraceManager.query([=] () { return new CTraceManager(componentName); }); + //return theTraceManager.query([=] () { return new CTraceManager(iHateThis.get()); }); + return *theTraceManager.query([] () { return new CTraceManager; }); } \ No newline at end of file diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 75ddcb6ed90..15656262906 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -18,13 +18,16 @@ #ifndef JTRACE_HPP #define JTRACE_HPP -#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED -#include "opentelemetry/trace/provider.h" //StartSpanOptions -#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) - -namespace context = opentelemetry::context; -namespace nostd = opentelemetry::nostd; -namespace opentel_trace = opentelemetry::trace; +/** + * @brief This follows open telemetry's span attribute naming conventions + * Known HPCC span Keys could be added here + * Specialized span keys can also be defined within the scope of a span + */ +namespace HPCCSemanticConventions +{ +static constexpr const char *kGLOBALIDHTTPHeader = "HPCC-Global-Id"; +static constexpr const char *kCallerIdHTTPHeader = "HPCC-Caller-Id"; +} class jlib_decl LogTrace { @@ -33,8 +36,8 @@ class jlib_decl LogTrace StringAttr callerId; StringAttr localId; - StringAttr globalIdHTTPHeaderName = "HPCC-Global-Id"; - StringAttr callerIdHTTPHeaderName = "HPCC-Caller-Id"; + StringAttr globalIdHTTPHeaderName = HPCCSemanticConventions::kGLOBALIDHTTPHeader; + StringAttr callerIdHTTPHeaderName = HPCCSemanticConventions::kCallerIdHTTPHeader; const char* assignLocalId(); @@ -63,177 +66,32 @@ class jlib_decl LogTrace void setLocalId(const char* id); }; -/** - * @brief This follows open telemetry's span attribute naming conventions - * Known HPCC span Keys could be added here - * Specialized span keys can also be defined within the scope of a span - */ -namespace HPCCSemanticConventions -{ -static constexpr const char *kGLOBALIDHTTPHeader = "HPCC-Global-Id"; -static constexpr const char *kCallerIdHTTPHeader = "HPCC-Caller-Id"; -} - interface ISpan : extends IInterface { - virtual void setAttribute(const char * key, const char * val) = 0; - virtual void setAttributes(const IProperties * attributes) = 0; - virtual void addEvent(const char * eventName) = 0; + virtual void setSpanAttribute(const char * key, const char * val) = 0; + virtual void setSpanAttributes(const IProperties * attributes) = 0; + virtual void addSpanEvent(const char * eventName) = 0; + virtual void setActive() = 0; virtual const char * queryHPCCGlobalID() = 0; virtual const char * queryHPCCCallerID() = 0; - virtual const char * queryOTSpanName() = 0; - virtual const char * queryOTTraceID() = 0; - virtual const char * queryOTSpanID() = 0; -}; - -class CNoOpSpan : public CInterfaceOf -{ - void setAttribute(const char * key, const char * val) override {}; - void setAttributes(const IProperties * attributes) override {}; - void addEvent(const char * eventName) override {}; - const char * queryOTTraceID() override { return ""; }; - const char * queryOTSpanID() override { return ""; }; - const char * queryOTSpanName() override { return ""; }; - const char * queryHPCCGlobalID() override { return ""; }; - const char * queryHPCCCallerID() override { return ""; }; -}; - -class CSpan : public CInterfaceOf -{ -public: - CSpan(); - CSpan(opentelemetry::trace::StartSpanOptions * options, const char * spanName, const char * tracerName_); - ~CSpan(); - - void setAttributes(const IProperties * attributes) override; - void setAttribute(const char * key, const char * val) override; - void addEvent(const char * eventName) override; - const char * queryOTTraceID() override { return openTelTraceID; }; - const char * queryOTSpanID() override { return openTelSpanID; }; - - const char * queryOTSpanName() override { return name.get(); } - const char * queryHPCCGlobalID() override { return hpccGlobalId.get(); } - const char * queryHPCCCallerID() override { return hpccCallerId.get(); } - -protected: - void setOTTraceID(); - void setOTSpanID(); - - StringAttr name; - StringAttr tracerName; - StringAttr openTelTraceID; - StringAttr openTelSpanID; - StringAttr hpccGlobalId; - StringAttr hpccCallerId; - StringAttr opentelTraceParent; - StringAttr opentelTraceState; - - //opentelemetry::trace::StartSpanOptions options; - nostd::shared_ptr span; -}; - -class CTransactionSpan : public CSpan -{ -private: - opentelemetry::v1::trace::SpanContext parentContext = opentelemetry::trace::SpanContext::GetInvalid(); - void setAttributesFromHTTPHeaders(const IProperties * httpHeaders, opentelemetry::trace::StartSpanOptions * options); - void setAttributesFromHTTPHeaders(StringArray & httpHeaders, opentelemetry::trace::StartSpanOptions * options); - -public: - CTransactionSpan(const char * spanName, const char * tracerName_, const IProperties * httpHeaders); - CTransactionSpan(const char * spanName, const char * tracerName_, StringArray & httpHeaders); - - bool queryOTParentSpanID(StringAttr & parentSpanId) //override - { - if (!parentContext.IsValid()) - return false; - - if (!parentContext.trace_id().IsValid()) - return false; - - char span_id[16] = {0}; - parentContext.span_id().ToLowerBase16(span_id); - parentSpanId = std::string(span_id, 16).c_str(); - - return true; - } -}; - -class CHPCCHttpTextMapCarrier; - -class CClientSpan : public CSpan -{ -public: - CClientSpan(const char * spanName, const char * tracerName_); - bool injectClientContext(IProperties * httpHeaders); - bool injectClientContext(CHPCCHttpTextMapCarrier * carrier); + virtual const char * querySpanName() = 0; + virtual const char * queryTraceID() = 0; + virtual const char * queryTraceFlags() = 0; + virtual const char * queryTraceName() = 0; + virtual const char * querySpanID() = 0; }; interface ITraceManager : extends IInterface { - virtual ISpan * createTransactionSpan(const char * name, StringArray & httpHeaders) = 0; - virtual ISpan * createTransactionSpan(const char * name, const IProperties * httpHeaders) = 0; - virtual ISpan * createClientSpan(const char * name) = 0; - virtual ISpan * createInternalSpan(const char * name) = 0; + virtual ISpan * createServerSpan(const char * name, StringArray & httpHeaders, ISpan * parentSpan) = 0; + virtual ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, ISpan * parentSpan) = 0; + virtual ISpan * createClientSpan(const char * name, ISpan * parentSpan) = 0; + virtual ISpan * createInternalSpan(const char * name, ISpan * parentSpan) = 0; }; -class CTraceManager : CInterfaceOf -{ -private: - void initTracer(); - void cleanupTracer(); - bool enabled = true; - - StringAttr moduleName; - nostd::shared_ptr tracer; - -public: - CTraceManager(const char * componentName) - { - moduleName.set(componentName); - initTracer(); - - auto provider = opentelemetry::trace::Provider::GetTracerProvider(); - tracer = provider->GetTracer(moduleName.get()); - } - - ISpan * createTransactionSpan(const char * name, StringArray & httpHeaders) override - { - if (!enabled) - return new CNoOpSpan(); - - return new CTransactionSpan(name, moduleName.get(), httpHeaders); - } - - ISpan * createTransactionSpan(const char * name, const IProperties * httpHeaders) override - { - if (!enabled) - return new CNoOpSpan(); - - return new CTransactionSpan(name, moduleName.get(), httpHeaders); - } - - ISpan * createClientSpan(const char * name) override - { - if (!enabled) - return new CNoOpSpan(); - - return new CClientSpan(name, moduleName.get()); - } - - ISpan * createInternalSpan(const char * name) override - { - if (!enabled) - return new CNoOpSpan(); - - opentelemetry::trace::StartSpanOptions options; - options.kind = opentelemetry::trace::SpanKind::kInternal; - return new CSpan(&options, name, moduleName.get()); - } -}; - -extern jlib_decl CTraceManager * queryTraceManager(const char * ); +extern jlib_decl void initTraceManager(const char * componentName, IPropertyTree * traceConfig); +extern jlib_decl ITraceManager & queryTraceManager(); /* To use feature-level tracing flags, protect the tracing with a test such as: