diff --git a/.github/workflows/nginx-otel-module-check.yml b/.github/workflows/nginx-otel-module-check.yml index 9493af5..ba81a95 100644 --- a/.github/workflows/nginx-otel-module-check.yml +++ b/.github/workflows/nginx-otel-module-check.yml @@ -29,3 +29,54 @@ jobs: with: name: nginx-otel-module path: build/ngx_otel_module.so + - name: Archive protoc and opentelemetry-proto + uses: actions/upload-artifact@v3 + with: + name: protoc-opentelemetry-proto + path: | + build/_deps/grpc-build/third_party/protobuf/protoc + build/_deps/otelcpp-src/third_party/opentelemetry-proto + test-module: + needs: build-module + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Download module + uses: actions/download-artifact@v3 + with: + name: nginx-otel-module + path: build + - name: Download protoc and opentelemetry-proto + uses: actions/download-artifact@v3 + with: + name: protoc-opentelemetry-proto + path: build/_deps + - name: List files + run: ls -laR . + - name: Fix protoc file permissions + run: chmod +x build/_deps/grpc-build/third_party/protobuf/protoc + - name: Install perl modules + run: sudo cpan IO::Socket::SSL Crypt::Misc + - name: Download otelcol + run: | + curl -LO https://github.com/\ + open-telemetry/opentelemetry-collector-releases/releases/download/\ + v0.76.1/otelcol_0.76.1_linux_amd64.tar.gz + tar -xzf otelcol_0.76.1_linux_amd64.tar.gz + - name: Checkout nginx and nginx-test + run: | + hg clone http://hg.nginx.org/nginx/ + hg clone http://hg.nginx.org/nginx-tests/ + - name: Build nginx + working-directory: nginx + run: | + auto/configure --with-compat --with-debug --with-http_ssl_module \ + --with-http_v2_module --with-http_v3_module + make -j 4 + - name: Run tests + working-directory: tests + run: | + PERL5LIB=../nginx-tests/lib TEST_NGINX_UNSAFE=1 \ + TEST_NGINX_VERBOSE=1 TEST_NGINX_GLOBALS="load_module \ + ${PWD}/../build/ngx_otel_module.so;" prove -v . diff --git a/tests/h2_otel.t b/tests/h2_otel.t new file mode 100644 index 0000000..b90deb2 --- /dev/null +++ b/tests/h2_otel.t @@ -0,0 +1,554 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for opentelemetry exporter in case HTTP/2. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new() + ->has(qw/http_v2 http_ssl rewrite mirror grpc socket_ssl_alpn/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:8083; + interval 1s; + batch_size 10; + batch_count 1; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080 http2; + listen 127.0.0.1:8081; + listen 127.0.0.1:8082 http2 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8084; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8084 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } + +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTEL module')->plan(69); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +(undef, my $t_off_resp) = http2_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1); + +(my $t_headers_ignore, undef) = http2_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http2_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http2_get('/context-extract'); +(my $tp_headers_extract, undef) = http2_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http2_get('/context-inject'); +(my $tp_headers_inject, undef) = http2_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http2_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http2_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http2_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), + 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '2.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "string_value", $$spans{span1}), '2.0', + 'http.flavor metric - trace on (https)'); +isnt(get_attr("http.user_agent", "string_value", $$spans{span1}), + 'nginx-tests', 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "int_value", $$spans{span1}), 8082, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr( + "http.response.header.content.type", "array_value", $$spans{span0} + )}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/2.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +isnt($t_headers_inject->{'x-otel-traceparent'}, undef, + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http2_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $port = $extra{port} || 8080; + + my $s = $extra{ssl} + ? Test::Nginx::HTTP2->new( + undef, socket => get_ssl_socket($port, ['h2'])) + : Test::Nginx::HTTP2->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests', mode => 2 }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01', + mode => 2 + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + mode => 2 + }]}) + : $s->new_stream({ path => $path }); + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_ssl_socket { + my ($port, $alpn) = @_; + + return http( + '', PeerAddr => '127.0.0.1:' . port($port), start => 1, + SSL => 1, + SSL_alpn_protocols => $alpn, + SSL_error_trap => sub { die $_[1] } + ); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + $protobuf = encode_base64($protobuf); + + local $/; + open CMD => "echo '$protobuf' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/h2_otel_collector.t b/tests/h2_otel_collector.t new file mode 100644 index 0000000..ba9a4f2 --- /dev/null +++ b/tests/h2_otel_collector.t @@ -0,0 +1,438 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for opentelemetry exporter in case HTTP/2 using otelcol. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http_ssl http_v2 rewrite/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:%%PORT_4317%%; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080 http2; + listen 127.0.0.1:8081; + listen 127.0.0.1:8082 http2 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } +} + +EOF + +$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging, file] + metrics: + receivers: [otlp] + exporters: [logging, file] + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +#suppress otel collector output +open OLDERR, ">&", \*STDERR; +open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; +$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); +open STDERR, ">&", \*OLDERR; +$t->waitforsocket('127.0.0.1:' . port(4317)) or + die 'No otel collector open socket'; + +$t->try_run('no OTEL module')->plan(69); + +############################################################################### + +#do requests +(undef, my $t_off_resp) = http2_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http2_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http2_get('/trace-on', port => 8082, ssl => 1); + +(my $t_headers_ignore, undef) = http2_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http2_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http2_get('/context-extract'); +(my $tp_headers_extract, undef) = http2_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http2_get('/context-inject'); +(my $tp_headers_inject, undef) = http2_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http2_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http2_get('/context-propagate', trace_headers => 1); + +#batch1 (5 reqeusts) +http2_get('/trace-on') for (1..5); + +#waiting batch1 is sent to collector for 1s +select undef, undef, undef, 1; + +my @batches = split /\n/, $t->read_file('otel.json'); +my $batch_json = JSON::PP::decode_json($batches[0]); +my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate amount of batches +is(scalar @batches, 2, 'amount of batches - trace on'); + +#validate batch size +is(scalar @{$spans}, 10, 'batch0 size - trace on'); +is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] + {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "stringValue", + $$batch_json{resourceSpans}[0]{resource}), + 'test_server', 'service.name - trace on'); +is($$spans[0]{name}, 'default_location', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "stringValue", $$spans[0]), '2.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "intValue", $$spans[0]), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "intValue", $$spans[0]), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "stringValue", $$spans[1]), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "stringValue", $$spans[1]), '2.0', + 'http.flavor metric - trace on (https)'); +isnt(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests', + 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "intValue", $$spans[1]), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "intValue", $$spans[1]), 8082, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'OK', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'text/plain', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "stringValue", $$spans[0]), + 'GET /trace-on HTTP/2.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); +is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace on'); +is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); +is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans[0]{traceId}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans[0]{spanId}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans[0]{parentSpanId}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +unlike($batches[0], + qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, + 'no metrics - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +isnt($t_headers_inject->{'x-otel-traceparent'}, undef, + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans[7]{traceId}-$$spans[7]{spanId}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans[8]{traceId}-$$spans[8]{spanId}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +$t->stop(); +my $log = $t->read_file("error.log"); + +unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: no error parsing metadata'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +############################################################################### + +sub http2_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $port = $extra{port} || 8080; + + my $s = $extra{ssl} + ? Test::Nginx::HTTP2->new( + undef, socket => get_ssl_socket($port, ['h2'])) + : Test::Nginx::HTTP2->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests', mode => 2 }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01', + mode => 2 + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + mode => 2 + }]}) + : $s->new_stream({ path => $path }); + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_ssl_socket { + my ($port, $alpn) = @_; + + return http( + '', PeerAddr => '127.0.0.1:' . port($port), start => 1, + SSL => 1, + SSL_alpn_protocols => $alpn, + SSL_error_trap => sub { die $_[1] } + ); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; + + return defined $res ? $res->{"value"}{$type} : undef; +} + +############################################################################### diff --git a/tests/h3_otel.t b/tests/h3_otel.t new file mode 100644 index 0000000..8ee7b92 --- /dev/null +++ b/tests/h3_otel.t @@ -0,0 +1,509 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for opentelemetry exporter in case HTTP/3. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use Test::Nginx::HTTP3; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite mirror grpc cryptx/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + ssl_protocols TLSv1.3; + + otel_exporter { + endpoint 127.0.0.1:8082; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:%%PORT_8980_UDP%% quic; + listen 127.0.0.1:8081; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8082 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8083; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } + +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTEL module')->plan(56); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +(undef, my $t_off_resp) = http3_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http3_get('/trace-on'); + +(my $t_headers_ignore, undef) = http3_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http3_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http3_get('/context-extract'); +(my $tp_headers_extract, undef) = http3_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http3_get('/context-inject'); +(my $tp_headers_inject, undef) = http3_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http3_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http3_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http3_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), + 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'https', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '3.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8980, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr( + "http.response.header.content.type", "array_value", $$spans{span0} + )}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/3.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans{span0}{trace_id}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans{span0}{span_id}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans{span0}{parent_span_id}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +isnt($t_headers_inject->{'x-otel-traceparent'}, undef, + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log content", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http3_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $s = Test::Nginx::HTTP3->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests' }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01' + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7' + }]}) + : $s->new_stream({ path => $path }); + + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + $protobuf = encode_base64($protobuf); + + local $/; + open CMD => "echo '$protobuf' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/h3_otel_collector.t b/tests/h3_otel_collector.t new file mode 100644 index 0000000..af2e264 --- /dev/null +++ b/tests/h3_otel_collector.t @@ -0,0 +1,394 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for opentelemetry exporter in case HTTP/3 using otelcol. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP3; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http_v2 http_v3 rewrite cryptx/) + ->has_daemon(qw/openssl/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + ssl_protocols TLSv1.3; + + otel_exporter { + endpoint 127.0.0.1:%%PORT_4317%%; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:%%PORT_8980_UDP%% quic; + listen 127.0.0.1:8081; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://127.0.0.1:8081/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } +} + +EOF + +$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging, file] + metrics: + receivers: [otlp] + exporters: [logging, file] + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +#suppress otel collector output +open OLDERR, ">&", \*STDERR; +open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; +$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); +open STDERR, ">&", \*OLDERR; +$t->waitforsocket('127.0.0.1:' . port(4317)) or + die 'No otel collector open socket'; + +$t->try_run('no OTEL module')->plan(56); + +############################################################################### + +#do requests +(undef, my $t_off_resp) = http3_get('/trace-off'); + +#batch0 (10 requests) +my ($tp_headers, $tp_resp) = http3_get('/trace-on', trace_headers => 1); +my ($t_headers, $t_resp) = http3_get('/trace-on'); + +(my $t_headers_ignore, undef) = http3_get('/context-ignore'); +(my $tp_headers_ignore, undef) = http3_get('/context-ignore', + trace_headers => 1); +(my $t_headers_extract, undef) = http3_get('/context-extract'); +(my $tp_headers_extract, undef) = http3_get('/context-extract', + trace_headers => 1); +(my $t_headers_inject, undef) = http3_get('/context-inject'); +(my $tp_headers_inject, undef) = http3_get('/context-inject', + trace_headers => 1); +(my $t_headers_propagate, undef) = http3_get('/context-propagate'); +(my $tp_headers_propagate, undef) = + http3_get('/context-propagate', trace_headers => 1); + +#batch1 (5 reqeusts) +http3_get('/trace-on') for (1..5); + +#waiting batch1 is sent to collector for 1s +select undef, undef, undef, 1; + +my @batches = split /\n/, $t->read_file('otel.json'); +my $batch_json = JSON::PP::decode_json($batches[0]); +my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate amount of batches +is(scalar @batches, 2, 'amount of batches - trace on'); + +#validate batch size +is(scalar @{$spans}, 10, 'batch0 size - trace on'); +is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] + {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "stringValue", + $$batch_json{resourceSpans}[0]{resource}), + 'test_server', 'service.name - trace on'); +is($$spans[0]{name}, 'default_location', 'span.name - trace on'); + +#validate metrics +is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "stringValue", $$spans[0]), 'https', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "stringValue", $$spans[0]), '3.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "intValue", $$spans[0]), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "intValue", $$spans[0]), 8980, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'OK', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'text/plain', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "stringValue", $$spans[0]), + 'GET /trace-on HTTP/3.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); +is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace on'); +is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); +is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); + +#variables +is($tp_headers->{'x-otel-trace-id'}, $$spans[0]{traceId}, + '$otel_trace_id variable - trace on'); +is($tp_headers->{'x-otel-span-id'}, $$spans[0]{spanId}, + '$otel_span_id variable - trace on'); +is($tp_headers->{'x-otel-parent-id'}, $$spans[0]{parentSpanId}, + '$otel_parent_id variable - trace on'); +is($tp_headers->{'x-otel-parent-sampled'}, 1, + '$otel_parent_sampled variable - trace on'); +is($t_headers->{'x-otel-parent-sampled'}, 0, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +unlike($batches[0], + qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, + 'no metrics - trace off'); + +#trace context: ignore +is($t_headers_ignore->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context ignore (no trace headers)'); +is($t_headers_ignore->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context ignore (no trace headers)'); + +is($tp_headers_ignore->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context ignore (trace headers)'); +is($tp_headers_ignore->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +is($t_headers_extract->{'x-otel-traceparent'}, undef, + 'no traceparent - trace context extract (no trace headers)'); +is($t_headers_extract->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context extract (no trace headers)'); + +is($tp_headers_extract->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent span id - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-traceparent'}, + '00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01', + 'traceparent - trace context extract (trace headers)'); +is($tp_headers_extract->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +isnt($t_headers_inject->{'x-otel-traceparent'}, undef, + 'traceparent - trace context inject (no trace headers)'); +is($t_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (no trace headers)'); + +is($tp_headers_inject->{'x-otel-parent-id'}, undef, + 'no parent span id - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-traceparent'}, + "00-$$spans[7]{traceId}-$$spans[7]{spanId}-01", + 'traceparent - trace context inject (trace headers)'); +is($tp_headers_inject->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +is($t_headers_propagate->{'x-otel-traceparent'}, + "00-$$spans[8]{traceId}-$$spans[8]{spanId}-01", + 'traceparent - trace context propagate (no trace headers)'); +is($t_headers_propagate->{'x-otel-tracestate'}, undef, + 'no tracestate - trace context propagate (no trace headers)'); + +is($tp_headers_propagate->{'x-otel-parent-id'}, 'b9c7c989f97918e1', + 'parent id - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-traceparent'}, + "00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01", + 'traceparent - trace context propagate (trace headers)'); +is($tp_headers_propagate->{'x-otel-tracestate'}, + 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace context propagate (trace headers)'); + +$t->stop(); +my $log = $t->read_file("error.log"); + +unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: no error parsing metadata'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +############################################################################### + +sub http3_get { + my ($path, %extra) = @_; + my ($frames, $frame); + + my $s = Test::Nginx::HTTP3->new(); + + my $sid = $extra{trace_headers} + ? $s->new_stream({ headers => [ + { name => ':method', value => 'GET' }, + { name => ':scheme', value => 'http' }, + { name => ':path', value => $path }, + { name => ':authority', value => 'localhost' }, + { name => 'user-agent', value => 'nginx-tests' }, + { name => 'traceparent', + value => '00-0af7651916cd43dd8448eb211c80319c-' . + 'b9c7c989f97918e1-01' + }, + { name => 'tracestate', + value => 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7' + }]}) + : $s->new_stream({ path => $path }); + + $frames = $s->read(all => [{ sid => $sid, fin => 1 }]); + + ($frame) = grep { $_->{type} eq "HEADERS" } @$frames; + my $headers = $frame->{headers}; + + ($frame) = grep { $_->{type} eq "DATA" } @$frames; + my $data = $frame->{data}; + + return $headers, $data; +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; + + return defined $res ? $res->{"value"}{$type} : undef; +} + +############################################################################### diff --git a/tests/otel.t b/tests/otel.t new file mode 100644 index 0000000..280557e --- /dev/null +++ b/tests/otel.t @@ -0,0 +1,515 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for opentelemetry exporter in case HTTP. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; +use Test::Nginx::HTTP2; +use MIME::Base64; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/http http_ssl http_v2 mirror rewrite/) + ->has_daemon(qw/openssl base64/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:8082; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080; + listen 127.0.0.1:8081 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } + + server { + listen 127.0.0.1:8082 http2; + server_name localhost; + otel_trace off; + + location / { + mirror /mirror; + grpc_pass 127.0.0.1:8083; + } + + location /mirror { + internal; + grpc_pass 127.0.0.1:%%PORT_4317%%; + } + } + + server { + listen 127.0.0.1:8083 http2; + server_name localhost; + otel_trace off; + + location / { + add_header content-type application/grpc; + add_header grpc-status 0; + add_header grpc-message ""; + return 200; + } + } +} + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +$t->try_run('no OTEL module')->plan(69); + +############################################################################### + +my $p = port(4317); +my $f = grpc(); + +#do requests +my $t_off_resp = http1_get('/trace-off'); + +#batch0 (10 requests) +my $tp_resp = http1_get('/trace-on', trace_headers => 1); +my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); + +my $t_resp_ignore = http1_get('/context-ignore'); +my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); +my $t_resp_extract = http1_get('/context-extract'); +my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); +my $t_resp_inject = http1_get('/context-inject'); +my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); +my $t_resp_propagate = http1_get('/context-propagate'); +my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); + +my ($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch0 = to_hash(decode_protobuf(substr $frame->{data}, 8)); +my $spans = $$batch0{scope_spans}; + +#batch1 (5 reqeusts) +http1_get('/trace-on') for (1..5); + +($frame) = grep { $_->{type} eq "DATA" } @{$f->{http_start}()}; +my $batch1 = to_hash(decode_protobuf(substr $frame->{data}, 8)); + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate batch size +delete $$spans{scope}; #remove 'scope' entry +is(scalar keys %{$spans}, 10, 'batch0 size - trace on'); +delete $$batch1{scope_spans}{scope}; #remove 'scope' entry +is(scalar keys %{$$batch1{scope_spans}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "string_value", + $$batch0{resource}), 'test_server', 'service.name - trace on'); +is($$spans{span0}{name}, '"default_location"', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "string_value", $$spans{span0}), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "string_value", $$spans{span0}), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "string_value", $$spans{span0}), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "string_value", $$spans{span0}), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "string_value", $$spans{span0}), '1.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "string_value", $$spans{span0}), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "int_value", $$spans{span0}), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "int_value", $$spans{span0}), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "int_value", $$spans{span0}), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "string_value", $$spans{span0}), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "int_value", $$spans{span0}), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span0}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span0}), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "string_value", $$spans{span1}), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "string_value", $$spans{span1}), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "string_value", $$spans{span1}), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "string_value", $$spans{span1}), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "string_value", $$spans{span1}), '1.0', + 'http.flavor metric - trace on (https)'); +is(get_attr("http.user_agent", "string_value", $$spans{span1}), + 'nginx-tests', 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "int_value", $$spans{span1}), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "int_value", $$spans{span1}), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "int_value", $$spans{span1}), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "string_value", $$spans{span1}), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "int_value", $$spans{span1}), 8081, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "string_value", $$spans{span1}), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "int_value", $$spans{span1}), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "array_value", $$spans{span0})} + {values}{string_value}, '"OK"', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", + "array_value", $$spans{span0})}{values}{string_value}, '"text/plain"', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "string_value", $$spans{span0}), + 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); + +#extract trace info +is($$spans{span0}{parent_span_id}, 'b9c7c989f97918e1', + 'traceparent - trace on'); +is($$spans{span0}{trace_state}, '"congo=ucfJifl5GOE,rojo=00f067aa0ba902b7"', + 'tracestate - trace on'); +is($$spans{span1}{parent_span_id}, undef, 'no traceparent - trace on'); +is($$spans{span1}{trace_state}, undef, 'no tracestate - trace on'); + +#variables +like($tp_resp, qr/X-Otel-Trace-Id: $$spans{span0}{trace_id}/, + '$otel_trace_id variable - trace on'); +like($tp_resp, qr/X-Otel-Span-Id: $$spans{span0}{span_id}/, + '$otel_span_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Id: $$spans{span0}{parent_span_id}/, + '$otel_parent_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, + '$otel_parent_sampled variable - trace on'); +like($t_resp, qr/X-Otel-Parent-Sampled: 0/, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$spans}), 0, 'no metric in batch0 - trace off'); +is((scalar grep { + get_attr("http.target", "string_value", $$spans{$_}) eq '/trace-off' + } keys %{$$batch1{scope_spans}}), 0, 'no metric in batch1 - trace off'); + +#trace context: ignore +unlike($t_resp_ignore, qr/X-Otel-Traceparent/, + 'no traceparent - trace context ignore (no trace headers)'); +unlike($t_resp_ignore, qr/X-Otel-Tracestate/, + 'no tracestate - trace context ignore (no trace headers)'); + +unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +unlike($t_resp_extract, qr/X-Otel-Traceparent/, + 'no traceparent - trace context extract (no trace headers)'); +unlike($t_resp_extract, qr/X-Otel-Tracestate/, + 'no tracestate - trace context extract (no trace headers)'); + +like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent span id - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +like($t_resp_inject, qr/X-Otel-Traceparent/, + 'traceparent - trace context inject (no trace headers)'); +unlike($t_resp_inject, qr/X-Otel-Tracestate/, + 'no tracestate - trace context inject (no trace headers)'); + +unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context inject (trace headers)'); +like($tp_resp_inject, + qr/Traceparent: 00-$$spans{span7}{trace_id}-$$spans{span7}{span_id}-01/, + 'traceparent - trace context inject (trace headers)'); +unlike($tp_resp_inject, qr/Tracestate:/, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +like($t_resp_propagate, + qr/Traceparent: 00-$$spans{span8}{trace_id}-$$spans{span8}{span_id}-01/, + 'traceparent - trace context propagate (no trace headers)'); +unlike($t_resp_propagate, qr/X-Otel-Tracestate/, + 'no tracestate - trace context propagate (no trace headers)'); + +like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent id - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/parent: 00-0af7651916cd43dd8448eb211c80319c-$$spans{span9}{span_id}-01/, + 'traceparent - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context propagate (trace headers)'); + +SKIP: { +skip "depends on error log contents", 2 unless $ENV{TEST_NGINX_UNSAFE}; + +$t->stop(); +my $log = $t->read_file("error.log"); + +like($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: error parsing metadata - no protobuf in response'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +} + +############################################################################### + +sub http1_get { + my ($path, %extra) = @_; + + my $port = $extra{port} || 8080; + + my $r = < '127.0.0.1:' . port($port), + SSL => $extra{ssl}); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { + $_ =~ /^attribute\d+/ && $$obj{$_}{key} eq '"' . $attr . '"' + } keys %{$obj}; + + if (defined $res) { + $$obj{$res}{value}{$type} =~ s/(^\")|(\"$)//g + if $type eq 'string_value'; + + return $$obj{$res}{value}{$type}; + } + + return undef; +} + +sub decode_protobuf { + my ($protobuf) = @_; + + $protobuf = encode_base64($protobuf); + + local $/; + open CMD => "echo '$protobuf' | base64 -d | " . + '$PWD/../build/_deps/grpc-build/third_party/protobuf/protoc '. + '--decode opentelemetry.proto.trace.v1.ResourceSpans -I ' . + '$PWD/../build/_deps/otelcpp-src/third_party/opentelemetry-proto ' . + 'opentelemetry/proto/collector/trace/v1/trace_service.proto |' + or die "Can't decode protobuf: $!\n"; + my $out = ; + close CMD; + + return $out; +} + +sub decode_bytes { + my ($bytes) = @_; + + my $c = sub { return chr oct(shift) }; + + $bytes =~ s/\\(\d{3})/$c->($1)/eg; + $bytes =~ s/(^\")|(\"$)//g; + $bytes =~ s/\\\\/\\/g; + $bytes =~ s/\\r/\r/g; + $bytes =~ s/\\n/\n/g; + $bytes =~ s/\\t/\t/g; + $bytes =~ s/\\"/\"/g; + $bytes =~ s/\\'/\'/g; + + return unpack("H*", unpack("a*", $bytes)); +} + +sub to_hash { + my ($textdata) = @_; + + my %out = (); + push my @stack, \%out; + my ($attr_count, $span_count) = (0, 0); + for my $line (split /\n/, $textdata) { + $line =~ s/(^\s+)|(\s+$)//g; + if ($line =~ /\:/) { + my ($k, $v) = split /\: /, $line; + $v = decode_bytes($v) if ($k =~ /trace_id|span_id|parent_span_id/); + $stack[$#stack]{$k} = $v; + } elsif ($line =~ /\{/) { + $line =~ s/\s\{//; + $line = 'attribute' . $attr_count++ if ($line eq 'attributes'); + if ($line eq 'spans') { + $line = 'span' . $span_count++; + $attr_count = 0; + } + my %new = (); + $stack[$#stack]{$line} = \%new; + push @stack, \%new; + } elsif ($line =~ /\}/) { + pop @stack; + } + } + + return \%out; +} + +sub grpc { + my ($server, $client, $f, $s, $c, $sid, $csid, $uri); + + $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalHost => '127.0.0.1', + LocalPort => $p, + Listen => 5, + Reuse => 1 + ) or die "Can't create listening socket: $!\n"; + + $f->{http_start} = sub { + if (IO::Select->new($server)->can_read(5)) { + $client = $server->accept(); + } else { + # connection could be unexpectedly reused + goto reused if $client; + return undef; + } + + $client->sysread($_, 24) == 24 or return; # preface + + $c = Test::Nginx::HTTP2->new(1, socket => $client, + pure => 1, preface => "") or return; + +reused: + my $frames = $c->read(all => [{ fin => 1 }]); + + $client->close(); + + return $frames; + }; + + return $f; +} + +############################################################################### diff --git a/tests/otel_collector.t b/tests/otel_collector.t new file mode 100644 index 0000000..8a8f8e9 --- /dev/null +++ b/tests/otel_collector.t @@ -0,0 +1,402 @@ +#!/usr/bin/perl + +# (C) Nginx, Inc. + +# Tests for opentelemetry exporter in case HTTP using otelcol. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +plan(skip_all => "depends on logs content") unless $ENV{TEST_NGINX_UNSAFE}; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http http_ssl rewrite/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + ssl_certificate_key localhost.key; + ssl_certificate localhost.crt; + + otel_exporter { + endpoint 127.0.0.1:%%PORT_4317%%; + interval 1s; + batch_size 10; + batch_count 2; + } + + otel_service_name test_server; + otel_trace on; + + server { + listen 127.0.0.1:8080; + listen 127.0.0.1:8081 ssl; + server_name localhost; + + location /trace-on { + otel_trace_context extract; + otel_span_name default_location; + otel_span_attr http.request.header.completion + $request_completion; + otel_span_attr http.response.header.content.type + $sent_http_content_type; + otel_span_attr http.request $request; + add_header "X-Otel-Trace-Id" $otel_trace_id; + add_header "X-Otel-Span-Id" $otel_span_id; + add_header "X-Otel-Parent-Id" $otel_parent_id; + add_header "X-Otel-Parent-Sampled" $otel_parent_sampled; + return 200 "TRACE-ON"; + } + + location /context-ignore { + otel_trace_context ignore; + otel_span_name context_ignore; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-extract { + otel_trace_context extract; + otel_span_name context_extract; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-inject { + otel_trace_context inject; + otel_span_name context_inject; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /context-propagate { + otel_trace_context propagate; + otel_span_name context_propogate; + add_header "X-Otel-Parent-Id" $otel_parent_id; + proxy_pass http://localhost:8080/trace-off; + } + + location /trace-off { + otel_trace off; + add_header "X-Otel-Traceparent" $http_traceparent; + add_header "X-Otel-Tracestate" $http_tracestate; + return 200 "TRACE-OFF"; + } + } +} + +EOF + +$t->write_file_expand('otel-config.yaml', <testdir() }/otel.json + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging, file] + metrics: + receivers: [otlp] + exporters: [logging, file] + +EOF + +$t->write_file('openssl.conf', <<'EOF'); +[ req ] +default_bits = 2048 +encrypt_key = no +distinguished_name = req_distinguished_name +[ req_distinguished_name ] + +EOF + +my $d = $t->testdir(); + +foreach my $name ('localhost') { + system('openssl req -x509 -new ' + . "-config $d/openssl.conf -subj /CN=$name/ " + . "-out $d/$name.crt -keyout $d/$name.key " + . ">>$d/openssl.out 2>&1") == 0 + or die "Can't create certificate for $name: $!\n"; +} + +#suppress otel collector output +open OLDERR, ">&", \*STDERR; +open STDERR, ">>" , $^O eq 'MSWin32' ? 'nul' : '/dev/null'; +$t->run_daemon('../otelcol', '--config', $t->testdir().'/otel-config.yaml'); +open STDERR, ">&", \*OLDERR; +$t->waitforsocket('127.0.0.1:' . port(4317)) or + die 'No otel collector open socket'; + +$t->try_run('no OTEL module')->plan(69); + +############################################################################### + +#do requests +my $t_off_resp = http1_get('/trace-off'); + +#batch0 (10 requests) +my $tp_resp = http1_get('/trace-on', trace_headers => 1); +my $t_resp = http1_get('/trace-on', port => 8081, ssl => 1); + +my $t_resp_ignore = http1_get('/context-ignore'); +my $tp_resp_ignore = http1_get('/context-ignore', trace_headers => 1); +my $t_resp_extract = http1_get('/context-extract'); +my $tp_resp_extract = http1_get('/context-extract', trace_headers => 1); +my $t_resp_inject = http1_get('/context-inject'); +my $tp_resp_inject = http1_get('/context-inject', trace_headers => 1); +my $t_resp_propagate = http1_get('/context-propagate'); +my $tp_resp_propagate = http1_get('/context-propagate', trace_headers => 1); + +#batch1 (5 reqeusts) +http1_get('/trace-on') for (1..5); + +#waiting batch1 is sent to collector for 1s +select undef, undef, undef, 1; + +my @batches = split /\n/, $t->read_file('otel.json'); +my $batch_json = JSON::PP::decode_json($batches[0]); +my $spans = $$batch_json{"resourceSpans"}[0]{"scopeSpans"}[0]{"spans"}; + +#validate responses +like($tp_resp, qr/TRACE-ON/, 'http request1 - trace on'); +like($t_resp, qr/TRACE-ON/, 'http request2 - trace on'); +like($t_off_resp, qr/TRACE-OFF/, 'http request - trace off'); + +#validate amount of batches +is(scalar @batches, 2, 'amount of batches - trace on'); + +#validate batch size +is(scalar @{$spans}, 10, 'batch0 size - trace on'); +is(scalar @{${JSON::PP::decode_json($batches[1])}{"resourceSpans"}[0] + {"scopeSpans"}[0]{"spans"}}, 5, 'batch1 size - trace on'); + +#validate general attributes +is(get_attr("service.name", "stringValue", + $$batch_json{resourceSpans}[0]{resource}), + 'test_server', 'service.name - trace on'); +is($$spans[0]{name}, 'default_location', 'span.name - trace on'); + +#validate http metrics +is(get_attr("http.method", "stringValue", $$spans[0]), 'GET', + 'http.method metric - trace on'); +is(get_attr("http.target", "stringValue", $$spans[0]), '/trace-on', + 'http.target metric - trace on'); +is(get_attr("http.route", "stringValue", $$spans[0]), '/trace-on', + 'http.route metric - trace on'); +is(get_attr("http.scheme", "stringValue", $$spans[0]), 'http', + 'http.scheme metric - trace on'); +is(get_attr("http.flavor", "stringValue", $$spans[0]), '1.0', + 'http.flavor metric - trace on'); +is(get_attr("http.user_agent", "stringValue", $$spans[0]), 'nginx-tests', + 'http.user_agent metric - trace on'); +is(get_attr("http.request_content_length", "intValue", $$spans[0]), 0, + 'http.request_content_length metric - trace on'); +is(get_attr("http.response_content_length", "intValue", $$spans[0]), 8, + 'http.response_content_length metric - trace on'); +is(get_attr("http.status_code", "intValue", $$spans[0]), 200, + 'http.status_code metric - trace on'); +is(get_attr("net.host.name", "stringValue", $$spans[0]), 'localhost', + 'net.host.name metric - trace on'); +is(get_attr("net.host.port", "intValue", $$spans[0]), 8080, + 'net.host.port metric - trace on'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[0]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[0]), qr/\d+/, + 'net.sock.peer.port metric - trace on'); + +#validate custom http metrics +is(${get_attr("http.request.header.completion", "arrayValue", $$spans[0])} + {values}[0]{stringValue}, 'OK', + 'http.request.header.completion metric - trace on'); +is(${get_attr("http.response.header.content.type", "arrayValue",$$spans[0])} + {values}[0]{stringValue}, 'text/plain', + 'http.response.header.content.type metric - trace on'); +is(get_attr("http.request", "stringValue", $$spans[0]), + 'GET /trace-on HTTP/1.0', 'http.request metric - trace on'); + +#validate https metrics +is(get_attr("http.method", "stringValue", $$spans[1]), 'GET', + 'http.method metric - trace on (https)'); +is(get_attr("http.target", "stringValue", $$spans[1]), '/trace-on', + 'http.target metric - trace on (https)'); +is(get_attr("http.route", "stringValue", $$spans[1]), '/trace-on', + 'http.route metric - trace on (https)'); +is(get_attr("http.scheme", "stringValue", $$spans[1]), 'https', + 'http.scheme metric - trace on (https)'); +is(get_attr("http.flavor", "stringValue", $$spans[1]), '1.0', + 'http.flavor metric - trace on (https)'); +is(get_attr("http.user_agent", "stringValue", $$spans[1]), 'nginx-tests', + 'http.user_agent metric - trace on (https)'); +is(get_attr("http.request_content_length", "intValue", $$spans[1]), 0, + 'http.request_content_length metric - trace on (https)'); +is(get_attr("http.response_content_length", "intValue", $$spans[1]), 8, + 'http.response_content_length metric - trace on (https)'); +is(get_attr("http.status_code", "intValue", $$spans[1]), 200, + 'http.status_code metric - trace on (https)'); +is(get_attr("net.host.name", "stringValue", $$spans[1]), 'localhost', + 'net.host.name metric - trace on (https)'); +is(get_attr("net.host.port", "intValue", $$spans[1]), 8081, + 'net.host.port metric - trace on (https)'); +is(get_attr("net.sock.peer.addr", "stringValue", $$spans[1]), '127.0.0.1', + 'net.sock.peer.addr metric - trace on (https)'); +like(get_attr("net.sock.peer.port", "intValue", $$spans[1]), qr/\d+/, + 'net.sock.peer.port metric - trace on (https)'); + +#extract trace info +is($$spans[0]{parentSpanId}, 'b9c7c989f97918e1', 'traceparent - trace on'); +is($$spans[0]{traceState}, 'congo=ucfJifl5GOE,rojo=00f067aa0ba902b7', + 'tracestate - trace on'); +is($$spans[1]{parentSpanId}, '', 'no traceparent - trace on'); +is($$spans[1]{traceState}, undef, 'no tracestate - trace on'); + +#variables +like($tp_resp, qr/X-Otel-Trace-Id: $$spans[0]{traceId}/, + '$otel_trace_id variable - trace on'); +like($tp_resp, qr/X-Otel-Span-Id: $$spans[0]{spanId}/, + '$otel_span_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Id: $$spans[0]{parentSpanId}/, + '$otel_parent_id variable - trace on'); +like($tp_resp, qr/X-Otel-Parent-Sampled: 1/, + '$otel_parent_sampled variable - trace on'); +like($t_resp, qr/X-Otel-Parent-Sampled: 0/, + '$otel_parent_sampled variable - trace on (no traceparent header)'); + +#trace off +unlike($batches[0].$batches[1], + qr/\Q{"key":"http.target","value":{"stringValue":"\/trace-off"}}\E/, + 'no metrics - trace off'); + +#trace context: ignore +unlike($t_resp_ignore, qr/X-Otel-Traceparent/, + 'no traceparent - trace context ignore (no trace headers)'); +unlike($t_resp_ignore, qr/X-Otel-Tracestate/, + 'no tracestate - trace context ignore (no trace headers)'); + +unlike($tp_resp_ignore, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context ignore (trace headers)'); +like($tp_resp_ignore, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context ignore (trace headers)'); + +#trace context: extract +unlike($t_resp_extract, qr/X-Otel-Traceparent/, + 'no traceparent - trace context extract (no trace headers)'); +unlike($t_resp_extract, qr/X-Otel-Tracestate/, + 'no tracestate - trace context extract (no trace headers)'); + +like($tp_resp_extract, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent span id - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01/, + 'traceparent - trace context extract (trace headers)'); +like($tp_resp_extract, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context extract (trace headers)'); + +#trace context: inject +like($t_resp_inject, qr/X-Otel-Traceparent/, + 'traceparent - trace context inject (no trace headers)'); +unlike($t_resp_inject, qr/X-Otel-Tracestate/, + 'no tracestate - trace context inject (no trace headers)'); + +unlike($tp_resp_inject, qr/X-Otel-Parent-Id/, + 'no parent span id - trace context inject (trace headers)'); +like($tp_resp_inject, + qr/Traceparent: 00-$$spans[7]{traceId}-$$spans[7]{spanId}-01/, + 'traceparent - trace context inject (trace headers)'); +unlike($tp_resp_inject, qr/Tracestate:/, + 'no tracestate - trace context inject (trace headers)'); + +#trace context: propagate +like($t_resp_propagate, + qr/X-Otel-Traceparent: 00-$$spans[8]{traceId}-$$spans[8]{spanId}-01/, + 'traceparent - trace context propagate (no trace headers)'); +unlike($t_resp_propagate, qr/X-Otel-Tracestate/, + 'no tracestate - trace context propagate (no trace headers)'); + +like($tp_resp_propagate, qr/X-Otel-Parent-Id: b9c7c989f97918e1/, + 'parent id - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Traceparent: 00-0af7651916cd43dd8448eb211c80319c-$$spans[9]{spanId}-01/, + 'traceparent - trace context propagate (trace headers)'); +like($tp_resp_propagate, + qr/Tracestate: congo=ucfJifl5GOE,rojo=00f067aa0ba902b7/, + 'tracestate - trace context propagate (trace headers)'); + +$t->stop(); +my $log = $t->read_file("error.log"); + +unlike($log, qr/OTel\/grpc: Error parsing metadata: error=invalid value/, + 'log: no error parsing metadata'); +unlike($log, qr/OTel export failure: No status received/, + 'log: no export failure'); + +############################################################################### + +sub http1_get { + my ($path, %extra) = @_; + + my $port = $extra{port} || 8080; + + my $r = < '127.0.0.1:' . port($port), + SSL => $extra{ssl}); +} + +sub get_attr { + my($attr, $type, $obj) = @_; + + my ($res) = grep { $$_{"key"} eq $attr } @{$$obj{"attributes"}}; + + return defined $res ? $res->{"value"}{$type} : undef; +} + +###############################################################################