From c54177c1022f370d5214a2d0942b3b76fbafb7fe Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Thu, 30 Nov 2017 09:44:27 +0100 Subject: [PATCH 1/4] [t] allow running with TEST_NGINX_RANDOMIZE Every test run needs unique randomized ports. --- CHANGELOG.md | 1 + gateway/conf/nginx.conf.liquid | 8 +-- t/TestAPIcast.pm | 69 ++++++++++++++++++++++ t/TestAPIcastBlackbox.pm | 10 +++- t/apicast-async-reporting.t | 8 +-- t/apicast-upstream-balancer.t | 8 +-- t/apicast.t | 4 +- t/configuration-loading-boot-with-config.t | 12 ++-- t/management.t | 8 +-- 9 files changed, 103 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f715a8c5e..7be63458b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Ability to load several environment configurations [PR #504](https://github.com/3scale/apicast/pull/504) - Ability to configure policy chain from the environment configuration [PR #496](https://github.com/3scale/apicast/pull/496) - Load environment variables defined in the configuration [PR #507](https://github.com/3scale/apicast/pull/507) +- Allow configuration of the echo/management/fake backend ports [PR #506](https://github.com/3scale/apicast/pull/506) ## Changed diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index 6666cfa2b..112e43eba 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -54,15 +54,15 @@ http { {% endfor %} server { - listen 8090; + listen {{ port.management | default: 8090 }}; - server_name _; + server_name management _; {% include "conf.d/management.conf" %} } server { - listen 8081; + listen {{ port.backend | default: 8081 }}; server_name backend; @@ -70,7 +70,7 @@ http { } server { - listen 8081 default_server; + listen {{ port.echo | default: 8081 }} default_server; server_name echo _; diff --git a/t/TestAPIcast.pm b/t/TestAPIcast.pm index 3de2611b2..da0c1fccb 100644 --- a/t/TestAPIcast.pm +++ b/t/TestAPIcast.pm @@ -26,12 +26,53 @@ if ($ENV{DEBUG}) { $ENV{TEST_NGINX_ERROR_LOG} ||= '/dev/stderr'; } +our @EXPORT = qw( get_random_port ); + + +sub get_random_port { + my $tries = 1000; + my $ServerPort; + + for (my $i = 0; $i < $tries; $i++) { + my $port = int(rand 60000) + 1025; + + my $sock = IO::Socket::INET->new( + LocalAddr => $Test::Nginx::Util::ServerAddr, + LocalPort => $port, + Proto => 'tcp', + Timeout => 0.1, + ); + + if (defined $sock) { + $sock->close(); + $ServerPort = $port; + last; + } + + if ($Test::Nginx::Util::Verbose) { + warn "Try again, port $port is already in use: $@\n"; + } + } + + if (!defined $ServerPort) { + bail_out "Cannot find an available listening port number after $tries attempts.\n"; + } + + return $ServerPort; +} + env_to_nginx("TEST_NGINX_SERVER_PORT=$Test::Nginx::Util::ServerPortForClient"); log_level('debug'); repeat_each($ENV{TEST_NGINX_REPEAT_EACH} || 2); no_root_location(); +add_block_preprocessor(sub { + my $block = shift; + + $ENV{TEST_NGINX_RANDOM_PORT} = $block->random_port; +}); + our $dns = sub ($$$) { my ($host, $ip, $ttl) = @_; @@ -66,6 +107,27 @@ our $dns = sub ($$$) { } }; +sub Test::Base::Filter::random_port { + my ($self) = @_; + + my $block = $self->current_block; + my $random_port = $block->random_port; + + if ( !defined $random_port ) { + if ($Test::Nginx::Util::Randomize) { + $random_port = get_random_port(); + } else { + $random_port = 1953; + } + } + + $block->set_value('random_port', $random_port); + + $ENV{TEST_NGINX_RANDOM_PORT} = $random_port; + + return $random_port +} + sub Test::Base::Filter::dns { my ($self, $code) = @_; @@ -79,4 +141,11 @@ sub Test::Base::Filter::dns { return $dns->(@$input) } + +sub Test::Base::Filter::env { + my ($self, $input) = @_; + + return Test::Nginx::Util::expand_env_in_config($input); +} + 1; diff --git a/t/TestAPIcastBlackbox.pm b/t/TestAPIcastBlackbox.pm index 3306aba57..a1c7730be 100644 --- a/t/TestAPIcastBlackbox.pm +++ b/t/TestAPIcastBlackbox.pm @@ -79,6 +79,9 @@ my $write_nginx_config = sub { my $PidFile = $Test::Nginx::Util::PidFile; my $AccLogFile = $Test::Nginx::Util::AccLogFile; my $ServerPort = $Test::Nginx::Util::ServerPort; + my $backend_port = TestAPIcast::get_random_port(); + my $management_port = TestAPIcast::get_random_port(); + my $echo_port = TestAPIcast::get_random_port(); my $sites_d = $block->sites_d; @@ -111,7 +114,12 @@ return { pid = '$PidFile', lua_code_cache = 'on', access_log = '$AccLogFile', - port = { apicast = '$ServerPort' }, + port = { + apicast = '$ServerPort', + management = '$management_port', + backend = '$backend_port', + echo = '$echo_port', + }, env = { THREESCALE_CONFIG_FILE = [[$configuration_file]], APICAST_CONFIGURATION_LOADER = 'boot', diff --git a/t/apicast-async-reporting.t b/t/apicast-async-reporting.t index 881616958..6c4bb868e 100644 --- a/t/apicast-async-reporting.t +++ b/t/apicast-async-reporting.t @@ -146,7 +146,7 @@ backend client uri: https://127.0.0.1:1953/transactions/authrep.xml?service_toke === TEST 3: uses endpoint host as Host header when connecting to the backend --- main_config -env RESOLVER=127.0.0.1:1953; +env RESOLVER=127.0.0.1:$TEST_NGINX_RANDOM_PORT; --- http_config include $TEST_NGINX_UPSTREAM_CONFIG; lua_package_path "$TEST_NGINX_LUA_PATH"; @@ -196,11 +196,11 @@ GET /t?user_key=val --- response_body all ok --- error_code: 200 ---- udp_listen: 1953 +--- udp_listen random_port --- udp_reply dns [ "localhost.example.com", "127.0.0.1", 60 ] --- no_error_log [error] ---- error_log -backend client uri: http://localhost.example.com:1984/transactions/authrep.xml?service_token=service-token&service_id=42&usage%5Bhits%5D=2&user_key=val ok: true status: 200 +--- error_log env +backend client uri: http://localhost.example.com:$TEST_NGINX_SERVER_PORT/transactions/authrep.xml?service_token=service-token&service_id=42&usage%5Bhits%5D=2&user_key=val ok: true status: 200 --- wait: 3 diff --git a/t/apicast-upstream-balancer.t b/t/apicast-upstream-balancer.t index 2d84f38ad..50399618c 100644 --- a/t/apicast-upstream-balancer.t +++ b/t/apicast-upstream-balancer.t @@ -15,7 +15,7 @@ location /t { local resty_resolver = require 'resty.resolver' local dns_client = require 'resty.dns.resolver' - local dns = dns_client:new{ nameservers = { { "127.0.0.1", 1953 } } } + local dns = dns_client:new{ nameservers = { { "127.0.0.1", $TEST_NGINX_RANDOM_PORT } } } local resolver = resty_resolver.new(dns) local servers = resolver:get_servers('3scale.net') @@ -31,7 +31,7 @@ location /t { ngx.say(require('cjson').encode(upstream)) } } ---- udp_listen: 1953 +--- udp_listen random_port --- udp_reply dns [ "localhost", "127.0.0.1" ] --- request @@ -116,7 +116,7 @@ location /t { local resty_resolver = require 'resty.resolver' local dns_client = require 'resty.dns.resolver' - local dns = dns_client:new{ nameservers = { { "127.0.0.1", 1953 } } } + local dns = dns_client:new{ nameservers = { { "127.0.0.1", $TEST_NGINX_RANDOM_PORT } } } local resolver = resty_resolver.new(dns) ngx.ctx.upstream = resolver:get_servers('localhost', { port = $TEST_NGINX_SERVER_PORT }) @@ -124,7 +124,7 @@ location /t { proxy_pass http://upstream/api; } ---- udp_listen: 1953 +--- udp_listen random_port --- udp_reply dns [ "localhost", "127.0.0.1" ] --- request diff --git a/t/apicast.t b/t/apicast.t index e15a97ff8..16fa91edd 100644 --- a/t/apicast.t +++ b/t/apicast.t @@ -557,7 +557,7 @@ X-3scale-usage: usage%5Bhits%5D=2 === TEST 16: uses endpoint host as Host header when connecting to the backend --- main_config -env RESOLVER=127.0.0.1:1953; +env RESOLVER=127.0.0.1:$TEST_NGINX_RANDOM_PORT; --- http_config include $TEST_NGINX_UPSTREAM_CONFIG; lua_package_path "$TEST_NGINX_LUA_PATH"; @@ -605,7 +605,7 @@ GET /t?user_key=val --- response_body all ok --- error_code: 200 ---- udp_listen: 1953 +--- udp_listen random_port --- udp_reply dns [ "localhost.example.com", "127.0.0.1", 3600 ] --- no_error_log diff --git a/t/configuration-loading-boot-with-config.t b/t/configuration-loading-boot-with-config.t index 6ecbf4cf7..94d7c2f04 100644 --- a/t/configuration-loading-boot-with-config.t +++ b/t/configuration-loading-boot-with-config.t @@ -15,8 +15,8 @@ __DATA__ === TEST 1: require configuration file to exist should exit when the config file is missing --- must_die ---- configuration_file -t/servroot/html/config.json +--- configuration_file env +$TEST_NGINX_SERVER_ROOT/html/config.json --- error_log config.json: No such file or directory --- user_files @@ -25,8 +25,8 @@ config.json: No such file or directory === TEST 2: require valid json file should exit when the file has invalid json --- must_die ---- configuration_file -t/servroot/html/config.json +--- configuration_file env +$TEST_NGINX_SERVER_ROOT/html/config.json --- error_log Expected value but found invalid token at character 1 --- user_files @@ -35,8 +35,8 @@ not valid json === TEST 3: empty json file should continue as empty json is enough ---- configuration_file -t/servroot/html/config.json +--- configuration_file env +$TEST_NGINX_SERVER_ROOT/html/config.json --- request GET --- error_code: 404 diff --git a/t/management.t b/t/management.t index c7ed548dc..5b142d368 100644 --- a/t/management.t +++ b/t/management.t @@ -152,7 +152,7 @@ Could not resolve GET /foobar - nil exposes boot function --- main_config env THREESCALE_PORTAL_ENDPOINT=http://localhost.local:$TEST_NGINX_SERVER_PORT/config/; -env RESOLVER=127.0.0.1:1953; +env RESOLVER=127.0.0.1:$TEST_NGINX_RANDOM_PORT; env APICAST_MANAGEMENT_API=debug; --- http_config lua_package_path "$TEST_NGINX_LUA_PATH"; @@ -166,7 +166,7 @@ POST /boot --- response_body {"status":"ok","config":{"services":[{"id":42}]}} --- error_code: 200 ---- udp_listen: 1953 +--- udp_listen random_port --- udp_reply dns [ "localhost.local", "127.0.0.1", 60 ] --- no_error_log @@ -176,7 +176,7 @@ POST /boot keeps the same configuration --- main_config env THREESCALE_PORTAL_ENDPOINT=http://localhost.local:$TEST_NGINX_SERVER_PORT/config/; -env RESOLVER=127.0.0.1:1953; +env RESOLVER=127.0.0.1:$TEST_NGINX_RANDOM_PORT; env APICAST_MANAGEMENT_API=debug; --- http_config lua_package_path "$TEST_NGINX_LUA_PATH"; @@ -195,7 +195,7 @@ POST /test {"status":"ok","config":{"services":[{"id":42}]}} {"status":"ok","config":{"services":[{"id":42}]}} --- error_code: 200 ---- udp_listen: 1953 +--- udp_listen random_port --- udp_reply dns [ "localhost.local", "127.0.0.1", 60 ] --- no_error_log From 33a6060dfe9f3202fb318df76dc755ab778ded83 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Thu, 30 Nov 2017 09:45:42 +0100 Subject: [PATCH 2/4] [make] prove run in parallel --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d2c309f20..d12d965f6 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ REGISTRY ?= quay.io/3scale export TEST_NGINX_BINARY ?= openresty NGINX = $(shell which $(TEST_NGINX_BINARY)) SHELL=/bin/bash -o pipefail + +NPROC ?= $(firstword $(shell nproc 2>/dev/null) 1) + SEPARATOR="\n=============================================\n" IMAGE_NAME ?= apicast-test @@ -48,8 +51,9 @@ nginx: @ ($(NGINX) -V 2>&1) > /dev/null prove: HARNESS ?= TAP::Harness +prove: export TEST_NGINX_RANDOMIZE=1 prove: $(ROVER) nginx ## Test nginx - $(ROVER) exec prove --harness=$(HARNESS) 2>&1 | awk '/found ONLY/ { print "FAIL: because found ONLY in test"; print; exit 1 }; { print }' + $(ROVER) exec prove -j$(NPROC) --harness=$(HARNESS) 2>&1 | awk '/found ONLY/ { print "FAIL: because found ONLY in test"; print; exit 1 }; { print }' prove-docker: apicast-source prove-docker: export IMAGE_NAME = apicast-test From c9a379373b94ae86466ca05627203f8e06d94d32 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Thu, 30 Nov 2017 09:46:00 +0100 Subject: [PATCH 3/4] [make] docker-prove runs make prove * so there is just one way to run tests * and it will be ran parallel automatically --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 63d5b44e8..e73864dac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: environment: TEST_NGINX_BINARY: openresty TEST_NGINX_REDIS_HOST: redis - command: "'$$TEST_NGINX_BINARY -V; cd ; rover exec prove; exit $$?'" + command: "'$$TEST_NGINX_BINARY -V; cd ; make prove; exit $$?'" dns_search: - example.com depends_on: From cb78ee35165c69f8e53eebd1a804735de647f9bb Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Thu, 30 Nov 2017 09:46:29 +0100 Subject: [PATCH 4/4] [circleci] cleanup the make prove command --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c0eb178d..488b099ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,11 @@ jobs: - lua_modules - run: mkdir -p tmp/junit - run: $(make rover) exec make busted - - run: JUNIT_OUTPUT_FILE=tmp/junit/prove.xml make prove HARNESS=TAP::Harness::JUnit + - run: + command: make prove + environment: + JUNIT_OUTPUT_FILE: tmp/junit/prove.xml + HARNESS: TAP::Harness::JUnit - store_test_results: path: tmp/junit - store_artifacts: