From f54c831be78f414dcc98401691643d906e51ae99 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 16 Nov 2023 16:31:08 +0800 Subject: [PATCH] Improve the developer experience when running tests Migrate the PRs from the pulsar-client-cpp repo: - https://github.com/apache/pulsar-client-cpp/pull/297 - https://github.com/apache/pulsar-client-cpp/pull/236 - https://github.com/apache/pulsar-client-cpp/pull/340 --- build-support/pulsar-test-service-start.sh | 2 +- .../start-test-service-inside-container.sh | 95 +++++++++++-------- tests/pulsar_test.py | 8 +- 3 files changed, 59 insertions(+), 46 deletions(-) diff --git a/build-support/pulsar-test-service-start.sh b/build-support/pulsar-test-service-start.sh index a44fafe..ed1e7ba 100755 --- a/build-support/pulsar-test-service-start.sh +++ b/build-support/pulsar-test-service-start.sh @@ -25,7 +25,7 @@ cd $SRC_DIR ./build-support/pulsar-test-service-stop.sh -CONTAINER_ID=$(docker run -i -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600) +CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600) echo $CONTAINER_ID > .tests-container-id.txt docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf diff --git a/build-support/start-test-service-inside-container.sh b/build-support/start-test-service-inside-container.sh index 658be03..f3bdd75 100755 --- a/build-support/start-test-service-inside-container.sh +++ b/build-support/start-test-service-inside-container.sh @@ -34,8 +34,23 @@ bin/pulsar tokens create \ --secret-key file:///pulsar/data/tokens/secret.key \ > /pulsar/data/tokens/token.txt +# Unset the HTTP proxy to avoid the REST requests being affected +export http_proxy= +TOKEN=$(bin/pulsar tokens create \ + --subject superUser \ + --secret-key file:///pulsar/data/tokens/secret.key) + +put() { + curl -H "Authorization: Bearer $TOKEN" \ + -L http://localhost:8080/admin/v2/$1 \ + -H 'Content-Type: application/json' \ + -X PUT \ + -d $(echo $2 | sed 's/ //g') +} + export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf export PULSAR_PID_DIR=/tmp +sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml bin/pulsar-daemon start standalone \ --no-functions-worker --no-stream-storage \ --bookkeeper-dir data/bookkeeper @@ -48,57 +63,53 @@ echo "-- Pulsar service is ready -- Configure permissions" export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf # Create "standalone" cluster if it does not exist -bin/pulsar-admin clusters list | grep -q '^standalone$' || - bin/pulsar-admin clusters create \ - standalone \ - --url http://localhost:8080/ \ - --url-secure https://localhost:8443/ \ - --broker-url pulsar://localhost:6650/ \ - --broker-url-secure pulsar+ssl://localhost:6651/ +put clusters/standalone '{ + "serviceUrl": "http://localhost:8080/", + "serviceUrlTls": "https://localhost:8443/", + "brokerServiceUrl": "pulsar://localhost:6650/", + "brokerServiceUrlTls": "pulsar+ssl://localhost:6651/" +}' # Update "public" tenant -bin/pulsar-admin tenants create public -r "anonymous" -c "standalone" +put tenants/public '{ + "adminRoles": ["anonymous"], + "allowedClusters": ["standalone"] +}' # Update "public/default" with no auth required -bin/pulsar-admin namespaces create public/default -c standalone -bin/pulsar-admin namespaces grant-permission public/default \ - --actions produce,consume \ - --role "anonymous" +put namespaces/public/default '{ + "auth_policies": { + "namespace_auth": { + "anonymous": ["produce", "consume"] + } + }, + "replication_clusters": ["standalone"] +}' # Create "public/default-2" with no auth required -bin/pulsar-admin namespaces create public/default-2 \ - --clusters standalone -bin/pulsar-admin namespaces grant-permission public/default-2 \ - --actions produce,consume \ - --role "anonymous" - -# Create "public/default-3" with no auth required -bin/pulsar-admin namespaces create public/default-3 \ - --clusters standalone -bin/pulsar-admin namespaces grant-permission public/default-3 \ - --actions produce,consume \ - --role "anonymous" - -# Create "public/default-4" with encryption required -bin/pulsar-admin namespaces create public/default-4 \ - --clusters standalone -bin/pulsar-admin namespaces grant-permission public/default-4 \ - --actions produce,consume \ - --role "anonymous" -bin/pulsar-admin namespaces set-encryption-required public/default-4 -e - -# Create "public/test-backlog-quotas" to test backlog quotas policy -bin/pulsar-admin namespaces create public/test-backlog-quotas \ - --clusters standalone +put namespaces/public/default-2 '{ + "auth_policies": { + "namespace_auth": { + "anonymous": ["produce", "consume"] + } + }, + "replication_clusters": ["standalone"] +}' # Create "private" tenant -bin/pulsar-admin tenants create private -r "" -c "standalone" +put tenants/private '{ + "adminRoles": [], + "allowedClusters": ["standalone"] +}' # Create "private/auth" with required authentication -bin/pulsar-admin namespaces create private/auth --clusters standalone - -bin/pulsar-admin namespaces grant-permission private/auth \ - --actions produce,consume \ - --role "token-principal" +put namespaces/private/auth '{ + "auth_policies": { + "namespace_auth": { + "token-principal": ["produce", "consume"] + } + }, + "replication_clusters": ["standalone"] +}' echo "-- Ready to start tests" diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py index 5e228bd..dbc356b 100755 --- a/tests/pulsar_test.py +++ b/tests/pulsar_test.py @@ -56,7 +56,9 @@ from urllib.request import urlopen, Request TM = 10000 # Do not wait forever in tests -CERTS_DIR = os.path.dirname(os.path.abspath(__file__)) + "/test-conf/" +TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +TOKEN_PATH = TESTS_DIR + "/.test-token.txt" +CERTS_DIR = TESTS_DIR + "/test-conf/" def doHttpPost(url, data): req = Request(url, data.encode()) @@ -1247,7 +1249,7 @@ def test_get_topics_partitions(self): client.close() def test_token_auth(self): - with open(".test-token.txt") as tf: + with open(TOKEN_PATH) as tf: token = tf.read().strip() # Use adminUrl to test both HTTP request and binary protocol @@ -1266,7 +1268,7 @@ def test_token_auth(self): def test_token_auth_supplier(self): def read_token(): - with open(".test-token.txt") as tf: + with open(TOKEN_PATH) as tf: return tf.read().strip() client = Client(self.serviceUrl, authentication=AuthenticationToken(read_token))