diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 927cbc3c2ce6..63f192940e3e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -88,7 +88,9 @@ registry: session-timeout: 60s connection-timeout: 15s block-until-connected: 15s - digest: ~ + authorization: + digest: ~ + metrics: enabled: true diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 9b0e94d64451..ee5ca199e6aa 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -126,7 +126,8 @@ registry: session-timeout: 60s connection-timeout: 15s block-until-connected: 15s - digest: ~ + authorization: + digest: ~ api: audit-enable: false diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index a4ce4b882837..dfd21429854f 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -80,7 +80,8 @@ registry: session-timeout: 60s connection-timeout: 15s block-until-connected: 15s - digest: ~ + authorization: + digest: ~ master: listen-port: 5678 diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/README.md b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/README.md index 7ee512a451d7..3fd0fdfc4fba 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/README.md +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/README.md @@ -20,7 +20,8 @@ registry: connection-timeout: 9s block-until-connected: 600ms # The following options are set according to personal needs - digest: ~ + authorization: + digest: ~ ``` After do this config, you can start your DolphinScheduler cluster, your cluster will use zookeeper as registry center to diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index d02b5f3c0c0b..e814b8ec45b4 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -52,8 +52,6 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import com.google.common.base.Strings; - @Slf4j final class ZookeeperRegistry implements Registry { @@ -80,9 +78,10 @@ final class ZookeeperRegistry implements Registry { .sessionTimeoutMs(DurationUtils.toMillisInt(properties.getSessionTimeout())) .connectionTimeoutMs(DurationUtils.toMillisInt(properties.getConnectionTimeout())); - final String digest = properties.getDigest(); - if (!Strings.isNullOrEmpty(digest)) { - builder.authorization("digest", digest.getBytes(StandardCharsets.UTF_8)) + if (properties.getAuthorization().size() > 0) { + final String schema = properties.getAuthorization().keySet().stream().findFirst().get(); + final String schemaValue = properties.getAuthorization().get(schema); + builder.authorization(schema.toLowerCase(), schemaValue.getBytes(StandardCharsets.UTF_8)) .aclProvider(new ACLProvider() { @Override @@ -96,6 +95,7 @@ public List getAclForPath(final String path) { } }); } + client = builder.build(); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java index c5b27d8b8807..327f8369b920 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java @@ -20,6 +20,8 @@ import org.apache.commons.lang3.StringUtils; import java.time.Duration; +import java.util.HashMap; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; @@ -79,6 +81,9 @@ public void validate(Object target, Errors errors) { || zookeeper.getBlockUntilConnected().isNegative()) { errors.rejectValue("zookeeper.blockUntilConnected", "", "zookeeper.blockUntilConnected should be positive"); } + if (zookeeper.getAuthorization() != null && zookeeper.getAuthorization().size() != 1) { + errors.rejectValue("zookeeper.authorization", "", "zookeeper.authorization should be unique"); + } printConfig(); } @@ -88,10 +93,11 @@ private void printConfig() { "\n namespace -> " + zookeeper.getNamespace() + "\n connectString -> " + zookeeper.getConnectString() + "\n retryPolicy -> " + zookeeper.getRetryPolicy() + - "\n digest -> " + zookeeper.getDigest() + + "\n authorization -> " + zookeeper.getAuthorization() + "\n sessionTimeout -> " + zookeeper.getSessionTimeout() + "\n connectionTimeout -> " + zookeeper.getConnectionTimeout() + "\n blockUntilConnected -> " + zookeeper.getBlockUntilConnected() + + "\n authorization -> " + zookeeper.getAuthorization() + "\n****************************ZookeeperRegistryProperties**************************************"; log.info(config); } @@ -102,7 +108,7 @@ public static final class ZookeeperProperties { private String namespace = "dolphinscheduler"; private String connectString; private RetryPolicy retryPolicy = new RetryPolicy(); - private String digest; + private Map authorization = new HashMap<>(); private Duration sessionTimeout = Duration.ofSeconds(60); private Duration connectionTimeout = Duration.ofSeconds(15); private Duration blockUntilConnected = Duration.ofSeconds(15); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryDigestTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryDigestTestCase.java new file mode 100644 index 000000000000..b3f00f2ca46a --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryDigestTestCase.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.registry.zookeeper; + +import org.apache.dolphinscheduler.plugin.registry.RegistryTestCase; + +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.DumbWatcher; +import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; + +import java.util.Collections; +import java.util.stream.Stream; + +import lombok.SneakyThrows; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; + +@ActiveProfiles("digest") +@SpringBootTest(classes = ZookeeperRegistryProperties.class) +@SpringBootApplication(scanBasePackageClasses = ZookeeperRegistryProperties.class) +public class ZookeeperRegistryDigestTestCase extends RegistryTestCase { + + @Autowired + private ZookeeperRegistryProperties zookeeperRegistryProperties; + + private static GenericContainer zookeeperContainer; + + private static final Network NETWORK = Network.newNetwork(); + + private static ZooKeeper zk; + + private static final String ROOT_USER = "root"; + + private static final String ROOT_PASSWORD = "root_passwd"; + + private static final String ID_PASSWORD = String.format("%s:%s", ROOT_USER, ROOT_PASSWORD); + + private static void setupRootACLForDigest(final ZooKeeper zk) throws Exception { + final String digest = DigestAuthenticationProvider.generateDigest(ID_PASSWORD); + final ACL acl = new ACL(ZooDefs.Perms.ALL, new Id("digest", digest)); + zk.setACL("/", Collections.singletonList(acl), -1); + } + + @SneakyThrows + @BeforeAll + public static void setUpTestingServer() { + zookeeperContainer = new GenericContainer<>(DockerImageName.parse("zookeeper:3.8")) + .withNetwork(NETWORK) + .withExposedPorts(2181); + Startables.deepStart(Stream.of(zookeeperContainer)).join(); + System.clearProperty("registry.zookeeper.connect-string"); + System.setProperty("registry.zookeeper.connect-string", "localhost:" + zookeeperContainer.getMappedPort(2181)); + zk = new ZooKeeper("localhost:" + zookeeperContainer.getMappedPort(2181), + 30000, new DumbWatcher(), new ZKClientConfig()); + setupRootACLForDigest(zk); + } + + @SneakyThrows + @Override + public ZookeeperRegistry createRegistry() { + return new ZookeeperRegistry(zookeeperRegistryProperties); + } + + @SneakyThrows + @AfterAll + public static void tearDownTestingServer() { + zk.close(); + zookeeperContainer.close(); + } +} diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application-digest.yaml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application-digest.yaml new file mode 100644 index 000000000000..c618466e06d5 --- /dev/null +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application-digest.yaml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: 127.0.0.1:2181 + retry-policy: + base-sleep-time: 60ms + max-sleep: 300ms + max-retries: 5 + session-timeout: 30s + connection-timeout: 9s + block-until-connected: 3s + authorization: + digest: root:root_passwd diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml index 92902a608cb4..b6ef4efa8901 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/resources/application.yaml @@ -27,4 +27,5 @@ registry: session-timeout: 30s connection-timeout: 9s block-until-connected: 3s - digest: ~ + authorization: + digest: ~ diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index 5cac4c29e5ae..0a2ae9743859 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -37,7 +37,8 @@ registry: session-timeout: 60s connection-timeout: 15s block-until-connected: 15s - digest: ~ + authorization: + digest: ~ worker: # worker listener port