From f6f756844cf6dcbebf95f17e7c5b8c5ee8d15c16 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 8 Dec 2023 18:25:13 +0530 Subject: [PATCH 01/16] feat: new commit --- .github/workflows/maven.yml | 2 +- .../java/org/apache/atlas/plugin/util/KeycloakUserStore.java | 5 +++-- .../apache/atlas/keycloak/client/AtlasKeycloakClient.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9e4cdcd20d..a20e0eed70 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - keycloaktest jobs: build: diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index a73fdcac8d..7bb2fbf934 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -81,6 +81,7 @@ public static ExecutorService getExecutorService(String namePattern) { public boolean isKeycloakSubjectsStoreUpdated(long cacheLastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getKeycloakSubjectsStoreUpdatedTime"); + cacheLastUpdatedTime = -1; if (cacheLastUpdatedTime == -1) { return true; } @@ -389,7 +390,7 @@ public RangerRole call() throws Exception { //get all groups for Roles Thread groupsFetcher = new Thread(() -> { int start = 0; - int size = AtlasConfiguration.KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE.getInt(); + int size = 2000; boolean found = true; Set ret = new HashSet<>(); @@ -416,7 +417,7 @@ public RangerRole call() throws Exception { //get all users for Roles Thread usersFetcher = new Thread(() -> { int start = 0; - int size = AtlasConfiguration.KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE.getInt(); + int size = 2000; boolean found = true; Set ret = new HashSet<>(); diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java index fe723bbce1..8a90fe966d 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java @@ -59,7 +59,7 @@ public List getRoleUserMembers(String roleName) throws Atlas public List getAllUsers() throws AtlasBaseException { int start = 0; - int size = 500; + int size = 2000; boolean found = true; List ret = new ArrayList<>(0); @@ -104,7 +104,7 @@ public void deleteRealmLevelRoleMappingsForGroup(String groupId, List getAllRoles() throws AtlasBaseException { int start = 0; - int size = 500; + int size = 2000; boolean found = true; List ret = new ArrayList<>(0); From c77cea8c80a2d7b2d49c7905db5f7a2f3bbdc6d7 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:21:23 +0530 Subject: [PATCH 02/16] Make it 500 --- .../java/org/apache/atlas/plugin/util/KeycloakUserStore.java | 4 ++-- .../org/apache/atlas/keycloak/client/AtlasKeycloakClient.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 7bb2fbf934..85017fc38b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -390,7 +390,7 @@ public RangerRole call() throws Exception { //get all groups for Roles Thread groupsFetcher = new Thread(() -> { int start = 0; - int size = 2000; + int size = 500; boolean found = true; Set ret = new HashSet<>(); @@ -417,7 +417,7 @@ public RangerRole call() throws Exception { //get all users for Roles Thread usersFetcher = new Thread(() -> { int start = 0; - int size = 2000; + int size = 500; boolean found = true; Set ret = new HashSet<>(); diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java index 8a90fe966d..fe723bbce1 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java @@ -59,7 +59,7 @@ public List getRoleUserMembers(String roleName) throws Atlas public List getAllUsers() throws AtlasBaseException { int start = 0; - int size = 2000; + int size = 500; boolean found = true; List ret = new ArrayList<>(0); @@ -104,7 +104,7 @@ public void deleteRealmLevelRoleMappingsForGroup(String groupId, List getAllRoles() throws AtlasBaseException { int start = 0; - int size = 2000; + int size = 500; boolean found = true; List ret = new ArrayList<>(0); From b414697c3f80c4011544913d00ea06b32d74063d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:26:28 +0530 Subject: [PATCH 03/16] Revert "Revert "PLT-372: Add Heracles client in Metastore for fetching users efficiently"" --- auth-agents-common/pom.xml | 2 +- .../atlas/plugin/util/KeycloakUserStore.java | 12 +- {client-keycloak => client-auth}/pom.xml | 2 +- .../auth/client/auth/AbstractAuthClient.java | 53 ++++---- .../auth/KeycloakAuthenticationService.java | 28 ++--- .../atlas/auth/client/config/AuthConfig.java | 100 ++++++++++++++++ .../auth/client/config/AuthConfigBuilder.java | 55 +++++++++ .../client/heracles/AtlasHeraclesClient.java | 71 +++++++++++ .../client/heracles/HeraclesRestClient.java | 18 +++ .../heracles/RetrofitHeraclesClient.java | 16 +++ .../models/HeraclesUsersRepresentation.java | 113 ++++++++++++++++++ .../client/keycloak}/AtlasKeycloakClient.java | 58 +-------- .../client/keycloak}/KeycloakRestClient.java | 67 ++++++----- .../keycloak}/RetrofitKeycloakClient.java | 2 +- client-heracles/pom.xml | 75 ++++++++++++ .../client/config/HeraclesConfigBuilder.java | 5 +- .../client/config/KeycloakConfigBuilder.java | 52 -------- .../org/apache/atlas/AtlasConfiguration.java | 3 +- pom.xml | 2 +- repository/pom.xml | 2 +- .../preprocessor/ConnectionPreProcessor.java | 2 +- .../accesscontrol/PersonaPreProcessor.java | 2 +- .../sql/QueryCollectionPreProcessor.java | 2 +- .../repository/store/users/KeycloakStore.java | 3 +- .../apache/atlas/web/rest/MigrationREST.java | 2 +- .../AtlasKeycloakAuthenticationProvider.java | 2 +- 26 files changed, 550 insertions(+), 199 deletions(-) rename {client-keycloak => client-auth}/pom.xml (98%) rename client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java => client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java (69%) rename client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java => client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java (75%) create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java rename {client-keycloak/src/main/java/org/apache/atlas/keycloak/client => client-auth/src/main/java/org/apache/atlas/auth/client/keycloak}/AtlasKeycloakClient.java (70%) rename {client-keycloak/src/main/java/org/apache/atlas/keycloak/client => client-auth/src/main/java/org/apache/atlas/auth/client/keycloak}/KeycloakRestClient.java (54%) rename {client-keycloak/src/main/java/org/apache/atlas/keycloak/client => client-auth/src/main/java/org/apache/atlas/auth/client/keycloak}/RetrofitKeycloakClient.java (99%) create mode 100644 client-heracles/pom.xml rename client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java => client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java (84%) delete mode 100644 client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java diff --git a/auth-agents-common/pom.xml b/auth-agents-common/pom.xml index 7d63912e3d..aa37156d5f 100644 --- a/auth-agents-common/pom.xml +++ b/auth-agents-common/pom.xml @@ -51,7 +51,7 @@ org.apache.atlas - client-keycloak + client-auth ${project.version} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 85017fc38b..2af3e9a44f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -39,7 +39,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.heracles.AtlasHeraclesClient.getHeraclesClient; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; import static org.apache.atlas.repository.util.AccessControlUtils.BACKEND_SERVICE_USER_NAME; @@ -277,10 +278,7 @@ public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws Atlas } Map> userGroupMapping = new HashMap<>(); - - List kUsers = getKeycloakClient().getAllUsers(); - LOG.info("Found {} keycloak users", kUsers.size()); - + List kUsers = getHeraclesClient().getAllUsers(); List> callables = new ArrayList<>(); kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); @@ -417,13 +415,13 @@ public RangerRole call() throws Exception { //get all users for Roles Thread usersFetcher = new Thread(() -> { int start = 0; - int size = 500; + int size = 100; boolean found = true; Set ret = new HashSet<>(); do { try { - Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); + Set userRepresentations = getHeraclesClient().getRoleUserMembers(kRole.getName(), start, size); if (CollectionUtils.isNotEmpty(userRepresentations)) { ret.addAll(userRepresentations); start += size; diff --git a/client-keycloak/pom.xml b/client-auth/pom.xml similarity index 98% rename from client-keycloak/pom.xml rename to client-auth/pom.xml index b0a231b0c3..9087a98bed 100644 --- a/client-keycloak/pom.xml +++ b/client-auth/pom.xml @@ -27,7 +27,7 @@ 4.0.0 - client-keycloak + client-auth 8 diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java similarity index 69% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java index 86bdf6fbf7..cb13431384 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java @@ -1,13 +1,15 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.auth; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.Timer; +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.heracles.RetrofitHeraclesClient; +import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient; import okhttp3.*; import okhttp3.logging.HttpLoggingInterceptor; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; -import org.apache.atlas.keycloak.client.service.AtlasKeycloakAuthService; import org.apache.atlas.service.metrics.MetricUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,22 +28,23 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; -abstract class AbstractKeycloakClient { +public class AbstractAuthClient { - private final static Logger LOG = LoggerFactory.getLogger(AbstractKeycloakClient.class); + private final static Logger LOG = LoggerFactory.getLogger(AbstractAuthClient.class); private static final Map ERROR_CODE_MAP = new HashMap<>(); - private static final int DEFAULT_KEYCLOAK_RETRY = 3; + private static final int DEFAULT_RETRY = 3; private static final String AUTHORIZATION = "Authorization"; private static final String BEARER = "Bearer "; private static final int TIMEOUT_IN_SEC = 60; private static final String INTEGRATION = "integration"; private static final String KEYCLOAK = "keycloak"; - protected final KeycloakConfig keycloakConfig; - protected final RetrofitKeycloakClient retrofit; + protected final AuthConfig authConfig; + protected final RetrofitKeycloakClient retrofitKeycloakClient; + protected final RetrofitHeraclesClient retrofitHeraclesClient; - private final AtlasKeycloakAuthService authService; + private final KeycloakAuthenticationService authService; private MetricUtils metricUtils = null; static { @@ -49,8 +52,8 @@ abstract class AbstractKeycloakClient { ERROR_CODE_MAP.put(HTTP_BAD_REQUEST, BAD_REQUEST); } - public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { - this.keycloakConfig = keycloakConfig; + public AbstractAuthClient(AuthConfig authConfig) { + this.authConfig = authConfig; this.metricUtils = new MetricUtils(); HttpLoggingInterceptor httpInterceptor = new HttpLoggingInterceptor(); httpInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); @@ -64,11 +67,15 @@ public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { .writeTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) .readTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) .build(); - this.retrofit = new Retrofit.Builder().client(okHttpClient) - .baseUrl(this.keycloakConfig.getAuthServerUrl()) + this.retrofitKeycloakClient = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.authConfig.getAuthServerUrl()) .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() .create(RetrofitKeycloakClient.class); - authService = new AtlasKeycloakAuthService(keycloakConfig); + this.retrofitHeraclesClient = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.authConfig.getHeraclesApiServerUrl()) + .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper().disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES))).build() + .create(RetrofitHeraclesClient.class); + authService = new KeycloakAuthenticationService(authConfig); } /** @@ -97,21 +104,20 @@ public Response intercept(@NonNull Chain chain) throws IOException { return chain.proceed(request); } }; - /** * Called only during auth failures. */ Authenticator authInterceptor = new Authenticator() { @Override public Request authenticate(Route route, @NonNull Response response) { - if (responseCount(response) > DEFAULT_KEYCLOAK_RETRY) { - LOG.warn("Keycloak: Falling back, retried {} times", DEFAULT_KEYCLOAK_RETRY); + if (responseCount(response) > DEFAULT_RETRY) { + LOG.warn("Auth Client: Falling back, retried {} times", DEFAULT_RETRY); return null; } - LOG.info("Keycloak: Current keycloak token status, Expired: {}", authService.isTokenExpired()); + LOG.info("Auth Client: Current keycloak token status, Expired: {}", authService.isTokenExpired()); return response.request().newBuilder() - .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) - .build(); + .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) + .build(); } private int responseCount(Response response) { @@ -134,13 +140,14 @@ protected retrofit2.Response processResponse(retrofit2.Call req) throw return response; } String errMsg = response.errorBody().string(); - LOG.error("Keycloak: Client request processing failed code {} message:{}, request: {} {}", + LOG.error("Auth Client: Client request processing failed code {} message:{}, request: {} {}", response.code(), errMsg, req.request().method(), req.request().url()); throw new AtlasBaseException(ERROR_CODE_MAP.getOrDefault(response.code(), BAD_REQUEST), errMsg); } catch (Exception e) { - LOG.error("Keycloak: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); - throw new AtlasBaseException(BAD_REQUEST, "Keycloak request failed"); + LOG.error("Auth Client: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); + throw new AtlasBaseException(BAD_REQUEST, "Auth request failed"); } } + } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java similarity index 75% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java index fc4d40df7e..a3b3e9255e 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java @@ -1,11 +1,11 @@ -package org.apache.atlas.keycloak.client.service; +package org.apache.atlas.auth.client.auth; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.*; import okhttp3.logging.HttpLoggingInterceptor; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.RetrofitKeycloakClient; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient; import org.jetbrains.annotations.NotNull; import org.keycloak.representations.AccessTokenResponse; import org.slf4j.Logger; @@ -19,9 +19,9 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -public final class AtlasKeycloakAuthService { +public final class KeycloakAuthenticationService { - public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakAuthService.class); + public final static Logger LOG = LoggerFactory.getLogger(KeycloakAuthenticationService.class); private final static String GRANT_TYPE = "grant_type"; private static final String CLIENT_ID = "client_id"; @@ -30,14 +30,14 @@ public final class AtlasKeycloakAuthService { private static final int TIMEOUT_IN_SECS = 60; private final RetrofitKeycloakClient retrofit; - private final KeycloakConfig keycloakConfig; + private final AuthConfig authConfig; private AccessTokenResponse currentAccessToken; private long expirationTime = -1; - public AtlasKeycloakAuthService(KeycloakConfig keycloakConfig) { - this.keycloakConfig = keycloakConfig; + public KeycloakAuthenticationService(AuthConfig authConfig) { + this.authConfig = authConfig; this.retrofit = new Retrofit.Builder().client(getOkHttpClient()) - .baseUrl(this.keycloakConfig.getAuthServerUrl()) + .baseUrl(this.authConfig.getAuthServerUrl()) .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() .create(RetrofitKeycloakClient.class); } @@ -59,7 +59,7 @@ private OkHttpClient getOkHttpClient() { Interceptor responseLoggingInterceptor = chain -> { Request request = chain.request(); okhttp3.Response response = chain.proceed(request); - LOG.info("Keycloak: Auth Request for url {} Status: {}", request.url(), response.code()); + LOG.info("Auth Client: Auth Request for url {} Status: {}", request.url(), response.code()); return response; }; @@ -70,16 +70,16 @@ public String getAuthToken() { synchronized (this) { if (isTokenExpired()) { try { - retrofit2.Response resp = this.retrofit.grantToken(this.keycloakConfig.getRealmId(), getTokenRequest()).execute(); + retrofit2.Response resp = this.retrofit.grantToken(this.authConfig.getRealmId(), getTokenRequest()).execute(); if (resp.isSuccessful()) { currentAccessToken = resp.body(); expirationTime = currentTime() + currentAccessToken.getExpiresIn() - EXPIRY_OFFSET_SEC; - LOG.info("Keycloak: Auth token fetched with expiry:{} sec", expirationTime); + LOG.info("Auth Client: Auth token fetched with expiry:{} sec", expirationTime); } else { throw new AtlasBaseException(BAD_REQUEST, resp.errorBody().string()); } } catch (Exception e) { - LOG.error("Keycloak: Error while fetching access token for keycloak client.", e); + LOG.error("Auth Client: Error while fetching access token for keycloak client.", e); throw new RuntimeException(e); } } @@ -97,7 +97,7 @@ public boolean isTokenExpired() { } private RequestBody getTokenRequest() { - return new FormBody.Builder().addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()).addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.keycloakConfig.getGrantType()).build(); + return new FormBody.Builder().addEncoded(CLIENT_ID, this.authConfig.getClientId()).addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.authConfig.getGrantType()).build(); } private long currentTime() { diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java new file mode 100644 index 0000000000..33d98de049 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java @@ -0,0 +1,100 @@ +package org.apache.atlas.auth.client.config; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Optional; + +import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; + +public class AuthConfig { + private static final Logger LOG = LoggerFactory.getLogger(AuthConfig.class); + + public String authServerUrl; + public String realmId; + public String clientId; + public String clientSecret; + public String grantType; + public String heraclesApiServerUrl; + + private static final String KEYCLOAK_PROPERTIES = "keycloak.json"; + private static final String DEFAULT_GRANT_TYPE = "client_credentials"; + private static final String KEY_REALM_ID = "realm"; + private static final String KEY_AUTH_SERVER_URL = "auth-server-url"; + private static final String KEY_CLIENT_ID = "resource"; + private static final String KEY_CREDENTIALS = "credentials"; + private static final String KEY_SECRET = "secret"; + + public String getAuthServerUrl() { + return authServerUrl; + } + + public String getRealmId() { + return realmId; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getGrantType() { + return grantType; + } + + public String getHeraclesApiServerUrl() { + return heraclesApiServerUrl; + } + + public static AuthConfig getConfig() throws AtlasBaseException { + String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); + Optional confFile = getConfigurationFile(confLocation); + + if (confFile.isPresent()) { + try { + JSONObject object = new JSONObject(readFileToString(confFile.get())); + return buildAuthConfigFromJson(object); + } catch (Exception e) { + LOG.error("Error parsing Keycloak configuration: ", e); + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Error parsing Keycloak configuration"); + } + } else { + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); + } + } + + private static Optional getConfigurationFile(String confLocation) { + if (StringUtils.isNotEmpty(confLocation)) { + File confFile = new File(confLocation, KEYCLOAK_PROPERTIES); + if (confFile.exists()) { + return Optional.of(confFile); + } + } + return Optional.empty(); + } + + private static String readFileToString(File file) throws Exception { + return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static AuthConfig buildAuthConfigFromJson(JSONObject object) throws Exception { + String realmId = object.getString(KEY_REALM_ID); + String authServerUrl = object.getString(KEY_AUTH_SERVER_URL) + "/"; + String clientId = object.getString(KEY_CLIENT_ID); + String grantType = DEFAULT_GRANT_TYPE; + String clientSecret = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); + + LOG.info("Keycloak configuration: REALM_ID:{}, AUTH_SERVER_URL:{}", realmId, authServerUrl); + return AuthConfigBuilder.builder().realId(realmId).authServerUrl(authServerUrl).clientId(clientId).grantType(grantType).clientSecret(clientSecret).build(); + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java new file mode 100644 index 0000000000..552033d2c1 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java @@ -0,0 +1,55 @@ +package org.apache.atlas.auth.client.config; + +import org.apache.atlas.AtlasConfiguration; + +public class AuthConfigBuilder { + + private String authServerUrl; + private String realmId; + private String clientId; + private String clientSecret; + private String grantType = "client_credentials"; + + private AuthConfigBuilder() { + } + + public static AuthConfigBuilder builder() { + return new AuthConfigBuilder(); + } + + public AuthConfigBuilder authServerUrl(String authServerUrl) { + this.authServerUrl = authServerUrl; + return this; + } + + public AuthConfigBuilder realId(String realId) { + this.realmId = realId; + return this; + } + + public AuthConfigBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public AuthConfigBuilder clientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public AuthConfigBuilder grantType(String grantType) { + this.grantType = grantType; + return this; + } + + public AuthConfig build() { + AuthConfig authConfig = new AuthConfig(); + authConfig.authServerUrl = authServerUrl; + authConfig.realmId = realmId; + authConfig.clientId = clientId; + authConfig.clientSecret = clientSecret; + authConfig.grantType = grantType; + authConfig.heraclesApiServerUrl= AtlasConfiguration.HERACLES_API_SERVER_URL.getString()+"/"; + return authConfig; + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java new file mode 100644 index 0000000000..6f265486e6 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -0,0 +1,71 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; +import org.keycloak.representations.idm.UserRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class AtlasHeraclesClient { + public final static Logger LOG = LoggerFactory.getLogger(AtlasHeraclesClient.class); + + private static HeraclesRestClient HERACLES; + private static AtlasHeraclesClient HERACLES_CLIENT; + + public AtlasHeraclesClient() {} + + public static AtlasHeraclesClient getHeraclesClient() { + if(Objects.isNull(HERACLES_CLIENT)) { + LOG.info("Initializing Heracles client.."); + try{ + init(AuthConfig.getConfig()); + } catch (Exception e) { + LOG.error("Error initializing Heracles client", e); + } + } + return HERACLES_CLIENT; + } + + private static void init(AuthConfig authConfig) { + synchronized (AtlasHeraclesClient.class) { + if (HERACLES == null) { + HERACLES = new HeraclesRestClient(authConfig); + HERACLES_CLIENT = new AtlasHeraclesClient(); + } + } + } + + public List getAllUsers() throws AtlasBaseException { + int start = 0; + int size = 100; + boolean found = true; + + List ret = new ArrayList<>(0); + do { + + List userRepresentations = HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, null, HeraclesUsersRepresentation.USER_SORT).body().toKeycloakUserRepresentations(); + if (userRepresentations != null && !userRepresentations.isEmpty()) { + ret.addAll(userRepresentations); + start += size; + } else { + found = false; + } + } while (found && ret.size() % size == 0); + + return ret; + } + + + public Set getRoleUserMembers(String roleName, int start, int size) throws AtlasBaseException { + String template = "{\"$and\":[{\"roles\":{\"$elemMatch\":[\"{0}\"]}}]}"; + String filter = template.replace("{0}", roleName); + return HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, filter,HeraclesUsersRepresentation.USER_SORT ).body().toKeycloakUserRepresentations().stream().collect(Collectors.toSet()); + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java new file mode 100644 index 0000000000..a9adc68b7e --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -0,0 +1,18 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.auth.AbstractAuthClient; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; +import retrofit2.Response; + +public class HeraclesRestClient extends AbstractAuthClient { + + public HeraclesRestClient(final AuthConfig authConfig) { + super(authConfig); + } + public Response getUsers(int offset,int limit, String columns, String filter, String sort) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getUsers(offset, columns, filter, limit,sort)); + } + +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java new file mode 100644 index 0000000000..c35451852c --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -0,0 +1,16 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; +import retrofit2.Call; +import retrofit2.http.GET; +import retrofit2.http.Headers; +import retrofit2.http.Query; + +public interface RetrofitHeraclesClient { + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/users") + Call getUsers(@Query("offset") Integer offset, @Query("columns") String columns, + @Query(value = "filter", encoded = true) String filter, @Query("limit") Integer limit, + @Query("sort") String sort); + +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java new file mode 100644 index 0000000000..4cea34f509 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java @@ -0,0 +1,113 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.keycloak.representations.idm.UserRepresentation; + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesUsersRepresentation { + protected int totalRecord; + protected int filterRecord; + protected List records; + public static final String USER_PROJECTIONS = "emailVerified,enabled,id,status,username"; + public static final String USER_SORT = "username"; + + public HeraclesUsersRepresentation() { + } + + public HeraclesUsersRepresentation(int totalRecord, int filterRecord, List records) { + this.totalRecord = totalRecord; + this.filterRecord = filterRecord; + this.records = records; + } + + public int getTotalRecord() { + return totalRecord; + } + + public void setTotalRecord(int totalRecord) { + this.totalRecord = totalRecord; + } + + public int getFilterRecord() { + return filterRecord; + } + + public void setFilterRecord(int filterRecord) { + this.filterRecord = filterRecord; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + public List toKeycloakUserRepresentations() { + List userRepresentations = new ArrayList<>(); + for (HeraclesUserRepresentation heraclesUserRepresentation : records) { + UserRepresentation userRepresentation = new UserRepresentation(); + userRepresentation.setEmailVerified(heraclesUserRepresentation.emailVerified); + userRepresentation.setEnabled(heraclesUserRepresentation.enabled); + userRepresentation.setUsername(heraclesUserRepresentation.username); + userRepresentation.setId(heraclesUserRepresentation.id); + userRepresentations.add(userRepresentation); + } + return userRepresentations; + } +} + +@JsonIgnoreProperties(ignoreUnknown = true) +class HeraclesUserRepresentation { + protected boolean emailVerified; + protected boolean enabled; + protected String username; + protected String id; + + public HeraclesUserRepresentation() { + } + + public HeraclesUserRepresentation(boolean emailVerified, boolean enabled, String username, String id) { + this.emailVerified = emailVerified; + this.enabled = enabled; + this.username = username; + this.id = id; + } + + public boolean isEmailVerified() { + return emailVerified; + } + + public void setEmailVerified(boolean emailVerified) { + this.emailVerified = emailVerified; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java similarity index 70% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java index fe723bbce1..9ec43428e6 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java @@ -1,29 +1,19 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; -import org.apache.atlas.keycloak.client.config.KeycloakConfigBuilder; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; import org.keycloak.representations.idm.*; import org.keycloak.representations.oidc.TokenMetadataRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; -import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; - /** * Keycloak client, deals with token creation refresh. */ @@ -31,14 +21,6 @@ public final class AtlasKeycloakClient { public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakClient.class); - private final static String KEYCLOAK_PROPERTIES = "keycloak.json"; - private final static String DEFAULT_GRANT_TYPE = "client_credentials"; - private final static String KEY_REALM_ID = "realm"; - private final static String KEY_AUTH_SERVER_URL = "auth-server-url"; - private final static String KEY_CLIENT_ID = "resource"; - private final static String KEY_CREDENTIALS = "credentials"; - private final static String KEY_SECRET = "secret"; - private static KeycloakRestClient KEYCLOAK; private static AtlasKeycloakClient KEYCLOAK_CLIENT; @@ -183,13 +165,7 @@ public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException if (Objects.isNull(KEYCLOAK_CLIENT)) { LOG.info("Initializing Keycloak client.."); try { - init(getConfig()); - } catch (IOException e) { - LOG.error("Failed to fetch Keycloak conf {}", e.getMessage()); - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); - } catch (JSONException e) { - LOG.error("Failed to parse Keycloak conf {}", e.getMessage()); - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); + init(AuthConfig.getConfig()); } catch (Exception e) { LOG.error("Failed to connect to Keycloak {}", e.getMessage()); throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); @@ -201,7 +177,7 @@ public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException return KEYCLOAK_CLIENT; } - private static void init(KeycloakConfig config) { + private static void init(AuthConfig config) { synchronized (AtlasKeycloakClient.class) { if (KEYCLOAK_CLIENT == null) { KEYCLOAK = new KeycloakRestClient(config); @@ -209,30 +185,4 @@ private static void init(KeycloakConfig config) { } } } - - private static KeycloakConfig getConfig() throws Exception { - String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); - File confFile; - if (StringUtils.isNotEmpty(confLocation)) { - confFile = new File(confLocation, KEYCLOAK_PROPERTIES); - - if (confFile.exists()) { - String keyConf = new String(Files.readAllBytes(confFile.toPath()), StandardCharsets.UTF_8); - JSONObject object = new JSONObject(keyConf); - - String REALM_ID = object.getString(KEY_REALM_ID); - String AUTH_SERVER_URL = object.getString(KEY_AUTH_SERVER_URL) + "/"; - String CLIENT_ID = object.getString(KEY_CLIENT_ID); - String GRANT_TYPE = DEFAULT_GRANT_TYPE; - String CLIENT_SECRET = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); - - LOG.info("Keycloak conf: REALM_ID:{}, AUTH_SERVER_URL:{}", REALM_ID, AUTH_SERVER_URL); - return KeycloakConfigBuilder.builder().realId(REALM_ID).authServerUrl(AUTH_SERVER_URL).clientId(CLIENT_ID).grantType(GRANT_TYPE).clientSecret(CLIENT_SECRET).build(); - } else { - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); - } - } else { - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Configuration location not found " + confLocation); - } - } } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java similarity index 54% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java index 7cc1eea22f..b6f58982e6 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java @@ -1,9 +1,10 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; +import org.apache.atlas.auth.client.auth.AbstractAuthClient; import okhttp3.FormBody; import okhttp3.RequestBody; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; import org.keycloak.representations.idm.*; import org.keycloak.representations.oidc.TokenMetadataRepresentation; import retrofit2.Response; @@ -14,129 +15,129 @@ /** * Keycloak Rest client wrapper used in atlas metastore */ -public final class KeycloakRestClient extends AbstractKeycloakClient { +public final class KeycloakRestClient extends AbstractAuthClient { private static final String TOKEN = "token"; private static final String CLIENT_ID = "client_id"; private static final String CLIENT_SECRET = "client_secret"; - public KeycloakRestClient(final KeycloakConfig keycloakConfig) { - super(keycloakConfig); + public KeycloakRestClient(final AuthConfig authConfig) { + super(authConfig); } public Response> searchUserByUserName(String username) throws AtlasBaseException { - return processResponse(this.retrofit.searchUserByUserName(this.keycloakConfig.getRealmId(), username)); + return processResponse(this.retrofitKeycloakClient.searchUserByUserName(this.authConfig.getRealmId(), username)); } public Response> getAllUsers(int start, int size) throws AtlasBaseException { - return processResponse(this.retrofit.getAllUsers(this.keycloakConfig.getRealmId(), start, size)); + return processResponse(this.retrofitKeycloakClient.getAllUsers(this.authConfig.getRealmId(), start, size)); } public Response> getRoleUserMembers(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleUserMembers(this.authConfig.getRealmId(), roleName)); } public Response> getRoleUserMembers(String roleName, Integer start, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName, start, size)); + return processResponse(this.retrofitKeycloakClient.getRoleUserMembers(this.authConfig.getRealmId(), roleName, start, size)); } public Response> searchGroupByName(String groupName, Integer start, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.searchGroupByName(this.keycloakConfig.getRealmId(), groupName, start, size)); + return processResponse(this.retrofitKeycloakClient.searchGroupByName(this.authConfig.getRealmId(), groupName, start, size)); } public Response> getRoleGroupMembers(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleGroupMembers(this.authConfig.getRealmId(), roleName)); } public Response> getRoleGroupMembers(String roleName, Integer first, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName, first, size)); + return processResponse(this.retrofitKeycloakClient.getRoleGroupMembers(this.authConfig.getRealmId(), roleName, first, size)); } public Response> getGroupsForUserById(String userId) throws AtlasBaseException { - return processResponse(this.retrofit.getGroupsForUserById(this.keycloakConfig.getRealmId(), userId)); + return processResponse(this.retrofitKeycloakClient.getGroupsForUserById(this.authConfig.getRealmId(), userId)); } public void addRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.addRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + processResponse(this.retrofitKeycloakClient.addRealmLevelRoleMappingsForGroup(this.authConfig.getRealmId(), groupId, roles)); } public void deleteRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + processResponse(this.retrofitKeycloakClient.deleteRealmLevelRoleMappingsForGroup(this.authConfig.getRealmId(), groupId, roles)); } public Response> getAllRoles(int start, int size) throws AtlasBaseException { - return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), start, size)); + return processResponse(this.retrofitKeycloakClient.getAllRoles(this.authConfig.getRealmId(), start, size)); } public void deleteRoleById(String roleId) throws AtlasBaseException { - processResponse(this.retrofit.deleteRoleById(this.keycloakConfig.getRealmId(), roleId)); + processResponse(this.retrofitKeycloakClient.deleteRoleById(this.authConfig.getRealmId(), roleId)); } public void deleteRoleByName(String roleName) throws AtlasBaseException { - processResponse(this.retrofit.deleteRoleByName(this.keycloakConfig.getRealmId(), roleName)); + processResponse(this.retrofitKeycloakClient.deleteRoleByName(this.authConfig.getRealmId(), roleName)); } public Response> addRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { - return processResponse(this.retrofit.addRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + return processResponse(this.retrofitKeycloakClient.addRealmLevelRoleMappingsForUser(this.authConfig.getRealmId(), userId, roles)); } public void deleteRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + processResponse(this.retrofitKeycloakClient.deleteRealmLevelRoleMappingsForUser(this.authConfig.getRealmId(), userId, roles)); } public void createRole(RoleRepresentation roleRepresentation) throws AtlasBaseException { - processResponse(this.retrofit.createRole(this.keycloakConfig.getRealmId(), roleRepresentation)); + processResponse(this.retrofitKeycloakClient.createRole(this.authConfig.getRealmId(), roleRepresentation)); } public void updateRole(String roleId, RoleRepresentation roleRepresentation) throws AtlasBaseException { - processResponse(this.retrofit.updateRole(this.keycloakConfig.getRealmId(), roleId, roleRepresentation)); + processResponse(this.retrofitKeycloakClient.updateRole(this.authConfig.getRealmId(), roleId, roleRepresentation)); } public Response getRoleById(String roleId) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleById(this.keycloakConfig.getRealmId(), roleId)); + return processResponse(this.retrofitKeycloakClient.getRoleById(this.authConfig.getRealmId(), roleId)); } public Response getRoleByName(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleByName(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleByName(this.authConfig.getRealmId(), roleName)); } public Response> getAllRoles(Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), first, max)); + return processResponse(this.retrofitKeycloakClient.getAllRoles(this.authConfig.getRealmId(), first, max)); } public Response> getRoleComposites(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleComposites(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleComposites(this.authConfig.getRealmId(), roleName)); } public void addComposites(String roleName, List roles) throws AtlasBaseException { - processResponse(this.retrofit.addComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + processResponse(this.retrofitKeycloakClient.addComposites(this.authConfig.getRealmId(), roleName, roles)); } public void deleteComposites(String roleName, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + processResponse(this.retrofitKeycloakClient.deleteComposites(this.authConfig.getRealmId(), roleName, roles)); } public Response> getAdminEvents(List operationTypes, String authRealm, String authClient, String authUser, String authIpAddress, String resourcePath, String dateFrom, String dateTo, Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getAdminEvents(this.keycloakConfig.getRealmId(), operationTypes, + return processResponse(this.retrofitKeycloakClient.getAdminEvents(this.authConfig.getRealmId(), operationTypes, authRealm, authClient, authUser, authIpAddress, resourcePath, dateFrom, dateTo, first, max)); } public Response> getEvents(List type, String client, String user, String dateFrom, String dateTo, String ipAddress, Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getEvents(this.keycloakConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); + return processResponse(this.retrofitKeycloakClient.getEvents(this.authConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); } public Response introspectToken(String token) throws AtlasBaseException { - return processResponse(this.retrofit.introspectToken(this.keycloakConfig.getRealmId(), getIntrospectTokenRequest(token))); + return processResponse(this.retrofitKeycloakClient.introspectToken(this.authConfig.getRealmId(), getIntrospectTokenRequest(token))); } private RequestBody getIntrospectTokenRequest(String token) { return new FormBody.Builder() .addEncoded(TOKEN, token) - .addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()) - .addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()) + .addEncoded(CLIENT_ID, this.authConfig.getClientId()) + .addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()) .build(); } } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java similarity index 99% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java index c396c9368d..4fe8bb6ff5 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java @@ -1,4 +1,4 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; import okhttp3.RequestBody; import org.keycloak.representations.AccessTokenResponse; diff --git a/client-heracles/pom.xml b/client-heracles/pom.xml new file mode 100644 index 0000000000..b812aefb65 --- /dev/null +++ b/client-heracles/pom.xml @@ -0,0 +1,75 @@ + + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + client-heracles + + + 8 + 8 + + + + + org.keycloak + keycloak-core + ${keycloak-admin-client.version} + + + * + * + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.retrofit2 + retrofit + ${retrofit.version} + + + com.squareup.retrofit2 + converter-jackson + ${retrofit.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + org.apache.atlas + atlas-common + + + + \ No newline at end of file diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java b/client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java similarity index 84% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java rename to client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java index 048d2fac71..863eec0021 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java +++ b/client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java @@ -1,6 +1,6 @@ -package org.apache.atlas.keycloak.client.config; +package main.java.heracles.client.config; -public final class KeycloakConfig { +public class HeraclesConfigBuilder { String authServerUrl; String realmId; @@ -27,5 +27,4 @@ public String getClientSecret() { public String getGrantType() { return grantType; } - } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java deleted file mode 100644 index 44583c59f3..0000000000 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.apache.atlas.keycloak.client.config; - -public final class KeycloakConfigBuilder { - - private String authServerUrl; - private String realmId; - private String clientId; - private String clientSecret; - private String grantType = "client_credentials"; - - private KeycloakConfigBuilder() { - } - - public static KeycloakConfigBuilder builder() { - return new KeycloakConfigBuilder(); - } - - public KeycloakConfigBuilder authServerUrl(String authServerUrl) { - this.authServerUrl = authServerUrl; - return this; - } - - public KeycloakConfigBuilder realId(String realId) { - this.realmId = realId; - return this; - } - - public KeycloakConfigBuilder clientId(String clientId) { - this.clientId = clientId; - return this; - } - - public KeycloakConfigBuilder clientSecret(String clientSecret) { - this.clientSecret = clientSecret; - return this; - } - - public KeycloakConfigBuilder grantType(String grantType) { - this.grantType = grantType; - return this; - } - - public KeycloakConfig build() { - KeycloakConfig keycloakConfig = new KeycloakConfig(); - keycloakConfig.authServerUrl = authServerUrl; - keycloakConfig.realmId = realmId; - keycloakConfig.clientId = clientId; - keycloakConfig.clientSecret = clientSecret; - keycloakConfig.grantType = grantType; - return keycloakConfig; - } -} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 889d5b413f..d10e4b13fc 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -108,7 +108,8 @@ public enum AtlasConfiguration { PERSONA_POLICY_ASSET_MAX_LIMIT("atlas.persona.policy.asset.maxlimit", 1000), ENABLE_KEYCLOAK_TOKEN_INTROSPECTION("atlas.canary.keycloak.token-introspection", false), - KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE("atlas.keycloak.admin.resource-pagination-size", 1500); + KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE("atlas.keycloak.admin.resource-pagination-size", 1500), + HERACLES_API_SERVER_URL("atlas.heracles.api.server.url", "http://heracles-service.heracles.svc.cluster.local"); private static final Configuration APPLICATION_PROPERTIES; diff --git a/pom.xml b/pom.xml index 92764dc646..29e3131c22 100644 --- a/pom.xml +++ b/pom.xml @@ -790,7 +790,7 @@ server-api notification client - client-keycloak + client-auth graphdb repository diff --git a/repository/pom.xml b/repository/pom.xml index 03e66a8310..10d8d876fb 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -318,7 +318,7 @@ org.apache.atlas - client-keycloak + client-auth 3.0.0-SNAPSHOT diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index b994398c23..02fb63bbc8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -55,7 +55,6 @@ import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; @@ -63,6 +62,7 @@ import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class ConnectionPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index f579dc4ff7..77078f5c53 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -20,7 +20,7 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java index 63e8ba4250..5dec4d3190 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java @@ -58,7 +58,7 @@ import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; import static org.apache.atlas.repository.Constants.ATTR_VIEWER_GROUPS; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java index 3a5f1782f0..22e04cd751 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java @@ -33,8 +33,7 @@ import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; -import static org.apache.atlas.repository.util.AccessControlUtils.INSTANCE_DOMAIN_KEY; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class KeycloakStore { private static final Logger LOG = LoggerFactory.getLogger(KeycloakStore.class); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index b1cb307550..b491bb88ce 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -33,7 +33,7 @@ import java.util.*; import java.util.stream.Collectors; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.*; @Path("migration") diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java index 88523998cd..1a22038398 100644 --- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java +++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java @@ -18,7 +18,7 @@ import io.micrometer.core.instrument.Counter; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.service.metrics.MetricUtils; import org.apache.atlas.ApplicationProperties; import org.apache.commons.configuration.Configuration; From a268fea91804549089285d10a8cb517853339db2 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 14 Dec 2023 00:06:04 +0530 Subject: [PATCH 04/16] feat: add Heracles based fetcher --- .../atlas/plugin/util/KeycloakUserStore.java | 108 +++++++++++++++++- .../plugin/util/RangerRolesProvider.java | 2 +- .../plugin/util/RangerUserStoreProvider.java | 2 +- .../client/heracles/AtlasHeraclesClient.java | 19 +++ .../client/heracles/HeraclesRestClient.java | 16 ++- .../heracles/RetrofitHeraclesClient.java | 12 ++ .../HeraclesRoleViewRepresentation.java | 60 ++++++++++ .../HeraclesUserViewRepresentation.java | 61 ++++++++++ .../org/apache/atlas/web/rest/AuthREST.java | 4 +- 9 files changed, 276 insertions(+), 8 deletions(-) create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 2af3e9a44f..dcd05bb295 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -20,8 +20,8 @@ package org.apache.atlas.plugin.util; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.plugin.model.RangerRole; import org.apache.atlas.utils.AtlasPerfMetrics; @@ -189,6 +189,108 @@ public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseExce return rangerRoles; } + public RangerUserStore loadUsersNew(long lastUpdatedTime) throws AtlasBaseException { + int userSize = 100; + int userFrom = 0; + boolean userFound = true; + Map> userGroupMapping = new HashMap<>(); + + List ret = new ArrayList<>(); + do { + List users = getHeraclesClient().getUsersView(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { + userFound = false; + } else { + ret.addAll(users); + userFrom += userSize; + } + for(UserRepresentation user : users) { + userGroupMapping.put(user.getUsername(), new HashSet<>(user.getGroups() == null ? Collections.emptyList() : user.getGroups())); + } + + } while (userFound && ret.size() % userSize == 0); + + RangerUserStore userStore = new RangerUserStore(); + userStore.setUserGroupMapping(userGroupMapping); + Date current = new Date(); + userStore.setUserStoreUpdateTime(current); + userStore.setServiceName(serviceName); + userStore.setUserStoreVersion(-1L); + return userStore; + } + + public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException { + RangerRoles rangerRoles = new RangerRoles(); + Map> roleUserMapping = new HashMap<>(); + Set roleSet = new HashSet<>(); + + int userSize = 100; + int userFrom = 0; + boolean userFound = true; + + List ret = new ArrayList<>(); + do { + List users = getHeraclesClient().getUsersView(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { + userFound = false; + } else { + ret.addAll(users); + userFrom += userSize; + } + for(UserRepresentation user : users) { + Set userRoles = new HashSet<>(user.getRealmRoles()); + for(String role : userRoles) { + List roleMembers = roleUserMapping.get(role); + if(roleMembers == null) { + roleMembers = new ArrayList<>(); + roleUserMapping.put(role, roleMembers); + } + roleMembers.add(new RangerRole.RoleMember(user.getUsername(), false)); + } + } + + } while (userFound && ret.size() % userSize == 0); + + int roleSize = 100; + int roleFrom = 0; + boolean roleFound = true; + + List retRole = new ArrayList<>(); + do { + List roles = getHeraclesClient().getRolesView(roleFrom, roleSize); + if (CollectionUtils.isEmpty(roles)) { + roleFound = false; + } else { + retRole.addAll(roles); + roleFrom += roleSize; + } + + for(HeraclesRoleViewRepresentation role : roles) { + RangerRole rangerRole = new RangerRole(); + rangerRole.setName(role.getName()); + rangerRole.setGroups(role.getGroups().stream().map(x -> new RangerRole.RoleMember(x, true)).collect(Collectors.toList())); + rangerRole.setUsers(roleUserMapping.get(role.getName())); + rangerRole.setRoles(role.getRoles().stream().map(x -> new RangerRole.RoleMember(x, false)).collect(Collectors.toList())); + + roleSet.add(rangerRole); + } + + } while (roleFound && retRole.size() % roleSize == 0); + + processDefaultRole(roleSet); + LOG.info("Inverting roles"); + invertRoles(roleSet); + + rangerRoles.setRangerRoles(roleSet); + rangerRoles.setServiceName(serviceName); + + Date current = new Date(); + rangerRoles.setRoleUpdateTime(current); + rangerRoles.setServiceName(serviceName); + rangerRoles.setRoleVersion(-1L); + return rangerRoles; + } + public void invertRoles(Set roleSet) { Map roleMap = new HashMap<>(); for (RangerRole role : roleSet) { @@ -278,7 +380,7 @@ public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws Atlas } Map> userGroupMapping = new HashMap<>(); - List kUsers = getHeraclesClient().getAllUsers(); + List kUsers = getKeycloakClient().getAllUsers(); List> callables = new ArrayList<>(); kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); @@ -421,7 +523,7 @@ public RangerRole call() throws Exception { do { try { - Set userRepresentations = getHeraclesClient().getRoleUserMembers(kRole.getName(), start, size); + Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); if (CollectionUtils.isNotEmpty(userRepresentations)) { ret.addAll(userRepresentations); start += size; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java index 4607d6192a..c26681952a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java @@ -195,7 +195,7 @@ private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundEx try { if ("atlas".equals(serviceName)) { - roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTimeInMillis); + roles = keycloakUserStore.loadRolesNew(lastUpdatedTimeInMillis); } else { roles = atlasAuthAdminClient.getRolesIfUpdated(lastUpdatedTimeInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java index 89980504b0..abed31ef03 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -196,7 +196,7 @@ private RangerUserStore loadUserStoreFromAdmin() throws RangerServiceNotFoundExc try { if ("atlas".equals(serviceName)) { - userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdateTimeInMillis); + userStore = keycloakUserStore.loadUsersNew(lastUpdateTimeInMillis); } else { userStore = atlasAuthAdminClient.getUserStoreIfUpdated(lastUpdateTimeInMillis); } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java index 6f265486e6..e178915e37 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -1,6 +1,8 @@ package org.apache.atlas.auth.client.heracles; import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import org.keycloak.representations.idm.UserRepresentation; @@ -68,4 +70,21 @@ public Set getRoleUserMembers(String roleName, int start, in String filter = template.replace("{0}", roleName); return HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, filter,HeraclesUsersRepresentation.USER_SORT ).body().toKeycloakUserRepresentations().stream().collect(Collectors.toSet()); } + + public List getUsersView(int start, int size) throws AtlasBaseException { + List views = HERACLES.getUsersView(start, size, HeraclesUserViewRepresentation.sortBy).body(); + List userRepresentations = views.stream().map(x -> { + UserRepresentation userRepresentation = new UserRepresentation(); + userRepresentation.setId(x.getId()); + userRepresentation.setUsername(x.getUsername()); + userRepresentation.setRealmRoles(x.getRoles()); + userRepresentation.setGroups(x.getGroups()); + return userRepresentation; + }).collect(Collectors.toList()); + return userRepresentations; + } + + public List getRolesView(int start, int size) throws AtlasBaseException { + return HERACLES.getRolesView(start, size, HeraclesRoleViewRepresentation.sortBy).body(); + } } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java index a9adc68b7e..0e76b0e4dd 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -2,10 +2,14 @@ import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.auth.client.auth.AbstractAuthClient; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import retrofit2.Response; +import java.util.List; + public class HeraclesRestClient extends AbstractAuthClient { public HeraclesRestClient(final AuthConfig authConfig) { @@ -15,4 +19,14 @@ public Response getUsers(int offset,int limit, Stri return processResponse(this.retrofitHeraclesClient.getUsers(offset, columns, filter, limit,sort)); } -} + public Response> getUsersView(int offset, int limit, String sort) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getUsersView(offset, limit,sort)); + } + + public Response> getRolesView(int offset, int limit, String sort) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getRolesView(offset, limit,sort)); + } + + + +} \ No newline at end of file diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java index c35451852c..f32f305495 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -1,11 +1,15 @@ package org.apache.atlas.auth.client.heracles; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import retrofit2.Call; import retrofit2.http.GET; import retrofit2.http.Headers; import retrofit2.http.Query; +import java.util.List; + public interface RetrofitHeraclesClient { @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) @GET("/users") @@ -13,4 +17,12 @@ Call getUsers(@Query("offset") Integer offset, @Que @Query(value = "filter", encoded = true) String filter, @Query("limit") Integer limit, @Query("sort") String sort); + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/users/view") + Call> getUsersView(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort); + + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/roles/view") + Call> getRolesView(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort); + } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java new file mode 100644 index 0000000000..5b695bcf71 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java @@ -0,0 +1,60 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesRoleViewRepresentation { + protected String id; + protected String name; + protected String realmId; + protected List roles; + protected List groups; + + public static String sortBy = "name"; + + public HeraclesRoleViewRepresentation() { + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getRealmId() { + return realmId; + } + + public List getRoles() { + return roles == null ? new ArrayList<>() : roles; + } + + public List getGroups() { + return groups == null ? new ArrayList<>() : groups; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setRealmId(String realmId) { + this.realmId = realmId; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setGroups(List groups) { + this.groups = groups; + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java new file mode 100644 index 0000000000..de3bc1e105 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java @@ -0,0 +1,61 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.keycloak.representations.idm.UserRepresentation; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesUserViewRepresentation { + protected String id; + protected String username; + protected boolean enabled; + protected List roles; + protected List groups; + + public static String sortBy = "username"; + + public HeraclesUserViewRepresentation() { + } + + public String getId() { + return id; + } + + public String getUsername() { + return username; + } + + public boolean isEnabled() { + return enabled; + } + + public List getRoles() { + return roles; + } + + public List getGroups() { + return groups; + } + + public void setId(String id) { + this.id = id; + } + + public void setUsername(String username) { + this.username = username; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setGroups(List groups) { + this.groups = groups; + } + +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c5868cbfa3..f2f7ac2167 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -100,7 +100,7 @@ public RangerRoles downloadRoles(@PathParam("serviceName") final String serviceN } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerRoles roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTime); + RangerRoles roles = keycloakUserStore.loadRolesNew(lastUpdatedTime); if (roles == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); @@ -127,7 +127,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerUserStore userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdatedTime); + RangerUserStore userStore = keycloakUserStore.loadUsersNew(lastUpdatedTime); if (userStore == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); From 4b19bdc5cc3205565b24573194287b16f33e9b6d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 8 Dec 2023 18:25:13 +0530 Subject: [PATCH 05/16] feat: new commit --- .github/workflows/maven.yml | 2 +- .../java/org/apache/atlas/plugin/util/KeycloakUserStore.java | 5 +++-- .../apache/atlas/keycloak/client/AtlasKeycloakClient.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9e4cdcd20d..a20e0eed70 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - keycloaktest jobs: build: diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index a73fdcac8d..7bb2fbf934 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -81,6 +81,7 @@ public static ExecutorService getExecutorService(String namePattern) { public boolean isKeycloakSubjectsStoreUpdated(long cacheLastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getKeycloakSubjectsStoreUpdatedTime"); + cacheLastUpdatedTime = -1; if (cacheLastUpdatedTime == -1) { return true; } @@ -389,7 +390,7 @@ public RangerRole call() throws Exception { //get all groups for Roles Thread groupsFetcher = new Thread(() -> { int start = 0; - int size = AtlasConfiguration.KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE.getInt(); + int size = 2000; boolean found = true; Set ret = new HashSet<>(); @@ -416,7 +417,7 @@ public RangerRole call() throws Exception { //get all users for Roles Thread usersFetcher = new Thread(() -> { int start = 0; - int size = AtlasConfiguration.KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE.getInt(); + int size = 2000; boolean found = true; Set ret = new HashSet<>(); diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java index fe723bbce1..8a90fe966d 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java @@ -59,7 +59,7 @@ public List getRoleUserMembers(String roleName) throws Atlas public List getAllUsers() throws AtlasBaseException { int start = 0; - int size = 500; + int size = 2000; boolean found = true; List ret = new ArrayList<>(0); @@ -104,7 +104,7 @@ public void deleteRealmLevelRoleMappingsForGroup(String groupId, List getAllRoles() throws AtlasBaseException { int start = 0; - int size = 500; + int size = 2000; boolean found = true; List ret = new ArrayList<>(0); From 6412e6dbb9ec96b78c435b0ee50a7906d9058dca Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Dec 2023 13:21:23 +0530 Subject: [PATCH 06/16] Make it 500 --- .../java/org/apache/atlas/plugin/util/KeycloakUserStore.java | 4 ++-- .../org/apache/atlas/keycloak/client/AtlasKeycloakClient.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 7bb2fbf934..85017fc38b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -390,7 +390,7 @@ public RangerRole call() throws Exception { //get all groups for Roles Thread groupsFetcher = new Thread(() -> { int start = 0; - int size = 2000; + int size = 500; boolean found = true; Set ret = new HashSet<>(); @@ -417,7 +417,7 @@ public RangerRole call() throws Exception { //get all users for Roles Thread usersFetcher = new Thread(() -> { int start = 0; - int size = 2000; + int size = 500; boolean found = true; Set ret = new HashSet<>(); diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java index 8a90fe966d..fe723bbce1 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java @@ -59,7 +59,7 @@ public List getRoleUserMembers(String roleName) throws Atlas public List getAllUsers() throws AtlasBaseException { int start = 0; - int size = 2000; + int size = 500; boolean found = true; List ret = new ArrayList<>(0); @@ -104,7 +104,7 @@ public void deleteRealmLevelRoleMappingsForGroup(String groupId, List getAllRoles() throws AtlasBaseException { int start = 0; - int size = 2000; + int size = 500; boolean found = true; List ret = new ArrayList<>(0); From 13398fe8158f27f971828496d05af6aeada3bded Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:26:28 +0530 Subject: [PATCH 07/16] Revert "Revert "PLT-372: Add Heracles client in Metastore for fetching users efficiently"" --- auth-agents-common/pom.xml | 2 +- .../atlas/plugin/util/KeycloakUserStore.java | 12 +- {client-keycloak => client-auth}/pom.xml | 2 +- .../auth/client/auth/AbstractAuthClient.java | 53 ++++---- .../auth/KeycloakAuthenticationService.java | 28 ++--- .../atlas/auth/client/config/AuthConfig.java | 100 ++++++++++++++++ .../auth/client/config/AuthConfigBuilder.java | 55 +++++++++ .../client/heracles/AtlasHeraclesClient.java | 71 +++++++++++ .../client/heracles/HeraclesRestClient.java | 18 +++ .../heracles/RetrofitHeraclesClient.java | 16 +++ .../models/HeraclesUsersRepresentation.java | 113 ++++++++++++++++++ .../client/keycloak}/AtlasKeycloakClient.java | 58 +-------- .../client/keycloak}/KeycloakRestClient.java | 67 ++++++----- .../keycloak}/RetrofitKeycloakClient.java | 2 +- client-heracles/pom.xml | 75 ++++++++++++ .../client/config/HeraclesConfigBuilder.java | 5 +- .../client/config/KeycloakConfigBuilder.java | 52 -------- .../org/apache/atlas/AtlasConfiguration.java | 3 +- pom.xml | 2 +- repository/pom.xml | 2 +- .../preprocessor/ConnectionPreProcessor.java | 2 +- .../accesscontrol/PersonaPreProcessor.java | 2 +- .../sql/QueryCollectionPreProcessor.java | 2 +- .../repository/store/users/KeycloakStore.java | 3 +- .../apache/atlas/web/rest/MigrationREST.java | 2 +- .../AtlasKeycloakAuthenticationProvider.java | 2 +- 26 files changed, 550 insertions(+), 199 deletions(-) rename {client-keycloak => client-auth}/pom.xml (98%) rename client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java => client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java (69%) rename client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java => client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java (75%) create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java rename {client-keycloak/src/main/java/org/apache/atlas/keycloak/client => client-auth/src/main/java/org/apache/atlas/auth/client/keycloak}/AtlasKeycloakClient.java (70%) rename {client-keycloak/src/main/java/org/apache/atlas/keycloak/client => client-auth/src/main/java/org/apache/atlas/auth/client/keycloak}/KeycloakRestClient.java (54%) rename {client-keycloak/src/main/java/org/apache/atlas/keycloak/client => client-auth/src/main/java/org/apache/atlas/auth/client/keycloak}/RetrofitKeycloakClient.java (99%) create mode 100644 client-heracles/pom.xml rename client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java => client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java (84%) delete mode 100644 client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java diff --git a/auth-agents-common/pom.xml b/auth-agents-common/pom.xml index 7d63912e3d..aa37156d5f 100644 --- a/auth-agents-common/pom.xml +++ b/auth-agents-common/pom.xml @@ -51,7 +51,7 @@ org.apache.atlas - client-keycloak + client-auth ${project.version} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 85017fc38b..2af3e9a44f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -39,7 +39,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.heracles.AtlasHeraclesClient.getHeraclesClient; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; import static org.apache.atlas.repository.util.AccessControlUtils.BACKEND_SERVICE_USER_NAME; @@ -277,10 +278,7 @@ public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws Atlas } Map> userGroupMapping = new HashMap<>(); - - List kUsers = getKeycloakClient().getAllUsers(); - LOG.info("Found {} keycloak users", kUsers.size()); - + List kUsers = getHeraclesClient().getAllUsers(); List> callables = new ArrayList<>(); kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); @@ -417,13 +415,13 @@ public RangerRole call() throws Exception { //get all users for Roles Thread usersFetcher = new Thread(() -> { int start = 0; - int size = 500; + int size = 100; boolean found = true; Set ret = new HashSet<>(); do { try { - Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); + Set userRepresentations = getHeraclesClient().getRoleUserMembers(kRole.getName(), start, size); if (CollectionUtils.isNotEmpty(userRepresentations)) { ret.addAll(userRepresentations); start += size; diff --git a/client-keycloak/pom.xml b/client-auth/pom.xml similarity index 98% rename from client-keycloak/pom.xml rename to client-auth/pom.xml index b0a231b0c3..9087a98bed 100644 --- a/client-keycloak/pom.xml +++ b/client-auth/pom.xml @@ -27,7 +27,7 @@ 4.0.0 - client-keycloak + client-auth 8 diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java similarity index 69% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java index 86bdf6fbf7..cb13431384 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java @@ -1,13 +1,15 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.auth; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.Timer; +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.heracles.RetrofitHeraclesClient; +import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient; import okhttp3.*; import okhttp3.logging.HttpLoggingInterceptor; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; -import org.apache.atlas.keycloak.client.service.AtlasKeycloakAuthService; import org.apache.atlas.service.metrics.MetricUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,22 +28,23 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; -abstract class AbstractKeycloakClient { +public class AbstractAuthClient { - private final static Logger LOG = LoggerFactory.getLogger(AbstractKeycloakClient.class); + private final static Logger LOG = LoggerFactory.getLogger(AbstractAuthClient.class); private static final Map ERROR_CODE_MAP = new HashMap<>(); - private static final int DEFAULT_KEYCLOAK_RETRY = 3; + private static final int DEFAULT_RETRY = 3; private static final String AUTHORIZATION = "Authorization"; private static final String BEARER = "Bearer "; private static final int TIMEOUT_IN_SEC = 60; private static final String INTEGRATION = "integration"; private static final String KEYCLOAK = "keycloak"; - protected final KeycloakConfig keycloakConfig; - protected final RetrofitKeycloakClient retrofit; + protected final AuthConfig authConfig; + protected final RetrofitKeycloakClient retrofitKeycloakClient; + protected final RetrofitHeraclesClient retrofitHeraclesClient; - private final AtlasKeycloakAuthService authService; + private final KeycloakAuthenticationService authService; private MetricUtils metricUtils = null; static { @@ -49,8 +52,8 @@ abstract class AbstractKeycloakClient { ERROR_CODE_MAP.put(HTTP_BAD_REQUEST, BAD_REQUEST); } - public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { - this.keycloakConfig = keycloakConfig; + public AbstractAuthClient(AuthConfig authConfig) { + this.authConfig = authConfig; this.metricUtils = new MetricUtils(); HttpLoggingInterceptor httpInterceptor = new HttpLoggingInterceptor(); httpInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); @@ -64,11 +67,15 @@ public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { .writeTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) .readTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) .build(); - this.retrofit = new Retrofit.Builder().client(okHttpClient) - .baseUrl(this.keycloakConfig.getAuthServerUrl()) + this.retrofitKeycloakClient = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.authConfig.getAuthServerUrl()) .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() .create(RetrofitKeycloakClient.class); - authService = new AtlasKeycloakAuthService(keycloakConfig); + this.retrofitHeraclesClient = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.authConfig.getHeraclesApiServerUrl()) + .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper().disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES))).build() + .create(RetrofitHeraclesClient.class); + authService = new KeycloakAuthenticationService(authConfig); } /** @@ -97,21 +104,20 @@ public Response intercept(@NonNull Chain chain) throws IOException { return chain.proceed(request); } }; - /** * Called only during auth failures. */ Authenticator authInterceptor = new Authenticator() { @Override public Request authenticate(Route route, @NonNull Response response) { - if (responseCount(response) > DEFAULT_KEYCLOAK_RETRY) { - LOG.warn("Keycloak: Falling back, retried {} times", DEFAULT_KEYCLOAK_RETRY); + if (responseCount(response) > DEFAULT_RETRY) { + LOG.warn("Auth Client: Falling back, retried {} times", DEFAULT_RETRY); return null; } - LOG.info("Keycloak: Current keycloak token status, Expired: {}", authService.isTokenExpired()); + LOG.info("Auth Client: Current keycloak token status, Expired: {}", authService.isTokenExpired()); return response.request().newBuilder() - .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) - .build(); + .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) + .build(); } private int responseCount(Response response) { @@ -134,13 +140,14 @@ protected retrofit2.Response processResponse(retrofit2.Call req) throw return response; } String errMsg = response.errorBody().string(); - LOG.error("Keycloak: Client request processing failed code {} message:{}, request: {} {}", + LOG.error("Auth Client: Client request processing failed code {} message:{}, request: {} {}", response.code(), errMsg, req.request().method(), req.request().url()); throw new AtlasBaseException(ERROR_CODE_MAP.getOrDefault(response.code(), BAD_REQUEST), errMsg); } catch (Exception e) { - LOG.error("Keycloak: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); - throw new AtlasBaseException(BAD_REQUEST, "Keycloak request failed"); + LOG.error("Auth Client: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); + throw new AtlasBaseException(BAD_REQUEST, "Auth request failed"); } } + } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java similarity index 75% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java index fc4d40df7e..a3b3e9255e 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java @@ -1,11 +1,11 @@ -package org.apache.atlas.keycloak.client.service; +package org.apache.atlas.auth.client.auth; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.*; import okhttp3.logging.HttpLoggingInterceptor; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.RetrofitKeycloakClient; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient; import org.jetbrains.annotations.NotNull; import org.keycloak.representations.AccessTokenResponse; import org.slf4j.Logger; @@ -19,9 +19,9 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -public final class AtlasKeycloakAuthService { +public final class KeycloakAuthenticationService { - public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakAuthService.class); + public final static Logger LOG = LoggerFactory.getLogger(KeycloakAuthenticationService.class); private final static String GRANT_TYPE = "grant_type"; private static final String CLIENT_ID = "client_id"; @@ -30,14 +30,14 @@ public final class AtlasKeycloakAuthService { private static final int TIMEOUT_IN_SECS = 60; private final RetrofitKeycloakClient retrofit; - private final KeycloakConfig keycloakConfig; + private final AuthConfig authConfig; private AccessTokenResponse currentAccessToken; private long expirationTime = -1; - public AtlasKeycloakAuthService(KeycloakConfig keycloakConfig) { - this.keycloakConfig = keycloakConfig; + public KeycloakAuthenticationService(AuthConfig authConfig) { + this.authConfig = authConfig; this.retrofit = new Retrofit.Builder().client(getOkHttpClient()) - .baseUrl(this.keycloakConfig.getAuthServerUrl()) + .baseUrl(this.authConfig.getAuthServerUrl()) .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() .create(RetrofitKeycloakClient.class); } @@ -59,7 +59,7 @@ private OkHttpClient getOkHttpClient() { Interceptor responseLoggingInterceptor = chain -> { Request request = chain.request(); okhttp3.Response response = chain.proceed(request); - LOG.info("Keycloak: Auth Request for url {} Status: {}", request.url(), response.code()); + LOG.info("Auth Client: Auth Request for url {} Status: {}", request.url(), response.code()); return response; }; @@ -70,16 +70,16 @@ public String getAuthToken() { synchronized (this) { if (isTokenExpired()) { try { - retrofit2.Response resp = this.retrofit.grantToken(this.keycloakConfig.getRealmId(), getTokenRequest()).execute(); + retrofit2.Response resp = this.retrofit.grantToken(this.authConfig.getRealmId(), getTokenRequest()).execute(); if (resp.isSuccessful()) { currentAccessToken = resp.body(); expirationTime = currentTime() + currentAccessToken.getExpiresIn() - EXPIRY_OFFSET_SEC; - LOG.info("Keycloak: Auth token fetched with expiry:{} sec", expirationTime); + LOG.info("Auth Client: Auth token fetched with expiry:{} sec", expirationTime); } else { throw new AtlasBaseException(BAD_REQUEST, resp.errorBody().string()); } } catch (Exception e) { - LOG.error("Keycloak: Error while fetching access token for keycloak client.", e); + LOG.error("Auth Client: Error while fetching access token for keycloak client.", e); throw new RuntimeException(e); } } @@ -97,7 +97,7 @@ public boolean isTokenExpired() { } private RequestBody getTokenRequest() { - return new FormBody.Builder().addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()).addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.keycloakConfig.getGrantType()).build(); + return new FormBody.Builder().addEncoded(CLIENT_ID, this.authConfig.getClientId()).addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.authConfig.getGrantType()).build(); } private long currentTime() { diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java new file mode 100644 index 0000000000..33d98de049 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java @@ -0,0 +1,100 @@ +package org.apache.atlas.auth.client.config; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Optional; + +import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; + +public class AuthConfig { + private static final Logger LOG = LoggerFactory.getLogger(AuthConfig.class); + + public String authServerUrl; + public String realmId; + public String clientId; + public String clientSecret; + public String grantType; + public String heraclesApiServerUrl; + + private static final String KEYCLOAK_PROPERTIES = "keycloak.json"; + private static final String DEFAULT_GRANT_TYPE = "client_credentials"; + private static final String KEY_REALM_ID = "realm"; + private static final String KEY_AUTH_SERVER_URL = "auth-server-url"; + private static final String KEY_CLIENT_ID = "resource"; + private static final String KEY_CREDENTIALS = "credentials"; + private static final String KEY_SECRET = "secret"; + + public String getAuthServerUrl() { + return authServerUrl; + } + + public String getRealmId() { + return realmId; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getGrantType() { + return grantType; + } + + public String getHeraclesApiServerUrl() { + return heraclesApiServerUrl; + } + + public static AuthConfig getConfig() throws AtlasBaseException { + String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); + Optional confFile = getConfigurationFile(confLocation); + + if (confFile.isPresent()) { + try { + JSONObject object = new JSONObject(readFileToString(confFile.get())); + return buildAuthConfigFromJson(object); + } catch (Exception e) { + LOG.error("Error parsing Keycloak configuration: ", e); + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Error parsing Keycloak configuration"); + } + } else { + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); + } + } + + private static Optional getConfigurationFile(String confLocation) { + if (StringUtils.isNotEmpty(confLocation)) { + File confFile = new File(confLocation, KEYCLOAK_PROPERTIES); + if (confFile.exists()) { + return Optional.of(confFile); + } + } + return Optional.empty(); + } + + private static String readFileToString(File file) throws Exception { + return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static AuthConfig buildAuthConfigFromJson(JSONObject object) throws Exception { + String realmId = object.getString(KEY_REALM_ID); + String authServerUrl = object.getString(KEY_AUTH_SERVER_URL) + "/"; + String clientId = object.getString(KEY_CLIENT_ID); + String grantType = DEFAULT_GRANT_TYPE; + String clientSecret = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); + + LOG.info("Keycloak configuration: REALM_ID:{}, AUTH_SERVER_URL:{}", realmId, authServerUrl); + return AuthConfigBuilder.builder().realId(realmId).authServerUrl(authServerUrl).clientId(clientId).grantType(grantType).clientSecret(clientSecret).build(); + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java new file mode 100644 index 0000000000..552033d2c1 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java @@ -0,0 +1,55 @@ +package org.apache.atlas.auth.client.config; + +import org.apache.atlas.AtlasConfiguration; + +public class AuthConfigBuilder { + + private String authServerUrl; + private String realmId; + private String clientId; + private String clientSecret; + private String grantType = "client_credentials"; + + private AuthConfigBuilder() { + } + + public static AuthConfigBuilder builder() { + return new AuthConfigBuilder(); + } + + public AuthConfigBuilder authServerUrl(String authServerUrl) { + this.authServerUrl = authServerUrl; + return this; + } + + public AuthConfigBuilder realId(String realId) { + this.realmId = realId; + return this; + } + + public AuthConfigBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public AuthConfigBuilder clientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public AuthConfigBuilder grantType(String grantType) { + this.grantType = grantType; + return this; + } + + public AuthConfig build() { + AuthConfig authConfig = new AuthConfig(); + authConfig.authServerUrl = authServerUrl; + authConfig.realmId = realmId; + authConfig.clientId = clientId; + authConfig.clientSecret = clientSecret; + authConfig.grantType = grantType; + authConfig.heraclesApiServerUrl= AtlasConfiguration.HERACLES_API_SERVER_URL.getString()+"/"; + return authConfig; + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java new file mode 100644 index 0000000000..6f265486e6 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -0,0 +1,71 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; +import org.keycloak.representations.idm.UserRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class AtlasHeraclesClient { + public final static Logger LOG = LoggerFactory.getLogger(AtlasHeraclesClient.class); + + private static HeraclesRestClient HERACLES; + private static AtlasHeraclesClient HERACLES_CLIENT; + + public AtlasHeraclesClient() {} + + public static AtlasHeraclesClient getHeraclesClient() { + if(Objects.isNull(HERACLES_CLIENT)) { + LOG.info("Initializing Heracles client.."); + try{ + init(AuthConfig.getConfig()); + } catch (Exception e) { + LOG.error("Error initializing Heracles client", e); + } + } + return HERACLES_CLIENT; + } + + private static void init(AuthConfig authConfig) { + synchronized (AtlasHeraclesClient.class) { + if (HERACLES == null) { + HERACLES = new HeraclesRestClient(authConfig); + HERACLES_CLIENT = new AtlasHeraclesClient(); + } + } + } + + public List getAllUsers() throws AtlasBaseException { + int start = 0; + int size = 100; + boolean found = true; + + List ret = new ArrayList<>(0); + do { + + List userRepresentations = HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, null, HeraclesUsersRepresentation.USER_SORT).body().toKeycloakUserRepresentations(); + if (userRepresentations != null && !userRepresentations.isEmpty()) { + ret.addAll(userRepresentations); + start += size; + } else { + found = false; + } + } while (found && ret.size() % size == 0); + + return ret; + } + + + public Set getRoleUserMembers(String roleName, int start, int size) throws AtlasBaseException { + String template = "{\"$and\":[{\"roles\":{\"$elemMatch\":[\"{0}\"]}}]}"; + String filter = template.replace("{0}", roleName); + return HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, filter,HeraclesUsersRepresentation.USER_SORT ).body().toKeycloakUserRepresentations().stream().collect(Collectors.toSet()); + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java new file mode 100644 index 0000000000..a9adc68b7e --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -0,0 +1,18 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.auth.AbstractAuthClient; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; +import retrofit2.Response; + +public class HeraclesRestClient extends AbstractAuthClient { + + public HeraclesRestClient(final AuthConfig authConfig) { + super(authConfig); + } + public Response getUsers(int offset,int limit, String columns, String filter, String sort) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getUsers(offset, columns, filter, limit,sort)); + } + +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java new file mode 100644 index 0000000000..c35451852c --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -0,0 +1,16 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; +import retrofit2.Call; +import retrofit2.http.GET; +import retrofit2.http.Headers; +import retrofit2.http.Query; + +public interface RetrofitHeraclesClient { + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/users") + Call getUsers(@Query("offset") Integer offset, @Query("columns") String columns, + @Query(value = "filter", encoded = true) String filter, @Query("limit") Integer limit, + @Query("sort") String sort); + +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java new file mode 100644 index 0000000000..4cea34f509 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java @@ -0,0 +1,113 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.keycloak.representations.idm.UserRepresentation; + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesUsersRepresentation { + protected int totalRecord; + protected int filterRecord; + protected List records; + public static final String USER_PROJECTIONS = "emailVerified,enabled,id,status,username"; + public static final String USER_SORT = "username"; + + public HeraclesUsersRepresentation() { + } + + public HeraclesUsersRepresentation(int totalRecord, int filterRecord, List records) { + this.totalRecord = totalRecord; + this.filterRecord = filterRecord; + this.records = records; + } + + public int getTotalRecord() { + return totalRecord; + } + + public void setTotalRecord(int totalRecord) { + this.totalRecord = totalRecord; + } + + public int getFilterRecord() { + return filterRecord; + } + + public void setFilterRecord(int filterRecord) { + this.filterRecord = filterRecord; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + public List toKeycloakUserRepresentations() { + List userRepresentations = new ArrayList<>(); + for (HeraclesUserRepresentation heraclesUserRepresentation : records) { + UserRepresentation userRepresentation = new UserRepresentation(); + userRepresentation.setEmailVerified(heraclesUserRepresentation.emailVerified); + userRepresentation.setEnabled(heraclesUserRepresentation.enabled); + userRepresentation.setUsername(heraclesUserRepresentation.username); + userRepresentation.setId(heraclesUserRepresentation.id); + userRepresentations.add(userRepresentation); + } + return userRepresentations; + } +} + +@JsonIgnoreProperties(ignoreUnknown = true) +class HeraclesUserRepresentation { + protected boolean emailVerified; + protected boolean enabled; + protected String username; + protected String id; + + public HeraclesUserRepresentation() { + } + + public HeraclesUserRepresentation(boolean emailVerified, boolean enabled, String username, String id) { + this.emailVerified = emailVerified; + this.enabled = enabled; + this.username = username; + this.id = id; + } + + public boolean isEmailVerified() { + return emailVerified; + } + + public void setEmailVerified(boolean emailVerified) { + this.emailVerified = emailVerified; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java similarity index 70% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java index fe723bbce1..9ec43428e6 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java @@ -1,29 +1,19 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; -import org.apache.atlas.keycloak.client.config.KeycloakConfigBuilder; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; import org.keycloak.representations.idm.*; import org.keycloak.representations.oidc.TokenMetadataRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; -import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; - /** * Keycloak client, deals with token creation refresh. */ @@ -31,14 +21,6 @@ public final class AtlasKeycloakClient { public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakClient.class); - private final static String KEYCLOAK_PROPERTIES = "keycloak.json"; - private final static String DEFAULT_GRANT_TYPE = "client_credentials"; - private final static String KEY_REALM_ID = "realm"; - private final static String KEY_AUTH_SERVER_URL = "auth-server-url"; - private final static String KEY_CLIENT_ID = "resource"; - private final static String KEY_CREDENTIALS = "credentials"; - private final static String KEY_SECRET = "secret"; - private static KeycloakRestClient KEYCLOAK; private static AtlasKeycloakClient KEYCLOAK_CLIENT; @@ -183,13 +165,7 @@ public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException if (Objects.isNull(KEYCLOAK_CLIENT)) { LOG.info("Initializing Keycloak client.."); try { - init(getConfig()); - } catch (IOException e) { - LOG.error("Failed to fetch Keycloak conf {}", e.getMessage()); - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); - } catch (JSONException e) { - LOG.error("Failed to parse Keycloak conf {}", e.getMessage()); - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); + init(AuthConfig.getConfig()); } catch (Exception e) { LOG.error("Failed to connect to Keycloak {}", e.getMessage()); throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); @@ -201,7 +177,7 @@ public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException return KEYCLOAK_CLIENT; } - private static void init(KeycloakConfig config) { + private static void init(AuthConfig config) { synchronized (AtlasKeycloakClient.class) { if (KEYCLOAK_CLIENT == null) { KEYCLOAK = new KeycloakRestClient(config); @@ -209,30 +185,4 @@ private static void init(KeycloakConfig config) { } } } - - private static KeycloakConfig getConfig() throws Exception { - String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); - File confFile; - if (StringUtils.isNotEmpty(confLocation)) { - confFile = new File(confLocation, KEYCLOAK_PROPERTIES); - - if (confFile.exists()) { - String keyConf = new String(Files.readAllBytes(confFile.toPath()), StandardCharsets.UTF_8); - JSONObject object = new JSONObject(keyConf); - - String REALM_ID = object.getString(KEY_REALM_ID); - String AUTH_SERVER_URL = object.getString(KEY_AUTH_SERVER_URL) + "/"; - String CLIENT_ID = object.getString(KEY_CLIENT_ID); - String GRANT_TYPE = DEFAULT_GRANT_TYPE; - String CLIENT_SECRET = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); - - LOG.info("Keycloak conf: REALM_ID:{}, AUTH_SERVER_URL:{}", REALM_ID, AUTH_SERVER_URL); - return KeycloakConfigBuilder.builder().realId(REALM_ID).authServerUrl(AUTH_SERVER_URL).clientId(CLIENT_ID).grantType(GRANT_TYPE).clientSecret(CLIENT_SECRET).build(); - } else { - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); - } - } else { - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Configuration location not found " + confLocation); - } - } } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java similarity index 54% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java index 7cc1eea22f..b6f58982e6 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java @@ -1,9 +1,10 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; +import org.apache.atlas.auth.client.auth.AbstractAuthClient; import okhttp3.FormBody; import okhttp3.RequestBody; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; import org.keycloak.representations.idm.*; import org.keycloak.representations.oidc.TokenMetadataRepresentation; import retrofit2.Response; @@ -14,129 +15,129 @@ /** * Keycloak Rest client wrapper used in atlas metastore */ -public final class KeycloakRestClient extends AbstractKeycloakClient { +public final class KeycloakRestClient extends AbstractAuthClient { private static final String TOKEN = "token"; private static final String CLIENT_ID = "client_id"; private static final String CLIENT_SECRET = "client_secret"; - public KeycloakRestClient(final KeycloakConfig keycloakConfig) { - super(keycloakConfig); + public KeycloakRestClient(final AuthConfig authConfig) { + super(authConfig); } public Response> searchUserByUserName(String username) throws AtlasBaseException { - return processResponse(this.retrofit.searchUserByUserName(this.keycloakConfig.getRealmId(), username)); + return processResponse(this.retrofitKeycloakClient.searchUserByUserName(this.authConfig.getRealmId(), username)); } public Response> getAllUsers(int start, int size) throws AtlasBaseException { - return processResponse(this.retrofit.getAllUsers(this.keycloakConfig.getRealmId(), start, size)); + return processResponse(this.retrofitKeycloakClient.getAllUsers(this.authConfig.getRealmId(), start, size)); } public Response> getRoleUserMembers(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleUserMembers(this.authConfig.getRealmId(), roleName)); } public Response> getRoleUserMembers(String roleName, Integer start, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName, start, size)); + return processResponse(this.retrofitKeycloakClient.getRoleUserMembers(this.authConfig.getRealmId(), roleName, start, size)); } public Response> searchGroupByName(String groupName, Integer start, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.searchGroupByName(this.keycloakConfig.getRealmId(), groupName, start, size)); + return processResponse(this.retrofitKeycloakClient.searchGroupByName(this.authConfig.getRealmId(), groupName, start, size)); } public Response> getRoleGroupMembers(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleGroupMembers(this.authConfig.getRealmId(), roleName)); } public Response> getRoleGroupMembers(String roleName, Integer first, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName, first, size)); + return processResponse(this.retrofitKeycloakClient.getRoleGroupMembers(this.authConfig.getRealmId(), roleName, first, size)); } public Response> getGroupsForUserById(String userId) throws AtlasBaseException { - return processResponse(this.retrofit.getGroupsForUserById(this.keycloakConfig.getRealmId(), userId)); + return processResponse(this.retrofitKeycloakClient.getGroupsForUserById(this.authConfig.getRealmId(), userId)); } public void addRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.addRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + processResponse(this.retrofitKeycloakClient.addRealmLevelRoleMappingsForGroup(this.authConfig.getRealmId(), groupId, roles)); } public void deleteRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + processResponse(this.retrofitKeycloakClient.deleteRealmLevelRoleMappingsForGroup(this.authConfig.getRealmId(), groupId, roles)); } public Response> getAllRoles(int start, int size) throws AtlasBaseException { - return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), start, size)); + return processResponse(this.retrofitKeycloakClient.getAllRoles(this.authConfig.getRealmId(), start, size)); } public void deleteRoleById(String roleId) throws AtlasBaseException { - processResponse(this.retrofit.deleteRoleById(this.keycloakConfig.getRealmId(), roleId)); + processResponse(this.retrofitKeycloakClient.deleteRoleById(this.authConfig.getRealmId(), roleId)); } public void deleteRoleByName(String roleName) throws AtlasBaseException { - processResponse(this.retrofit.deleteRoleByName(this.keycloakConfig.getRealmId(), roleName)); + processResponse(this.retrofitKeycloakClient.deleteRoleByName(this.authConfig.getRealmId(), roleName)); } public Response> addRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { - return processResponse(this.retrofit.addRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + return processResponse(this.retrofitKeycloakClient.addRealmLevelRoleMappingsForUser(this.authConfig.getRealmId(), userId, roles)); } public void deleteRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + processResponse(this.retrofitKeycloakClient.deleteRealmLevelRoleMappingsForUser(this.authConfig.getRealmId(), userId, roles)); } public void createRole(RoleRepresentation roleRepresentation) throws AtlasBaseException { - processResponse(this.retrofit.createRole(this.keycloakConfig.getRealmId(), roleRepresentation)); + processResponse(this.retrofitKeycloakClient.createRole(this.authConfig.getRealmId(), roleRepresentation)); } public void updateRole(String roleId, RoleRepresentation roleRepresentation) throws AtlasBaseException { - processResponse(this.retrofit.updateRole(this.keycloakConfig.getRealmId(), roleId, roleRepresentation)); + processResponse(this.retrofitKeycloakClient.updateRole(this.authConfig.getRealmId(), roleId, roleRepresentation)); } public Response getRoleById(String roleId) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleById(this.keycloakConfig.getRealmId(), roleId)); + return processResponse(this.retrofitKeycloakClient.getRoleById(this.authConfig.getRealmId(), roleId)); } public Response getRoleByName(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleByName(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleByName(this.authConfig.getRealmId(), roleName)); } public Response> getAllRoles(Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), first, max)); + return processResponse(this.retrofitKeycloakClient.getAllRoles(this.authConfig.getRealmId(), first, max)); } public Response> getRoleComposites(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleComposites(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleComposites(this.authConfig.getRealmId(), roleName)); } public void addComposites(String roleName, List roles) throws AtlasBaseException { - processResponse(this.retrofit.addComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + processResponse(this.retrofitKeycloakClient.addComposites(this.authConfig.getRealmId(), roleName, roles)); } public void deleteComposites(String roleName, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + processResponse(this.retrofitKeycloakClient.deleteComposites(this.authConfig.getRealmId(), roleName, roles)); } public Response> getAdminEvents(List operationTypes, String authRealm, String authClient, String authUser, String authIpAddress, String resourcePath, String dateFrom, String dateTo, Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getAdminEvents(this.keycloakConfig.getRealmId(), operationTypes, + return processResponse(this.retrofitKeycloakClient.getAdminEvents(this.authConfig.getRealmId(), operationTypes, authRealm, authClient, authUser, authIpAddress, resourcePath, dateFrom, dateTo, first, max)); } public Response> getEvents(List type, String client, String user, String dateFrom, String dateTo, String ipAddress, Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getEvents(this.keycloakConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); + return processResponse(this.retrofitKeycloakClient.getEvents(this.authConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); } public Response introspectToken(String token) throws AtlasBaseException { - return processResponse(this.retrofit.introspectToken(this.keycloakConfig.getRealmId(), getIntrospectTokenRequest(token))); + return processResponse(this.retrofitKeycloakClient.introspectToken(this.authConfig.getRealmId(), getIntrospectTokenRequest(token))); } private RequestBody getIntrospectTokenRequest(String token) { return new FormBody.Builder() .addEncoded(TOKEN, token) - .addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()) - .addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()) + .addEncoded(CLIENT_ID, this.authConfig.getClientId()) + .addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()) .build(); } } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java similarity index 99% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java index c396c9368d..4fe8bb6ff5 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java @@ -1,4 +1,4 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; import okhttp3.RequestBody; import org.keycloak.representations.AccessTokenResponse; diff --git a/client-heracles/pom.xml b/client-heracles/pom.xml new file mode 100644 index 0000000000..b812aefb65 --- /dev/null +++ b/client-heracles/pom.xml @@ -0,0 +1,75 @@ + + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + client-heracles + + + 8 + 8 + + + + + org.keycloak + keycloak-core + ${keycloak-admin-client.version} + + + * + * + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.retrofit2 + retrofit + ${retrofit.version} + + + com.squareup.retrofit2 + converter-jackson + ${retrofit.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + org.apache.atlas + atlas-common + + + + \ No newline at end of file diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java b/client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java similarity index 84% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java rename to client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java index 048d2fac71..863eec0021 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java +++ b/client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java @@ -1,6 +1,6 @@ -package org.apache.atlas.keycloak.client.config; +package main.java.heracles.client.config; -public final class KeycloakConfig { +public class HeraclesConfigBuilder { String authServerUrl; String realmId; @@ -27,5 +27,4 @@ public String getClientSecret() { public String getGrantType() { return grantType; } - } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java deleted file mode 100644 index 44583c59f3..0000000000 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.apache.atlas.keycloak.client.config; - -public final class KeycloakConfigBuilder { - - private String authServerUrl; - private String realmId; - private String clientId; - private String clientSecret; - private String grantType = "client_credentials"; - - private KeycloakConfigBuilder() { - } - - public static KeycloakConfigBuilder builder() { - return new KeycloakConfigBuilder(); - } - - public KeycloakConfigBuilder authServerUrl(String authServerUrl) { - this.authServerUrl = authServerUrl; - return this; - } - - public KeycloakConfigBuilder realId(String realId) { - this.realmId = realId; - return this; - } - - public KeycloakConfigBuilder clientId(String clientId) { - this.clientId = clientId; - return this; - } - - public KeycloakConfigBuilder clientSecret(String clientSecret) { - this.clientSecret = clientSecret; - return this; - } - - public KeycloakConfigBuilder grantType(String grantType) { - this.grantType = grantType; - return this; - } - - public KeycloakConfig build() { - KeycloakConfig keycloakConfig = new KeycloakConfig(); - keycloakConfig.authServerUrl = authServerUrl; - keycloakConfig.realmId = realmId; - keycloakConfig.clientId = clientId; - keycloakConfig.clientSecret = clientSecret; - keycloakConfig.grantType = grantType; - return keycloakConfig; - } -} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 889d5b413f..d10e4b13fc 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -108,7 +108,8 @@ public enum AtlasConfiguration { PERSONA_POLICY_ASSET_MAX_LIMIT("atlas.persona.policy.asset.maxlimit", 1000), ENABLE_KEYCLOAK_TOKEN_INTROSPECTION("atlas.canary.keycloak.token-introspection", false), - KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE("atlas.keycloak.admin.resource-pagination-size", 1500); + KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE("atlas.keycloak.admin.resource-pagination-size", 1500), + HERACLES_API_SERVER_URL("atlas.heracles.api.server.url", "http://heracles-service.heracles.svc.cluster.local"); private static final Configuration APPLICATION_PROPERTIES; diff --git a/pom.xml b/pom.xml index 6ffea247db..1cc9aa70dc 100644 --- a/pom.xml +++ b/pom.xml @@ -790,7 +790,7 @@ server-api notification client - client-keycloak + client-auth graphdb repository diff --git a/repository/pom.xml b/repository/pom.xml index 03e66a8310..10d8d876fb 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -318,7 +318,7 @@ org.apache.atlas - client-keycloak + client-auth 3.0.0-SNAPSHOT diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index b994398c23..02fb63bbc8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -55,7 +55,6 @@ import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; @@ -63,6 +62,7 @@ import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class ConnectionPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index f579dc4ff7..77078f5c53 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -20,7 +20,7 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java index 63e8ba4250..5dec4d3190 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java @@ -58,7 +58,7 @@ import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; import static org.apache.atlas.repository.Constants.ATTR_VIEWER_GROUPS; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java index 3a5f1782f0..22e04cd751 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java @@ -33,8 +33,7 @@ import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; -import static org.apache.atlas.repository.util.AccessControlUtils.INSTANCE_DOMAIN_KEY; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class KeycloakStore { private static final Logger LOG = LoggerFactory.getLogger(KeycloakStore.class); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index b1cb307550..b491bb88ce 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -33,7 +33,7 @@ import java.util.*; import java.util.stream.Collectors; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.*; @Path("migration") diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java index 88523998cd..1a22038398 100644 --- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java +++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java @@ -18,7 +18,7 @@ import io.micrometer.core.instrument.Counter; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.service.metrics.MetricUtils; import org.apache.atlas.ApplicationProperties; import org.apache.commons.configuration.Configuration; From 87ba9fe0ec26933716a72afd9bfdc4605344b53a Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 14 Dec 2023 00:06:04 +0530 Subject: [PATCH 08/16] feat: add Heracles based fetcher --- .../atlas/plugin/util/KeycloakUserStore.java | 108 +++++++++++++++++- .../plugin/util/RangerRolesProvider.java | 2 +- .../plugin/util/RangerUserStoreProvider.java | 2 +- .../client/heracles/AtlasHeraclesClient.java | 19 +++ .../client/heracles/HeraclesRestClient.java | 16 ++- .../heracles/RetrofitHeraclesClient.java | 12 ++ .../HeraclesRoleViewRepresentation.java | 60 ++++++++++ .../HeraclesUserViewRepresentation.java | 61 ++++++++++ .../org/apache/atlas/web/rest/AuthREST.java | 4 +- 9 files changed, 276 insertions(+), 8 deletions(-) create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java create mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 2af3e9a44f..dcd05bb295 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -20,8 +20,8 @@ package org.apache.atlas.plugin.util; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.plugin.model.RangerRole; import org.apache.atlas.utils.AtlasPerfMetrics; @@ -189,6 +189,108 @@ public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseExce return rangerRoles; } + public RangerUserStore loadUsersNew(long lastUpdatedTime) throws AtlasBaseException { + int userSize = 100; + int userFrom = 0; + boolean userFound = true; + Map> userGroupMapping = new HashMap<>(); + + List ret = new ArrayList<>(); + do { + List users = getHeraclesClient().getUsersView(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { + userFound = false; + } else { + ret.addAll(users); + userFrom += userSize; + } + for(UserRepresentation user : users) { + userGroupMapping.put(user.getUsername(), new HashSet<>(user.getGroups() == null ? Collections.emptyList() : user.getGroups())); + } + + } while (userFound && ret.size() % userSize == 0); + + RangerUserStore userStore = new RangerUserStore(); + userStore.setUserGroupMapping(userGroupMapping); + Date current = new Date(); + userStore.setUserStoreUpdateTime(current); + userStore.setServiceName(serviceName); + userStore.setUserStoreVersion(-1L); + return userStore; + } + + public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException { + RangerRoles rangerRoles = new RangerRoles(); + Map> roleUserMapping = new HashMap<>(); + Set roleSet = new HashSet<>(); + + int userSize = 100; + int userFrom = 0; + boolean userFound = true; + + List ret = new ArrayList<>(); + do { + List users = getHeraclesClient().getUsersView(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { + userFound = false; + } else { + ret.addAll(users); + userFrom += userSize; + } + for(UserRepresentation user : users) { + Set userRoles = new HashSet<>(user.getRealmRoles()); + for(String role : userRoles) { + List roleMembers = roleUserMapping.get(role); + if(roleMembers == null) { + roleMembers = new ArrayList<>(); + roleUserMapping.put(role, roleMembers); + } + roleMembers.add(new RangerRole.RoleMember(user.getUsername(), false)); + } + } + + } while (userFound && ret.size() % userSize == 0); + + int roleSize = 100; + int roleFrom = 0; + boolean roleFound = true; + + List retRole = new ArrayList<>(); + do { + List roles = getHeraclesClient().getRolesView(roleFrom, roleSize); + if (CollectionUtils.isEmpty(roles)) { + roleFound = false; + } else { + retRole.addAll(roles); + roleFrom += roleSize; + } + + for(HeraclesRoleViewRepresentation role : roles) { + RangerRole rangerRole = new RangerRole(); + rangerRole.setName(role.getName()); + rangerRole.setGroups(role.getGroups().stream().map(x -> new RangerRole.RoleMember(x, true)).collect(Collectors.toList())); + rangerRole.setUsers(roleUserMapping.get(role.getName())); + rangerRole.setRoles(role.getRoles().stream().map(x -> new RangerRole.RoleMember(x, false)).collect(Collectors.toList())); + + roleSet.add(rangerRole); + } + + } while (roleFound && retRole.size() % roleSize == 0); + + processDefaultRole(roleSet); + LOG.info("Inverting roles"); + invertRoles(roleSet); + + rangerRoles.setRangerRoles(roleSet); + rangerRoles.setServiceName(serviceName); + + Date current = new Date(); + rangerRoles.setRoleUpdateTime(current); + rangerRoles.setServiceName(serviceName); + rangerRoles.setRoleVersion(-1L); + return rangerRoles; + } + public void invertRoles(Set roleSet) { Map roleMap = new HashMap<>(); for (RangerRole role : roleSet) { @@ -278,7 +380,7 @@ public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws Atlas } Map> userGroupMapping = new HashMap<>(); - List kUsers = getHeraclesClient().getAllUsers(); + List kUsers = getKeycloakClient().getAllUsers(); List> callables = new ArrayList<>(); kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); @@ -421,7 +523,7 @@ public RangerRole call() throws Exception { do { try { - Set userRepresentations = getHeraclesClient().getRoleUserMembers(kRole.getName(), start, size); + Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); if (CollectionUtils.isNotEmpty(userRepresentations)) { ret.addAll(userRepresentations); start += size; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java index 4607d6192a..c26681952a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java @@ -195,7 +195,7 @@ private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundEx try { if ("atlas".equals(serviceName)) { - roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTimeInMillis); + roles = keycloakUserStore.loadRolesNew(lastUpdatedTimeInMillis); } else { roles = atlasAuthAdminClient.getRolesIfUpdated(lastUpdatedTimeInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java index 89980504b0..abed31ef03 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -196,7 +196,7 @@ private RangerUserStore loadUserStoreFromAdmin() throws RangerServiceNotFoundExc try { if ("atlas".equals(serviceName)) { - userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdateTimeInMillis); + userStore = keycloakUserStore.loadUsersNew(lastUpdateTimeInMillis); } else { userStore = atlasAuthAdminClient.getUserStoreIfUpdated(lastUpdateTimeInMillis); } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java index 6f265486e6..e178915e37 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -1,6 +1,8 @@ package org.apache.atlas.auth.client.heracles; import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import org.keycloak.representations.idm.UserRepresentation; @@ -68,4 +70,21 @@ public Set getRoleUserMembers(String roleName, int start, in String filter = template.replace("{0}", roleName); return HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, filter,HeraclesUsersRepresentation.USER_SORT ).body().toKeycloakUserRepresentations().stream().collect(Collectors.toSet()); } + + public List getUsersView(int start, int size) throws AtlasBaseException { + List views = HERACLES.getUsersView(start, size, HeraclesUserViewRepresentation.sortBy).body(); + List userRepresentations = views.stream().map(x -> { + UserRepresentation userRepresentation = new UserRepresentation(); + userRepresentation.setId(x.getId()); + userRepresentation.setUsername(x.getUsername()); + userRepresentation.setRealmRoles(x.getRoles()); + userRepresentation.setGroups(x.getGroups()); + return userRepresentation; + }).collect(Collectors.toList()); + return userRepresentations; + } + + public List getRolesView(int start, int size) throws AtlasBaseException { + return HERACLES.getRolesView(start, size, HeraclesRoleViewRepresentation.sortBy).body(); + } } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java index a9adc68b7e..0e76b0e4dd 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -2,10 +2,14 @@ import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.auth.client.auth.AbstractAuthClient; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import retrofit2.Response; +import java.util.List; + public class HeraclesRestClient extends AbstractAuthClient { public HeraclesRestClient(final AuthConfig authConfig) { @@ -15,4 +19,14 @@ public Response getUsers(int offset,int limit, Stri return processResponse(this.retrofitHeraclesClient.getUsers(offset, columns, filter, limit,sort)); } -} + public Response> getUsersView(int offset, int limit, String sort) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getUsersView(offset, limit,sort)); + } + + public Response> getRolesView(int offset, int limit, String sort) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getRolesView(offset, limit,sort)); + } + + + +} \ No newline at end of file diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java index c35451852c..f32f305495 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -1,11 +1,15 @@ package org.apache.atlas.auth.client.heracles; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import retrofit2.Call; import retrofit2.http.GET; import retrofit2.http.Headers; import retrofit2.http.Query; +import java.util.List; + public interface RetrofitHeraclesClient { @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) @GET("/users") @@ -13,4 +17,12 @@ Call getUsers(@Query("offset") Integer offset, @Que @Query(value = "filter", encoded = true) String filter, @Query("limit") Integer limit, @Query("sort") String sort); + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/users/view") + Call> getUsersView(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort); + + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/roles/view") + Call> getRolesView(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort); + } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java new file mode 100644 index 0000000000..5b695bcf71 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java @@ -0,0 +1,60 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesRoleViewRepresentation { + protected String id; + protected String name; + protected String realmId; + protected List roles; + protected List groups; + + public static String sortBy = "name"; + + public HeraclesRoleViewRepresentation() { + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getRealmId() { + return realmId; + } + + public List getRoles() { + return roles == null ? new ArrayList<>() : roles; + } + + public List getGroups() { + return groups == null ? new ArrayList<>() : groups; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setRealmId(String realmId) { + this.realmId = realmId; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setGroups(List groups) { + this.groups = groups; + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java new file mode 100644 index 0000000000..de3bc1e105 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java @@ -0,0 +1,61 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.keycloak.representations.idm.UserRepresentation; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesUserViewRepresentation { + protected String id; + protected String username; + protected boolean enabled; + protected List roles; + protected List groups; + + public static String sortBy = "username"; + + public HeraclesUserViewRepresentation() { + } + + public String getId() { + return id; + } + + public String getUsername() { + return username; + } + + public boolean isEnabled() { + return enabled; + } + + public List getRoles() { + return roles; + } + + public List getGroups() { + return groups; + } + + public void setId(String id) { + this.id = id; + } + + public void setUsername(String username) { + this.username = username; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setGroups(List groups) { + this.groups = groups; + } + +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c5868cbfa3..f2f7ac2167 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -100,7 +100,7 @@ public RangerRoles downloadRoles(@PathParam("serviceName") final String serviceN } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerRoles roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTime); + RangerRoles roles = keycloakUserStore.loadRolesNew(lastUpdatedTime); if (roles == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); @@ -127,7 +127,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerUserStore userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdatedTime); + RangerUserStore userStore = keycloakUserStore.loadUsersNew(lastUpdatedTime); if (userStore == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); From 589810c0dffa6950d0beff5650e200fc64b0e01c Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:00:55 +0530 Subject: [PATCH 09/16] feat: add user fetching from Heracles --- .github/workflows/maven.yml | 2 +- .../atlas/plugin/util/KeycloakUserStore.java | 123 ++++++------------ .../plugin/util/RangerRolesProvider.java | 2 +- .../plugin/util/RangerUserStoreProvider.java | 2 +- .../client/heracles/AtlasHeraclesClient.java | 10 +- .../client/heracles/HeraclesRestClient.java | 10 +- .../heracles/RetrofitHeraclesClient.java | 11 ++ .../org/apache/atlas/web/rest/AuthREST.java | 4 +- 8 files changed, 66 insertions(+), 98 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a20e0eed70..9e4cdcd20d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - keycloaktest + - lineageondemand jobs: build: diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index dcd05bb295..51f1ddf849 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -158,68 +158,6 @@ public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseExce if (!isKeycloakUpdated) { return null; } - - List kRoles = getKeycloakClient().getAllRoles(); - LOG.info("Found {} keycloak roles", kRoles.size()); - - Set roleSet = new HashSet<>(); - RangerRoles rangerRoles = new RangerRoles(); - List userNamesList = new ArrayList<>(); - - submitCallablesAndWaitToFinish("RoleSubjectsFetcher", - kRoles.stream() - .map(x -> new RoleSubjectsFetcher(x, roleSet, userNamesList)) - .collect(Collectors.toList())); - - processDefaultRole(roleSet); - - LOG.info("Inverting roles"); - invertRoles(roleSet); - - rangerRoles.setRangerRoles(roleSet); - rangerRoles.setServiceName(serviceName); - - Date current = new Date(); - rangerRoles.setRoleUpdateTime(current); - rangerRoles.setServiceName(serviceName); - rangerRoles.setRoleVersion(-1L); - - RequestContext.get().endMetricRecord(recorder); - - return rangerRoles; - } - - public RangerUserStore loadUsersNew(long lastUpdatedTime) throws AtlasBaseException { - int userSize = 100; - int userFrom = 0; - boolean userFound = true; - Map> userGroupMapping = new HashMap<>(); - - List ret = new ArrayList<>(); - do { - List users = getHeraclesClient().getUsersView(userFrom, userSize); - if (CollectionUtils.isEmpty(users)) { - userFound = false; - } else { - ret.addAll(users); - userFrom += userSize; - } - for(UserRepresentation user : users) { - userGroupMapping.put(user.getUsername(), new HashSet<>(user.getGroups() == null ? Collections.emptyList() : user.getGroups())); - } - - } while (userFound && ret.size() % userSize == 0); - - RangerUserStore userStore = new RangerUserStore(); - userStore.setUserGroupMapping(userGroupMapping); - Date current = new Date(); - userStore.setUserStoreUpdateTime(current); - userStore.setServiceName(serviceName); - userStore.setUserStoreVersion(-1L); - return userStore; - } - - public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException { RangerRoles rangerRoles = new RangerRoles(); Map> roleUserMapping = new HashMap<>(); Set roleSet = new HashSet<>(); @@ -230,22 +168,21 @@ public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException List ret = new ArrayList<>(); do { - List users = getHeraclesClient().getUsersView(userFrom, userSize); + List users = getHeraclesClient().getUsersMappings(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { userFound = false; } else { ret.addAll(users); userFrom += userSize; } - for(UserRepresentation user : users) { + + for (UserRepresentation user : users) { Set userRoles = new HashSet<>(user.getRealmRoles()); - for(String role : userRoles) { - List roleMembers = roleUserMapping.get(role); - if(roleMembers == null) { - roleMembers = new ArrayList<>(); - roleUserMapping.put(role, roleMembers); - } - roleMembers.add(new RangerRole.RoleMember(user.getUsername(), false)); + + for (String role : userRoles) { + roleUserMapping.computeIfAbsent(role, k -> new ArrayList<>()) + .add(new RangerRole.RoleMember(user.getUsername(), false)); } } @@ -257,7 +194,9 @@ public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException List retRole = new ArrayList<>(); do { - List roles = getHeraclesClient().getRolesView(roleFrom, roleSize); + + List roles = getHeraclesClient().getRolesMappings(roleFrom, roleSize); + if (CollectionUtils.isEmpty(roles)) { roleFound = false; } else { @@ -265,7 +204,7 @@ public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException roleFrom += roleSize; } - for(HeraclesRoleViewRepresentation role : roles) { + for (HeraclesRoleViewRepresentation role : roles) { RangerRole rangerRole = new RangerRole(); rangerRole.setName(role.getName()); rangerRole.setGroups(role.getGroups().stream().map(x -> new RangerRole.RoleMember(x, true)).collect(Collectors.toList())); @@ -288,9 +227,18 @@ public RangerRoles loadRolesNew(long lastUpdatedTime) throws AtlasBaseException rangerRoles.setRoleUpdateTime(current); rangerRoles.setServiceName(serviceName); rangerRoles.setRoleVersion(-1L); + + RequestContext.get().endMetricRecord(recorder); + return rangerRoles; } + private void extractUserGroupMapping(List users, Map> userGroupMapping) { + for (UserRepresentation user : users) { + userGroupMapping.put(user.getUsername(), new HashSet<>(user.getGroups() == null ? Collections.emptyList() : user.getGroups())); + } + } + public void invertRoles(Set roleSet) { Map roleMap = new HashMap<>(); for (RangerRole role : roleSet) { @@ -373,28 +321,33 @@ private void processDefaultRole(Set roleSet) { public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("loadUserStoreIfUpdated"); - - boolean isKeycloakUpdated = isKeycloakSubjectsStoreUpdated(lastUpdatedTime); - if (!isKeycloakUpdated) { - return null; - } - + int userSize = 100; + int userFrom = 0; + boolean userFound = true; Map> userGroupMapping = new HashMap<>(); - List kUsers = getKeycloakClient().getAllUsers(); - List> callables = new ArrayList<>(); - kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); + List ret = new ArrayList<>(); + + do { + List users = getHeraclesClient().getUsersMappings(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { + userFound = false; + } else { + ret.addAll(users); + userFrom += userSize; + } + extractUserGroupMapping(users, userGroupMapping); - submitCallablesAndWaitToFinish("UserGroupsFetcher", callables); + } while (userFound && ret.size() % userSize == 0); RangerUserStore userStore = new RangerUserStore(); userStore.setUserGroupMapping(userGroupMapping); - Date current = new Date(); - userStore.setUserStoreUpdateTime(current); + userStore.setUserStoreUpdateTime(new Date()); userStore.setServiceName(serviceName); userStore.setUserStoreVersion(-1L); RequestContext.get().endMetricRecord(recorder); + return userStore; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java index c26681952a..4607d6192a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java @@ -195,7 +195,7 @@ private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundEx try { if ("atlas".equals(serviceName)) { - roles = keycloakUserStore.loadRolesNew(lastUpdatedTimeInMillis); + roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTimeInMillis); } else { roles = atlasAuthAdminClient.getRolesIfUpdated(lastUpdatedTimeInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java index abed31ef03..89980504b0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -196,7 +196,7 @@ private RangerUserStore loadUserStoreFromAdmin() throws RangerServiceNotFoundExc try { if ("atlas".equals(serviceName)) { - userStore = keycloakUserStore.loadUsersNew(lastUpdateTimeInMillis); + userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdateTimeInMillis); } else { userStore = atlasAuthAdminClient.getUserStoreIfUpdated(lastUpdateTimeInMillis); } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java index e178915e37..44e11660a9 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -71,8 +71,9 @@ public Set getRoleUserMembers(String roleName, int start, in return HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, filter,HeraclesUsersRepresentation.USER_SORT ).body().toKeycloakUserRepresentations().stream().collect(Collectors.toSet()); } - public List getUsersView(int start, int size) throws AtlasBaseException { - List views = HERACLES.getUsersView(start, size, HeraclesUserViewRepresentation.sortBy).body(); + public List getUsersMappings(int start, int size) throws AtlasBaseException { + String[] columns = {"roles", "groups"}; + List views = HERACLES.getUsersMappings(start, size, HeraclesUserViewRepresentation.sortBy, columns).body(); List userRepresentations = views.stream().map(x -> { UserRepresentation userRepresentation = new UserRepresentation(); userRepresentation.setId(x.getId()); @@ -84,7 +85,8 @@ public List getUsersView(int start, int size) throws AtlasBa return userRepresentations; } - public List getRolesView(int start, int size) throws AtlasBaseException { - return HERACLES.getRolesView(start, size, HeraclesRoleViewRepresentation.sortBy).body(); + public List getRolesMappings(int start, int size) throws AtlasBaseException { + String[] columns = {"composite_roles","groups"}; + return HERACLES.getRolesMappings(start, size, HeraclesRoleViewRepresentation.sortBy, columns).body(); } } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java index 0e76b0e4dd..a247c4eb82 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -19,14 +19,16 @@ public Response getUsers(int offset,int limit, Stri return processResponse(this.retrofitHeraclesClient.getUsers(offset, columns, filter, limit,sort)); } - public Response> getUsersView(int offset, int limit, String sort) throws AtlasBaseException { - return processResponse(this.retrofitHeraclesClient.getUsersView(offset, limit,sort)); + public Response> getUsersMappings(int offset, int limit, String sort, String[] columns) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getUsersMapping(offset, limit,sort, columns)); } - public Response> getRolesView(int offset, int limit, String sort) throws AtlasBaseException { - return processResponse(this.retrofitHeraclesClient.getRolesView(offset, limit,sort)); + public Response> getRolesMappings(int offset, int limit, String sort, String[] columns) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getRolesMapping(offset, limit, sort, columns)); } + + } \ No newline at end of file diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java index f32f305495..09bdee0626 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -25,4 +25,15 @@ Call getUsers(@Query("offset") Integer offset, @Que @GET("/roles/view") Call> getRolesView(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort); + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/users/mappings") + Call> getUsersMapping(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort, + @Query("columns") String[] columns); + + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/roles/mappings") + Call> getRolesMapping(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort, + @Query("columns") String[] columns); + + } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index f2f7ac2167..c5868cbfa3 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -100,7 +100,7 @@ public RangerRoles downloadRoles(@PathParam("serviceName") final String serviceN } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerRoles roles = keycloakUserStore.loadRolesNew(lastUpdatedTime); + RangerRoles roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTime); if (roles == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); @@ -127,7 +127,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerUserStore userStore = keycloakUserStore.loadUsersNew(lastUpdatedTime); + RangerUserStore userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdatedTime); if (userStore == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); From 51a31f030a4c72e3ae3eac6e1aad775fa174e58c Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:14:32 +0530 Subject: [PATCH 10/16] fix: use existing functions to fetch roles, users --- .../org/apache/atlas/plugin/util/RangerRolesProvider.java | 2 +- .../org/apache/atlas/plugin/util/RangerUserStoreProvider.java | 2 +- webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java index c26681952a..4607d6192a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerRolesProvider.java @@ -195,7 +195,7 @@ private RangerRoles loadUserGroupRolesFromAdmin() throws RangerServiceNotFoundEx try { if ("atlas".equals(serviceName)) { - roles = keycloakUserStore.loadRolesNew(lastUpdatedTimeInMillis); + roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTimeInMillis); } else { roles = atlasAuthAdminClient.getRolesIfUpdated(lastUpdatedTimeInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java index abed31ef03..89980504b0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -196,7 +196,7 @@ private RangerUserStore loadUserStoreFromAdmin() throws RangerServiceNotFoundExc try { if ("atlas".equals(serviceName)) { - userStore = keycloakUserStore.loadUsersNew(lastUpdateTimeInMillis); + userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdateTimeInMillis); } else { userStore = atlasAuthAdminClient.getUserStoreIfUpdated(lastUpdateTimeInMillis); } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index f2f7ac2167..c5868cbfa3 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -100,7 +100,7 @@ public RangerRoles downloadRoles(@PathParam("serviceName") final String serviceN } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerRoles roles = keycloakUserStore.loadRolesNew(lastUpdatedTime); + RangerRoles roles = keycloakUserStore.loadRolesIfUpdated(lastUpdatedTime); if (roles == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); @@ -127,7 +127,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String } KeycloakUserStore keycloakUserStore = new KeycloakUserStore(serviceName); - RangerUserStore userStore = keycloakUserStore.loadUsersNew(lastUpdatedTime); + RangerUserStore userStore = keycloakUserStore.loadUserStoreIfUpdated(lastUpdatedTime); if (userStore == null) { response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); From c3ca255d4aca899637a13c2b113f284f3587cffb Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:33:44 +0530 Subject: [PATCH 11/16] Update maven.yml --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a20e0eed70..9e4cdcd20d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - keycloaktest + - lineageondemand jobs: build: From e838fcf9fe3883be2934c1db72322594e3898016 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:13:27 +0530 Subject: [PATCH 12/16] feat: refactor few functions and remove some unused func. deps --- .../atlas/plugin/util/KeycloakUserStore.java | 294 +++--------------- .../plugin/util/RangerUserStoreProvider.java | 2 - .../org/apache/atlas/AtlasConfiguration.java | 2 +- 3 files changed, 39 insertions(+), 259 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 51f1ddf849..2f9470c3fc 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -19,7 +19,7 @@ package org.apache.atlas.plugin.util; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; @@ -33,10 +33,6 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; @@ -73,13 +69,6 @@ public KeycloakUserStore(String serviceName) { } } - public static ExecutorService getExecutorService(String namePattern) { - ExecutorService service = Executors.newFixedThreadPool(NUM_THREADS, - new ThreadFactoryBuilder().setNameFormat(namePattern + Thread.currentThread().getName()) - .build()); - return service; - } - public boolean isKeycloakSubjectsStoreUpdated(long cacheLastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getKeycloakSubjectsStoreUpdatedTime"); cacheLastUpdatedTime = -1; @@ -162,59 +151,55 @@ public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseExce Map> roleUserMapping = new HashMap<>(); Set roleSet = new HashSet<>(); - int userSize = 100; + int userSize = AtlasConfiguration.HERACLES_CLIENT_PAGINATION_SIZE.getInt(); int userFrom = 0; - boolean userFound = true; + List userRetrievalResult; - List ret = new ArrayList<>(); do { - List users = getHeraclesClient().getUsersMappings(userFrom, userSize); + userRetrievalResult = getHeraclesClient().getUsersMappings(userFrom, userSize); - if (CollectionUtils.isEmpty(users)) { - userFound = false; - } else { - ret.addAll(users); - userFrom += userSize; - } + if (!CollectionUtils.isEmpty(userRetrievalResult)) { + userRetrievalResult.forEach(user -> { + Set userRoles = new HashSet<>(user.getRealmRoles()); - for (UserRepresentation user : users) { - Set userRoles = new HashSet<>(user.getRealmRoles()); + userRoles.forEach(role -> roleUserMapping + .computeIfAbsent(role, k -> new ArrayList<>()) + .add(new RangerRole.RoleMember(user.getUsername(), false)) + ); + }); - for (String role : userRoles) { - roleUserMapping.computeIfAbsent(role, k -> new ArrayList<>()) - .add(new RangerRole.RoleMember(user.getUsername(), false)); - } + userFrom += userSize; } - } while (userFound && ret.size() % userSize == 0); + } while (!CollectionUtils.isEmpty(userRetrievalResult) && userRetrievalResult.size() % userSize == 0); - int roleSize = 100; + int roleSize = AtlasConfiguration.HERACLES_CLIENT_PAGINATION_SIZE.getInt(); int roleFrom = 0; - boolean roleFound = true; + List roleRetrievalResult; - List retRole = new ArrayList<>(); do { + roleRetrievalResult = getHeraclesClient().getRolesMappings(roleFrom, roleSize); + + if (!CollectionUtils.isEmpty(roleRetrievalResult)) { + roleRetrievalResult.forEach(role -> { + RangerRole rangerRole = new RangerRole(); + rangerRole.setName(role.getName()); + rangerRole.setGroups(role.getGroups().stream() + .map(x -> new RangerRole.RoleMember(x, false)) + .collect(Collectors.toList())); + rangerRole.setUsers(roleUserMapping.get(role.getName())); + rangerRole.setRoles(role.getRoles().stream() + .map(x -> new RangerRole.RoleMember(x, false)) + .collect(Collectors.toList())); + + roleSet.add(rangerRole); + }); - List roles = getHeraclesClient().getRolesMappings(roleFrom, roleSize); - - if (CollectionUtils.isEmpty(roles)) { - roleFound = false; - } else { - retRole.addAll(roles); roleFrom += roleSize; } - for (HeraclesRoleViewRepresentation role : roles) { - RangerRole rangerRole = new RangerRole(); - rangerRole.setName(role.getName()); - rangerRole.setGroups(role.getGroups().stream().map(x -> new RangerRole.RoleMember(x, true)).collect(Collectors.toList())); - rangerRole.setUsers(roleUserMapping.get(role.getName())); - rangerRole.setRoles(role.getRoles().stream().map(x -> new RangerRole.RoleMember(x, false)).collect(Collectors.toList())); - - roleSet.add(rangerRole); - } + } while (!CollectionUtils.isEmpty(roleRetrievalResult) && roleRetrievalResult.size() % roleSize == 0); - } while (roleFound && retRole.size() % roleSize == 0); processDefaultRole(roleSet); LOG.info("Inverting roles"); @@ -321,6 +306,11 @@ private void processDefaultRole(Set roleSet) { public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("loadUserStoreIfUpdated"); + boolean isKeycloakUpdated = isKeycloakSubjectsStoreUpdated(lastUpdatedTime); + if (!isKeycloakUpdated) { + return null; + } + int userSize = 100; int userFrom = 0; boolean userFound = true; @@ -350,212 +340,4 @@ public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws Atlas return userStore; } - - private static RangerRole keycloakRoleToRangerRole(RoleRepresentation kRole) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakRolesToRangerRoles"); - - RangerRole rangerRole = new RangerRole(); - rangerRole.setName(kRole.getName()); - rangerRole.setDescription(kRole.getDescription() + " " + kRole.getId()); - - RequestContext.get().endMetricRecord(recorder); - return rangerRole; - } - - private static List keycloakGroupsToRangerRoleMember(Set kGroups) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakGroupsToRangerRoleMember"); - List rangerGroups = new ArrayList<>(); - - for (GroupRepresentation kGroup : kGroups) { - //TODO: Revisit isAdmin flag - rangerGroups.add(new RangerRole.RoleMember(kGroup.getName(), false)); - } - - RequestContext.get().endMetricRecord(recorder); - return rangerGroups; - } - - private static List keycloakUsersToRangerRoleMember(Set kUsers) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakUsersToRangerRoleMember"); - List rangerUsers = new ArrayList<>(); - - for (UserRepresentation kUser : kUsers) { - //TODO: Revisit isAdmin flag - rangerUsers.add(new RangerRole.RoleMember(kUser.getUsername(), false)); - } - - RequestContext.get().endMetricRecord(recorder); - return rangerUsers; - } - - private static List keycloakRolesToRangerRoleMember(Set kRoles) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakRolesToRangerRoleMember"); - List rangerRoles = new ArrayList<>(); - - for (RoleRepresentation kRole : kRoles) { - //TODO: Revisit isAdmin flag - rangerRoles.add(new RangerRole.RoleMember(kRole.getName(), false)); - } - - RequestContext.get().endMetricRecord(recorder); - return rangerRoles; - } - - protected static void submitCallablesAndWaitToFinish(String threadName, List> callables) throws AtlasBaseException { - ExecutorService service = getExecutorService(threadName + "-%d-"); - try { - - LOG.info("Submitting callables: {}", threadName); - callables.forEach(service::submit); - - service.shutdown(); - - boolean terminated = service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - LOG.info("awaitTermination done: {}", threadName); - - if (!terminated) { - LOG.warn("Time out occurred while waiting to complete {}", threadName); - } - } catch (InterruptedException e) { - throw new AtlasBaseException(); - } - } - - static class RoleSubjectsFetcher implements Callable { - private Set roleSet; - private RoleRepresentation kRole; - List userNamesList; - - public RoleSubjectsFetcher(RoleRepresentation kRole, - Set roleSet, - List userNamesList) { - this.kRole = kRole; - this.roleSet = roleSet; - this.userNamesList = userNamesList; - } - - @Override - public RangerRole call() throws Exception { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("roleSubjectsFetcher"); - final RangerRole rangerRole = keycloakRoleToRangerRole(kRole); - - try { - //get all groups for Roles - Thread groupsFetcher = new Thread(() -> { - int start = 0; - int size = 500; - boolean found = true; - Set ret = new HashSet<>(); - - do { - try { - Set kGroups = getKeycloakClient().getRoleGroupMembers(kRole.getName(), start, size); - if (CollectionUtils.isNotEmpty(kGroups)) { - ret.addAll(kGroups); - start += size; - } else { - found = false; - } - } catch (Exception e) { - LOG.error("Failed to get group members with role", e); - throw new RuntimeException(e); - } - - } while (found && ret.size() % size == 0); - - rangerRole.setGroups(keycloakGroupsToRangerRoleMember(ret)); - }); - groupsFetcher.start(); - - //get all users for Roles - Thread usersFetcher = new Thread(() -> { - int start = 0; - int size = 100; - boolean found = true; - Set ret = new HashSet<>(); - - do { - try { - Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); - if (CollectionUtils.isNotEmpty(userRepresentations)) { - ret.addAll(userRepresentations); - start += size; - } else { - found = false; - } - } catch (Exception e) { - LOG.error("Failed to get users for role {}", kRole.getName(), e); - throw new RuntimeException(e); - } - - } while (found && ret.size() % size == 0); - - rangerRole.setUsers(keycloakUsersToRangerRoleMember(ret)); - userNamesList.addAll(ret); - }); - usersFetcher.start(); - - //get all roles for Roles - Thread subRolesFetcher = new Thread(() -> { - Set kSubRoles = null; - try { - kSubRoles = getKeycloakClient().getRoleComposites(kRole.getName()); - rangerRole.setRoles(keycloakRolesToRangerRoleMember(kSubRoles)); - } catch (AtlasBaseException e) { - LOG.error("Failed to get composite for role {}", kRole.getName(), e); - throw new RuntimeException(e); - } - }); - subRolesFetcher.start(); - - try { - groupsFetcher.join(); - usersFetcher.join(); - subRolesFetcher.join(); - } catch (InterruptedException e) { - LOG.error("Failed to wait for threads to complete: {}", kRole.getName()); - e.printStackTrace(); - } - - RequestContext.get().endMetricRecord(recorder); - roleSet.add(rangerRole); - } catch (Exception e) { - LOG.error("RoleSubjectsFetcher: Failed to process role {}: {}", kRole.getName(), e.getMessage()); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - - return rangerRole; - } - } - - static class UserGroupsFetcher implements Callable { - private Map> userGroupMapping; - private UserRepresentation kUser; - - public UserGroupsFetcher(UserRepresentation kUser, Map> userGroupMapping) { - this.kUser = kUser; - this.userGroupMapping = userGroupMapping; - } - - @Override - public Object call() throws Exception { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("userGroupsFetcher"); - - try { - List kGroups = getKeycloakClient().getGroupsForUserById(kUser.getId()); - userGroupMapping.put(kUser.getUsername(), - kGroups.stream() - .map(GroupRepresentation::getName) - .collect(Collectors.toSet())); - - } catch (Exception e) { - LOG.error("UserGroupsFetcher: Failed to process user {}: {}", kUser.getUsername(), e.getMessage()); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - - return null; - } - } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java index 89980504b0..3dd97bcee8 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -26,7 +26,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.atlas.admin.client.RangerAdminClient; import org.apache.atlas.plugin.service.RangerBasePlugin; import java.io.File; @@ -34,7 +33,6 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; -import java.util.Date; import java.util.HashMap; import java.util.Set; diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index d10e4b13fc..7ae882b245 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -108,7 +108,7 @@ public enum AtlasConfiguration { PERSONA_POLICY_ASSET_MAX_LIMIT("atlas.persona.policy.asset.maxlimit", 1000), ENABLE_KEYCLOAK_TOKEN_INTROSPECTION("atlas.canary.keycloak.token-introspection", false), - KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE("atlas.keycloak.admin.resource-pagination-size", 1500), + HERACLES_CLIENT_PAGINATION_SIZE("atlas.heracles.admin.resource-pagination-size", 100), HERACLES_API_SERVER_URL("atlas.heracles.api.server.url", "http://heracles-service.heracles.svc.cluster.local"); From e20b4a1ee3d7ff6a3e0da0627b47ab666f024fc8 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:26:02 +0530 Subject: [PATCH 13/16] fix: remove redundunt bug --- .../java/org/apache/atlas/plugin/util/KeycloakUserStore.java | 1 - 1 file changed, 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index 2f9470c3fc..d915510d47 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -71,7 +71,6 @@ public KeycloakUserStore(String serviceName) { public boolean isKeycloakSubjectsStoreUpdated(long cacheLastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getKeycloakSubjectsStoreUpdatedTime"); - cacheLastUpdatedTime = -1; if (cacheLastUpdatedTime == -1) { return true; } From 820a8e1ba599c976a08c61992e9d44dac9f37b97 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:28:35 +0530 Subject: [PATCH 14/16] fix: remove redundunt funcs --- .../client/heracles/AtlasHeraclesClient.java | 27 ------------------- .../client/heracles/HeraclesRestClient.java | 4 --- .../heracles/RetrofitHeraclesClient.java | 6 ----- 3 files changed, 37 deletions(-) diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java index 44e11660a9..0c599e8078 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -44,33 +44,6 @@ private static void init(AuthConfig authConfig) { } } - public List getAllUsers() throws AtlasBaseException { - int start = 0; - int size = 100; - boolean found = true; - - List ret = new ArrayList<>(0); - do { - - List userRepresentations = HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, null, HeraclesUsersRepresentation.USER_SORT).body().toKeycloakUserRepresentations(); - if (userRepresentations != null && !userRepresentations.isEmpty()) { - ret.addAll(userRepresentations); - start += size; - } else { - found = false; - } - } while (found && ret.size() % size == 0); - - return ret; - } - - - public Set getRoleUserMembers(String roleName, int start, int size) throws AtlasBaseException { - String template = "{\"$and\":[{\"roles\":{\"$elemMatch\":[\"{0}\"]}}]}"; - String filter = template.replace("{0}", roleName); - return HERACLES.getUsers(start, size, HeraclesUsersRepresentation.USER_PROJECTIONS, filter,HeraclesUsersRepresentation.USER_SORT ).body().toKeycloakUserRepresentations().stream().collect(Collectors.toSet()); - } - public List getUsersMappings(int start, int size) throws AtlasBaseException { String[] columns = {"roles", "groups"}; List views = HERACLES.getUsersMappings(start, size, HeraclesUserViewRepresentation.sortBy, columns).body(); diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java index ae686b70fa..e7548138d3 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -15,10 +15,6 @@ public class HeraclesRestClient extends AbstractAuthClient { public HeraclesRestClient(final AuthConfig authConfig) { super(authConfig); } - public Response getUsers(int offset,int limit, String columns, String filter, String sort) throws AtlasBaseException { - return processResponse(this.retrofitHeraclesClient.getUsers(offset, columns, filter, limit,sort)); - } - public Response> getUsersMappings(int offset, int limit, String sort, String[] columns) throws AtlasBaseException { return processResponse(this.retrofitHeraclesClient.getUsersMapping(offset, limit,sort, columns)); } diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java index 22601f9467..5a78ed4225 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -11,12 +11,6 @@ import java.util.List; public interface RetrofitHeraclesClient { - @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) - @GET("/users") - Call getUsers(@Query("offset") Integer offset, @Query("columns") String columns, - @Query(value = "filter", encoded = true) String filter, @Query("limit") Integer limit, - @Query("sort") String sort); - @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) @GET("/users/mappings") Call> getUsersMapping(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort, From 712a4e28f5f183a45ee0c4863a998fb64d9db4a9 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:35:29 +0530 Subject: [PATCH 15/16] fix: remove redundunt classes --- .../client/heracles/AtlasHeraclesClient.java | 3 - .../client/heracles/HeraclesRestClient.java | 1 - .../heracles/RetrofitHeraclesClient.java | 1 - .../models/HeraclesUsersRepresentation.java | 113 ------------------ 4 files changed, 118 deletions(-) delete mode 100644 client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java index 0c599e8078..dd212794ca 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -4,15 +4,12 @@ import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import org.keycloak.representations.idm.UserRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; public class AtlasHeraclesClient { diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java index e7548138d3..c6fbbcb935 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -5,7 +5,6 @@ import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import retrofit2.Response; import java.util.List; diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java index 5a78ed4225..b9fd8de6ca 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -2,7 +2,6 @@ import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; -import org.apache.atlas.auth.client.heracles.models.HeraclesUsersRepresentation; import retrofit2.Call; import retrofit2.http.GET; import retrofit2.http.Headers; diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java deleted file mode 100644 index 4cea34f509..0000000000 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUsersRepresentation.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.apache.atlas.auth.client.heracles.models; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.keycloak.representations.idm.UserRepresentation; - -import java.util.ArrayList; -import java.util.List; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class HeraclesUsersRepresentation { - protected int totalRecord; - protected int filterRecord; - protected List records; - public static final String USER_PROJECTIONS = "emailVerified,enabled,id,status,username"; - public static final String USER_SORT = "username"; - - public HeraclesUsersRepresentation() { - } - - public HeraclesUsersRepresentation(int totalRecord, int filterRecord, List records) { - this.totalRecord = totalRecord; - this.filterRecord = filterRecord; - this.records = records; - } - - public int getTotalRecord() { - return totalRecord; - } - - public void setTotalRecord(int totalRecord) { - this.totalRecord = totalRecord; - } - - public int getFilterRecord() { - return filterRecord; - } - - public void setFilterRecord(int filterRecord) { - this.filterRecord = filterRecord; - } - - public List getRecords() { - return records; - } - - public void setRecords(List records) { - this.records = records; - } - - public List toKeycloakUserRepresentations() { - List userRepresentations = new ArrayList<>(); - for (HeraclesUserRepresentation heraclesUserRepresentation : records) { - UserRepresentation userRepresentation = new UserRepresentation(); - userRepresentation.setEmailVerified(heraclesUserRepresentation.emailVerified); - userRepresentation.setEnabled(heraclesUserRepresentation.enabled); - userRepresentation.setUsername(heraclesUserRepresentation.username); - userRepresentation.setId(heraclesUserRepresentation.id); - userRepresentations.add(userRepresentation); - } - return userRepresentations; - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class HeraclesUserRepresentation { - protected boolean emailVerified; - protected boolean enabled; - protected String username; - protected String id; - - public HeraclesUserRepresentation() { - } - - public HeraclesUserRepresentation(boolean emailVerified, boolean enabled, String username, String id) { - this.emailVerified = emailVerified; - this.enabled = enabled; - this.username = username; - this.id = id; - } - - public boolean isEmailVerified() { - return emailVerified; - } - - public void setEmailVerified(boolean emailVerified) { - this.emailVerified = emailVerified; - } - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - -} From 641fae7c3f8317af0839901d0da3906f76166abe Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 10 Jan 2024 19:32:15 +0530 Subject: [PATCH 16/16] nit: adding function refactoring --- .../apache/atlas/auth/client/heracles/AtlasHeraclesClient.java | 3 +-- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java index dd212794ca..507f0dbf0c 100644 --- a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -44,7 +44,7 @@ private static void init(AuthConfig authConfig) { public List getUsersMappings(int start, int size) throws AtlasBaseException { String[] columns = {"roles", "groups"}; List views = HERACLES.getUsersMappings(start, size, HeraclesUserViewRepresentation.sortBy, columns).body(); - List userRepresentations = views.stream().map(x -> { + return views.stream().map(x -> { UserRepresentation userRepresentation = new UserRepresentation(); userRepresentation.setId(x.getId()); userRepresentation.setUsername(x.getUsername()); @@ -52,7 +52,6 @@ public List getUsersMappings(int start, int size) throws Atl userRepresentation.setGroups(x.getGroups()); return userRepresentation; }).collect(Collectors.toList()); - return userRepresentations; } public List getRolesMappings(int start, int size) throws AtlasBaseException { diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 7ae882b245..c9452ebf0d 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -109,7 +109,7 @@ public enum AtlasConfiguration { PERSONA_POLICY_ASSET_MAX_LIMIT("atlas.persona.policy.asset.maxlimit", 1000), ENABLE_KEYCLOAK_TOKEN_INTROSPECTION("atlas.canary.keycloak.token-introspection", false), HERACLES_CLIENT_PAGINATION_SIZE("atlas.heracles.admin.resource-pagination-size", 100), - HERACLES_API_SERVER_URL("atlas.heracles.api.server.url", "http://heracles-service.heracles.svc.cluster.local"); + HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"); private static final Configuration APPLICATION_PROPERTIES;