diff --git a/auth-agents-common/pom.xml b/auth-agents-common/pom.xml index 7d63912e3d..aa37156d5f 100644 --- a/auth-agents-common/pom.xml +++ b/auth-agents-common/pom.xml @@ -51,7 +51,7 @@ org.apache.atlas - client-keycloak + client-auth ${project.version} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java index a73fdcac8d..d915510d47 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/KeycloakUserStore.java @@ -19,9 +19,9 @@ package org.apache.atlas.plugin.util; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.RequestContext; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.plugin.model.RangerRole; import org.apache.atlas.utils.AtlasPerfMetrics; @@ -33,13 +33,10 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.heracles.AtlasHeraclesClient.getHeraclesClient; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; import static org.apache.atlas.repository.util.AccessControlUtils.BACKEND_SERVICE_USER_NAME; @@ -72,13 +69,6 @@ public KeycloakUserStore(String serviceName) { } } - public static ExecutorService getExecutorService(String namePattern) { - ExecutorService service = Executors.newFixedThreadPool(NUM_THREADS, - new ThreadFactoryBuilder().setNameFormat(namePattern + Thread.currentThread().getName()) - .build()); - return service; - } - public boolean isKeycloakSubjectsStoreUpdated(long cacheLastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getKeycloakSubjectsStoreUpdatedTime"); if (cacheLastUpdatedTime == -1) { @@ -156,21 +146,61 @@ public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseExce if (!isKeycloakUpdated) { return null; } + RangerRoles rangerRoles = new RangerRoles(); + Map> roleUserMapping = new HashMap<>(); + Set roleSet = new HashSet<>(); - List kRoles = getKeycloakClient().getAllRoles(); - LOG.info("Found {} keycloak roles", kRoles.size()); + int userSize = AtlasConfiguration.HERACLES_CLIENT_PAGINATION_SIZE.getInt(); + int userFrom = 0; + List userRetrievalResult; - Set roleSet = new HashSet<>(); - RangerRoles rangerRoles = new RangerRoles(); - List userNamesList = new ArrayList<>(); + do { + userRetrievalResult = getHeraclesClient().getUsersMappings(userFrom, userSize); + + if (!CollectionUtils.isEmpty(userRetrievalResult)) { + userRetrievalResult.forEach(user -> { + Set userRoles = new HashSet<>(user.getRealmRoles()); + + userRoles.forEach(role -> roleUserMapping + .computeIfAbsent(role, k -> new ArrayList<>()) + .add(new RangerRole.RoleMember(user.getUsername(), false)) + ); + }); + + userFrom += userSize; + } + + } while (!CollectionUtils.isEmpty(userRetrievalResult) && userRetrievalResult.size() % userSize == 0); + + int roleSize = AtlasConfiguration.HERACLES_CLIENT_PAGINATION_SIZE.getInt(); + int roleFrom = 0; + List roleRetrievalResult; + + do { + roleRetrievalResult = getHeraclesClient().getRolesMappings(roleFrom, roleSize); + + if (!CollectionUtils.isEmpty(roleRetrievalResult)) { + roleRetrievalResult.forEach(role -> { + RangerRole rangerRole = new RangerRole(); + rangerRole.setName(role.getName()); + rangerRole.setGroups(role.getGroups().stream() + .map(x -> new RangerRole.RoleMember(x, false)) + .collect(Collectors.toList())); + rangerRole.setUsers(roleUserMapping.get(role.getName())); + rangerRole.setRoles(role.getRoles().stream() + .map(x -> new RangerRole.RoleMember(x, false)) + .collect(Collectors.toList())); + + roleSet.add(rangerRole); + }); + + roleFrom += roleSize; + } + + } while (!CollectionUtils.isEmpty(roleRetrievalResult) && roleRetrievalResult.size() % roleSize == 0); - submitCallablesAndWaitToFinish("RoleSubjectsFetcher", - kRoles.stream() - .map(x -> new RoleSubjectsFetcher(x, roleSet, userNamesList)) - .collect(Collectors.toList())); processDefaultRole(roleSet); - LOG.info("Inverting roles"); invertRoles(roleSet); @@ -187,6 +217,12 @@ public RangerRoles loadRolesIfUpdated(long lastUpdatedTime) throws AtlasBaseExce return rangerRoles; } + private void extractUserGroupMapping(List users, Map> userGroupMapping) { + for (UserRepresentation user : users) { + userGroupMapping.put(user.getUsername(), new HashSet<>(user.getGroups() == null ? Collections.emptyList() : user.getGroups())); + } + } + public void invertRoles(Set roleSet) { Map roleMap = new HashMap<>(); for (RangerRole role : roleSet) { @@ -269,239 +305,38 @@ private void processDefaultRole(Set roleSet) { public RangerUserStore loadUserStoreIfUpdated(long lastUpdatedTime) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("loadUserStoreIfUpdated"); - boolean isKeycloakUpdated = isKeycloakSubjectsStoreUpdated(lastUpdatedTime); if (!isKeycloakUpdated) { return null; } + int userSize = 100; + int userFrom = 0; + boolean userFound = true; Map> userGroupMapping = new HashMap<>(); + List ret = new ArrayList<>(); - List kUsers = getKeycloakClient().getAllUsers(); - LOG.info("Found {} keycloak users", kUsers.size()); - - List> callables = new ArrayList<>(); - kUsers.forEach(x -> callables.add(new UserGroupsFetcher(x, userGroupMapping))); + do { + List users = getHeraclesClient().getUsersMappings(userFrom, userSize); + if (CollectionUtils.isEmpty(users)) { + userFound = false; + } else { + ret.addAll(users); + userFrom += userSize; + } + extractUserGroupMapping(users, userGroupMapping); - submitCallablesAndWaitToFinish("UserGroupsFetcher", callables); + } while (userFound && ret.size() % userSize == 0); RangerUserStore userStore = new RangerUserStore(); userStore.setUserGroupMapping(userGroupMapping); - Date current = new Date(); - userStore.setUserStoreUpdateTime(current); + userStore.setUserStoreUpdateTime(new Date()); userStore.setServiceName(serviceName); userStore.setUserStoreVersion(-1L); RequestContext.get().endMetricRecord(recorder); - return userStore; - } - - private static RangerRole keycloakRoleToRangerRole(RoleRepresentation kRole) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakRolesToRangerRoles"); - - RangerRole rangerRole = new RangerRole(); - rangerRole.setName(kRole.getName()); - rangerRole.setDescription(kRole.getDescription() + " " + kRole.getId()); - - RequestContext.get().endMetricRecord(recorder); - return rangerRole; - } - - private static List keycloakGroupsToRangerRoleMember(Set kGroups) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakGroupsToRangerRoleMember"); - List rangerGroups = new ArrayList<>(); - - for (GroupRepresentation kGroup : kGroups) { - //TODO: Revisit isAdmin flag - rangerGroups.add(new RangerRole.RoleMember(kGroup.getName(), false)); - } - - RequestContext.get().endMetricRecord(recorder); - return rangerGroups; - } - - private static List keycloakUsersToRangerRoleMember(Set kUsers) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakUsersToRangerRoleMember"); - List rangerUsers = new ArrayList<>(); - - for (UserRepresentation kUser : kUsers) { - //TODO: Revisit isAdmin flag - rangerUsers.add(new RangerRole.RoleMember(kUser.getUsername(), false)); - } - - RequestContext.get().endMetricRecord(recorder); - return rangerUsers; - } - - private static List keycloakRolesToRangerRoleMember(Set kRoles) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("keycloakRolesToRangerRoleMember"); - List rangerRoles = new ArrayList<>(); - - for (RoleRepresentation kRole : kRoles) { - //TODO: Revisit isAdmin flag - rangerRoles.add(new RangerRole.RoleMember(kRole.getName(), false)); - } - - RequestContext.get().endMetricRecord(recorder); - return rangerRoles; - } - - protected static void submitCallablesAndWaitToFinish(String threadName, List> callables) throws AtlasBaseException { - ExecutorService service = getExecutorService(threadName + "-%d-"); - try { - - LOG.info("Submitting callables: {}", threadName); - callables.forEach(service::submit); - - service.shutdown(); - - boolean terminated = service.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - LOG.info("awaitTermination done: {}", threadName); - - if (!terminated) { - LOG.warn("Time out occurred while waiting to complete {}", threadName); - } - } catch (InterruptedException e) { - throw new AtlasBaseException(); - } - } - - static class RoleSubjectsFetcher implements Callable { - private Set roleSet; - private RoleRepresentation kRole; - List userNamesList; - - public RoleSubjectsFetcher(RoleRepresentation kRole, - Set roleSet, - List userNamesList) { - this.kRole = kRole; - this.roleSet = roleSet; - this.userNamesList = userNamesList; - } - @Override - public RangerRole call() throws Exception { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("roleSubjectsFetcher"); - final RangerRole rangerRole = keycloakRoleToRangerRole(kRole); - - try { - //get all groups for Roles - Thread groupsFetcher = new Thread(() -> { - int start = 0; - int size = AtlasConfiguration.KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE.getInt(); - boolean found = true; - Set ret = new HashSet<>(); - - do { - try { - Set kGroups = getKeycloakClient().getRoleGroupMembers(kRole.getName(), start, size); - if (CollectionUtils.isNotEmpty(kGroups)) { - ret.addAll(kGroups); - start += size; - } else { - found = false; - } - } catch (Exception e) { - LOG.error("Failed to get group members with role", e); - throw new RuntimeException(e); - } - - } while (found && ret.size() % size == 0); - - rangerRole.setGroups(keycloakGroupsToRangerRoleMember(ret)); - }); - groupsFetcher.start(); - - //get all users for Roles - Thread usersFetcher = new Thread(() -> { - int start = 0; - int size = AtlasConfiguration.KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE.getInt(); - boolean found = true; - Set ret = new HashSet<>(); - - do { - try { - Set userRepresentations = getKeycloakClient().getRoleUserMembers(kRole.getName(), start, size); - if (CollectionUtils.isNotEmpty(userRepresentations)) { - ret.addAll(userRepresentations); - start += size; - } else { - found = false; - } - } catch (Exception e) { - LOG.error("Failed to get users for role {}", kRole.getName(), e); - throw new RuntimeException(e); - } - - } while (found && ret.size() % size == 0); - - rangerRole.setUsers(keycloakUsersToRangerRoleMember(ret)); - userNamesList.addAll(ret); - }); - usersFetcher.start(); - - //get all roles for Roles - Thread subRolesFetcher = new Thread(() -> { - Set kSubRoles = null; - try { - kSubRoles = getKeycloakClient().getRoleComposites(kRole.getName()); - rangerRole.setRoles(keycloakRolesToRangerRoleMember(kSubRoles)); - } catch (AtlasBaseException e) { - LOG.error("Failed to get composite for role {}", kRole.getName(), e); - throw new RuntimeException(e); - } - }); - subRolesFetcher.start(); - - try { - groupsFetcher.join(); - usersFetcher.join(); - subRolesFetcher.join(); - } catch (InterruptedException e) { - LOG.error("Failed to wait for threads to complete: {}", kRole.getName()); - e.printStackTrace(); - } - - RequestContext.get().endMetricRecord(recorder); - roleSet.add(rangerRole); - } catch (Exception e) { - LOG.error("RoleSubjectsFetcher: Failed to process role {}: {}", kRole.getName(), e.getMessage()); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - - return rangerRole; - } - } - - static class UserGroupsFetcher implements Callable { - private Map> userGroupMapping; - private UserRepresentation kUser; - - public UserGroupsFetcher(UserRepresentation kUser, Map> userGroupMapping) { - this.kUser = kUser; - this.userGroupMapping = userGroupMapping; - } - - @Override - public Object call() throws Exception { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("userGroupsFetcher"); - - try { - List kGroups = getKeycloakClient().getGroupsForUserById(kUser.getId()); - userGroupMapping.put(kUser.getUsername(), - kGroups.stream() - .map(GroupRepresentation::getName) - .collect(Collectors.toSet())); - - } catch (Exception e) { - LOG.error("UserGroupsFetcher: Failed to process user {}: {}", kUser.getUsername(), e.getMessage()); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - - return null; - } + return userStore; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java index 91be2ab704..163a9a5078 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java @@ -26,7 +26,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.atlas.admin.client.RangerAdminClient; import org.apache.atlas.plugin.service.RangerBasePlugin; import java.io.File; @@ -34,7 +33,6 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; -import java.util.Date; import java.util.HashMap; import java.util.Set; diff --git a/client-keycloak/pom.xml b/client-auth/pom.xml similarity index 98% rename from client-keycloak/pom.xml rename to client-auth/pom.xml index b0a231b0c3..9087a98bed 100644 --- a/client-keycloak/pom.xml +++ b/client-auth/pom.xml @@ -27,7 +27,7 @@ 4.0.0 - client-keycloak + client-auth 8 diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java similarity index 69% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java index 86bdf6fbf7..cb13431384 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java @@ -1,13 +1,15 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.auth; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.Timer; +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.heracles.RetrofitHeraclesClient; +import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient; import okhttp3.*; import okhttp3.logging.HttpLoggingInterceptor; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; -import org.apache.atlas.keycloak.client.service.AtlasKeycloakAuthService; import org.apache.atlas.service.metrics.MetricUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,22 +28,23 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; -abstract class AbstractKeycloakClient { +public class AbstractAuthClient { - private final static Logger LOG = LoggerFactory.getLogger(AbstractKeycloakClient.class); + private final static Logger LOG = LoggerFactory.getLogger(AbstractAuthClient.class); private static final Map ERROR_CODE_MAP = new HashMap<>(); - private static final int DEFAULT_KEYCLOAK_RETRY = 3; + private static final int DEFAULT_RETRY = 3; private static final String AUTHORIZATION = "Authorization"; private static final String BEARER = "Bearer "; private static final int TIMEOUT_IN_SEC = 60; private static final String INTEGRATION = "integration"; private static final String KEYCLOAK = "keycloak"; - protected final KeycloakConfig keycloakConfig; - protected final RetrofitKeycloakClient retrofit; + protected final AuthConfig authConfig; + protected final RetrofitKeycloakClient retrofitKeycloakClient; + protected final RetrofitHeraclesClient retrofitHeraclesClient; - private final AtlasKeycloakAuthService authService; + private final KeycloakAuthenticationService authService; private MetricUtils metricUtils = null; static { @@ -49,8 +52,8 @@ abstract class AbstractKeycloakClient { ERROR_CODE_MAP.put(HTTP_BAD_REQUEST, BAD_REQUEST); } - public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { - this.keycloakConfig = keycloakConfig; + public AbstractAuthClient(AuthConfig authConfig) { + this.authConfig = authConfig; this.metricUtils = new MetricUtils(); HttpLoggingInterceptor httpInterceptor = new HttpLoggingInterceptor(); httpInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY); @@ -64,11 +67,15 @@ public AbstractKeycloakClient(KeycloakConfig keycloakConfig) { .writeTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) .readTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS) .build(); - this.retrofit = new Retrofit.Builder().client(okHttpClient) - .baseUrl(this.keycloakConfig.getAuthServerUrl()) + this.retrofitKeycloakClient = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.authConfig.getAuthServerUrl()) .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() .create(RetrofitKeycloakClient.class); - authService = new AtlasKeycloakAuthService(keycloakConfig); + this.retrofitHeraclesClient = new Retrofit.Builder().client(okHttpClient) + .baseUrl(this.authConfig.getHeraclesApiServerUrl()) + .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper().disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES))).build() + .create(RetrofitHeraclesClient.class); + authService = new KeycloakAuthenticationService(authConfig); } /** @@ -97,21 +104,20 @@ public Response intercept(@NonNull Chain chain) throws IOException { return chain.proceed(request); } }; - /** * Called only during auth failures. */ Authenticator authInterceptor = new Authenticator() { @Override public Request authenticate(Route route, @NonNull Response response) { - if (responseCount(response) > DEFAULT_KEYCLOAK_RETRY) { - LOG.warn("Keycloak: Falling back, retried {} times", DEFAULT_KEYCLOAK_RETRY); + if (responseCount(response) > DEFAULT_RETRY) { + LOG.warn("Auth Client: Falling back, retried {} times", DEFAULT_RETRY); return null; } - LOG.info("Keycloak: Current keycloak token status, Expired: {}", authService.isTokenExpired()); + LOG.info("Auth Client: Current keycloak token status, Expired: {}", authService.isTokenExpired()); return response.request().newBuilder() - .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) - .build(); + .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken()) + .build(); } private int responseCount(Response response) { @@ -134,13 +140,14 @@ protected retrofit2.Response processResponse(retrofit2.Call req) throw return response; } String errMsg = response.errorBody().string(); - LOG.error("Keycloak: Client request processing failed code {} message:{}, request: {} {}", + LOG.error("Auth Client: Client request processing failed code {} message:{}, request: {} {}", response.code(), errMsg, req.request().method(), req.request().url()); throw new AtlasBaseException(ERROR_CODE_MAP.getOrDefault(response.code(), BAD_REQUEST), errMsg); } catch (Exception e) { - LOG.error("Keycloak: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); - throw new AtlasBaseException(BAD_REQUEST, "Keycloak request failed"); + LOG.error("Auth Client: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e); + throw new AtlasBaseException(BAD_REQUEST, "Auth request failed"); } } + } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java similarity index 75% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java index fc4d40df7e..a3b3e9255e 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java @@ -1,11 +1,11 @@ -package org.apache.atlas.keycloak.client.service; +package org.apache.atlas.auth.client.auth; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.*; import okhttp3.logging.HttpLoggingInterceptor; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.RetrofitKeycloakClient; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; +import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient; import org.jetbrains.annotations.NotNull; import org.keycloak.representations.AccessTokenResponse; import org.slf4j.Logger; @@ -19,9 +19,9 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -public final class AtlasKeycloakAuthService { +public final class KeycloakAuthenticationService { - public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakAuthService.class); + public final static Logger LOG = LoggerFactory.getLogger(KeycloakAuthenticationService.class); private final static String GRANT_TYPE = "grant_type"; private static final String CLIENT_ID = "client_id"; @@ -30,14 +30,14 @@ public final class AtlasKeycloakAuthService { private static final int TIMEOUT_IN_SECS = 60; private final RetrofitKeycloakClient retrofit; - private final KeycloakConfig keycloakConfig; + private final AuthConfig authConfig; private AccessTokenResponse currentAccessToken; private long expirationTime = -1; - public AtlasKeycloakAuthService(KeycloakConfig keycloakConfig) { - this.keycloakConfig = keycloakConfig; + public KeycloakAuthenticationService(AuthConfig authConfig) { + this.authConfig = authConfig; this.retrofit = new Retrofit.Builder().client(getOkHttpClient()) - .baseUrl(this.keycloakConfig.getAuthServerUrl()) + .baseUrl(this.authConfig.getAuthServerUrl()) .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build() .create(RetrofitKeycloakClient.class); } @@ -59,7 +59,7 @@ private OkHttpClient getOkHttpClient() { Interceptor responseLoggingInterceptor = chain -> { Request request = chain.request(); okhttp3.Response response = chain.proceed(request); - LOG.info("Keycloak: Auth Request for url {} Status: {}", request.url(), response.code()); + LOG.info("Auth Client: Auth Request for url {} Status: {}", request.url(), response.code()); return response; }; @@ -70,16 +70,16 @@ public String getAuthToken() { synchronized (this) { if (isTokenExpired()) { try { - retrofit2.Response resp = this.retrofit.grantToken(this.keycloakConfig.getRealmId(), getTokenRequest()).execute(); + retrofit2.Response resp = this.retrofit.grantToken(this.authConfig.getRealmId(), getTokenRequest()).execute(); if (resp.isSuccessful()) { currentAccessToken = resp.body(); expirationTime = currentTime() + currentAccessToken.getExpiresIn() - EXPIRY_OFFSET_SEC; - LOG.info("Keycloak: Auth token fetched with expiry:{} sec", expirationTime); + LOG.info("Auth Client: Auth token fetched with expiry:{} sec", expirationTime); } else { throw new AtlasBaseException(BAD_REQUEST, resp.errorBody().string()); } } catch (Exception e) { - LOG.error("Keycloak: Error while fetching access token for keycloak client.", e); + LOG.error("Auth Client: Error while fetching access token for keycloak client.", e); throw new RuntimeException(e); } } @@ -97,7 +97,7 @@ public boolean isTokenExpired() { } private RequestBody getTokenRequest() { - return new FormBody.Builder().addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()).addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.keycloakConfig.getGrantType()).build(); + return new FormBody.Builder().addEncoded(CLIENT_ID, this.authConfig.getClientId()).addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.authConfig.getGrantType()).build(); } private long currentTime() { diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java new file mode 100644 index 0000000000..33d98de049 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java @@ -0,0 +1,100 @@ +package org.apache.atlas.auth.client.config; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Optional; + +import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; + +public class AuthConfig { + private static final Logger LOG = LoggerFactory.getLogger(AuthConfig.class); + + public String authServerUrl; + public String realmId; + public String clientId; + public String clientSecret; + public String grantType; + public String heraclesApiServerUrl; + + private static final String KEYCLOAK_PROPERTIES = "keycloak.json"; + private static final String DEFAULT_GRANT_TYPE = "client_credentials"; + private static final String KEY_REALM_ID = "realm"; + private static final String KEY_AUTH_SERVER_URL = "auth-server-url"; + private static final String KEY_CLIENT_ID = "resource"; + private static final String KEY_CREDENTIALS = "credentials"; + private static final String KEY_SECRET = "secret"; + + public String getAuthServerUrl() { + return authServerUrl; + } + + public String getRealmId() { + return realmId; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getGrantType() { + return grantType; + } + + public String getHeraclesApiServerUrl() { + return heraclesApiServerUrl; + } + + public static AuthConfig getConfig() throws AtlasBaseException { + String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); + Optional confFile = getConfigurationFile(confLocation); + + if (confFile.isPresent()) { + try { + JSONObject object = new JSONObject(readFileToString(confFile.get())); + return buildAuthConfigFromJson(object); + } catch (Exception e) { + LOG.error("Error parsing Keycloak configuration: ", e); + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Error parsing Keycloak configuration"); + } + } else { + throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); + } + } + + private static Optional getConfigurationFile(String confLocation) { + if (StringUtils.isNotEmpty(confLocation)) { + File confFile = new File(confLocation, KEYCLOAK_PROPERTIES); + if (confFile.exists()) { + return Optional.of(confFile); + } + } + return Optional.empty(); + } + + private static String readFileToString(File file) throws Exception { + return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static AuthConfig buildAuthConfigFromJson(JSONObject object) throws Exception { + String realmId = object.getString(KEY_REALM_ID); + String authServerUrl = object.getString(KEY_AUTH_SERVER_URL) + "/"; + String clientId = object.getString(KEY_CLIENT_ID); + String grantType = DEFAULT_GRANT_TYPE; + String clientSecret = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); + + LOG.info("Keycloak configuration: REALM_ID:{}, AUTH_SERVER_URL:{}", realmId, authServerUrl); + return AuthConfigBuilder.builder().realId(realmId).authServerUrl(authServerUrl).clientId(clientId).grantType(grantType).clientSecret(clientSecret).build(); + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java new file mode 100644 index 0000000000..552033d2c1 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java @@ -0,0 +1,55 @@ +package org.apache.atlas.auth.client.config; + +import org.apache.atlas.AtlasConfiguration; + +public class AuthConfigBuilder { + + private String authServerUrl; + private String realmId; + private String clientId; + private String clientSecret; + private String grantType = "client_credentials"; + + private AuthConfigBuilder() { + } + + public static AuthConfigBuilder builder() { + return new AuthConfigBuilder(); + } + + public AuthConfigBuilder authServerUrl(String authServerUrl) { + this.authServerUrl = authServerUrl; + return this; + } + + public AuthConfigBuilder realId(String realId) { + this.realmId = realId; + return this; + } + + public AuthConfigBuilder clientId(String clientId) { + this.clientId = clientId; + return this; + } + + public AuthConfigBuilder clientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public AuthConfigBuilder grantType(String grantType) { + this.grantType = grantType; + return this; + } + + public AuthConfig build() { + AuthConfig authConfig = new AuthConfig(); + authConfig.authServerUrl = authServerUrl; + authConfig.realmId = realmId; + authConfig.clientId = clientId; + authConfig.clientSecret = clientSecret; + authConfig.grantType = grantType; + authConfig.heraclesApiServerUrl= AtlasConfiguration.HERACLES_API_SERVER_URL.getString()+"/"; + return authConfig; + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java new file mode 100644 index 0000000000..507f0dbf0c --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/AtlasHeraclesClient.java @@ -0,0 +1,61 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; +import org.apache.atlas.exception.AtlasBaseException; +import org.keycloak.representations.idm.UserRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class AtlasHeraclesClient { + public final static Logger LOG = LoggerFactory.getLogger(AtlasHeraclesClient.class); + + private static HeraclesRestClient HERACLES; + private static AtlasHeraclesClient HERACLES_CLIENT; + + public AtlasHeraclesClient() {} + + public static AtlasHeraclesClient getHeraclesClient() { + if(Objects.isNull(HERACLES_CLIENT)) { + LOG.info("Initializing Heracles client.."); + try{ + init(AuthConfig.getConfig()); + } catch (Exception e) { + LOG.error("Error initializing Heracles client", e); + } + } + return HERACLES_CLIENT; + } + + private static void init(AuthConfig authConfig) { + synchronized (AtlasHeraclesClient.class) { + if (HERACLES == null) { + HERACLES = new HeraclesRestClient(authConfig); + HERACLES_CLIENT = new AtlasHeraclesClient(); + } + } + } + + public List getUsersMappings(int start, int size) throws AtlasBaseException { + String[] columns = {"roles", "groups"}; + List views = HERACLES.getUsersMappings(start, size, HeraclesUserViewRepresentation.sortBy, columns).body(); + return views.stream().map(x -> { + UserRepresentation userRepresentation = new UserRepresentation(); + userRepresentation.setId(x.getId()); + userRepresentation.setUsername(x.getUsername()); + userRepresentation.setRealmRoles(x.getRoles()); + userRepresentation.setGroups(x.getGroups()); + return userRepresentation; + }).collect(Collectors.toList()); + } + + public List getRolesMappings(int start, int size) throws AtlasBaseException { + String[] columns = {"composite_roles","groups"}; + return HERACLES.getRolesMappings(start, size, HeraclesRoleViewRepresentation.sortBy, columns).body(); + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java new file mode 100644 index 0000000000..c6fbbcb935 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/HeraclesRestClient.java @@ -0,0 +1,25 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.config.AuthConfig; +import org.apache.atlas.auth.client.auth.AbstractAuthClient; +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; +import org.apache.atlas.exception.AtlasBaseException; +import retrofit2.Response; + +import java.util.List; + +public class HeraclesRestClient extends AbstractAuthClient { + + public HeraclesRestClient(final AuthConfig authConfig) { + super(authConfig); + } + public Response> getUsersMappings(int offset, int limit, String sort, String[] columns) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getUsersMapping(offset, limit,sort, columns)); + } + + public Response> getRolesMappings(int offset, int limit, String sort, String[] columns) throws AtlasBaseException { + return processResponse(this.retrofitHeraclesClient.getRolesMapping(offset, limit, sort, columns)); + } + +} \ No newline at end of file diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java new file mode 100644 index 0000000000..b9fd8de6ca --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/RetrofitHeraclesClient.java @@ -0,0 +1,24 @@ +package org.apache.atlas.auth.client.heracles; + +import org.apache.atlas.auth.client.heracles.models.HeraclesRoleViewRepresentation; +import org.apache.atlas.auth.client.heracles.models.HeraclesUserViewRepresentation; +import retrofit2.Call; +import retrofit2.http.GET; +import retrofit2.http.Headers; +import retrofit2.http.Query; + +import java.util.List; + +public interface RetrofitHeraclesClient { + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/users/mappings") + Call> getUsersMapping(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort, + @Query("columns") String[] columns); + + @Headers({"Accept: application/json,text/plain", "Cache-Control: no-store", "Cache-Control: no-cache"}) + @GET("/roles/mappings") + Call> getRolesMapping(@Query("offset") Integer offset, @Query("limit") Integer limit, @Query("sort") String sort, + @Query("columns") String[] columns); + + +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java new file mode 100644 index 0000000000..5b695bcf71 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesRoleViewRepresentation.java @@ -0,0 +1,60 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.util.ArrayList; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesRoleViewRepresentation { + protected String id; + protected String name; + protected String realmId; + protected List roles; + protected List groups; + + public static String sortBy = "name"; + + public HeraclesRoleViewRepresentation() { + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getRealmId() { + return realmId; + } + + public List getRoles() { + return roles == null ? new ArrayList<>() : roles; + } + + public List getGroups() { + return groups == null ? new ArrayList<>() : groups; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setRealmId(String realmId) { + this.realmId = realmId; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setGroups(List groups) { + this.groups = groups; + } +} diff --git a/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java new file mode 100644 index 0000000000..de3bc1e105 --- /dev/null +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/heracles/models/HeraclesUserViewRepresentation.java @@ -0,0 +1,61 @@ +package org.apache.atlas.auth.client.heracles.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.keycloak.representations.idm.UserRepresentation; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class HeraclesUserViewRepresentation { + protected String id; + protected String username; + protected boolean enabled; + protected List roles; + protected List groups; + + public static String sortBy = "username"; + + public HeraclesUserViewRepresentation() { + } + + public String getId() { + return id; + } + + public String getUsername() { + return username; + } + + public boolean isEnabled() { + return enabled; + } + + public List getRoles() { + return roles; + } + + public List getGroups() { + return groups; + } + + public void setId(String id) { + this.id = id; + } + + public void setUsername(String username) { + this.username = username; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public void setGroups(List groups) { + this.groups = groups; + } + +} diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java similarity index 70% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java index fe723bbce1..9ec43428e6 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AtlasKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/AtlasKeycloakClient.java @@ -1,29 +1,19 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; -import org.apache.atlas.keycloak.client.config.KeycloakConfigBuilder; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; import org.keycloak.representations.idm.*; import org.keycloak.representations.oidc.TokenMetadataRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; -import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY; - /** * Keycloak client, deals with token creation refresh. */ @@ -31,14 +21,6 @@ public final class AtlasKeycloakClient { public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakClient.class); - private final static String KEYCLOAK_PROPERTIES = "keycloak.json"; - private final static String DEFAULT_GRANT_TYPE = "client_credentials"; - private final static String KEY_REALM_ID = "realm"; - private final static String KEY_AUTH_SERVER_URL = "auth-server-url"; - private final static String KEY_CLIENT_ID = "resource"; - private final static String KEY_CREDENTIALS = "credentials"; - private final static String KEY_SECRET = "secret"; - private static KeycloakRestClient KEYCLOAK; private static AtlasKeycloakClient KEYCLOAK_CLIENT; @@ -183,13 +165,7 @@ public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException if (Objects.isNull(KEYCLOAK_CLIENT)) { LOG.info("Initializing Keycloak client.."); try { - init(getConfig()); - } catch (IOException e) { - LOG.error("Failed to fetch Keycloak conf {}", e.getMessage()); - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); - } catch (JSONException e) { - LOG.error("Failed to parse Keycloak conf {}", e.getMessage()); - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); + init(AuthConfig.getConfig()); } catch (Exception e) { LOG.error("Failed to connect to Keycloak {}", e.getMessage()); throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, e.getMessage()); @@ -201,7 +177,7 @@ public static AtlasKeycloakClient getKeycloakClient() throws AtlasBaseException return KEYCLOAK_CLIENT; } - private static void init(KeycloakConfig config) { + private static void init(AuthConfig config) { synchronized (AtlasKeycloakClient.class) { if (KEYCLOAK_CLIENT == null) { KEYCLOAK = new KeycloakRestClient(config); @@ -209,30 +185,4 @@ private static void init(KeycloakConfig config) { } } } - - private static KeycloakConfig getConfig() throws Exception { - String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY); - File confFile; - if (StringUtils.isNotEmpty(confLocation)) { - confFile = new File(confLocation, KEYCLOAK_PROPERTIES); - - if (confFile.exists()) { - String keyConf = new String(Files.readAllBytes(confFile.toPath()), StandardCharsets.UTF_8); - JSONObject object = new JSONObject(keyConf); - - String REALM_ID = object.getString(KEY_REALM_ID); - String AUTH_SERVER_URL = object.getString(KEY_AUTH_SERVER_URL) + "/"; - String CLIENT_ID = object.getString(KEY_CLIENT_ID); - String GRANT_TYPE = DEFAULT_GRANT_TYPE; - String CLIENT_SECRET = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET); - - LOG.info("Keycloak conf: REALM_ID:{}, AUTH_SERVER_URL:{}", REALM_ID, AUTH_SERVER_URL); - return KeycloakConfigBuilder.builder().realId(REALM_ID).authServerUrl(AUTH_SERVER_URL).clientId(CLIENT_ID).grantType(GRANT_TYPE).clientSecret(CLIENT_SECRET).build(); - } else { - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation); - } - } else { - throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Configuration location not found " + confLocation); - } - } } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java similarity index 54% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java index 7cc1eea22f..b6f58982e6 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/KeycloakRestClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/KeycloakRestClient.java @@ -1,9 +1,10 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; +import org.apache.atlas.auth.client.auth.AbstractAuthClient; import okhttp3.FormBody; import okhttp3.RequestBody; +import org.apache.atlas.auth.client.config.AuthConfig; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.config.KeycloakConfig; import org.keycloak.representations.idm.*; import org.keycloak.representations.oidc.TokenMetadataRepresentation; import retrofit2.Response; @@ -14,129 +15,129 @@ /** * Keycloak Rest client wrapper used in atlas metastore */ -public final class KeycloakRestClient extends AbstractKeycloakClient { +public final class KeycloakRestClient extends AbstractAuthClient { private static final String TOKEN = "token"; private static final String CLIENT_ID = "client_id"; private static final String CLIENT_SECRET = "client_secret"; - public KeycloakRestClient(final KeycloakConfig keycloakConfig) { - super(keycloakConfig); + public KeycloakRestClient(final AuthConfig authConfig) { + super(authConfig); } public Response> searchUserByUserName(String username) throws AtlasBaseException { - return processResponse(this.retrofit.searchUserByUserName(this.keycloakConfig.getRealmId(), username)); + return processResponse(this.retrofitKeycloakClient.searchUserByUserName(this.authConfig.getRealmId(), username)); } public Response> getAllUsers(int start, int size) throws AtlasBaseException { - return processResponse(this.retrofit.getAllUsers(this.keycloakConfig.getRealmId(), start, size)); + return processResponse(this.retrofitKeycloakClient.getAllUsers(this.authConfig.getRealmId(), start, size)); } public Response> getRoleUserMembers(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleUserMembers(this.authConfig.getRealmId(), roleName)); } public Response> getRoleUserMembers(String roleName, Integer start, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleUserMembers(this.keycloakConfig.getRealmId(), roleName, start, size)); + return processResponse(this.retrofitKeycloakClient.getRoleUserMembers(this.authConfig.getRealmId(), roleName, start, size)); } public Response> searchGroupByName(String groupName, Integer start, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.searchGroupByName(this.keycloakConfig.getRealmId(), groupName, start, size)); + return processResponse(this.retrofitKeycloakClient.searchGroupByName(this.authConfig.getRealmId(), groupName, start, size)); } public Response> getRoleGroupMembers(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleGroupMembers(this.authConfig.getRealmId(), roleName)); } public Response> getRoleGroupMembers(String roleName, Integer first, Integer size) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleGroupMembers(this.keycloakConfig.getRealmId(), roleName, first, size)); + return processResponse(this.retrofitKeycloakClient.getRoleGroupMembers(this.authConfig.getRealmId(), roleName, first, size)); } public Response> getGroupsForUserById(String userId) throws AtlasBaseException { - return processResponse(this.retrofit.getGroupsForUserById(this.keycloakConfig.getRealmId(), userId)); + return processResponse(this.retrofitKeycloakClient.getGroupsForUserById(this.authConfig.getRealmId(), userId)); } public void addRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.addRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + processResponse(this.retrofitKeycloakClient.addRealmLevelRoleMappingsForGroup(this.authConfig.getRealmId(), groupId, roles)); } public void deleteRealmLevelRoleMappingsForGroup(String groupId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteRealmLevelRoleMappingsForGroup(this.keycloakConfig.getRealmId(), groupId, roles)); + processResponse(this.retrofitKeycloakClient.deleteRealmLevelRoleMappingsForGroup(this.authConfig.getRealmId(), groupId, roles)); } public Response> getAllRoles(int start, int size) throws AtlasBaseException { - return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), start, size)); + return processResponse(this.retrofitKeycloakClient.getAllRoles(this.authConfig.getRealmId(), start, size)); } public void deleteRoleById(String roleId) throws AtlasBaseException { - processResponse(this.retrofit.deleteRoleById(this.keycloakConfig.getRealmId(), roleId)); + processResponse(this.retrofitKeycloakClient.deleteRoleById(this.authConfig.getRealmId(), roleId)); } public void deleteRoleByName(String roleName) throws AtlasBaseException { - processResponse(this.retrofit.deleteRoleByName(this.keycloakConfig.getRealmId(), roleName)); + processResponse(this.retrofitKeycloakClient.deleteRoleByName(this.authConfig.getRealmId(), roleName)); } public Response> addRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { - return processResponse(this.retrofit.addRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + return processResponse(this.retrofitKeycloakClient.addRealmLevelRoleMappingsForUser(this.authConfig.getRealmId(), userId, roles)); } public void deleteRealmLevelRoleMappingsForUser(String userId, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteRealmLevelRoleMappingsForUser(this.keycloakConfig.getRealmId(), userId, roles)); + processResponse(this.retrofitKeycloakClient.deleteRealmLevelRoleMappingsForUser(this.authConfig.getRealmId(), userId, roles)); } public void createRole(RoleRepresentation roleRepresentation) throws AtlasBaseException { - processResponse(this.retrofit.createRole(this.keycloakConfig.getRealmId(), roleRepresentation)); + processResponse(this.retrofitKeycloakClient.createRole(this.authConfig.getRealmId(), roleRepresentation)); } public void updateRole(String roleId, RoleRepresentation roleRepresentation) throws AtlasBaseException { - processResponse(this.retrofit.updateRole(this.keycloakConfig.getRealmId(), roleId, roleRepresentation)); + processResponse(this.retrofitKeycloakClient.updateRole(this.authConfig.getRealmId(), roleId, roleRepresentation)); } public Response getRoleById(String roleId) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleById(this.keycloakConfig.getRealmId(), roleId)); + return processResponse(this.retrofitKeycloakClient.getRoleById(this.authConfig.getRealmId(), roleId)); } public Response getRoleByName(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleByName(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleByName(this.authConfig.getRealmId(), roleName)); } public Response> getAllRoles(Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getAllRoles(this.keycloakConfig.getRealmId(), first, max)); + return processResponse(this.retrofitKeycloakClient.getAllRoles(this.authConfig.getRealmId(), first, max)); } public Response> getRoleComposites(String roleName) throws AtlasBaseException { - return processResponse(this.retrofit.getRoleComposites(this.keycloakConfig.getRealmId(), roleName)); + return processResponse(this.retrofitKeycloakClient.getRoleComposites(this.authConfig.getRealmId(), roleName)); } public void addComposites(String roleName, List roles) throws AtlasBaseException { - processResponse(this.retrofit.addComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + processResponse(this.retrofitKeycloakClient.addComposites(this.authConfig.getRealmId(), roleName, roles)); } public void deleteComposites(String roleName, List roles) throws AtlasBaseException { - processResponse(this.retrofit.deleteComposites(this.keycloakConfig.getRealmId(), roleName, roles)); + processResponse(this.retrofitKeycloakClient.deleteComposites(this.authConfig.getRealmId(), roleName, roles)); } public Response> getAdminEvents(List operationTypes, String authRealm, String authClient, String authUser, String authIpAddress, String resourcePath, String dateFrom, String dateTo, Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getAdminEvents(this.keycloakConfig.getRealmId(), operationTypes, + return processResponse(this.retrofitKeycloakClient.getAdminEvents(this.authConfig.getRealmId(), operationTypes, authRealm, authClient, authUser, authIpAddress, resourcePath, dateFrom, dateTo, first, max)); } public Response> getEvents(List type, String client, String user, String dateFrom, String dateTo, String ipAddress, Integer first, Integer max) throws AtlasBaseException { - return processResponse(this.retrofit.getEvents(this.keycloakConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); + return processResponse(this.retrofitKeycloakClient.getEvents(this.authConfig.getRealmId(), type, client, user, dateFrom, dateTo, ipAddress, first, max)); } public Response introspectToken(String token) throws AtlasBaseException { - return processResponse(this.retrofit.introspectToken(this.keycloakConfig.getRealmId(), getIntrospectTokenRequest(token))); + return processResponse(this.retrofitKeycloakClient.introspectToken(this.authConfig.getRealmId(), getIntrospectTokenRequest(token))); } private RequestBody getIntrospectTokenRequest(String token) { return new FormBody.Builder() .addEncoded(TOKEN, token) - .addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()) - .addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()) + .addEncoded(CLIENT_ID, this.authConfig.getClientId()) + .addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()) .build(); } } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java similarity index 99% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java rename to client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java index c396c9368d..4fe8bb6ff5 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/RetrofitKeycloakClient.java +++ b/client-auth/src/main/java/org/apache/atlas/auth/client/keycloak/RetrofitKeycloakClient.java @@ -1,4 +1,4 @@ -package org.apache.atlas.keycloak.client; +package org.apache.atlas.auth.client.keycloak; import okhttp3.RequestBody; import org.keycloak.representations.AccessTokenResponse; diff --git a/client-heracles/pom.xml b/client-heracles/pom.xml new file mode 100644 index 0000000000..b812aefb65 --- /dev/null +++ b/client-heracles/pom.xml @@ -0,0 +1,75 @@ + + + + + + apache-atlas + org.apache.atlas + 3.0.0-SNAPSHOT + + 4.0.0 + + client-heracles + + + 8 + 8 + + + + + org.keycloak + keycloak-core + ${keycloak-admin-client.version} + + + * + * + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.retrofit2 + retrofit + ${retrofit.version} + + + com.squareup.retrofit2 + converter-jackson + ${retrofit.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + org.apache.atlas + atlas-common + + + + \ No newline at end of file diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java b/client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java similarity index 84% rename from client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java rename to client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java index 048d2fac71..863eec0021 100644 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfig.java +++ b/client-heracles/src/main/java/heracles/client/config/HeraclesConfigBuilder.java @@ -1,6 +1,6 @@ -package org.apache.atlas.keycloak.client.config; +package main.java.heracles.client.config; -public final class KeycloakConfig { +public class HeraclesConfigBuilder { String authServerUrl; String realmId; @@ -27,5 +27,4 @@ public String getClientSecret() { public String getGrantType() { return grantType; } - } diff --git a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java b/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java deleted file mode 100644 index 44583c59f3..0000000000 --- a/client-keycloak/src/main/java/org/apache/atlas/keycloak/client/config/KeycloakConfigBuilder.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.apache.atlas.keycloak.client.config; - -public final class KeycloakConfigBuilder { - - private String authServerUrl; - private String realmId; - private String clientId; - private String clientSecret; - private String grantType = "client_credentials"; - - private KeycloakConfigBuilder() { - } - - public static KeycloakConfigBuilder builder() { - return new KeycloakConfigBuilder(); - } - - public KeycloakConfigBuilder authServerUrl(String authServerUrl) { - this.authServerUrl = authServerUrl; - return this; - } - - public KeycloakConfigBuilder realId(String realId) { - this.realmId = realId; - return this; - } - - public KeycloakConfigBuilder clientId(String clientId) { - this.clientId = clientId; - return this; - } - - public KeycloakConfigBuilder clientSecret(String clientSecret) { - this.clientSecret = clientSecret; - return this; - } - - public KeycloakConfigBuilder grantType(String grantType) { - this.grantType = grantType; - return this; - } - - public KeycloakConfig build() { - KeycloakConfig keycloakConfig = new KeycloakConfig(); - keycloakConfig.authServerUrl = authServerUrl; - keycloakConfig.realmId = realmId; - keycloakConfig.clientId = clientId; - keycloakConfig.clientSecret = clientSecret; - keycloakConfig.grantType = grantType; - return keycloakConfig; - } -} \ No newline at end of file diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 889d5b413f..c9452ebf0d 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -108,7 +108,8 @@ public enum AtlasConfiguration { PERSONA_POLICY_ASSET_MAX_LIMIT("atlas.persona.policy.asset.maxlimit", 1000), ENABLE_KEYCLOAK_TOKEN_INTROSPECTION("atlas.canary.keycloak.token-introspection", false), - KEYCLOAK_ADMIN_CLIENT_PAGINATION_SIZE("atlas.keycloak.admin.resource-pagination-size", 1500); + HERACLES_CLIENT_PAGINATION_SIZE("atlas.heracles.admin.resource-pagination-size", 100), + HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"); private static final Configuration APPLICATION_PROPERTIES; diff --git a/pom.xml b/pom.xml index 6ffea247db..1cc9aa70dc 100644 --- a/pom.xml +++ b/pom.xml @@ -790,7 +790,7 @@ server-api notification client - client-keycloak + client-auth graphdb repository diff --git a/repository/pom.xml b/repository/pom.xml index 03e66a8310..10d8d876fb 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -318,7 +318,7 @@ org.apache.atlas - client-keycloak + client-auth 3.0.0-SNAPSHOT diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index b994398c23..02fb63bbc8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -55,7 +55,6 @@ import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; @@ -63,6 +62,7 @@ import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class ConnectionPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index ff0b0ddc9d..c0741f65f1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -20,7 +20,7 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java index 63e8ba4250..5dec4d3190 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java @@ -58,7 +58,7 @@ import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; import static org.apache.atlas.repository.Constants.ATTR_VIEWER_GROUPS; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java index 3a5f1782f0..22e04cd751 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/users/KeycloakStore.java @@ -33,8 +33,7 @@ import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; -import static org.apache.atlas.repository.util.AccessControlUtils.INSTANCE_DOMAIN_KEY; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class KeycloakStore { private static final Logger LOG = LoggerFactory.getLogger(KeycloakStore.class); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 11a56758ed..6bb026752c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -33,7 +33,7 @@ import java.util.*; import java.util.stream.Collectors; -import static org.apache.atlas.keycloak.client.AtlasKeycloakClient.getKeycloakClient; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.*; @Path("migration") diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java index 88523998cd..1a22038398 100644 --- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java +++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasKeycloakAuthenticationProvider.java @@ -18,7 +18,7 @@ import io.micrometer.core.instrument.Counter; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.keycloak.client.AtlasKeycloakClient; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.service.metrics.MetricUtils; import org.apache.atlas.ApplicationProperties; import org.apache.commons.configuration.Configuration;