feat: Policy Cache mechanism improvements[staging]

auth-agents-common/pom.xml

@@ -51,7 +51,7 @@
 
         <dependency>
             <groupId>org.apache.atlas</groupId>
-            <artifactId>client-keycloak</artifactId>
+            <artifactId>client-auth</artifactId>
             <version>${project.version}</version>
         </dependency>
 

auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerUserStoreProvider.java

@@ -26,15 +26,13 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.atlas.admin.client.RangerAdminClient;
 import org.apache.atlas.plugin.service.RangerBasePlugin;
 
 import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.Reader;
 import java.io.Writer;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.Set;
 

client-keycloak/pom.xml → client-auth/pom.xml

@@ -27,7 +27,7 @@
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>client-keycloak</artifactId>
+    <artifactId>client-auth</artifactId>
 
     <properties>
         <maven.compiler.source>8</maven.compiler.source>

client-keycloak/src/main/java/org/apache/atlas/keycloak/client/AbstractKeycloakClient.java → client-auth/src/main/java/org/apache/atlas/auth/client/auth/AbstractAuthClient.java

@@ -1,13 +1,15 @@
-package org.apache.atlas.keycloak.client;
+package org.apache.atlas.auth.client.auth;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.micrometer.core.instrument.Timer;
+import org.apache.atlas.auth.client.config.AuthConfig;
+import org.apache.atlas.auth.client.heracles.RetrofitHeraclesClient;
+import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient;
 import okhttp3.*;
 import okhttp3.logging.HttpLoggingInterceptor;
 import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.keycloak.client.config.KeycloakConfig;
-import org.apache.atlas.keycloak.client.service.AtlasKeycloakAuthService;
 import org.apache.atlas.service.metrics.MetricUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,31 +28,32 @@
 import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST;
 import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND;
 
-abstract class AbstractKeycloakClient {
+public class AbstractAuthClient {
 
-    private final static Logger LOG = LoggerFactory.getLogger(AbstractKeycloakClient.class);
+    private final static Logger LOG = LoggerFactory.getLogger(AbstractAuthClient.class);
     private static final Map<Integer, AtlasErrorCode> ERROR_CODE_MAP = new HashMap<>();
 
-    private static final int DEFAULT_KEYCLOAK_RETRY = 3;
+    private static final int DEFAULT_RETRY = 3;
     private static final String AUTHORIZATION = "Authorization";
     private static final String BEARER = "Bearer ";
     private static final int TIMEOUT_IN_SEC = 60;
     private static final String INTEGRATION = "integration";
     private static final String KEYCLOAK = "keycloak";
 
-    protected final KeycloakConfig keycloakConfig;
-    protected final RetrofitKeycloakClient retrofit;
+    protected final AuthConfig authConfig;
+    protected final RetrofitKeycloakClient retrofitKeycloakClient;
+    protected final RetrofitHeraclesClient retrofitHeraclesClient;
 
-    private final AtlasKeycloakAuthService authService;
+    private final KeycloakAuthenticationService authService;
     private MetricUtils metricUtils = null;
 
     static {
         ERROR_CODE_MAP.put(HTTP_NOT_FOUND, RESOURCE_NOT_FOUND);
         ERROR_CODE_MAP.put(HTTP_BAD_REQUEST, BAD_REQUEST);
     }
 
-    public AbstractKeycloakClient(KeycloakConfig keycloakConfig) {
-        this.keycloakConfig = keycloakConfig;
+    public AbstractAuthClient(AuthConfig authConfig) {
+        this.authConfig = authConfig;
         this.metricUtils = new MetricUtils();
         HttpLoggingInterceptor httpInterceptor = new HttpLoggingInterceptor();
         httpInterceptor.setLevel(HttpLoggingInterceptor.Level.BODY);
@@ -64,11 +67,15 @@ public AbstractKeycloakClient(KeycloakConfig keycloakConfig) {
                 .writeTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS)
                 .readTimeout(TIMEOUT_IN_SEC, TimeUnit.SECONDS)
                 .build();
-        this.retrofit = new Retrofit.Builder().client(okHttpClient)
-                .baseUrl(this.keycloakConfig.getAuthServerUrl())
+        this.retrofitKeycloakClient = new Retrofit.Builder().client(okHttpClient)
+                .baseUrl(this.authConfig.getAuthServerUrl())
                 .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build()
                 .create(RetrofitKeycloakClient.class);
-        authService = new AtlasKeycloakAuthService(keycloakConfig);
+        this.retrofitHeraclesClient = new Retrofit.Builder().client(okHttpClient)
+                .baseUrl(this.authConfig.getHeraclesApiServerUrl())
+                .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper().disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES))).build()
+                .create(RetrofitHeraclesClient.class);
+        authService = new KeycloakAuthenticationService(authConfig);
     }
 
     /**
@@ -97,21 +104,20 @@ public Response intercept(@NonNull Chain chain) throws IOException {
             return chain.proceed(request);
         }
     };
-
     /**
      * Called only during auth failures.
      */
     Authenticator authInterceptor = new Authenticator() {
         @Override
         public Request authenticate(Route route, @NonNull Response response) {
-            if (responseCount(response) > DEFAULT_KEYCLOAK_RETRY) {
-                LOG.warn("Keycloak: Falling back, retried {} times", DEFAULT_KEYCLOAK_RETRY);
+            if (responseCount(response) > DEFAULT_RETRY) {
+                LOG.warn("Auth Client: Falling back, retried {} times", DEFAULT_RETRY);
                 return null;
             }
-            LOG.info("Keycloak: Current keycloak token status, Expired: {}", authService.isTokenExpired());
+            LOG.info("Auth Client: Current keycloak token status, Expired: {}", authService.isTokenExpired());
             return response.request().newBuilder()
-                        .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken())
-                        .build();
+                    .addHeader(AUTHORIZATION, BEARER + authService.getAuthToken())
+                    .build();
         }
 
         private int responseCount(Response response) {
@@ -134,13 +140,14 @@ protected <T> retrofit2.Response<T> processResponse(retrofit2.Call<T> req) throw
                 return response;
             }
             String errMsg = response.errorBody().string();
-            LOG.error("Keycloak: Client request processing failed code {} message:{}, request: {} {}",
+            LOG.error("Auth Client: Client request processing failed code {} message:{}, request: {} {}",
                     response.code(), errMsg, req.request().method(), req.request().url());
             throw new AtlasBaseException(ERROR_CODE_MAP.getOrDefault(response.code(), BAD_REQUEST), errMsg);
         } catch (Exception e) {
-            LOG.error("Keycloak: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e);
-            throw new AtlasBaseException(BAD_REQUEST, "Keycloak request failed");
+            LOG.error("Auth Client: request failed, request: {} {}, Exception: {}", req.request().method(), req.request().url(), e);
+            throw new AtlasBaseException(BAD_REQUEST, "Auth request failed");
         }
     }
 
+
 }

client-keycloak/src/main/java/org/apache/atlas/keycloak/client/service/AtlasKeycloakAuthService.java → client-auth/src/main/java/org/apache/atlas/auth/client/auth/KeycloakAuthenticationService.java

@@ -1,11 +1,11 @@
-package org.apache.atlas.keycloak.client.service;
+package org.apache.atlas.auth.client.auth;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import okhttp3.*;
 import okhttp3.logging.HttpLoggingInterceptor;
+import org.apache.atlas.auth.client.config.AuthConfig;
 import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.keycloak.client.RetrofitKeycloakClient;
-import org.apache.atlas.keycloak.client.config.KeycloakConfig;
+import org.apache.atlas.auth.client.keycloak.RetrofitKeycloakClient;
 import org.jetbrains.annotations.NotNull;
 import org.keycloak.representations.AccessTokenResponse;
 import org.slf4j.Logger;
@@ -19,9 +19,9 @@
 
 import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST;
 
-public final class AtlasKeycloakAuthService {
+public final class KeycloakAuthenticationService {
 
-    public final static Logger LOG = LoggerFactory.getLogger(AtlasKeycloakAuthService.class);
+    public final static Logger LOG = LoggerFactory.getLogger(KeycloakAuthenticationService.class);
 
     private final static String GRANT_TYPE = "grant_type";
     private static final String CLIENT_ID = "client_id";
@@ -30,14 +30,14 @@ public final class AtlasKeycloakAuthService {
     private static final int TIMEOUT_IN_SECS = 60;
 
     private final RetrofitKeycloakClient retrofit;
-    private final KeycloakConfig keycloakConfig;
+    private final AuthConfig authConfig;
     private AccessTokenResponse currentAccessToken;
     private long expirationTime = -1;
 
-    public AtlasKeycloakAuthService(KeycloakConfig keycloakConfig) {
-        this.keycloakConfig = keycloakConfig;
+    public KeycloakAuthenticationService(AuthConfig authConfig) {
+        this.authConfig = authConfig;
         this.retrofit = new Retrofit.Builder().client(getOkHttpClient())
-                .baseUrl(this.keycloakConfig.getAuthServerUrl())
+                .baseUrl(this.authConfig.getAuthServerUrl())
                 .addConverterFactory(JacksonConverterFactory.create(new ObjectMapper())).build()
                 .create(RetrofitKeycloakClient.class);
     }
@@ -59,7 +59,7 @@ private OkHttpClient getOkHttpClient() {
     Interceptor responseLoggingInterceptor = chain -> {
         Request request = chain.request();
         okhttp3.Response response = chain.proceed(request);
-        LOG.info("Keycloak: Auth Request for url {} Status: {}", request.url(), response.code());
+        LOG.info("Auth Client: Auth Request for url {} Status: {}", request.url(), response.code());
         return response;
     };
 
@@ -70,16 +70,16 @@ public String getAuthToken() {
         synchronized (this) {
             if (isTokenExpired()) {
                 try {
-                    retrofit2.Response<AccessTokenResponse> resp = this.retrofit.grantToken(this.keycloakConfig.getRealmId(), getTokenRequest()).execute();
+                    retrofit2.Response<AccessTokenResponse> resp = this.retrofit.grantToken(this.authConfig.getRealmId(), getTokenRequest()).execute();
                     if (resp.isSuccessful()) {
                         currentAccessToken = resp.body();
                         expirationTime = currentTime() + currentAccessToken.getExpiresIn() - EXPIRY_OFFSET_SEC;
-                        LOG.info("Keycloak: Auth token fetched with expiry:{} sec", expirationTime);
+                        LOG.info("Auth Client: Auth token fetched with expiry:{} sec", expirationTime);
                     } else {
                         throw new AtlasBaseException(BAD_REQUEST, resp.errorBody().string());
                     }
                 } catch (Exception e) {
-                    LOG.error("Keycloak: Error while fetching access token for keycloak client.", e);
+                    LOG.error("Auth Client: Error while fetching access token for keycloak client.", e);
                     throw new RuntimeException(e);
                 }
             }
@@ -97,7 +97,7 @@ public boolean isTokenExpired() {
     }
 
     private RequestBody getTokenRequest() {
-        return new FormBody.Builder().addEncoded(CLIENT_ID, this.keycloakConfig.getClientId()).addEncoded(CLIENT_SECRET, this.keycloakConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.keycloakConfig.getGrantType()).build();
+        return new FormBody.Builder().addEncoded(CLIENT_ID, this.authConfig.getClientId()).addEncoded(CLIENT_SECRET, this.authConfig.getClientSecret()).addEncoded(GRANT_TYPE, this.authConfig.getGrantType()).build();
     }
 
     private long currentTime() {

client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfig.java

@@ -0,0 +1,100 @@
+package org.apache.atlas.auth.client.config;
+
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Optional;
+
+import static org.apache.atlas.ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY;
+
+public class AuthConfig {
+    private static final Logger LOG = LoggerFactory.getLogger(AuthConfig.class);
+
+    public String authServerUrl;
+    public String realmId;
+    public String clientId;
+    public String clientSecret;
+    public String grantType;
+    public String heraclesApiServerUrl;
+
+    private static final String KEYCLOAK_PROPERTIES = "keycloak.json";
+    private static final String DEFAULT_GRANT_TYPE = "client_credentials";
+    private static final String KEY_REALM_ID = "realm";
+    private static final String KEY_AUTH_SERVER_URL = "auth-server-url";
+    private static final String KEY_CLIENT_ID = "resource";
+    private static final String KEY_CREDENTIALS = "credentials";
+    private static final String KEY_SECRET = "secret";
+
+    public String getAuthServerUrl() {
+        return authServerUrl;
+    }
+
+    public String getRealmId() {
+        return realmId;
+    }
+
+    public String getClientId() {
+        return clientId;
+    }
+
+    public String getClientSecret() {
+        return clientSecret;
+    }
+
+    public String getGrantType() {
+        return grantType;
+    }
+
+    public String getHeraclesApiServerUrl() {
+        return heraclesApiServerUrl;
+    }
+
+    public static AuthConfig getConfig() throws AtlasBaseException {
+        String confLocation = System.getProperty(ATLAS_CONFIGURATION_DIRECTORY_PROPERTY);
+        Optional<File> confFile = getConfigurationFile(confLocation);
+
+        if (confFile.isPresent()) {
+            try {
+                JSONObject object = new JSONObject(readFileToString(confFile.get()));
+                return buildAuthConfigFromJson(object);
+            } catch (Exception e) {
+                LOG.error("Error parsing Keycloak configuration: ", e);
+                throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Error parsing Keycloak configuration");
+            }
+        } else {
+            throw new AtlasBaseException(AtlasErrorCode.KEYCLOAK_INIT_FAILED, "Keycloak configuration file not found in location " + confLocation);
+        }
+    }
+
+    private static Optional<File> getConfigurationFile(String confLocation) {
+        if (StringUtils.isNotEmpty(confLocation)) {
+            File confFile = new File(confLocation, KEYCLOAK_PROPERTIES);
+            if (confFile.exists()) {
+                return Optional.of(confFile);
+            }
+        }
+        return Optional.empty();
+    }
+
+    private static String readFileToString(File file) throws Exception {
+        return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
+    }
+
+    private static AuthConfig buildAuthConfigFromJson(JSONObject object) throws Exception {
+        String realmId = object.getString(KEY_REALM_ID);
+        String authServerUrl = object.getString(KEY_AUTH_SERVER_URL) + "/";
+        String clientId = object.getString(KEY_CLIENT_ID);
+        String grantType = DEFAULT_GRANT_TYPE;
+        String clientSecret = object.getJSONObject(KEY_CREDENTIALS).getString(KEY_SECRET);
+
+        LOG.info("Keycloak configuration: REALM_ID:{}, AUTH_SERVER_URL:{}", realmId, authServerUrl);
+        return AuthConfigBuilder.builder().realId(realmId).authServerUrl(authServerUrl).clientId(clientId).grantType(grantType).clientSecret(clientSecret).build();
+    }
+}

client-auth/src/main/java/org/apache/atlas/auth/client/config/AuthConfigBuilder.java

@@ -0,0 +1,55 @@
+package org.apache.atlas.auth.client.config;
+
+import org.apache.atlas.AtlasConfiguration;
+
+public class AuthConfigBuilder {
+
+    private String authServerUrl;
+    private String realmId;
+    private String clientId;
+    private String clientSecret;
+    private String grantType = "client_credentials";
+
+    private AuthConfigBuilder() {
+    }
+
+    public static AuthConfigBuilder builder() {
+        return new AuthConfigBuilder();
+    }
+
+    public AuthConfigBuilder authServerUrl(String authServerUrl) {
+        this.authServerUrl = authServerUrl;
+        return this;
+    }
+
+    public AuthConfigBuilder realId(String realId) {
+        this.realmId = realId;
+        return this;
+    }
+
+    public AuthConfigBuilder clientId(String clientId) {
+        this.clientId = clientId;
+        return this;
+    }
+
+    public AuthConfigBuilder clientSecret(String clientSecret) {
+        this.clientSecret = clientSecret;
+        return this;
+    }
+
+    public AuthConfigBuilder grantType(String grantType) {
+        this.grantType = grantType;
+        return this;
+    }
+
+    public AuthConfig build() {
+        AuthConfig authConfig = new AuthConfig();
+        authConfig.authServerUrl = authServerUrl;
+        authConfig.realmId = realmId;
+        authConfig.clientId = clientId;
+        authConfig.clientSecret = clientSecret;
+        authConfig.grantType = grantType;
+        authConfig.heraclesApiServerUrl= AtlasConfiguration.HERACLES_API_SERVER_URL.getString()+"/";
+        return authConfig;
+    }
+}