From a107b3a4f5c029d9ab728f956ff963386c0fa55a Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 01/83] Enable OAuth2 authentication. --- .../opal-client/opal_client/callbacks/api.py | 4 +- packages/opal-client/opal_client/client.py | 30 ++-- .../opal-client/opal_client/data/updater.py | 38 ++++- .../opal-client/opal_client/policy/fetcher.py | 24 ++- .../opal-client/opal_client/policy/updater.py | 26 ++- .../opal_client/policy_store/api.py | 4 +- .../authentication/authenticator.py | 50 ++++++ .../opal_common/authentication/authz.py | 6 +- .../opal_common/authentication/jwk.py | 46 +++++ .../opal_common/authentication/oauth2.py | 157 ++++++++++++++++++ packages/opal-common/opal_common/config.py | 22 +++ .../fetcher/providers/http_fetch_provider.py | 13 +- .../opal_server/authentication/__init__.py | 0 .../authentication/authenticator.py | 55 ++++++ packages/opal-server/opal_server/data/api.py | 5 +- .../opal_server/policy/webhook/api.py | 4 +- packages/opal-server/opal_server/pubsub.py | 10 +- .../opal-server/opal_server/scopes/api.py | 5 +- .../opal-server/opal_server/security/api.py | 5 +- .../opal-server/opal_server/security/jwks.py | 5 +- packages/opal-server/opal_server/server.py | 52 +++--- packages/requires.txt | 1 + 22 files changed, 468 insertions(+), 94 deletions(-) create mode 100644 packages/opal-common/opal_common/authentication/authenticator.py create mode 100644 packages/opal-common/opal_common/authentication/jwk.py create mode 100644 packages/opal-common/opal_common/authentication/oauth2.py create mode 100644 packages/opal-server/opal_server/authentication/__init__.py create mode 100644 packages/opal-server/opal_server/authentication/authenticator.py diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..9b828cced 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -29,8 +29,7 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +48,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[ClientAuthenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +63,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = ClientAuthenticator() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +122,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -140,6 +144,7 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +167,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +242,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index d2c81c9ed..bc8460706 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,6 +24,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig from opal_common.http import is_http_error_response @@ -54,6 +55,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[ClientAuthenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -110,17 +112,18 @@ def __init__( self._callbacks_register, ) self._token = token + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None self._shard_id = shard_id self._server_url = pubsub_url self._data_sources_config_url = data_sources_config_url self._opal_client_id = opal_client_id - self._extra_headers = [] + self._extra_headers = {} if self._token is not None: - self._extra_headers.append(get_authorization_header(self._token)) + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] if self._shard_id is not None: - self._extra_headers.append(("X-Shard-ID", self._shard_id)) - if len(self._extra_headers) == 0: - self._extra_headers = None + self._extra_headers['X-Shard-ID'] = self._shard_id self._stopping = False # custom SSL context (for self-signed certificates) self._custom_ssl_context = get_custom_ssl_context() @@ -132,6 +135,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() async def __aenter__(self): await self.start() @@ -177,8 +184,14 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + try: - async with ClientSession(headers=self._extra_headers) as session: + async with ClientSession(headers=headers) as session: response = await session.get(url, **self._ssl_context_kwargs) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) @@ -274,12 +287,19 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( - self._data_topics, - self._update_policy_data_callback, + topics=self._data_topics, + callback=self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + on_disconnect=[self.on_disconnect], + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5ae9d93b6 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +29,26 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[ClientAuthenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + self._auth_headers = {} + if self._token != "THIS_IS_A_DEV_SECRET": + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +94,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, @@ -95,7 +112,6 @@ async def _fetch_policy_bundle( self._policy_endpoint_url, headers={ "content-type": "text/plain", - **self._auth_headers, }, params=params, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..d505c52f5 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +44,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[ClientAuthenticator] = None, ): """inits the policy updater. @@ -64,15 +66,21 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data self._server_url = pubsub_url self._token = token - if self._token is None: - self._extra_headers = None - else: - self._extra_headers = [get_authorization_header(self._token)] + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None + self._extra_headers = {} + if self._token is not None: + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] # Pub/Sub topics we subscribe to for policy updates if self._scope_id == "default": self._topics = pubsub_topics_from_directories( @@ -87,7 +95,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +248,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index 3c7428644..c7353624d 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,15 +1,15 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..938ac001f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,50 @@ +from abc import abstractmethod +from typing import Optional + +from fastapi import Header +from opal_common.config import opal_common_config +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + +class Authenticator: + @property + def enabled(self): + return self._delegate().enabled + + async def authenticate(self, headers): + if hasattr(self._delegate(), "authenticate") and callable(getattr(self._delegate(), "authenticate")): + await self._delegate().authenticate(headers) + + @abstractmethod + def _delegate(self) -> dict: + pass + +class _ClientAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will authenticate API requests.") + else: + self.__delegate = JWTAuthenticator(self.__verifier()) + + def __verifier(self) -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info("API authentication disabled (public encryption key was not provided)") + + return verifier + + def _delegate(self) -> dict: + return self.__delegate + +class ClientAuthenticator(_ClientAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..9b0ec207f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,46 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + #TODO TODO: maxsize, ttl + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..645e8f9ea --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,157 @@ +import asyncio +import httpx +import jwt +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator: + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return True + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index 7666d47e4..175c289b7 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -160,6 +160,28 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str("AUTH_TYPE", None, description="Authentication type.") + OAUTH2_CLIENT_ID = confi.str("OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID.") + OAUTH2_CLIENT_SECRET = confi.str("OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret.") + OAUTH2_TOKEN_URL = confi.str("OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL.") + OAUTH2_INTROSPECT_URL = confi.str("OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL.") + OAUTH2_OPENID_CONFIGURATION_URL = confi.str("OAUTH2_OPENID_CONFIGURATION_URL", None, description="OAuth2 OpenID configuration URL.") + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", 100, description="OAuth2 token validation cache maxsize.") + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_TTL", 5 * 60, description="OAuth2 token validation cache TTL.") + + OAUTH2_EXP_MARGIN = confi.int("OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin.") + OAUTH2_EXACT_MATCH_CLAIMS = confi.str("OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims.") + OAUTH2_REQUIRED_CLAIMS = confi.str("OAUTH2_REQUIRED_CLAIMS", None, description="Comma separated list of required claims.") + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str("OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience") + OAUTH2_JWT_ISSUER = confi.str("OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer") + OAUTH2_JWK_CACHE_MAXSIZE = confi.int("OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize.") + OAUTH2_JWK_CACHE_TTL = confi.int("OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL.") ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7261b538b..36f004ff4 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,11 +1,12 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator from ...http import is_http_error_response @@ -53,6 +54,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[dict] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -65,6 +68,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = ClientAuthenticator() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -72,7 +78,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..6d8773a1e --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,55 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from opal_server.config import opal_server_config + +class _ServerAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will verify API requests with OAuth2 tokens.") + else: + self.__delegate = JWTAuthenticator(self.__signer()) + + def __signer(self) -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info("OPAL is running in secure mode - will verify API requests with JWT tokens.") + else: + logger.info("OPAL was not provided with JWT encryption keys, cannot verify api requests!") + return signer + + def _delegate(self) -> dict: + return self.__delegate + + def signer(self) -> Optional[JWTSigner]: + if hasattr(self._delegate(), "verifier"): + return self._delegate().verifier + else: + return None + +class ServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) + +class WebsocketServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate()(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..60836994a 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -20,8 +20,9 @@ require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +79,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/security/api.py b/packages/opal-server/opal_server/security/api.py index a17235163..2a562405a 100644 --- a/packages/opal-server/opal_server/security/api.py +++ b/packages/opal-server/opal_server/security/api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status from opal_common.authentication.deps import StaticBearerAuthenticator @@ -7,7 +8,7 @@ from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails -def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): +def init_security_router(signer: Optional[JWTSigner], authenticator: StaticBearerAuthenticator): router = APIRouter() @router.post( @@ -17,7 +18,7 @@ def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthentic dependencies=[Depends(authenticator)], ) async def generate_new_access_token(req: AccessTokenRequest): - if not signer.enabled: + if signer is None or not signer.enabled: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="opal server was not configured with security, cannot generate tokens!", diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index b286fd796..ebfc3cb7f 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,7 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +21,7 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import ServerAuthenticator, WebsocketServerAuthenticator from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -49,7 +49,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[ServerAuthenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +118,22 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer + if authenticator is not None: + self.authenticator = authenticator else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) - else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticator() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), jwks_url=jwks_url, jwks_static_dir=jwks_static_dir ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticator() + self.pubsub = PubSub(broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +209,17 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) + webhook_router = init_git_webhook_router(self.pubsub.endpoint, self.authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +228,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +237,22 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router(self._scopes, self.authenticator, self.pubsub.endpoint), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..37dd369cb 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -9,3 +9,4 @@ typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 From fe6c816de80a0964ccb9878c969ea8abab0f0453 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 02/83] Enable OAuth2 authentication. --- .../opal-client/opal_client/callbacks/api.py | 4 +- packages/opal-client/opal_client/client.py | 30 ++-- .../opal-client/opal_client/data/updater.py | 38 ++++- .../opal-client/opal_client/policy/fetcher.py | 24 ++- .../opal-client/opal_client/policy/updater.py | 26 ++- .../opal_client/policy_store/api.py | 4 +- .../authentication/authenticator.py | 50 ++++++ .../opal_common/authentication/authz.py | 6 +- .../opal_common/authentication/jwk.py | 46 +++++ .../opal_common/authentication/oauth2.py | 157 ++++++++++++++++++ packages/opal-common/opal_common/config.py | 22 +++ .../fetcher/providers/http_fetch_provider.py | 13 +- .../opal_server/authentication/__init__.py | 0 .../authentication/authenticator.py | 55 ++++++ packages/opal-server/opal_server/data/api.py | 5 +- .../opal_server/policy/webhook/api.py | 4 +- packages/opal-server/opal_server/pubsub.py | 10 +- .../opal-server/opal_server/scopes/api.py | 5 +- .../opal-server/opal_server/security/api.py | 5 +- .../opal-server/opal_server/security/jwks.py | 5 +- packages/opal-server/opal_server/server.py | 52 +++--- packages/requires.txt | 1 + 22 files changed, 468 insertions(+), 94 deletions(-) create mode 100644 packages/opal-common/opal_common/authentication/authenticator.py create mode 100644 packages/opal-common/opal_common/authentication/jwk.py create mode 100644 packages/opal-common/opal_common/authentication/oauth2.py create mode 100644 packages/opal-server/opal_server/authentication/__init__.py create mode 100644 packages/opal-server/opal_server/authentication/authenticator.py diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..9b828cced 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -29,8 +29,7 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +48,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[ClientAuthenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +63,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = ClientAuthenticator() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +122,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -140,6 +144,7 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +167,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +242,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index d2c81c9ed..bc8460706 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,6 +24,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig from opal_common.http import is_http_error_response @@ -54,6 +55,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[ClientAuthenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -110,17 +112,18 @@ def __init__( self._callbacks_register, ) self._token = token + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None self._shard_id = shard_id self._server_url = pubsub_url self._data_sources_config_url = data_sources_config_url self._opal_client_id = opal_client_id - self._extra_headers = [] + self._extra_headers = {} if self._token is not None: - self._extra_headers.append(get_authorization_header(self._token)) + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] if self._shard_id is not None: - self._extra_headers.append(("X-Shard-ID", self._shard_id)) - if len(self._extra_headers) == 0: - self._extra_headers = None + self._extra_headers['X-Shard-ID'] = self._shard_id self._stopping = False # custom SSL context (for self-signed certificates) self._custom_ssl_context = get_custom_ssl_context() @@ -132,6 +135,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() async def __aenter__(self): await self.start() @@ -177,8 +184,14 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + try: - async with ClientSession(headers=self._extra_headers) as session: + async with ClientSession(headers=headers) as session: response = await session.get(url, **self._ssl_context_kwargs) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) @@ -274,12 +287,19 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( - self._data_topics, - self._update_policy_data_callback, + topics=self._data_topics, + callback=self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + on_disconnect=[self.on_disconnect], + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5ae9d93b6 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +29,26 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[ClientAuthenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + self._auth_headers = {} + if self._token != "THIS_IS_A_DEV_SECRET": + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +94,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, @@ -95,7 +112,6 @@ async def _fetch_policy_bundle( self._policy_endpoint_url, headers={ "content-type": "text/plain", - **self._auth_headers, }, params=params, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..d505c52f5 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +44,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[ClientAuthenticator] = None, ): """inits the policy updater. @@ -64,15 +66,21 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data self._server_url = pubsub_url self._token = token - if self._token is None: - self._extra_headers = None - else: - self._extra_headers = [get_authorization_header(self._token)] + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None + self._extra_headers = {} + if self._token is not None: + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] # Pub/Sub topics we subscribe to for policy updates if self._scope_id == "default": self._topics = pubsub_topics_from_directories( @@ -87,7 +95,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +248,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index 3c7428644..c7353624d 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,15 +1,15 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..938ac001f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,50 @@ +from abc import abstractmethod +from typing import Optional + +from fastapi import Header +from opal_common.config import opal_common_config +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + +class Authenticator: + @property + def enabled(self): + return self._delegate().enabled + + async def authenticate(self, headers): + if hasattr(self._delegate(), "authenticate") and callable(getattr(self._delegate(), "authenticate")): + await self._delegate().authenticate(headers) + + @abstractmethod + def _delegate(self) -> dict: + pass + +class _ClientAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will authenticate API requests.") + else: + self.__delegate = JWTAuthenticator(self.__verifier()) + + def __verifier(self) -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info("API authentication disabled (public encryption key was not provided)") + + return verifier + + def _delegate(self) -> dict: + return self.__delegate + +class ClientAuthenticator(_ClientAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..9b0ec207f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,46 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + #TODO TODO: maxsize, ttl + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..645e8f9ea --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,157 @@ +import asyncio +import httpx +import jwt +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator: + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return True + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index 7666d47e4..175c289b7 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -160,6 +160,28 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str("AUTH_TYPE", None, description="Authentication type.") + OAUTH2_CLIENT_ID = confi.str("OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID.") + OAUTH2_CLIENT_SECRET = confi.str("OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret.") + OAUTH2_TOKEN_URL = confi.str("OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL.") + OAUTH2_INTROSPECT_URL = confi.str("OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL.") + OAUTH2_OPENID_CONFIGURATION_URL = confi.str("OAUTH2_OPENID_CONFIGURATION_URL", None, description="OAuth2 OpenID configuration URL.") + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", 100, description="OAuth2 token validation cache maxsize.") + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_TTL", 5 * 60, description="OAuth2 token validation cache TTL.") + + OAUTH2_EXP_MARGIN = confi.int("OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin.") + OAUTH2_EXACT_MATCH_CLAIMS = confi.str("OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims.") + OAUTH2_REQUIRED_CLAIMS = confi.str("OAUTH2_REQUIRED_CLAIMS", None, description="Comma separated list of required claims.") + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str("OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience") + OAUTH2_JWT_ISSUER = confi.str("OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer") + OAUTH2_JWK_CACHE_MAXSIZE = confi.int("OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize.") + OAUTH2_JWK_CACHE_TTL = confi.int("OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL.") ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7261b538b..36f004ff4 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,11 +1,12 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator from ...http import is_http_error_response @@ -53,6 +54,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[dict] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -65,6 +68,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = ClientAuthenticator() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -72,7 +78,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..6d8773a1e --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,55 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from opal_server.config import opal_server_config + +class _ServerAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will verify API requests with OAuth2 tokens.") + else: + self.__delegate = JWTAuthenticator(self.__signer()) + + def __signer(self) -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info("OPAL is running in secure mode - will verify API requests with JWT tokens.") + else: + logger.info("OPAL was not provided with JWT encryption keys, cannot verify api requests!") + return signer + + def _delegate(self) -> dict: + return self.__delegate + + def signer(self) -> Optional[JWTSigner]: + if hasattr(self._delegate(), "verifier"): + return self._delegate().verifier + else: + return None + +class ServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) + +class WebsocketServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate()(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..60836994a 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -20,8 +20,9 @@ require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +79,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/security/api.py b/packages/opal-server/opal_server/security/api.py index a17235163..2a562405a 100644 --- a/packages/opal-server/opal_server/security/api.py +++ b/packages/opal-server/opal_server/security/api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status from opal_common.authentication.deps import StaticBearerAuthenticator @@ -7,7 +8,7 @@ from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails -def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): +def init_security_router(signer: Optional[JWTSigner], authenticator: StaticBearerAuthenticator): router = APIRouter() @router.post( @@ -17,7 +18,7 @@ def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthentic dependencies=[Depends(authenticator)], ) async def generate_new_access_token(req: AccessTokenRequest): - if not signer.enabled: + if signer is None or not signer.enabled: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="opal server was not configured with security, cannot generate tokens!", diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 6a946a8c0..10de584fd 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,7 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +21,7 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import ServerAuthenticator, WebsocketServerAuthenticator from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -49,7 +49,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[ServerAuthenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +118,22 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer + if authenticator is not None: + self.authenticator = authenticator else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) - else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticator() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), jwks_url=jwks_url, jwks_static_dir=jwks_static_dir ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticator() + self.pubsub = PubSub(broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +209,17 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) + webhook_router = init_git_webhook_router(self.pubsub.endpoint, self.authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +228,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +237,22 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router(self._scopes, self.authenticator, self.pubsub.endpoint), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..37dd369cb 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -9,3 +9,4 @@ typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 From cfb206ca63bfe0628b0375f234de9e382ac910db Mon Sep 17 00:00:00 2001 From: Oded Date: Tue, 25 Jun 2024 17:45:15 +0300 Subject: [PATCH 03/83] Removing secrets from policy_store/config route and deprecating this route --- packages/opal-client/opal_client/policy_store/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index 3c7428644..12db7e62e 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -16,6 +16,8 @@ def init_policy_store_router(authenticator: JWTAuthenticator): "/policy-store/config", response_model=PolicyStoreDetails, response_model_exclude_none=True, + # Deprecating this route + deprecated=True, ) async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): try: @@ -28,12 +30,11 @@ async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): return PolicyStoreDetails( url=opal_client_config.POLICY_STORE_URL, - token=opal_client_config.POLICY_STORE_AUTH_TOKEN or None, + token=None, auth_type=opal_client_config.POLICY_STORE_AUTH_TYPE or PolicyStoreAuth.NONE, oauth_client_id=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_ID or None, - oauth_client_secret=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET - or None, + oauth_client_secret=None, oauth_server=opal_client_config.POLICY_STORE_AUTH_OAUTH_SERVER or None, ) From fe3ed7022ddd2be61c72c568fbe704f7fa6e28bd Mon Sep 17 00:00:00 2001 From: Oded Date: Tue, 25 Jun 2024 18:08:17 +0300 Subject: [PATCH 04/83] make exclude API params configureable to not break OSS --- packages/opal-client/opal_client/config.py | 6 ++++++ packages/opal-client/opal_client/policy_store/api.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/opal-client/opal_client/config.py b/packages/opal-client/opal_client/config.py index 3f9f72125..fcc441a24 100644 --- a/packages/opal-client/opal_client/config.py +++ b/packages/opal-client/opal_client/config.py @@ -115,6 +115,12 @@ class OpalClientConfig(Confi): description="path to the file containing the ca certificate(s) used for tls authentication with the policy store", ) + EXCLUDE_POLICY_STORE_SECRETS = confi.bool( + "EXCLUDE_POLICY_STORE_SECRETS", + False, + description="If set, policy store secrets will be excluded from the logs", + ) + # create an instance of a policy store upon load def load_policy_store(): from opal_client.policy_store.policy_store_client_factory import ( diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index 12db7e62e..08130cf4a 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -28,13 +28,19 @@ async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): logger.error(f"Unauthorized to publish update: {repr(e)}") raise + token = None + oauth_client_secret = None + if not opal_client_config.EXCLUDE_POLICY_STORE_SECRETS: + token = opal_client_config.POLICY_STORE_AUTH_TOKEN + oauth_client_secret = opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET return PolicyStoreDetails( url=opal_client_config.POLICY_STORE_URL, - token=None, + token=token or None, auth_type=opal_client_config.POLICY_STORE_AUTH_TYPE or PolicyStoreAuth.NONE, oauth_client_id=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_ID or None, - oauth_client_secret=None, + oauth_client_secret=oauth_client_secret + or None, oauth_server=opal_client_config.POLICY_STORE_AUTH_OAUTH_SERVER or None, ) From e49bc2652103289b30510a79748178b62017d89a Mon Sep 17 00:00:00 2001 From: Oded Date: Wed, 26 Jun 2024 11:54:56 +0300 Subject: [PATCH 05/83] fix lint --- packages/opal-client/opal_client/policy_store/api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index 08130cf4a..b27d83d70 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -16,7 +16,7 @@ def init_policy_store_router(authenticator: JWTAuthenticator): "/policy-store/config", response_model=PolicyStoreDetails, response_model_exclude_none=True, - # Deprecating this route + # Deprecating this route deprecated=True, ) async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): @@ -32,15 +32,16 @@ async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): oauth_client_secret = None if not opal_client_config.EXCLUDE_POLICY_STORE_SECRETS: token = opal_client_config.POLICY_STORE_AUTH_TOKEN - oauth_client_secret = opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET + oauth_client_secret = ( + opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET + ) return PolicyStoreDetails( url=opal_client_config.POLICY_STORE_URL, token=token or None, auth_type=opal_client_config.POLICY_STORE_AUTH_TYPE or PolicyStoreAuth.NONE, oauth_client_id=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_ID or None, - oauth_client_secret=oauth_client_secret - or None, + oauth_client_secret=oauth_client_secret or None, oauth_server=opal_client_config.POLICY_STORE_AUTH_OAUTH_SERVER or None, ) From bca6cb596be0392367c1fddc1138fb3597ab8d0c Mon Sep 17 00:00:00 2001 From: Oded Date: Wed, 26 Jun 2024 14:35:15 +0300 Subject: [PATCH 06/83] Fix env var description --- packages/opal-client/opal_client/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-client/opal_client/config.py b/packages/opal-client/opal_client/config.py index fcc441a24..58d7ae2c8 100644 --- a/packages/opal-client/opal_client/config.py +++ b/packages/opal-client/opal_client/config.py @@ -118,7 +118,7 @@ class OpalClientConfig(Confi): EXCLUDE_POLICY_STORE_SECRETS = confi.bool( "EXCLUDE_POLICY_STORE_SECRETS", False, - description="If set, policy store secrets will be excluded from the logs", + description="If set, policy store secrets will be excluded from the /policy-store/config route", ) # create an instance of a policy store upon load From 0d34580db28c9bf4ea753daee98a49d65c04c458 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Wed, 26 Jun 2024 18:43:23 +0300 Subject: [PATCH 07/83] dan/per-10181-release-a-new-opal-client-cedar-version (#605) * Added missing build steps for permitio/opal-client-cedar docker image * Added missing test steps for permitio/opal-client-cedar docker image --- .github/workflows/on_release.yml | 28 ++++++++++++++++++++++++++++ .github/workflows/tests.yml | 13 +++++++++++++ 2 files changed, 41 insertions(+) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 542b8a16e..8403dfa61 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -76,6 +76,19 @@ jobs: tags: | permitio/opal-client:test + - name: Build client for testing + id: build_client + uses: docker/build-push-action@v4 + with: + file: docker/Dockerfile + push: false + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + load: true + tags: | + permitio/opal-client-cedar:test + - name: Build client-standalone for testing id: build_client_standalone uses: docker/build-push-action@v4 @@ -136,6 +149,21 @@ jobs: permitio/opal-client:latest permitio/opal-client:${{ env.opal_version_tag }} + - name: Build & Push client cedar + if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + id: build_push_client_cedar + uses: docker/build-push-action@v4 + with: + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + tags: | + permitio/opal-client-cedar:latest + permitio/opal-client-cedar:${{ env.opal_version_tag }} + - name: Build client-standalone if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client_standalone diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 610cdd05d..bcab1fe7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,6 +79,19 @@ jobs: tags: | permitio/opal-client:test + - name: Build client cedar + id: build_client + uses: docker/build-push-action@v2 + with: + file: docker/Dockerfile + push: false + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + load: true + tags: | + permitio/opal-client-cedar:test + - name: Build server id: build_server uses: docker/build-push-action@v2 From 5857f6415055e3db6dc706e884621c3959f74f50 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Thu, 27 Jun 2024 16:34:49 +0300 Subject: [PATCH 08/83] Update tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bcab1fe7e..c044a2aa7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -80,7 +80,7 @@ jobs: permitio/opal-client:test - name: Build client cedar - id: build_client + id: build_client_cedar uses: docker/build-push-action@v2 with: file: docker/Dockerfile From 3fba32e54ecaa9b3b7b8797480519c2f336b8cfc Mon Sep 17 00:00:00 2001 From: roekatz Date: Wed, 26 Jun 2024 17:49:34 +0300 Subject: [PATCH 09/83] BasePolicyWatcherTask: Signal stop if broadcaster fails to connect --- .../opal-server/opal_server/policy/watcher/task.py | 11 ++++++----- packages/requires.txt | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/opal-server/opal_server/policy/watcher/task.py b/packages/opal-server/opal_server/policy/watcher/task.py index a2ba57558..e2b630eb5 100644 --- a/packages/opal-server/opal_server/policy/watcher/task.py +++ b/packages/opal-server/opal_server/policy/watcher/task.py @@ -50,11 +50,12 @@ async def _subscribe_internal(): ) if self._pubsub_endpoint.broadcaster is not None: - async with self._pubsub_endpoint.broadcaster.get_listening_context(): - await _subscribe_internal() - await self._pubsub_endpoint.broadcaster.get_reader_task() - - # Stop the watcher if broadcaster disconnects + try: + async with self._pubsub_endpoint.broadcaster.get_listening_context(): + await _subscribe_internal() + await self._pubsub_endpoint.broadcaster.get_reader_task() + finally: + # Stop the watcher if broadcaster disconnects / fails to connect self.signal_stop() else: # If no broadcaster is configured, just subscribe, no need to wait on anything diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..e077c4bf3 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -1,7 +1,7 @@ idna>=3.3,<4 typer>=0.4.1,<1 fastapi>=0.109.1,<1 -fastapi_websocket_pubsub==0.3.7 +fastapi_websocket_pubsub==0.3.9 fastapi_websocket_rpc>=0.1.21,<1 gunicorn>=22.0.0,<23 pydantic[email]>=1.9.1,<2 From 73ea4d5f0b95e0c5af27c280824e9ec1ebc83341 Mon Sep 17 00:00:00 2001 From: roekatz Date: Wed, 26 Jun 2024 17:50:31 +0300 Subject: [PATCH 10/83] Random documentation fixes --- packages/opal-common/opal_common/async_utils.py | 9 ++++++--- packages/opal-server/opal_server/git_fetcher.py | 9 +-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/opal-common/opal_common/async_utils.py b/packages/opal-common/opal_common/async_utils.py index e7f6029fd..a2df90c69 100644 --- a/packages/opal-common/opal_common/async_utils.py +++ b/packages/opal-common/opal_common/async_utils.py @@ -22,9 +22,12 @@ async def run_sync( """Shorthand for running a sync function in an executor within an async context. - For example: def sync_function_that_takes_time_to_run(arg1, - arg2): time.sleep(5) async def async_function(): - await run_sync(sync_function_that_takes_time_to_run, 1, arg2=5) + For example: + def sync_function_that_takes_time_to_run(arg1, arg2): + time.sleep(5) + + async def async_function(): + await run_sync(sync_function_that_takes_time_to_run, 1, arg2=5) """ return await asyncio.get_event_loop().run_in_executor( None, partial(func, *args, **kwargs) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 0887c9160..5ea85c047 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -139,14 +139,7 @@ def __init__( ) async def _get_repo_lock(self): - # # This implementation works across multiple processes/threads, but is not fair (next acquiree is random) - # locks_dir = self._base_dir / ".locks" - # await aiofiles.os.makedirs(str(locks_dir), exist_ok=True) - - # return NamedLock( - # locks_dir / GitPolicyFetcher.source_id(self._source), attempt_interval=0.1 - # ) - + # Previous file based implementation worked across multiple processes/threads, but wasn't fair (next acquiree is random) # This implementation works only within the same process/thread, but is fair (next acquiree is the earliest to enter the lock) src_id = GitPolicyFetcher.source_id(self._source) lock = GitPolicyFetcher.repo_locks[src_id] = GitPolicyFetcher.repo_locks.get( From 36e6dc1a80c17460558e797c24250c37a327cbbe Mon Sep 17 00:00:00 2001 From: roekatz Date: Thu, 27 Jun 2024 16:20:40 +0300 Subject: [PATCH 11/83] Fix tests to explicitly choose 'master' as default branch The tests rely on that, and this value also depends on local git configuration --- packages/opal-common/opal_common/git/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/git/tests/conftest.py b/packages/opal-common/opal_common/git/tests/conftest.py index c60099a5c..e3e5e0e14 100644 --- a/packages/opal-common/opal_common/git/tests/conftest.py +++ b/packages/opal-common/opal_common/git/tests/conftest.py @@ -98,7 +98,7 @@ def local_repo(tmp_path, helpers: Helpers) -> Repo: """ root: Path = tmp_path / "myrepo" root.mkdir() - repo = Repo.init(root) + repo = Repo.init(root, initial_branch="master") # create file to delete later helpers.create_new_file_commit(repo, root / "deleted.rego") From 03c011fabeae7e28f11965657e23f51cccbd1f0b Mon Sep 17 00:00:00 2001 From: roekatz Date: Thu, 27 Jun 2024 16:58:58 +0300 Subject: [PATCH 12/83] Tests: Change test server ports to avoid collisions --- .../opal_client/tests/server_to_client_intergation_test.py | 2 +- .../opal-server/opal_server/tests/policy_repo_webhook_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py b/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py index 3e6522277..a3372c56f 100644 --- a/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py +++ b/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py @@ -31,7 +31,7 @@ from opal_server.server import OpalServer # Server settings -PORT = int(os.environ.get("PORT") or "9123") +PORT = int(os.environ.get("PORT") or "9124") UPDATES_URL = f"ws://localhost:{PORT}/ws" DATA_ROUTE = "/fetchable_data" DATA_URL = f"http://localhost:{PORT}{DATA_ROUTE}" diff --git a/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py b/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py index a6633f984..8c8fd0bb7 100644 --- a/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py +++ b/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py @@ -29,7 +29,7 @@ from opal_common.utils import get_authorization_header from opal_server.config import PolicySourceTypes, opal_server_config -PORT = int(os.environ.get("PORT") or "9123") +PORT = int(os.environ.get("PORT") or "9125") # Basic server route config WEBHOOK_ROUTE = "/webhook" From 6d8063053f8ac07195caf74693a2129906586bbb Mon Sep 17 00:00:00 2001 From: roekatz Date: Thu, 27 Jun 2024 18:05:26 +0300 Subject: [PATCH 13/83] Dokcer test: No need to build test image for client cedar since we don't test it --- .github/workflows/tests.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c044a2aa7..610cdd05d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,19 +79,6 @@ jobs: tags: | permitio/opal-client:test - - name: Build client cedar - id: build_client_cedar - uses: docker/build-push-action@v2 - with: - file: docker/Dockerfile - push: false - target: client-cedar - cache-from: type=registry,ref=permitio/opal-client-cedar:latest - cache-to: type=inline - load: true - tags: | - permitio/opal-client-cedar:test - - name: Build server id: build_server uses: docker/build-push-action@v2 From c3cf66b97893648a53d3839639bb8690bbe0f31a Mon Sep 17 00:00:00 2001 From: roekatz Date: Wed, 3 Jul 2024 16:22:23 +0200 Subject: [PATCH 14/83] Revert "BasePolicyWatcherTask: Signal stop if broadcaster fails to connect" This reverts commit 3fba32e54ecaa9b3b7b8797480519c2f336b8cfc. --- .../opal-server/opal_server/policy/watcher/task.py | 11 +++++------ packages/requires.txt | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/opal-server/opal_server/policy/watcher/task.py b/packages/opal-server/opal_server/policy/watcher/task.py index e2b630eb5..a2ba57558 100644 --- a/packages/opal-server/opal_server/policy/watcher/task.py +++ b/packages/opal-server/opal_server/policy/watcher/task.py @@ -50,12 +50,11 @@ async def _subscribe_internal(): ) if self._pubsub_endpoint.broadcaster is not None: - try: - async with self._pubsub_endpoint.broadcaster.get_listening_context(): - await _subscribe_internal() - await self._pubsub_endpoint.broadcaster.get_reader_task() - finally: - # Stop the watcher if broadcaster disconnects / fails to connect + async with self._pubsub_endpoint.broadcaster.get_listening_context(): + await _subscribe_internal() + await self._pubsub_endpoint.broadcaster.get_reader_task() + + # Stop the watcher if broadcaster disconnects self.signal_stop() else: # If no broadcaster is configured, just subscribe, no need to wait on anything diff --git a/packages/requires.txt b/packages/requires.txt index e077c4bf3..5096c6000 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -1,7 +1,7 @@ idna>=3.3,<4 typer>=0.4.1,<1 fastapi>=0.109.1,<1 -fastapi_websocket_pubsub==0.3.9 +fastapi_websocket_pubsub==0.3.7 fastapi_websocket_rpc>=0.1.21,<1 gunicorn>=22.0.0,<23 pydantic[email]>=1.9.1,<2 From 94f98746c9442c1b947433b4a509062c2c009f40 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 7 Jul 2024 16:50:49 +0300 Subject: [PATCH 15/83] dan/per-10201-fix-opal-server-077-failures (#607) * Changed pygit2 requirement * dan/per-10181-release-a-new-opal-client-cedar-version (#605) * Added missing build steps for permitio/opal-client-cedar docker image * Added missing test steps for permitio/opal-client-cedar docker image * Update tests.yml * Fix env var description * BasePolicyWatcherTask: Signal stop if broadcaster fails to connect * Random documentation fixes * Fix tests to explicitly choose 'master' as default branch The tests rely on that, and this value also depends on local git configuration * Tests: Change test server ports to avoid collisions * Dokcer test: No need to build test image for client cedar since we don't test it --------- Co-authored-by: Oded Co-authored-by: roekatz --- packages/opal-server/requires.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-server/requires.txt b/packages/opal-server/requires.txt index 59c30201f..3a78295dc 100644 --- a/packages/opal-server/requires.txt +++ b/packages/opal-server/requires.txt @@ -5,6 +5,6 @@ pyjwt[crypto]>=2.1.0,<3 websockets>=10.3,<11 slowapi>=0.1.5,<1 # slowapi is stuck on and old `redis`, so fix that and switch from aioredis to redis -pygit2>=1.13.3,<2 +pygit2>=1.14.1,<1.15 asgiref>=3.5.2,<4 redis>=4.3.4,<5 From 93161a07ba1387680b04f71dc087c41ae84339c8 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 10 Jul 2024 03:53:32 +0000 Subject: [PATCH 16/83] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-ZIPP-7430899 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3c0d2e61e..69533e9fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pytest-rerunfailures wheel>=0.38.0 twine setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability From 9f3efba9785c3189537de903ab2b1ff53d63ea2d Mon Sep 17 00:00:00 2001 From: roekatz Date: Sat, 13 Jul 2024 13:45:31 +0300 Subject: [PATCH 17/83] Bump version to 0.7.8 --- packages/__packaging__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/__packaging__.py b/packages/__packaging__.py index 727191f80..a0b868fcf 100644 --- a/packages/__packaging__.py +++ b/packages/__packaging__.py @@ -6,9 +6,10 @@ Project homepage: https://github.com/permitio/opal """ + import os -VERSION = (0, 7, 7) +VERSION = (0, 7, 8) VERSION_STRING = ".".join(map(str, VERSION)) __version__ = VERSION_STRING From 5f0a8213344ab701d85e6683711eaa84d93ce304 Mon Sep 17 00:00:00 2001 From: roekatz Date: Sat, 13 Jul 2024 14:18:07 +0300 Subject: [PATCH 18/83] CI: Remove unnecessary (and buggy) build of cedar & standalone client for testing --- .github/workflows/on_release.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 8403dfa61..753a93cc2 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -76,32 +76,6 @@ jobs: tags: | permitio/opal-client:test - - name: Build client for testing - id: build_client - uses: docker/build-push-action@v4 - with: - file: docker/Dockerfile - push: false - target: client-cedar - cache-from: type=registry,ref=permitio/opal-client-cedar:latest - cache-to: type=inline - load: true - tags: | - permitio/opal-client-cedar:test - - - name: Build client-standalone for testing - id: build_client_standalone - uses: docker/build-push-action@v4 - with: - file: docker/Dockerfile - push: false - target: client-standalone - cache-from: type=registry,ref=permitio/opal-client-standalone:latest - cache-to: type=inline - load: true - tags: | - permitio/opal-client-standalone:test - - name: Build server for testing id: build_server uses: docker/build-push-action@v4 From 0f79bcd74fb74d983634cdb7fa45f867d8ba11b1 Mon Sep 17 00:00:00 2001 From: roekatz Date: Sat, 13 Jul 2024 15:14:50 +0300 Subject: [PATCH 19/83] CI: Comment out failing opal cedar client build --- .github/workflows/on_release.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 753a93cc2..736a64f0b 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -123,20 +123,20 @@ jobs: permitio/opal-client:latest permitio/opal-client:${{ env.opal_version_tag }} - - name: Build & Push client cedar - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} - id: build_push_client_cedar - uses: docker/build-push-action@v4 - with: - file: docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - target: client-cedar - cache-from: type=registry,ref=permitio/opal-client-cedar:latest - cache-to: type=inline - tags: | - permitio/opal-client-cedar:latest - permitio/opal-client-cedar:${{ env.opal_version_tag }} +# - name: Build & Push client cedar +# if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} +# id: build_push_client_cedar +# uses: docker/build-push-action@v4 +# with: +# file: docker/Dockerfile +# platforms: linux/amd64,linux/arm64 +# push: true +# target: client-cedar +# cache-from: type=registry,ref=permitio/opal-client-cedar:latest +# cache-to: type=inline +# tags: | +# permitio/opal-client-cedar:latest +# permitio/opal-client-cedar:${{ env.opal_version_tag }} - name: Build client-standalone if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} From 8b691d9d86f7f05973ebf75d1159f6c9819ba904 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:24:28 +0300 Subject: [PATCH 20/83] Changed relative imports to absolute --- packages/opal-client/opal_client/__init__.py | 2 +- packages/opal-client/opal_client/main.py | 2 +- .../policy_store/mock_policy_store_client.py | 2 +- .../opal_common/authentication/casting.py | 1 - packages/opal-common/opal_common/cli/typer_app.py | 2 +- packages/opal-common/opal_common/confi/__init__.py | 2 +- packages/opal-common/opal_common/confi/cli.py | 2 +- packages/opal-common/opal_common/confi/confi.py | 4 ++-- packages/opal-common/opal_common/config.py | 5 ++--- .../opal-common/opal_common/engine/__init__.py | 4 ++-- .../opal-common/opal_common/fetcher/__init__.py | 6 +++--- .../fetcher/engine/base_fetching_engine.py | 6 +++--- .../opal_common/fetcher/engine/core_callbacks.py | 2 +- .../opal_common/fetcher/engine/fetch_worker.py | 8 ++++---- .../opal_common/fetcher/engine/fetching_engine.py | 14 +++++++------- .../opal_common/fetcher/fetch_provider.py | 4 ++-- .../opal_common/fetcher/fetcher_register.py | 10 +++++----- .../opal_common/fetcher/providers/__init__.py | 2 +- .../providers/fastapi_rpc_fetch_provider.py | 6 +++--- .../fetcher/providers/http_fetch_provider.py | 10 +++++----- packages/opal-common/opal_common/logger.py | 12 ++++++------ packages/opal-server/opal_server/main.py | 2 +- scripts/gunicorn_conf.py | 2 -- 23 files changed, 53 insertions(+), 57 deletions(-) diff --git a/packages/opal-client/opal_client/__init__.py b/packages/opal-client/opal_client/__init__.py index c2810c5f7..a1eb3e09d 100644 --- a/packages/opal-client/opal_client/__init__.py +++ b/packages/opal-client/opal_client/__init__.py @@ -1 +1 @@ -from .client import OpalClient +from opal_client.client import OpalClient diff --git a/packages/opal-client/opal_client/main.py b/packages/opal-client/opal_client/main.py index 611cdd741..65f3bb665 100644 --- a/packages/opal-client/opal_client/main.py +++ b/packages/opal-client/opal_client/main.py @@ -1,4 +1,4 @@ -from .client import OpalClient +from opal_client.client import OpalClient client = OpalClient() # expose app for Uvicorn diff --git a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py index 4aa27f0d0..8d6742d4a 100644 --- a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py +++ b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py @@ -8,7 +8,7 @@ from opal_common.schemas.store import JSONPatchAction, StoreTransaction from pydantic import BaseModel -from .base_policy_store_client import BasePolicyStoreClient, JsonableValue +from opal_client.policy_store.base_policy_store_client import BasePolicyStoreClient, JsonableValue class MockPolicyStoreClient(BasePolicyStoreClient): diff --git a/packages/opal-common/opal_common/authentication/casting.py b/packages/opal-common/opal_common/authentication/casting.py index 9713ed2d5..d14a04fc7 100644 --- a/packages/opal-common/opal_common/authentication/casting.py +++ b/packages/opal-common/opal_common/authentication/casting.py @@ -5,7 +5,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception logger = logging.getLogger("opal.authentication") diff --git a/packages/opal-common/opal_common/cli/typer_app.py b/packages/opal-common/opal_common/cli/typer_app.py index 45de0d594..a1d70ff24 100644 --- a/packages/opal-common/opal_common/cli/typer_app.py +++ b/packages/opal-common/opal_common/cli/typer_app.py @@ -1,6 +1,6 @@ import typer -from .commands import all_commands +from opal_common.cli.commands import all_commands def get_typer_app(): diff --git a/packages/opal-common/opal_common/confi/__init__.py b/packages/opal-common/opal_common/confi/__init__.py index ccd3d49ff..114be01c7 100644 --- a/packages/opal-common/opal_common/confi/__init__.py +++ b/packages/opal-common/opal_common/confi/__init__.py @@ -1 +1 @@ -from .confi import * +from opal_common.confi.confi import * diff --git a/packages/opal-common/opal_common/confi/cli.py b/packages/opal-common/opal_common/confi/cli.py index 00e3097ee..cfca25f1c 100644 --- a/packages/opal-common/opal_common/confi/cli.py +++ b/packages/opal-common/opal_common/confi/cli.py @@ -4,7 +4,7 @@ import typer from typer.main import Typer -from .types import ConfiEntry +from opal_common.confi.types import ConfiEntry def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index f391c26a2..8b376be9d 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -19,8 +19,8 @@ from pydantic import BaseModel, ValidationError from typer import Typer -from .cli import get_cli_object_for_config_objects -from .types import ConfiDelay, ConfiEntry, no_cast +from opal_common.confi.cli import get_cli_object_for_config_objects +from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast class Placeholder(object): diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index 7666d47e4..b7d9395b6 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -2,8 +2,7 @@ from sys import prefix from opal_common.authentication.types import EncryptionKeyFormat, JWTAlgorithm - -from .confi import Confi, confi +from opal_common.confi import Confi, confi _LOG_FORMAT_WITHOUT_PID = "{time} | {name: <40}|{level:^6} | {message}\n{exception}" _LOG_FORMAT_WITH_PID = "{time} | {process} | {name: <40}|{level:^6} | {message}\n{exception}" @@ -173,7 +172,7 @@ class OpalCommonConfig(Confi): "HTTP_FETCHER_PROVIDER_CLIENT", "aiohttp", description="The client to use for fetching data, can be either aiohttp or httpx." - "if provided different value, aiohttp will be used.", + "if provided different value, aiohttp will be used.", ) diff --git a/packages/opal-common/opal_common/engine/__init__.py b/packages/opal-common/opal_common/engine/__init__.py index bbc306e62..33d1be247 100644 --- a/packages/opal-common/opal_common/engine/__init__.py +++ b/packages/opal-common/opal_common/engine/__init__.py @@ -1,2 +1,2 @@ -from .parsing import get_rego_package -from .paths import is_data_module, is_policy_module +from opal_common.engine.parsing import get_rego_package +from opal_common.engine.paths import is_data_module, is_policy_module diff --git a/packages/opal-common/opal_common/fetcher/__init__.py b/packages/opal-common/opal_common/fetcher/__init__.py index 84232e236..70a1f643c 100644 --- a/packages/opal-common/opal_common/fetcher/__init__.py +++ b/packages/opal-common/opal_common/fetcher/__init__.py @@ -1,3 +1,3 @@ -from .engine.fetching_engine import FetchingEngine -from .events import FetcherConfig, FetchEvent -from .fetcher_register import FetcherRegister +from opal_common.fetcher.engine.fetching_engine import FetchingEngine +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister diff --git a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py index a30f033d2..22f9325f9 100644 --- a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py @@ -1,8 +1,8 @@ from typing import Coroutine -from ..events import FetcherConfig, FetchEvent -from ..fetcher_register import FetcherRegister -from .core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback class BaseFetchingEngine: diff --git a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py index b083e779e..3da152f14 100644 --- a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py +++ b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py @@ -1,4 +1,4 @@ -from ..events import FetchEvent +from opal_common.fetcher.events import FetchEvent # Callback signatures diff --git a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py index eb816ecf2..460ee1465 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py @@ -1,10 +1,10 @@ import asyncio from typing import Coroutine -from ..events import FetchEvent -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine logger = get_logger("fetch_worker") diff --git a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py index fd50a9f14..cb03693bf 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py @@ -2,13 +2,13 @@ import uuid from typing import Coroutine, Dict, List, Union -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine -from .core_callbacks import OnFetchFailureCallback -from .fetch_worker import fetch_worker +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.fetch_worker import fetch_worker logger = get_logger("engine") diff --git a/packages/opal-common/opal_common/fetcher/fetch_provider.py b/packages/opal-common/opal_common/fetcher/fetch_provider.py index 62ad97532..70b91ea59 100644 --- a/packages/opal-common/opal_common/fetcher/fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/fetch_provider.py @@ -1,7 +1,7 @@ from tenacity import retry, stop, wait -from .events import FetchEvent -from .logger import get_logger +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.logger import get_logger logger = get_logger("opal.providers") diff --git a/packages/opal-common/opal_common/fetcher/fetcher_register.py b/packages/opal-common/opal_common/fetcher/fetcher_register.py index 5bf925160..18ed32f81 100644 --- a/packages/opal-common/opal_common/fetcher/fetcher_register.py +++ b/packages/opal-common/opal_common/fetcher/fetcher_register.py @@ -2,10 +2,10 @@ from opal_common.fetcher.logger import get_logger -from ..config import opal_common_config -from .events import FetchEvent -from .fetch_provider import BaseFetchProvider -from .providers.http_fetch_provider import HttpFetchProvider +from opal_common.config import opal_common_config +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.providers.http_fetch_provider import HttpFetchProvider logger = get_logger("opal.fetcher_register") @@ -30,7 +30,7 @@ def __init__(self, config: Optional[Dict[str, BaseFetchProvider]] = None) -> Non if config is not None: self._config = config else: - from ..emport import emport_objects_by_class + from opal_common.emport import emport_objects_by_class # load fetchers fetchers = [] diff --git a/packages/opal-common/opal_common/fetcher/providers/__init__.py b/packages/opal-common/opal_common/fetcher/providers/__init__.py index 8e1f6bf77..ff1078ced 100644 --- a/packages/opal-common/opal_common/fetcher/providers/__init__.py +++ b/packages/opal-common/opal_common/fetcher/providers/__init__.py @@ -1,3 +1,3 @@ -from ...emport import dynamic_all +from opal_common.emport import dynamic_all __all__ = dynamic_all(__file__) diff --git a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py index 61432b751..94513f9d2 100644 --- a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py @@ -3,9 +3,9 @@ from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger logger = get_logger("rpc_fetch_provider") diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7261b538b..9083f8aa1 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -8,11 +8,11 @@ from opal_common.config import opal_common_config from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger +from opal_common.http_utils import is_http_error_response +from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger logger = get_logger("http_fetch_provider") diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index 5f1229a80..a494491ef 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -3,12 +3,12 @@ from loguru import logger -from .config import opal_common_config -from .logging.filter import ModuleFilter -from .logging.formatter import Formatter -from .logging.intercept import InterceptHandler -from .logging.thirdparty import hijack_uvicorn_logs -from .monitoring.apm import fix_ddtrace_logging +from opal_common.config import opal_common_config +from opal_common.logging.filter import ModuleFilter +from opal_common.logging.formatter import Formatter +from opal_common.logging.intercept import InterceptHandler +from opal_common.logging.thirdparty import hijack_uvicorn_logs +from opal_common.monitoring.apm import fix_ddtrace_logging def configure_logs(): diff --git a/packages/opal-server/opal_server/main.py b/packages/opal-server/opal_server/main.py index 908c4561d..7e61e2a66 100644 --- a/packages/opal-server/opal_server/main.py +++ b/packages/opal-server/opal_server/main.py @@ -1,5 +1,5 @@ def create_app(*args, **kwargs): - from .server import OpalServer + from opal_server.server import OpalServer server = OpalServer(*args, **kwargs) return server.app diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index b40546ae6..0c2c15cb8 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,5 +1,3 @@ -import os - from opal_common.logger import logger From 38446a803ee0cf8ac941ac5444afa34a1c9f911c Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:30:28 +0300 Subject: [PATCH 21/83] Renamed redis_utils module --- packages/opal-server/opal_server/{redis.py => redis_utils.py} | 0 packages/opal-server/opal_server/scopes/scope_repository.py | 2 +- packages/opal-server/opal_server/scopes/task.py | 2 +- packages/opal-server/opal_server/server.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/opal-server/opal_server/{redis.py => redis_utils.py} (100%) diff --git a/packages/opal-server/opal_server/redis.py b/packages/opal-server/opal_server/redis_utils.py similarity index 100% rename from packages/opal-server/opal_server/redis.py rename to packages/opal-server/opal_server/redis_utils.py diff --git a/packages/opal-server/opal_server/scopes/scope_repository.py b/packages/opal-server/opal_server/scopes/scope_repository.py index b3627741c..d9f5d9d20 100644 --- a/packages/opal-server/opal_server/scopes/scope_repository.py +++ b/packages/opal-server/opal_server/scopes/scope_repository.py @@ -1,7 +1,7 @@ from typing import List from opal_common.schemas.scopes import Scope -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB class ScopeNotFoundError(Exception): diff --git a/packages/opal-server/opal_server/scopes/task.py b/packages/opal-server/opal_server/scopes/task.py index b3a577161..83b2b10f0 100644 --- a/packages/opal-server/opal_server/scopes/task.py +++ b/packages/opal-server/opal_server/scopes/task.py @@ -7,7 +7,7 @@ from opal_common.logger import logger from opal_server.config import opal_server_config from opal_server.policy.watcher.task import BasePolicyWatcherTask -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.scope_repository import ScopeRepository from opal_server.scopes.service import ScopesService diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 6a946a8c0..34d9905c3 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -32,7 +32,7 @@ from opal_server.policy.webhook.api import init_git_webhook_router from opal_server.publisher import setup_broadcaster_keepalive_task from opal_server.pubsub import PubSub -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.api import init_scope_router from opal_server.scopes.loader import load_scopes from opal_server.scopes.scope_repository import ScopeRepository From 040162cc11579e51daa8a23be1143b1311378961 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:32:13 +0300 Subject: [PATCH 22/83] Renamed http_utils module --- packages/opal-client/opal_client/callbacks/reporter.py | 2 +- packages/opal-client/opal_client/data/updater.py | 2 +- packages/opal-common/opal_common/{http.py => http_utils.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/opal-common/opal_common/{http.py => http_utils.py} (100%) diff --git a/packages/opal-client/opal_client/callbacks/reporter.py b/packages/opal-client/opal_client/callbacks/reporter.py index 264f45b51..c9f2987b6 100644 --- a/packages/opal-client/opal_client/callbacks/reporter.py +++ b/packages/opal-client/opal_client/callbacks/reporter.py @@ -5,7 +5,7 @@ from opal_client.callbacks.register import CallbackConfig, CallbacksRegister from opal_client.data.fetcher import DataFetcher from opal_common.fetcher.providers.http_fetch_provider import HttpFetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.logger import logger from opal_common.schemas.data import DataUpdateReport diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index d2c81c9ed..e288b5963 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -26,7 +26,7 @@ from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.schemas.data import ( DataEntryReport, DataSourceConfig, diff --git a/packages/opal-common/opal_common/http.py b/packages/opal-common/opal_common/http_utils.py similarity index 100% rename from packages/opal-common/opal_common/http.py rename to packages/opal-common/opal_common/http_utils.py From 8dd6cc1cb9ade1f97e4a3c57e8bcfd956cd122e9 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:32:41 +0300 Subject: [PATCH 23/83] Renamed logging_utils module --- packages/opal-common/opal_common/confi/confi.py | 2 +- packages/opal-common/opal_common/logger.py | 8 ++++---- .../opal_common/{logging => logging_utils}/__init__.py | 0 .../opal_common/{logging => logging_utils}/decorators.py | 0 .../opal_common/{logging => logging_utils}/filter.py | 0 .../opal_common/{logging => logging_utils}/formatter.py | 0 .../opal_common/{logging => logging_utils}/intercept.py | 0 .../opal_common/{logging => logging_utils}/thirdparty.py | 0 8 files changed, 5 insertions(+), 5 deletions(-) rename packages/opal-common/opal_common/{logging => logging_utils}/__init__.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/decorators.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/filter.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/formatter.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/intercept.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/thirdparty.py (100%) diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index 8b376be9d..f62576dfa 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -15,7 +15,7 @@ from decouple import Csv, UndefinedValueError, config, text_type, undefined from opal_common.authentication.casting import cast_private_key, cast_public_key from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception +from opal_common.logging_utils.decorators import log_exception from pydantic import BaseModel, ValidationError from typer import Typer diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index a494491ef..2a45a4dc2 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -4,10 +4,10 @@ from loguru import logger from opal_common.config import opal_common_config -from opal_common.logging.filter import ModuleFilter -from opal_common.logging.formatter import Formatter -from opal_common.logging.intercept import InterceptHandler -from opal_common.logging.thirdparty import hijack_uvicorn_logs +from opal_common.logging_utils.filter import ModuleFilter +from opal_common.logging_utils.formatter import Formatter +from opal_common.logging_utils.intercept import InterceptHandler +from opal_common.logging_utils.thirdparty import hijack_uvicorn_logs from opal_common.monitoring.apm import fix_ddtrace_logging diff --git a/packages/opal-common/opal_common/logging/__init__.py b/packages/opal-common/opal_common/logging_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/logging/__init__.py rename to packages/opal-common/opal_common/logging_utils/__init__.py diff --git a/packages/opal-common/opal_common/logging/decorators.py b/packages/opal-common/opal_common/logging_utils/decorators.py similarity index 100% rename from packages/opal-common/opal_common/logging/decorators.py rename to packages/opal-common/opal_common/logging_utils/decorators.py diff --git a/packages/opal-common/opal_common/logging/filter.py b/packages/opal-common/opal_common/logging_utils/filter.py similarity index 100% rename from packages/opal-common/opal_common/logging/filter.py rename to packages/opal-common/opal_common/logging_utils/filter.py diff --git a/packages/opal-common/opal_common/logging/formatter.py b/packages/opal-common/opal_common/logging_utils/formatter.py similarity index 100% rename from packages/opal-common/opal_common/logging/formatter.py rename to packages/opal-common/opal_common/logging_utils/formatter.py diff --git a/packages/opal-common/opal_common/logging/intercept.py b/packages/opal-common/opal_common/logging_utils/intercept.py similarity index 100% rename from packages/opal-common/opal_common/logging/intercept.py rename to packages/opal-common/opal_common/logging_utils/intercept.py diff --git a/packages/opal-common/opal_common/logging/thirdparty.py b/packages/opal-common/opal_common/logging_utils/thirdparty.py similarity index 100% rename from packages/opal-common/opal_common/logging/thirdparty.py rename to packages/opal-common/opal_common/logging_utils/thirdparty.py From c1c6d0feab346fe3a9594b763ab6d05423bea70a Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:32:50 +0300 Subject: [PATCH 24/83] Renamed git_utils module --- .../opal-client/opal_client/policy_store/opa_client.py | 2 +- .../opal_common/{git => git_utils}/__init__.py | 0 .../opal_common/{git => git_utils}/branch_tracker.py | 4 ++-- .../opal_common/{git => git_utils}/bundle_maker.py | 4 ++-- .../opal_common/{git => git_utils}/bundle_utils.py | 0 .../opal_common/{git => git_utils}/commit_viewer.py | 0 .../opal_common/{git => git_utils}/diff_viewer.py | 2 +- .../opal-common/opal_common/{git => git_utils}/env.py | 0 .../opal_common/{git => git_utils}/exceptions.py | 0 .../opal_common/{git => git_utils}/repo_cloner.py | 4 ++-- .../tar_file_to_local_git_extractor.py | 0 .../{git => git_utils}/tests/branch_tracker_test.py | 4 ++-- .../{git => git_utils}/tests/bundle_maker_test.py | 4 ++-- .../{git => git_utils}/tests/commit_viewer_test.py | 2 +- .../opal_common/{git => git_utils}/tests/conftest.py | 0 .../{git => git_utils}/tests/diff_viewer_test.py | 4 ++-- .../{git => git_utils}/tests/repo_cloner_test.py | 4 ++-- .../{git => git_utils}/tests/repo_watcher_test.py | 0 .../opal-common/opal_common/sources/api_policy_source.py | 2 +- .../opal-common/opal_common/sources/git_policy_source.py | 6 +++--- packages/opal-server/opal_server/git_fetcher.py | 2 +- packages/opal-server/opal_server/policy/bundles/api.py | 9 +++++---- .../opal-server/opal_server/policy/watcher/callbacks.py | 4 ++-- .../opal-server/opal_server/policy/watcher/factory.py | 2 +- packages/opal-server/opal_server/scopes/service.py | 2 +- 25 files changed, 31 insertions(+), 30 deletions(-) rename packages/opal-common/opal_common/{git => git_utils}/__init__.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/branch_tracker.py (97%) rename packages/opal-common/opal_common/{git => git_utils}/bundle_maker.py (99%) rename packages/opal-common/opal_common/{git => git_utils}/bundle_utils.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/commit_viewer.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/diff_viewer.py (99%) rename packages/opal-common/opal_common/{git => git_utils}/env.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/exceptions.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/repo_cloner.py (98%) rename packages/opal-common/opal_common/{git => git_utils}/tar_file_to_local_git_extractor.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/tests/branch_tracker_test.py (95%) rename packages/opal-common/opal_common/{git => git_utils}/tests/bundle_maker_test.py (99%) rename packages/opal-common/opal_common/{git => git_utils}/tests/commit_viewer_test.py (98%) rename packages/opal-common/opal_common/{git => git_utils}/tests/conftest.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/tests/diff_viewer_test.py (97%) rename packages/opal-common/opal_common/{git => git_utils}/tests/repo_cloner_test.py (96%) rename packages/opal-common/opal_common/{git => git_utils}/tests/repo_watcher_test.py (100%) diff --git a/packages/opal-client/opal_client/policy_store/opa_client.py b/packages/opal-client/opal_client/policy_store/opa_client.py index 1d8ec0211..54bc94dac 100644 --- a/packages/opal-client/opal_client/policy_store/opa_client.py +++ b/packages/opal-client/opal_client/policy_store/opa_client.py @@ -20,7 +20,7 @@ from opal_client.policy_store.schemas import PolicyStoreAuth from opal_client.utils import exclude_none_fields, proxy_response from opal_common.engine.parsing import get_rego_package -from opal_common.git.bundle_utils import BundleUtils +from opal_common.git_utils.bundle_utils import BundleUtils from opal_common.paths import PathUtils from opal_common.schemas.policy import DataModule, PolicyBundle, RegoModule from opal_common.schemas.store import JSONPatchAction, StoreTransaction, TransactionType diff --git a/packages/opal-common/opal_common/git/__init__.py b/packages/opal-common/opal_common/git_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/git/__init__.py rename to packages/opal-common/opal_common/git_utils/__init__.py diff --git a/packages/opal-common/opal_common/git/branch_tracker.py b/packages/opal-common/opal_common/git_utils/branch_tracker.py similarity index 97% rename from packages/opal-common/opal_common/git/branch_tracker.py rename to packages/opal-common/opal_common/git_utils/branch_tracker.py index 19bba8770..28f692e15 100644 --- a/packages/opal-common/opal_common/git/branch_tracker.py +++ b/packages/opal-common/opal_common/git_utils/branch_tracker.py @@ -3,8 +3,8 @@ from git import GitCommandError, Head, Remote, Repo from git.objects.commit import Commit -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from tenacity import retry, stop_after_attempt, wait_fixed diff --git a/packages/opal-common/opal_common/git/bundle_maker.py b/packages/opal-common/opal_common/git_utils/bundle_maker.py similarity index 99% rename from packages/opal-common/opal_common/git/bundle_maker.py rename to packages/opal-common/opal_common/git_utils/bundle_maker.py index 0b006b6a7..f9d621a89 100644 --- a/packages/opal-common/opal_common/git/bundle_maker.py +++ b/packages/opal-common/opal_common/git_utils/bundle_maker.py @@ -6,7 +6,7 @@ from git import Repo from git.objects import Commit from opal_common.engine import get_rego_package, is_data_module, is_policy_module -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, VersionedDirectory, VersionedFile, @@ -14,7 +14,7 @@ has_extension, is_under_directories, ) -from opal_common.git.diff_viewer import ( +from opal_common.git_utils.diff_viewer import ( DiffViewer, diffed_file_has_extension, diffed_file_is_under_directories, diff --git a/packages/opal-common/opal_common/git/bundle_utils.py b/packages/opal-common/opal_common/git_utils/bundle_utils.py similarity index 100% rename from packages/opal-common/opal_common/git/bundle_utils.py rename to packages/opal-common/opal_common/git_utils/bundle_utils.py diff --git a/packages/opal-common/opal_common/git/commit_viewer.py b/packages/opal-common/opal_common/git_utils/commit_viewer.py similarity index 100% rename from packages/opal-common/opal_common/git/commit_viewer.py rename to packages/opal-common/opal_common/git_utils/commit_viewer.py diff --git a/packages/opal-common/opal_common/git/diff_viewer.py b/packages/opal-common/opal_common/git_utils/diff_viewer.py similarity index 99% rename from packages/opal-common/opal_common/git/diff_viewer.py rename to packages/opal-common/opal_common/git_utils/diff_viewer.py index af5720f5c..ec6dff9d0 100644 --- a/packages/opal-common/opal_common/git/diff_viewer.py +++ b/packages/opal-common/opal_common/git_utils/diff_viewer.py @@ -4,7 +4,7 @@ from git import Repo from git.diff import Diff, DiffIndex from git.objects.commit import Commit -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.paths import PathUtils DiffFilter = Callable[[Diff], bool] diff --git a/packages/opal-common/opal_common/git/env.py b/packages/opal-common/opal_common/git_utils/env.py similarity index 100% rename from packages/opal-common/opal_common/git/env.py rename to packages/opal-common/opal_common/git_utils/env.py diff --git a/packages/opal-common/opal_common/git/exceptions.py b/packages/opal-common/opal_common/git_utils/exceptions.py similarity index 100% rename from packages/opal-common/opal_common/git/exceptions.py rename to packages/opal-common/opal_common/git_utils/exceptions.py diff --git a/packages/opal-common/opal_common/git/repo_cloner.py b/packages/opal-common/opal_common/git_utils/repo_cloner.py similarity index 98% rename from packages/opal-common/opal_common/git/repo_cloner.py rename to packages/opal-common/opal_common/git_utils/repo_cloner.py index 43bda1ba2..76ebb4949 100644 --- a/packages/opal-common/opal_common/git/repo_cloner.py +++ b/packages/opal-common/opal_common/git_utils/repo_cloner.py @@ -8,8 +8,8 @@ from git import GitCommandError, GitError, Repo from opal_common.config import opal_common_config -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from opal_common.utils import get_filepaths_with_glob from tenacity import RetryError, retry, stop, wait diff --git a/packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py b/packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py similarity index 100% rename from packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py rename to packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py diff --git a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py similarity index 95% rename from packages/opal-common/opal_common/git/tests/branch_tracker_test.py rename to packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py index bc17d6b09..751231a0d 100644 --- a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py @@ -18,8 +18,8 @@ from git import Repo from git.objects.commit import Commit -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed def test_pull_with_no_changes(local_repo_clone: Repo): diff --git a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py similarity index 99% rename from packages/opal-common/opal_common/git/tests/bundle_maker_test.py rename to packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py index 63624f6eb..5e77ad0e5 100644 --- a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py @@ -19,8 +19,8 @@ from git import Repo from git.objects import Commit -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer from opal_common.schemas.policy import PolicyBundle, RegoModule OPA_FILE_EXTENSIONS = (".rego", ".json") diff --git a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py similarity index 98% rename from packages/opal-common/opal_common/git/tests/commit_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py index 91d19fcf1..1f9ca522a 100644 --- a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py @@ -19,7 +19,7 @@ from git import Repo from git.objects import Commit -from opal_common.git.commit_viewer import CommitViewer, VersionedNode +from opal_common.git_utils.commit_viewer import CommitViewer, VersionedNode def node_paths(nodes: List[VersionedNode]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/conftest.py b/packages/opal-common/opal_common/git_utils/tests/conftest.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/conftest.py rename to packages/opal-common/opal_common/git_utils/tests/conftest.py diff --git a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py similarity index 97% rename from packages/opal-common/opal_common/git/tests/diff_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py index 6dc77ec4b..974ffaa7e 100644 --- a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py @@ -20,8 +20,8 @@ from git import Diff, Repo from git.objects import Commit -from opal_common.git.commit_viewer import VersionedFile -from opal_common.git.diff_viewer import DiffViewer, diffed_file_is_under_directories +from opal_common.git_utils.commit_viewer import VersionedFile +from opal_common.git_utils.diff_viewer import DiffViewer, diffed_file_is_under_directories def diff_paths(diffs: List[Diff]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py similarity index 96% rename from packages/opal-common/opal_common/git/tests/repo_cloner_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py index ffe6a02fa..567f3707b 100644 --- a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py @@ -18,8 +18,8 @@ from git import Repo from opal_common.confi import Confi -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner VALID_REPO_REMOTE_URL_HTTPS = "https://github.com/permitio/fastapi_websocket_pubsub.git" diff --git a/packages/opal-common/opal_common/git/tests/repo_watcher_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/repo_watcher_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 9b6487907..596e8aba0 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -6,7 +6,7 @@ import aiohttp from fastapi import status from fastapi.exceptions import HTTPException -from opal_common.git.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor +from opal_common.git_utils.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_common.utils import ( diff --git a/packages/opal-common/opal_common/sources/git_policy_source.py b/packages/opal-common/opal_common/sources/git_policy_source.py index bffe8517d..8252cd4ce 100644 --- a/packages/opal-common/opal_common/sources/git_policy_source.py +++ b/packages/opal-common/opal_common/sources/git_policy_source.py @@ -1,9 +1,9 @@ from typing import Optional from git import Repo -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 5ea85c047..36932ee30 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -12,7 +12,7 @@ from ddtrace import tracer from git import Repo from opal_common.async_utils import run_sync -from opal_common.git.bundle_maker import BundleMaker +from opal_common.git_utils.bundle_maker import BundleMaker from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.policy_source import ( diff --git a/packages/opal-server/opal_server/policy/bundles/api.py b/packages/opal-server/opal_server/policy/bundles/api.py index 223e72001..7e25e8ffc 100644 --- a/packages/opal-server/opal_server/policy/bundles/api.py +++ b/packages/opal-server/opal_server/policy/bundles/api.py @@ -4,11 +4,12 @@ import fastapi.responses from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status -from git import Repo +from git.repo import Repo + from opal_common.confi.confi import load_conf_if_none -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_server.config import opal_server_config diff --git a/packages/opal-server/opal_server/policy/watcher/callbacks.py b/packages/opal-server/opal_server/policy/watcher/callbacks.py index 62a30f16b..1b5f65590 100644 --- a/packages/opal-server/opal_server/policy/watcher/callbacks.py +++ b/packages/opal-server/opal_server/policy/watcher/callbacks.py @@ -3,13 +3,13 @@ from typing import List, Optional from git.objects import Commit -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, FileFilter, find_ignore_match, has_extension, ) -from opal_common.git.diff_viewer import DiffViewer +from opal_common.git_utils.diff_viewer import DiffViewer from opal_common.logger import logger from opal_common.paths import PathUtils from opal_common.schemas.policy import ( diff --git a/packages/opal-server/opal_server/policy/watcher/factory.py b/packages/opal-server/opal_server/policy/watcher/factory.py index 10fb1b19c..6d94d6fc4 100644 --- a/packages/opal-server/opal_server/policy/watcher/factory.py +++ b/packages/opal-server/opal_server/policy/watcher/factory.py @@ -3,7 +3,7 @@ from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint from opal_common.confi.confi import load_conf_if_none -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.sources.api_policy_source import ApiPolicySource from opal_common.sources.git_policy_source import GitPolicySource diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 533b397a8..f0104e7bf 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -7,7 +7,7 @@ import git from ddtrace import tracer from fastapi_websocket_pubsub import PubSubEndpoint -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.logger import logger from opal_common.schemas.policy import PolicyUpdateMessageNotification from opal_common.schemas.policy_source import GitPolicyScopeSource From 8d0952611f10d00a25cbff50da4bc34a7587b5b3 Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 16 Jul 2024 04:09:58 +0000 Subject: [PATCH 25/83] fix: packages/requires.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-SETUPTOOLS-7448482 --- packages/requires.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..43f5a740d 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -8,4 +8,4 @@ pydantic[email]>=1.9.1,<2 typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability From 3801481b0cc209bfc1c2c7951abf722e3b786bdc Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Tue, 16 Jul 2024 14:52:11 +0300 Subject: [PATCH 26/83] Fixed statistics API (#622) --- packages/opal-server/opal_server/statistics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/opal-server/opal_server/statistics.py b/packages/opal-server/opal_server/statistics.py index 6ae5ad619..14ea97f0a 100644 --- a/packages/opal-server/opal_server/statistics.py +++ b/packages/opal-server/opal_server/statistics.py @@ -124,9 +124,9 @@ async def _expire_old_servers(self): now = datetime.utcnow() still_alive = {} for server_id, last_seen in self._seen_servers.items(): - if ( - now - last_seen - ).total_seconds() < opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT: + if (now - last_seen).total_seconds() < float( + opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT + ): still_alive[server_id] = last_seen self._seen_servers = still_alive self._state.servers = {self._worker_id} | set(self._seen_servers.keys()) @@ -140,7 +140,7 @@ async def _periodic_server_keepalive(self): ServerKeepalive(worker_id=self._worker_id).dict(), ) await asyncio.sleep( - opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT / 2 + float(opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT) / 2 ) except asyncio.CancelledError: logger.debug("Statistics: periodic server keepalive cancelled") From e5f5dba6170197ec49b59c6aa8120d8176f1be12 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Tue, 16 Jul 2024 14:55:57 +0300 Subject: [PATCH 27/83] dan/per-10200-write-docs-on-how-to-use-opal (#612) * Added OPAL+ docs * Fixed invalid links * Changed OPAL+ docs * Added OPAL+ features * Added OPAL+ support * Added troubleshooting docs * Fixed pre-commit * Fixed broken link * Added OPAL+ custom data fetcher providers feature --- documentation/docs/opal-plus/deploy.mdx | 34 ++++++++ documentation/docs/opal-plus/features.mdx | 80 +++++++++++++++++++ .../introduction.mdx} | 13 +-- .../docs/opal-plus/troubleshooting.mdx | 43 ++++++++++ documentation/docs/release-updates.mdx | 4 +- documentation/sidebars.js | 16 ++-- 6 files changed, 177 insertions(+), 13 deletions(-) create mode 100644 documentation/docs/opal-plus/deploy.mdx create mode 100644 documentation/docs/opal-plus/features.mdx rename documentation/docs/{OPAL_PLUS.mdx => opal-plus/introduction.mdx} (84%) create mode 100644 documentation/docs/opal-plus/troubleshooting.mdx diff --git a/documentation/docs/opal-plus/deploy.mdx b/documentation/docs/opal-plus/deploy.mdx new file mode 100644 index 000000000..7448b37f7 --- /dev/null +++ b/documentation/docs/opal-plus/deploy.mdx @@ -0,0 +1,34 @@ +--- +sidebar_position: 2 +title: Deploy OPAL+ +--- + +With OPAL+, you get access to private Docker images that include additional features and capabilities. +To apply for Permit OPAL+, [fill in the form available here](https://hello.permit.io/opal-plus) + +In order to access the OPAL+ Docker images, you need to have Docker Hub credentials with an access token. +Those should be received from your Customer Success manager. +Reach out to us [on Slack](https://bit.ly/permit-slack) if you need assistance. + +## Accessing the OPAL+ Docker Images + +To access the OPAL+ Docker images, you need to log in to Docker Hub with your credentials. +You can do this by running the [docker login](https://docs.docker.com/reference/cli/docker/login/) command: + +```bash +docker login -u -p +``` + +If you are using Kubernetes, check out the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) on how to pull images from a private registry. + +After logging in, you can pull the OPAL+ Docker images using the following commands: + +```bash +docker pull permitio/opal-plus:latest +``` + +## Running the OPAL+ Docker Images + +Running the OPAL+ Docker images is similar to running the open-source OPAL images. + +Check out the [OPAL Docker documentation](/getting-started/running-opal/run-docker-containers) for more information. diff --git a/documentation/docs/opal-plus/features.mdx b/documentation/docs/opal-plus/features.mdx new file mode 100644 index 000000000..bc3a0249f --- /dev/null +++ b/documentation/docs/opal-plus/features.mdx @@ -0,0 +1,80 @@ +--- +sidebar_position: 3 +title: Advanced Features +--- + +OPAL+ has a number of advanced features extending the capabilities of the open-source OPAL. +These features are available to OPAL+ users only. + +To apply for Permit OPAL+, [fill in the form available here](https://hello.permit.io/opal-plus) + + +## Support and SLA + +OPAL+ provides dedicated support and a custom SLA to help you get the most out of your OPAL+ deployment. +Reach out to us [on Slack](https://bit.ly/permit-slack) for more information. + +## Licensing Capabilities + +OPAL+ provides additional licensing capabilities to help you manage your OPAL+ deployment. +Reach out to us [on Slack](https://bit.ly/permit-slack) for more information. + +## Custom Data Fetcher Providers + +OPAL+ provides custom data fetcher providers to help you fetch data from your private data sources. + +## Logging and Monitoring + +OPAL+ provides advanced logging and monitoring capabilities to help you track and debug your OPAL+ deployment. + +#### Connect to logging system + +On production, we advise you to connect OPAL+ to your logging system to collect and store the logs. +Configure the [OPAL_LOG_SERIALIZE](/getting-started/configuration) environment variable to `true` to serialize logs in JSON format. + +#### Monitor OPAL Servers and Clients + +OPAL+ provides monitoring endpoints to help you track the health of your OPAL+ servers and clients. +Configure the [OPAL_STATISTICS_ENABLED=true](/getting-started/configuration) environment variable to enable the statistics APIs. + +You can then monitor the state of your OPAL+ cluster by calling the `/stats` API route on the server. +```bash +curl http://opal-server:8181/stats -H "Authorization: Bearer " +# { "uptime": "2024-07-14T14:55:02.710Z", "version": "0.7.8", "client_count": 1, "server_count": 1 } +``` + +You can also get detailed information about the OPAL+ clients and servers by calling the `/statistics` API route on the server. +```bash +curl http://opal-server:8181/statistics -H "Authorization: Bear " +``` +```json +{ + "uptime": "2024-07-14T14:54:09.809Z", + "version": "0.7.8", + "clients": { + "opal-client-1": [ + { + "rpc_id": "7ba198b1329d439faaa79dd7447401dc", + "client_id": "693ac1b4d060416eaad50c2bf04121b1", + "topics": [ + "string" + ] + } + ], + "opal-client-2": [ + { + "rpc_id": "d343d92292794630994a8a077bcb413a", + "client_id": "4d71d88ba16f49e1a0ae89f16c5a55d5", + "topics": [ + "string" + ] + } + ] + }, + "servers": [ + "774b376fbead49b79f6a9fd42cef2cfd" + ] +} +``` + +For more information on monitoring OPAL, see the [Monitoring OPAL](/tutorials/monitoring_opal) tutorial. diff --git a/documentation/docs/OPAL_PLUS.mdx b/documentation/docs/opal-plus/introduction.mdx similarity index 84% rename from documentation/docs/OPAL_PLUS.mdx rename to documentation/docs/opal-plus/introduction.mdx index 8ca739539..08b609d8a 100644 --- a/documentation/docs/OPAL_PLUS.mdx +++ b/documentation/docs/opal-plus/introduction.mdx @@ -1,8 +1,9 @@ --- sidebar_position: 1 -title: Permit OPAL+ (Extended OPAL License) +title: Introduction --- +
Date: Wed, 17 Jul 2024 13:25:50 +0300 Subject: [PATCH 28/83] Bump version to 0.7.9 --- packages/__packaging__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/__packaging__.py b/packages/__packaging__.py index a0b868fcf..d16711e2c 100644 --- a/packages/__packaging__.py +++ b/packages/__packaging__.py @@ -9,7 +9,7 @@ import os -VERSION = (0, 7, 8) +VERSION = (0, 7, 9) VERSION_STRING = ".".join(map(str, VERSION)) __version__ = VERSION_STRING From 208253b3096fcc53ee2b0c9b94cc5ceee3e451de Mon Sep 17 00:00:00 2001 From: roekatz Date: Wed, 10 Jul 2024 19:45:41 +0300 Subject: [PATCH 29/83] Fix opal-server addr to opal-server rather than host.docker.internal --- docker/docker-compose-api-policy-source-example.yml | 2 +- docker/docker-compose-with-callbacks.yml | 2 +- docker/docker-compose-with-kafka-example.yml | 2 +- docker/docker-compose-with-rate-limiting.yml | 2 +- docker/docker-compose-with-security.yml | 2 +- docker/docker-compose-with-statistics.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml index 466689a5a..a91997e4e 100644 --- a/docker/docker-compose-api-policy-source-example.yml +++ b/docker/docker-compose-api-policy-source-example.yml @@ -37,7 +37,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-callbacks.yml b/docker/docker-compose-with-callbacks.yml index 7ce3eacce..b47e39bcb 100644 --- a/docker/docker-compose-with-callbacks.yml +++ b/docker/docker-compose-with-callbacks.yml @@ -32,7 +32,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml index 99b4129c7..ec4f113e2 100644 --- a/docker/docker-compose-with-kafka-example.yml +++ b/docker/docker-compose-with-kafka-example.yml @@ -70,7 +70,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml index a0fd3bbcc..b5fb169b4 100644 --- a/docker/docker-compose-with-rate-limiting.yml +++ b/docker/docker-compose-with-rate-limiting.yml @@ -31,7 +31,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # Turns on rate limiting in the server # supported formats documented here: https://limits.readthedocs.io/en/stable/quickstart.html#rate-limit-string-notation diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml index 8a80ad2a6..ad3ae6186 100644 --- a/docker/docker-compose-with-security.yml +++ b/docker/docker-compose-with-security.yml @@ -46,7 +46,7 @@ services: # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. # please notice - since we fetch data entries from the OPAL server itself, we need to authenticate to that endpoint # with the client's token (JWT). - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # -------------------------------------------------------------------------------- # the jwt audience and jwt issuer are not typically necessary in real setups diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml index 0c15d5a95..daf10fa02 100644 --- a/docker/docker-compose-with-statistics.yml +++ b/docker/docker-compose-with-statistics.yml @@ -32,7 +32,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # turning on statistics collection on the server side - OPAL_STATISTICS_ENABLED=true From ba89632f9ab6a6079dfada51c7eebeecc8973544 Mon Sep 17 00:00:00 2001 From: roekatz Date: Thu, 18 Jul 2024 12:41:47 +0300 Subject: [PATCH 30/83] CI test-docker: Also make sure opal-client fetched data sources --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 610cdd05d..d708b16e0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -105,7 +105,8 @@ jobs: - name: Output container logs run: docker-compose -f docker/docker-compose-test.yml logs - - name: check if opal-client was brought up + - name: check if opal-client was brought up successfully run: | docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Connected to PubSub server" docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Got policy bundle" + docker-compose -f docker/docker-compose-test.yml logs opal_client | grep 'PUT /v1/data/static -> 204' From 8c30c89b90171274f855f639e3667466cd6794ed Mon Sep 17 00:00:00 2001 From: roekatz Date: Thu, 18 Jul 2024 12:44:52 +0300 Subject: [PATCH 31/83] Docker-compose: Remove obsolete `version` field --- docker/docker-compose-api-policy-source-example.yml | 1 - docker/docker-compose-example-cedar.yml | 1 - docker/docker-compose-example.yml | 1 - docker/docker-compose-git-webhook.yml | 1 - docker/docker-compose-scopes-example.yml | 1 - docker/docker-compose-with-callbacks.yml | 1 - docker/docker-compose-with-kafka-example.yml | 1 - docker/docker-compose-with-oauth-initial.yml | 1 - docker/docker-compose-with-rate-limiting.yml | 1 - docker/docker-compose-with-security.yml | 1 - docker/docker-compose-with-statistics.yml | 1 - .../quickstart/docker-compose-config/overview.mdx | 1 - .../quickstart/opal-playground/updating-the-policy.mdx | 1 - 13 files changed, 13 deletions(-) diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml index a91997e4e..219381598 100644 --- a/docker/docker-compose-api-policy-source-example.yml +++ b/docker/docker-compose-api-policy-source-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-example-cedar.yml b/docker/docker-compose-example-cedar.yml index a3170e575..38e5509a6 100644 --- a/docker/docker-compose-example-cedar.yml +++ b/docker/docker-compose-example-cedar.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-example.yml b/docker/docker-compose-example.yml index 13855734b..36c52db58 100644 --- a/docker/docker-compose-example.yml +++ b/docker/docker-compose-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-git-webhook.yml b/docker/docker-compose-git-webhook.yml index c5a394270..388ced755 100644 --- a/docker/docker-compose-git-webhook.yml +++ b/docker/docker-compose-git-webhook.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-scopes-example.yml b/docker/docker-compose-scopes-example.yml index 789ebea5a..9a3c1f162 100644 --- a/docker/docker-compose-scopes-example.yml +++ b/docker/docker-compose-scopes-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: redis: image: redis diff --git a/docker/docker-compose-with-callbacks.yml b/docker/docker-compose-with-callbacks.yml index b47e39bcb..ca75903e6 100644 --- a/docker/docker-compose-with-callbacks.yml +++ b/docker/docker-compose-with-callbacks.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml index ec4f113e2..1289e0592 100644 --- a/docker/docker-compose-with-kafka-example.yml +++ b/docker/docker-compose-with-kafka-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # Based on: https://developer.confluent.io/quickstart/kafka-docker/ diff --git a/docker/docker-compose-with-oauth-initial.yml b/docker/docker-compose-with-oauth-initial.yml index 98b647e88..6a121e719 100644 --- a/docker/docker-compose-with-oauth-initial.yml +++ b/docker/docker-compose-with-oauth-initial.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml index b5fb169b4..6f10caf5e 100644 --- a/docker/docker-compose-with-rate-limiting.yml +++ b/docker/docker-compose-with-rate-limiting.yml @@ -1,5 +1,4 @@ # This docker compose example shows how to configure OPAL's rate limiting feature -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml index ad3ae6186..2c27a711f 100644 --- a/docker/docker-compose-with-security.yml +++ b/docker/docker-compose-with-security.yml @@ -1,6 +1,5 @@ # this docker compose file is relying on external environment variables! # run it by running the script: ./run-example-with-security.sh -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml index daf10fa02..eb26daef4 100644 --- a/docker/docker-compose-with-statistics.yml +++ b/docker/docker-compose-with-statistics.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx b/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx index 1d11235e3..f404a6c3b 100644 --- a/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx +++ b/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx @@ -11,7 +11,6 @@ This example is running three containers that we have mentioned at the beginning Here is an overview of the whole `docker-compose.yml` file, but don't worry, we will be referring to each section separately. ```yml showLineNumbers -version: "3.8" services: broadcast_channel: image: postgres:alpine diff --git a/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx b/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx index f0c632f58..24e9a3461 100644 --- a/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx +++ b/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx @@ -35,7 +35,6 @@ opal_server: You can also simply change the tracked repo in the example `docker-compose.yml` file by editing these variables: ```yml {7,9,11} showLineNumbers -version: "3.8" services: ... opal_server: From 5f01375cdd4626bd076794e7152328b96d9c9a07 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 19:27:17 +0300 Subject: [PATCH 32/83] Fixed OPAL Cedar Client build CI --- .github/workflows/on_release.yml | 54 +++++++++++++++++++------------- cedar-agent | 2 +- docker/Dockerfile | 8 ++--- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 736a64f0b..e616e11d0 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -109,13 +109,12 @@ jobs: # pushes the *same* docker images that were previously tested as part of e2e sanity test. # each image is pushed with the versioned tag first, if it succeeds the image is pushed with the latest tag as well. - name: Build & Push client - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: client cache-from: type=registry,ref=permitio/opal-client:latest cache-to: type=inline @@ -123,29 +122,13 @@ jobs: permitio/opal-client:latest permitio/opal-client:${{ env.opal_version_tag }} -# - name: Build & Push client cedar -# if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} -# id: build_push_client_cedar -# uses: docker/build-push-action@v4 -# with: -# file: docker/Dockerfile -# platforms: linux/amd64,linux/arm64 -# push: true -# target: client-cedar -# cache-from: type=registry,ref=permitio/opal-client-cedar:latest -# cache-to: type=inline -# tags: | -# permitio/opal-client-cedar:latest -# permitio/opal-client-cedar:${{ env.opal_version_tag }} - - name: Build client-standalone - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client_standalone uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: client-standalone cache-from: type=registry,ref=permitio/opal-client-standalone:latest cache-to: type=inline @@ -154,16 +137,45 @@ jobs: permitio/opal-client-standalone:${{ env.opal_version_tag }} - name: Build server - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_server uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: server cache-from: type=registry,ref=permitio/opal-server:latest cache-to: type=inline tags: | permitio/opal-server:latest permitio/opal-server:${{ env.opal_version_tag }} + + - name: Check if cedar-agent directory exists + id: check_cedar_agent + run: | + if [ -d "cedar-agent" ]; then + echo "exists=true" >> $GITHUB_ENV + else + echo "exists=false" >> $GITHUB_ENV + fi + + - name: Clone cedar-agent repository + if: steps.check_cedar_agent.outputs.exists == 'false' + id: clone_cedar_agent + working-directory: . + run: | + git clone https://github.com/permitio/cedar-agent.git cedar-agent + + - name: Build & Push client cedar + id: build_push_client_cedar + uses: docker/build-push-action@v4 + with: + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + tags: | + permitio/opal-client-cedar:latest + permitio/opal-client-cedar:${{ env.opal_version_tag }} diff --git a/cedar-agent b/cedar-agent index 1838635f1..687efc59e 160000 --- a/cedar-agent +++ b/cedar-agent @@ -1 +1 @@ -Subproject commit 1838635f16ba6db60d16c2ca28cb257e970bdff0 +Subproject commit 687efc59ecc732d1b98fc7789ab803abfc45b94c diff --git a/docker/Dockerfile b/docker/Dockerfile index bccdf3d2c..4723e37a7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,10 +17,10 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # --------------------------------------------------- FROM rust:1.69.0 as cedar-builder COPY cedar-agent /tmp/cedar-agent/ -ARG cargo_flags="-r" -RUN cd /tmp/cedar-agent && \ - cargo build ${cargo_flags} && \ - cp /tmp/cedar-agent/target/*/cedar-agent / +ARG cargo_flags="--release" +RUN cd /tmp/cedar-agent +RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build ${cargo_flags} +RUN cp /tmp/cedar-agent/target/*/cedar-agent / # COMMON IMAGE -------------------------------------- # --------------------------------------------------- From 6ae6b18ccfb63cb632a079883420a5cc6a5c76c3 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 19:45:21 +0300 Subject: [PATCH 33/83] Updated rust version for OPAL Cedar Client docker --- .github/workflows/on_release.yml | 16 ---------------- docker/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index e616e11d0..3733c0fd7 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -150,22 +150,6 @@ jobs: permitio/opal-server:latest permitio/opal-server:${{ env.opal_version_tag }} - - name: Check if cedar-agent directory exists - id: check_cedar_agent - run: | - if [ -d "cedar-agent" ]; then - echo "exists=true" >> $GITHUB_ENV - else - echo "exists=false" >> $GITHUB_ENV - fi - - - name: Clone cedar-agent repository - if: steps.check_cedar_agent.outputs.exists == 'false' - id: clone_cedar_agent - working-directory: . - run: | - git clone https://github.com/permitio/cedar-agent.git cedar-agent - - name: Build & Push client cedar id: build_push_client_cedar uses: docker/build-push-action@v4 diff --git a/docker/Dockerfile b/docker/Dockerfile index 4723e37a7..31ca18012 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # CEDAR AGENT BUILD STAGE --------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM rust:1.69.0 as cedar-builder +FROM rust:1.77-bullseye as cedar-builder COPY cedar-agent /tmp/cedar-agent/ ARG cargo_flags="--release" RUN cd /tmp/cedar-agent From 63c27c8395121af93fa26eaf6f4c2addbcf69adc Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 20:17:05 +0300 Subject: [PATCH 34/83] Fixed docker warnings --- docker/Dockerfile | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 31ca18012..89af2eba0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # BUILD STAGE --------------------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM python:3.10-bookworm as BuildStage +FROM python:3.10-bookworm AS build-stage # from now on, work in the /app directory WORKDIR /app/ # Layer dependency install (for caching) @@ -15,19 +15,18 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # CEDAR AGENT BUILD STAGE --------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM rust:1.77-bullseye as cedar-builder -COPY cedar-agent /tmp/cedar-agent/ -ARG cargo_flags="--release" +FROM rust:1.79 AS cedar-builder +COPY ./cedar-agent /tmp/cedar-agent/ RUN cd /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build ${cargo_flags} RUN cp /tmp/cedar-agent/target/*/cedar-agent / # COMMON IMAGE -------------------------------------- # --------------------------------------------------- -FROM python:3.10-slim-bookworm as common +FROM python:3.10-slim-bookworm AS common -# copy libraries from build stage (This won't copy redundant libraries we used in BuildStage) -COPY --from=BuildStage /usr/local /usr/local +# copy libraries from build stage (This won't copy redundant libraries we used in build-stage) +COPY --from=build-stage /usr/local /usr/local # Add non-root user (with home dir at /opal) RUN useradd -m -b / -s /bin/bash opal @@ -61,7 +60,7 @@ CMD ["./start.sh"] # STANDALONE IMAGE ---------------------------------- # --------------------------------------------------- -FROM common as client-standalone +FROM common AS client-standalone # uvicorn config ------------------------------------ # install the opal-client package RUN cd ./packages/opal-client && python setup.py install @@ -88,7 +87,7 @@ VOLUME /opal/backup # IMAGE to extract OPA from official image ---------- # --------------------------------------------------- -FROM alpine:latest as opa-extractor +FROM alpine:latest AS opa-extractor USER root RUN apk update && apk add skopeo tar @@ -106,7 +105,7 @@ RUN skopeo copy "docker://${opa_image}:${opa_tag}" docker-archive:./image.tar && # OPA CLIENT IMAGE ---------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client +FROM client-standalone AS client # Temporarily move back to root for additional setup USER root @@ -123,7 +122,7 @@ USER opal # CEDAR CLIENT IMAGE -------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client-cedar +FROM client-standalone AS client-cedar # Temporarily move back to root for additional setup USER root @@ -142,7 +141,7 @@ USER opal # SERVER IMAGE -------------------------------------- # --------------------------------------------------- -FROM common as server +FROM common AS server RUN apt-get update && apt-get install -y openssh-client git && apt-get clean From ed70c746fe4c2494df14260e1061ed6a49e39144 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 20:40:58 +0300 Subject: [PATCH 35/83] Fixed cedar build --- docker/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 89af2eba0..74ac1916b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,10 +16,9 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # split this stage to save time and reduce image size # --------------------------------------------------- FROM rust:1.79 AS cedar-builder -COPY ./cedar-agent /tmp/cedar-agent/ -RUN cd /tmp/cedar-agent -RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build ${cargo_flags} -RUN cp /tmp/cedar-agent/target/*/cedar-agent / +COPY ./cedar-agent /tmp/cedar-agent +WORKDIR /tmp/cedar-agent +RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release # COMMON IMAGE -------------------------------------- # --------------------------------------------------- @@ -128,7 +127,7 @@ FROM client-standalone AS client-cedar USER root # Copy cedar from its build stage -COPY --from=cedar-builder /cedar-agent /bin/cedar-agent +COPY --from=cedar-builder /tmp/cedar-agent/target/*/cedar-agent /bin/cedar-agent # enable inline Cedar agent ENV OPAL_POLICY_STORE_TYPE=CEDAR From 031070144c521af98d19f89e7fe507667199db3a Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Tue, 23 Jul 2024 14:53:02 +0300 Subject: [PATCH 36/83] Fixed pre-commit --- .../opal_client/policy_store/mock_policy_store_client.py | 6 ++++-- packages/opal-common/opal_common/cli/typer_app.py | 1 - packages/opal-common/opal_common/confi/cli.py | 3 +-- packages/opal-common/opal_common/confi/confi.py | 5 ++--- packages/opal-common/opal_common/config.py | 2 +- .../opal_common/fetcher/engine/base_fetching_engine.py | 2 +- .../opal-common/opal_common/fetcher/engine/fetch_worker.py | 2 +- .../opal_common/fetcher/engine/fetching_engine.py | 6 +++--- packages/opal-common/opal_common/fetcher/fetch_provider.py | 3 +-- .../opal-common/opal_common/fetcher/fetcher_register.py | 3 +-- .../fetcher/providers/fastapi_rpc_fetch_provider.py | 1 - .../opal_common/fetcher/providers/http_fetch_provider.py | 7 +++---- .../opal_common/git_utils/tests/diff_viewer_test.py | 5 ++++- packages/opal-common/opal_common/logger.py | 1 - .../opal-common/opal_common/sources/api_policy_source.py | 4 +++- packages/opal-server/opal_server/policy/bundles/api.py | 1 - 16 files changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py index 8d6742d4a..549dd8435 100644 --- a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py +++ b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py @@ -3,13 +3,15 @@ from typing import Any, Dict, List, Optional import jsonpatch +from opal_client.policy_store.base_policy_store_client import ( + BasePolicyStoreClient, + JsonableValue, +) from opal_client.utils import exclude_none_fields from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.store import JSONPatchAction, StoreTransaction from pydantic import BaseModel -from opal_client.policy_store.base_policy_store_client import BasePolicyStoreClient, JsonableValue - class MockPolicyStoreClient(BasePolicyStoreClient): """A naive mock policy and policy-data store for tests.""" diff --git a/packages/opal-common/opal_common/cli/typer_app.py b/packages/opal-common/opal_common/cli/typer_app.py index a1d70ff24..47d38dd39 100644 --- a/packages/opal-common/opal_common/cli/typer_app.py +++ b/packages/opal-common/opal_common/cli/typer_app.py @@ -1,5 +1,4 @@ import typer - from opal_common.cli.commands import all_commands diff --git a/packages/opal-common/opal_common/confi/cli.py b/packages/opal-common/opal_common/confi/cli.py index cfca25f1c..0ab88e55a 100644 --- a/packages/opal-common/opal_common/confi/cli.py +++ b/packages/opal-common/opal_common/confi/cli.py @@ -2,9 +2,8 @@ import click import typer -from typer.main import Typer - from opal_common.confi.types import ConfiEntry +from typer.main import Typer def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index f62576dfa..cbaa9a587 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -15,13 +15,12 @@ from decouple import Csv, UndefinedValueError, config, text_type, undefined from opal_common.authentication.casting import cast_private_key, cast_public_key from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey +from opal_common.confi.cli import get_cli_object_for_config_objects +from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast from opal_common.logging_utils.decorators import log_exception from pydantic import BaseModel, ValidationError from typer import Typer -from opal_common.confi.cli import get_cli_object_for_config_objects -from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast - class Placeholder(object): """Placeholder instead of default value for decouple.""" diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index b7d9395b6..ab18dd0cb 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -172,7 +172,7 @@ class OpalCommonConfig(Confi): "HTTP_FETCHER_PROVIDER_CLIENT", "aiohttp", description="The client to use for fetching data, can be either aiohttp or httpx." - "if provided different value, aiohttp will be used.", + "if provided different value, aiohttp will be used.", ) diff --git a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py index 22f9325f9..19a636a35 100644 --- a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py @@ -1,8 +1,8 @@ from typing import Coroutine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetcher_register import FetcherRegister -from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback class BaseFetchingEngine: diff --git a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py index 460ee1465..6db97b338 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py @@ -1,10 +1,10 @@ import asyncio from typing import Coroutine +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine from opal_common.fetcher.events import FetchEvent from opal_common.fetcher.fetcher_register import FetcherRegister from opal_common.fetcher.logger import get_logger -from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine logger = get_logger("fetch_worker") diff --git a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py index cb03693bf..b439d4b8d 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py @@ -2,13 +2,13 @@ import uuid from typing import Coroutine, Dict, List, Union +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.fetch_worker import fetch_worker from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.fetcher_register import FetcherRegister from opal_common.fetcher.logger import get_logger -from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine -from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback -from opal_common.fetcher.engine.fetch_worker import fetch_worker logger = get_logger("engine") diff --git a/packages/opal-common/opal_common/fetcher/fetch_provider.py b/packages/opal-common/opal_common/fetcher/fetch_provider.py index 70b91ea59..c05008fcd 100644 --- a/packages/opal-common/opal_common/fetcher/fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/fetch_provider.py @@ -1,7 +1,6 @@ -from tenacity import retry, stop, wait - from opal_common.fetcher.events import FetchEvent from opal_common.fetcher.logger import get_logger +from tenacity import retry, stop, wait logger = get_logger("opal.providers") diff --git a/packages/opal-common/opal_common/fetcher/fetcher_register.py b/packages/opal-common/opal_common/fetcher/fetcher_register.py index 18ed32f81..9abf1322c 100644 --- a/packages/opal-common/opal_common/fetcher/fetcher_register.py +++ b/packages/opal-common/opal_common/fetcher/fetcher_register.py @@ -1,10 +1,9 @@ from typing import Dict, Optional, Type -from opal_common.fetcher.logger import get_logger - from opal_common.config import opal_common_config from opal_common.fetcher.events import FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger from opal_common.fetcher.providers.http_fetch_provider import HttpFetchProvider logger = get_logger("opal.fetcher_register") diff --git a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py index 94513f9d2..4b574a8ea 100644 --- a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py @@ -2,7 +2,6 @@ from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient - from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 9083f8aa1..fc74223ed 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -6,13 +6,12 @@ import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config -from pydantic import validator - -from opal_common.http_utils import is_http_error_response -from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger +from opal_common.http_utils import is_http_error_response +from opal_common.security.sslcontext import get_custom_ssl_context +from pydantic import validator logger = get_logger("http_fetch_provider") diff --git a/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py index 974ffaa7e..bcfbb93be 100644 --- a/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py @@ -21,7 +21,10 @@ from git import Diff, Repo from git.objects import Commit from opal_common.git_utils.commit_viewer import VersionedFile -from opal_common.git_utils.diff_viewer import DiffViewer, diffed_file_is_under_directories +from opal_common.git_utils.diff_viewer import ( + DiffViewer, + diffed_file_is_under_directories, +) def diff_paths(diffs: List[Diff]) -> List[Path]: diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index 2a45a4dc2..8e826abd6 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -2,7 +2,6 @@ import sys from loguru import logger - from opal_common.config import opal_common_config from opal_common.logging_utils.filter import ModuleFilter from opal_common.logging_utils.formatter import Formatter diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 596e8aba0..7adc9ad70 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -6,7 +6,9 @@ import aiohttp from fastapi import status from fastapi.exceptions import HTTPException -from opal_common.git_utils.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor +from opal_common.git_utils.tar_file_to_local_git_extractor import ( + TarFileToLocalGitExtractor, +) from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_common.utils import ( diff --git a/packages/opal-server/opal_server/policy/bundles/api.py b/packages/opal-server/opal_server/policy/bundles/api.py index 7e25e8ffc..ae1da68ef 100644 --- a/packages/opal-server/opal_server/policy/bundles/api.py +++ b/packages/opal-server/opal_server/policy/bundles/api.py @@ -5,7 +5,6 @@ import fastapi.responses from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status from git.repo import Repo - from opal_common.confi.confi import load_conf_if_none from opal_common.git_utils.bundle_maker import BundleMaker from opal_common.git_utils.commit_viewer import CommitViewer From 3eee3e77714fffdfa3faddc4e0efcbafa558fd02 Mon Sep 17 00:00:00 2001 From: roekatz Date: Wed, 24 Jul 2024 21:28:34 +0300 Subject: [PATCH 37/83] Bump version to 0.7.10 --- packages/__packaging__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/__packaging__.py b/packages/__packaging__.py index d16711e2c..452772e5a 100644 --- a/packages/__packaging__.py +++ b/packages/__packaging__.py @@ -9,7 +9,7 @@ import os -VERSION = (0, 7, 9) +VERSION = (0, 7, 10) VERSION_STRING = ".".join(map(str, VERSION)) __version__ = VERSION_STRING From 82b41493e283a5ed1e0c3d4949f600d2f1229b25 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 11:55:08 +0200 Subject: [PATCH 38/83] Hardcode peer_type = datasource --- .../opal-common/opal_common/authentication/oauth2.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 645e8f9ea..4dd4b9cf0 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -1,8 +1,8 @@ import asyncio -import httpx -import jwt import time +from typing import Optional +import httpx from cachetools import cached, TTLCache from fastapi import Header from httpx import AsyncClient, BasicAuth @@ -10,13 +10,16 @@ from opal_common.authentication.jwk import JWKManager from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.config import opal_common_config -from typing import Optional + class _OAuth2Authenticator: async def authenticate(self, headers): if "Authorization" not in headers: token = await self.token() headers['Authorization'] = f"Bearer {token}" +# logger.info(f".....*****..... Adding headers: {headers}") +# else: +# logger.info(f".....*****..... Authorization header already exists") class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): @@ -72,6 +75,9 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) + #TODO TODO + claims["peer_type"] = "datasource" + return claims def _verify_opaque(self, token: str) -> {}: From e53f4a6a64727c165b44ae428165fd70831c7d63 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 12:02:08 +0200 Subject: [PATCH 39/83] Rebase feature branch onto updated master --- .../opal-client/opal_client/callbacks/api.py | 4 +- packages/opal-client/opal_client/client.py | 30 ++-- .../opal-client/opal_client/data/updater.py | 38 ++++- .../opal-client/opal_client/policy/fetcher.py | 24 ++- .../opal-client/opal_client/policy/updater.py | 26 ++- .../opal_client/policy_store/api.py | 4 +- .../authentication/authenticator.py | 50 ++++++ .../opal_common/authentication/authz.py | 6 +- .../opal_common/authentication/jwk.py | 46 +++++ .../opal_common/authentication/oauth2.py | 157 ++++++++++++++++++ packages/opal-common/opal_common/config.py | 22 +++ .../fetcher/providers/http_fetch_provider.py | 13 +- .../opal_server/authentication/__init__.py | 0 .../authentication/authenticator.py | 55 ++++++ packages/opal-server/opal_server/data/api.py | 5 +- .../opal_server/policy/webhook/api.py | 4 +- packages/opal-server/opal_server/pubsub.py | 10 +- .../opal-server/opal_server/scopes/api.py | 5 +- .../opal-server/opal_server/security/api.py | 5 +- .../opal-server/opal_server/security/jwks.py | 5 +- packages/opal-server/opal_server/server.py | 52 +++--- packages/requires.txt | 1 + 22 files changed, 468 insertions(+), 94 deletions(-) create mode 100644 packages/opal-common/opal_common/authentication/authenticator.py create mode 100644 packages/opal-common/opal_common/authentication/jwk.py create mode 100644 packages/opal-common/opal_common/authentication/oauth2.py create mode 100644 packages/opal-server/opal_server/authentication/__init__.py create mode 100644 packages/opal-server/opal_server/authentication/authenticator.py diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..9b828cced 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -29,8 +29,7 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +48,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[ClientAuthenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +63,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = ClientAuthenticator() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +122,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -140,6 +144,7 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +167,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +242,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index e288b5963..444219e34 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,6 +24,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig from opal_common.http_utils import is_http_error_response @@ -54,6 +55,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[ClientAuthenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -110,17 +112,18 @@ def __init__( self._callbacks_register, ) self._token = token + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None self._shard_id = shard_id self._server_url = pubsub_url self._data_sources_config_url = data_sources_config_url self._opal_client_id = opal_client_id - self._extra_headers = [] + self._extra_headers = {} if self._token is not None: - self._extra_headers.append(get_authorization_header(self._token)) + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] if self._shard_id is not None: - self._extra_headers.append(("X-Shard-ID", self._shard_id)) - if len(self._extra_headers) == 0: - self._extra_headers = None + self._extra_headers['X-Shard-ID'] = self._shard_id self._stopping = False # custom SSL context (for self-signed certificates) self._custom_ssl_context = get_custom_ssl_context() @@ -132,6 +135,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() async def __aenter__(self): await self.start() @@ -177,8 +184,14 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + try: - async with ClientSession(headers=self._extra_headers) as session: + async with ClientSession(headers=headers) as session: response = await session.get(url, **self._ssl_context_kwargs) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) @@ -274,12 +287,19 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( - self._data_topics, - self._update_policy_data_callback, + topics=self._data_topics, + callback=self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + on_disconnect=[self.on_disconnect], + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5ae9d93b6 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +29,26 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[ClientAuthenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + self._auth_headers = {} + if self._token != "THIS_IS_A_DEV_SECRET": + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +94,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, @@ -95,7 +112,6 @@ async def _fetch_policy_bundle( self._policy_endpoint_url, headers={ "content-type": "text/plain", - **self._auth_headers, }, params=params, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..d505c52f5 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +44,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[ClientAuthenticator] = None, ): """inits the policy updater. @@ -64,15 +66,21 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data self._server_url = pubsub_url self._token = token - if self._token is None: - self._extra_headers = None - else: - self._extra_headers = [get_authorization_header(self._token)] + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None + self._extra_headers = {} + if self._token is not None: + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] # Pub/Sub topics we subscribe to for policy updates if self._scope_id == "default": self._topics = pubsub_topics_from_directories( @@ -87,7 +95,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +248,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index b27d83d70..97113f109 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,15 +1,15 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..938ac001f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,50 @@ +from abc import abstractmethod +from typing import Optional + +from fastapi import Header +from opal_common.config import opal_common_config +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + +class Authenticator: + @property + def enabled(self): + return self._delegate().enabled + + async def authenticate(self, headers): + if hasattr(self._delegate(), "authenticate") and callable(getattr(self._delegate(), "authenticate")): + await self._delegate().authenticate(headers) + + @abstractmethod + def _delegate(self) -> dict: + pass + +class _ClientAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will authenticate API requests.") + else: + self.__delegate = JWTAuthenticator(self.__verifier()) + + def __verifier(self) -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info("API authentication disabled (public encryption key was not provided)") + + return verifier + + def _delegate(self) -> dict: + return self.__delegate + +class ClientAuthenticator(_ClientAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..9b0ec207f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,46 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + #TODO TODO: maxsize, ttl + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..645e8f9ea --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,157 @@ +import asyncio +import httpx +import jwt +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator: + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return True + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index ab18dd0cb..64c0ea093 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -159,6 +159,28 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str("AUTH_TYPE", None, description="Authentication type.") + OAUTH2_CLIENT_ID = confi.str("OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID.") + OAUTH2_CLIENT_SECRET = confi.str("OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret.") + OAUTH2_TOKEN_URL = confi.str("OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL.") + OAUTH2_INTROSPECT_URL = confi.str("OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL.") + OAUTH2_OPENID_CONFIGURATION_URL = confi.str("OAUTH2_OPENID_CONFIGURATION_URL", None, description="OAuth2 OpenID configuration URL.") + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", 100, description="OAuth2 token validation cache maxsize.") + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_TTL", 5 * 60, description="OAuth2 token validation cache TTL.") + + OAUTH2_EXP_MARGIN = confi.int("OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin.") + OAUTH2_EXACT_MATCH_CLAIMS = confi.str("OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims.") + OAUTH2_REQUIRED_CLAIMS = confi.str("OAUTH2_REQUIRED_CLAIMS", None, description="Comma separated list of required claims.") + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str("OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience") + OAUTH2_JWT_ISSUER = confi.str("OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer") + OAUTH2_JWK_CACHE_MAXSIZE = confi.int("OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize.") + OAUTH2_JWK_CACHE_TTL = confi.int("OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL.") ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fc74223ed..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession @@ -11,6 +11,7 @@ from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") @@ -52,6 +53,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[dict] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -64,6 +67,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = ClientAuthenticator() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -71,7 +77,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..6d8773a1e --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,55 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from opal_server.config import opal_server_config + +class _ServerAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will verify API requests with OAuth2 tokens.") + else: + self.__delegate = JWTAuthenticator(self.__signer()) + + def __signer(self) -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info("OPAL is running in secure mode - will verify API requests with JWT tokens.") + else: + logger.info("OPAL was not provided with JWT encryption keys, cannot verify api requests!") + return signer + + def _delegate(self) -> dict: + return self.__delegate + + def signer(self) -> Optional[JWTSigner]: + if hasattr(self._delegate(), "verifier"): + return self._delegate().verifier + else: + return None + +class ServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) + +class WebsocketServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate()(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..60836994a 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -20,8 +20,9 @@ require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +79,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/security/api.py b/packages/opal-server/opal_server/security/api.py index a17235163..2a562405a 100644 --- a/packages/opal-server/opal_server/security/api.py +++ b/packages/opal-server/opal_server/security/api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status from opal_common.authentication.deps import StaticBearerAuthenticator @@ -7,7 +8,7 @@ from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails -def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): +def init_security_router(signer: Optional[JWTSigner], authenticator: StaticBearerAuthenticator): router = APIRouter() @router.post( @@ -17,7 +18,7 @@ def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthentic dependencies=[Depends(authenticator)], ) async def generate_new_access_token(req: AccessTokenRequest): - if not signer.enabled: + if signer is None or not signer.enabled: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="opal server was not configured with security, cannot generate tokens!", diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 34d9905c3..81a4d5722 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,7 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +21,7 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import ServerAuthenticator, WebsocketServerAuthenticator from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -49,7 +49,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[ServerAuthenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +118,22 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer + if authenticator is not None: + self.authenticator = authenticator else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) - else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticator() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), jwks_url=jwks_url, jwks_static_dir=jwks_static_dir ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticator() + self.pubsub = PubSub(broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +209,17 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) + webhook_router = init_git_webhook_router(self.pubsub.endpoint, self.authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +228,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +237,22 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router(self._scopes, self.authenticator, self.pubsub.endpoint), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..37dd369cb 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -9,3 +9,4 @@ typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 From d45ba120bf6125e52d759a44792077b1e58820c1 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 40/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..10731adfd 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 917ea66312942f194142b900aa386a081782f9cf Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 11:55:08 +0200 Subject: [PATCH 41/83] Hardcode peer_type = datasource --- .../opal-common/opal_common/authentication/oauth2.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 645e8f9ea..4dd4b9cf0 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -1,8 +1,8 @@ import asyncio -import httpx -import jwt import time +from typing import Optional +import httpx from cachetools import cached, TTLCache from fastapi import Header from httpx import AsyncClient, BasicAuth @@ -10,13 +10,16 @@ from opal_common.authentication.jwk import JWKManager from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.config import opal_common_config -from typing import Optional + class _OAuth2Authenticator: async def authenticate(self, headers): if "Authorization" not in headers: token = await self.token() headers['Authorization'] = f"Bearer {token}" +# logger.info(f".....*****..... Adding headers: {headers}") +# else: +# logger.info(f".....*****..... Authorization header already exists") class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): @@ -72,6 +75,9 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) + #TODO TODO + claims["peer_type"] = "datasource" + return claims def _verify_opaque(self, token: str) -> {}: From bd0777c23d35d07873dcdcd5cc2365807bc583b9 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 28 Jul 2024 12:22:03 +0300 Subject: [PATCH 42/83] Add sync workflow for OPAL+ repository (#630) * Added sync OPAL+ workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow * Changed OPAL+ sync workflow --- .github/workflows/sync_opal_plus.yml | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/sync_opal_plus.yml diff --git a/.github/workflows/sync_opal_plus.yml b/.github/workflows/sync_opal_plus.yml new file mode 100644 index 000000000..ab92edaa6 --- /dev/null +++ b/.github/workflows/sync_opal_plus.yml @@ -0,0 +1,65 @@ +name: Sync branch to OPAL Plus + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + sync: + name: Sync branch to OPAL Plus + if: github.repository == 'permitio/opal' + runs-on: ubuntu-latest + steps: + - name: Set up Git configuration + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Get Token + id: get_workflow_token + uses: peter-murray/workflow-application-token-action@v1 + with: + application_id: ${{ secrets.APPLICATION_ID }} + application_private_key: ${{ secrets.APPLICATION_PRIVATE_KEY }} + + - name: Checkout permitio/opal repository + uses: actions/checkout@v4 + with: + repository: permitio/opal + ref: ${{ github.ref_name }} + path: opal + fetch-depth: 0 + + - name: Checkout permitio/opal-plus repository + uses: actions/checkout@v4 + with: + repository: permitio/opal-plus + path: opal-plus + token: ${{ steps.get_workflow_token.outputs.token }} + + - name: Create public-${{ github.ref_name }} branch in opal repository + working-directory: opal + run: | + git checkout -b public-${{ github.ref_name }} + + - name: Rebase opal-plus/public-${{ github.ref_name }} onto opal/${{ github.ref_name }} + working-directory: opal-plus + run: | + git remote add opal ../opal + git fetch opal + git checkout public-${{ github.ref_name }} + git rebase opal/${{ github.ref_name }} + + - name: Push changes to opal-plus/public-${{ github.ref_name }} branch + working-directory: opal-plus + run: | + git push origin public-${{ github.ref_name }} + + - name: Create Pull Request for opal-plus + working-directory: opal-plus + run: | + gh pr create --repo permitio/opal-plus --assignee "$GITHUB_ACTOR" --reviewer "$GITHUB_ACTOR" --base master --head public-${{ github.ref_name }} --title "Sync changes from public OPAL repository" --body "This PR synchronizes changes from the public OPAL repository to the private OPAL Plus repository." + env: + GITHUB_TOKEN: ${{ steps.get_workflow_token.outputs.token }} From 84beb45a82723883119b971aea86e4391d59530c Mon Sep 17 00:00:00 2001 From: Eli Moshkovich Date: Wed, 31 Jul 2024 17:17:36 -0700 Subject: [PATCH 43/83] added pypi release (#629) * added pypi release * push to master * pre commit * if condition added * wip1 * wip2 * wip3 * wip4 * wip5 * wip6 * wip7 * wip7 * check bump version * another version check * another version check with build * fixed with version push * test pypi * test pypi * Remove unwanted files before publishing * with dist at path * test legacy * final * fix if conditions and change trigger to publish * all assets within one single step * check1 * without workflow_dispatch --- .github/workflows/on_release.yml | 130 ++++++++++++++++++++++++------- 1 file changed, 101 insertions(+), 29 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 3733c0fd7..2b9461ecc 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -2,15 +2,8 @@ name: Build and publish to Docker Hub on: release: # job will automatically run after a new "release" is create on github. - types: [created] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - inputs: - dry_run: - description: 'If true, will not push the built images to docker hub.' - required: false - default: 'false' + types: [published] + #Allows you to run this workflow manually from the Actions tab jobs: # this job will build, test and (potentially) push the docker images to docker hub @@ -29,6 +22,13 @@ jobs: # - Pushes images (built at BUILD PHASE) to docker hub. docker_build_and_publish: runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.TOKEN_GITHUB }} + github_token: ${{ secrets.TOKEN_GITHUB }} + permissions: + id-token: write + contents: write # 'write' access to repository contents + pull-requests: write # 'write' access to pull requests steps: # BUILD PHASE - name: Checkout @@ -43,25 +43,14 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Get version tag from github release - if: github.event_name == 'release' && github.event.action == 'created' - run: | - echo "opal_version_tag=${{ github.event.release.tag_name }}" >> $GITHUB_ENV - - - name: Get version tag from git history - if: ${{ !(github.event_name == 'release' && github.event.action == 'created') }} - run: | - echo "opal_version_tag=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV - - name: Echo version tag run: | - echo "The version tag that will be published to docker hub is: ${{ env.opal_version_tag }}" + echo "The version tag that will be published to docker hub is: ${{ github.event.release.tag_name }}" - name: Build client for testing id: build_client @@ -114,13 +103,13 @@ jobs: with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + push: true target: client cache-from: type=registry,ref=permitio/opal-client:latest cache-to: type=inline tags: | permitio/opal-client:latest - permitio/opal-client:${{ env.opal_version_tag }} + permitio/opal-client:${{ github.event.release.tag_name }} - name: Build client-standalone id: build_push_client_standalone @@ -128,13 +117,13 @@ jobs: with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + push: true target: client-standalone cache-from: type=registry,ref=permitio/opal-client-standalone:latest cache-to: type=inline tags: | permitio/opal-client-standalone:latest - permitio/opal-client-standalone:${{ env.opal_version_tag }} + permitio/opal-client-standalone:${{ github.event.release.tag_name }} - name: Build server id: build_push_server @@ -142,13 +131,13 @@ jobs: with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + push: true target: server cache-from: type=registry,ref=permitio/opal-server:latest cache-to: type=inline tags: | permitio/opal-server:latest - permitio/opal-server:${{ env.opal_version_tag }} + permitio/opal-server:${{ github.event.release.tag_name }} - name: Build & Push client cedar id: build_push_client_cedar @@ -156,10 +145,93 @@ jobs: with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + push: true target: client-cedar cache-from: type=registry,ref=permitio/opal-client-cedar:latest cache-to: type=inline tags: | permitio/opal-client-cedar:latest - permitio/opal-client-cedar:${{ env.opal_version_tag }} + permitio/opal-client-cedar:${{ github.event.release.tag_name }} + + - name: Python setup + uses: actions/setup-python@v5 + with: + python-version: '3.11.8' + + # This is the root file representing the package for all the sub-packages. + - name: Bump version - packaging__.py + run: | + version_tag=${{ github.event.release.tag_name }} + version_tuple=$(echo $version_tag | sed 's/\./, /g') + sed -i "s/VERSION = (.*/VERSION = (${version_tuple})/" packages/__packaging__.py + cat packages/__packaging__.py + git config --local user.email "eli@permit.io" + git config --local user.name "elimoshkovich" + git add packages/__packaging__.py + git commit -m "Bump version to ${version_tag}" + + - name: Cleanup setup.py and Build every sub-packages + run: | + pip install wheel + cd packages/opal-common/ ; rm -rf *.egg-info build/ dist/ + python setup.py sdist bdist_wheel + cd ../.. + cd packages/opal-client/ ; rm -rf *.egg-info build/ dist/ + python setup.py sdist bdist_wheel + cd ../.. + cd packages/opal-server/ ; rm -rf *.egg-info build/ dist/ + python setup.py sdist bdist_wheel + cd ../.. + + # Upload package distributions to the release - All assets in one step + - name: Upload assets to release + uses: shogo82148/actions-upload-release-asset@v1.7.5 + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: | + packages/opal-common/dist/* + packages/opal-client/dist/* + packages/opal-server/dist/* + + # Publish package distributions to PyPI + - name: Publish package distributions to PyPI - Opal-Common + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + packages-dir: packages/opal-common/dist/ + # For Test only ! + # password: ${{ secrets.TEST_PYPI_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + env: + name: pypi + url: https://pypi.org/p/opal-common + + - name: Publish package distributions to PyPI - Opal-Client + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + packages-dir: packages/opal-client/dist/ + # For Test only ! + # password: ${{ secrets.TEST_PYPI_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + env: + name: pypi + url: https://pypi.org/p/opal-client + + - name: Publish package distributions to PyPI - Opal-Server + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + packages-dir: packages/opal-server/dist/ + # For Test only ! + # password: ${{ secrets.TEST_PYPI_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + env: + name: pypi + url: https://pypi.org/p/opal-server + + - name: Push changes of packages/__packaging__.py to GitHub + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.TOKEN_GITHUB }} + branch: master From 41ea4e44c124659d52182985b7290e5aea4fd5d7 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Thu, 1 Aug 2024 18:53:53 +0200 Subject: [PATCH 44/83] Add Docker Compose examples with OAuth2 token validations --- .../docker-compose-with-oauth-jwt-token.yml | 93 +++++++++++++++++++ ...docker-compose-with-oauth-opaque-token.yml | 83 +++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 docker/docker-compose-with-oauth-jwt-token.yml create mode 100644 docker/docker-compose-with-oauth-opaque-token.yml diff --git a/docker/docker-compose-with-oauth-jwt-token.yml b/docker/docker-compose-with-oauth-jwt-token.yml new file mode 100644 index 000000000..b62197241 --- /dev/null +++ b/docker/docker-compose-with-oauth-jwt-token.yml @@ -0,0 +1,93 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-oauth-opaque-token.yml b/docker/docker-compose-with-oauth-opaque-token.yml new file mode 100644 index 000000000..7641cd0e8 --- /dev/null +++ b/docker/docker-compose-with-oauth-opaque-token.yml @@ -0,0 +1,83 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" From 29139c4cfa29880dc98e360aa8fb059fc121920e Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Thu, 1 Aug 2024 19:04:35 +0200 Subject: [PATCH 45/83] Remove hardcoded peer_type claim --- packages/opal-common/opal_common/authentication/oauth2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 4dd4b9cf0..b54c45f97 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -75,9 +75,6 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) - #TODO TODO - claims["peer_type"] = "datasource" - return claims def _verify_opaque(self, token: str) -> {}: From 22f0ee037df7cc635e20ea697e903081c94fb2d3 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 12:02:08 +0200 Subject: [PATCH 46/83] Rebase feature branch onto updated master --- .../opal-client/opal_client/callbacks/api.py | 4 +- packages/opal-client/opal_client/client.py | 30 ++-- .../opal-client/opal_client/data/updater.py | 38 ++++- .../opal-client/opal_client/policy/fetcher.py | 24 ++- .../opal-client/opal_client/policy/updater.py | 26 ++- .../opal_client/policy_store/api.py | 4 +- .../authentication/authenticator.py | 50 ++++++ .../opal_common/authentication/authz.py | 6 +- .../opal_common/authentication/jwk.py | 46 +++++ .../opal_common/authentication/oauth2.py | 157 ++++++++++++++++++ packages/opal-common/opal_common/config.py | 22 +++ .../fetcher/providers/http_fetch_provider.py | 13 +- .../opal_server/authentication/__init__.py | 0 .../authentication/authenticator.py | 55 ++++++ packages/opal-server/opal_server/data/api.py | 5 +- .../opal_server/policy/webhook/api.py | 4 +- packages/opal-server/opal_server/pubsub.py | 10 +- .../opal-server/opal_server/scopes/api.py | 5 +- .../opal-server/opal_server/security/api.py | 5 +- .../opal-server/opal_server/security/jwks.py | 5 +- packages/opal-server/opal_server/server.py | 52 +++--- packages/requires.txt | 1 + 22 files changed, 468 insertions(+), 94 deletions(-) create mode 100644 packages/opal-common/opal_common/authentication/authenticator.py create mode 100644 packages/opal-common/opal_common/authentication/jwk.py create mode 100644 packages/opal-common/opal_common/authentication/oauth2.py create mode 100644 packages/opal-server/opal_server/authentication/__init__.py create mode 100644 packages/opal-server/opal_server/authentication/authenticator.py diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..9b828cced 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -29,8 +29,7 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +48,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[ClientAuthenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +63,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = ClientAuthenticator() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +122,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -140,6 +144,7 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +167,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +242,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index e288b5963..444219e34 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,6 +24,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig from opal_common.http_utils import is_http_error_response @@ -54,6 +55,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[ClientAuthenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -110,17 +112,18 @@ def __init__( self._callbacks_register, ) self._token = token + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None self._shard_id = shard_id self._server_url = pubsub_url self._data_sources_config_url = data_sources_config_url self._opal_client_id = opal_client_id - self._extra_headers = [] + self._extra_headers = {} if self._token is not None: - self._extra_headers.append(get_authorization_header(self._token)) + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] if self._shard_id is not None: - self._extra_headers.append(("X-Shard-ID", self._shard_id)) - if len(self._extra_headers) == 0: - self._extra_headers = None + self._extra_headers['X-Shard-ID'] = self._shard_id self._stopping = False # custom SSL context (for self-signed certificates) self._custom_ssl_context = get_custom_ssl_context() @@ -132,6 +135,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() async def __aenter__(self): await self.start() @@ -177,8 +184,14 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + try: - async with ClientSession(headers=self._extra_headers) as session: + async with ClientSession(headers=headers) as session: response = await session.get(url, **self._ssl_context_kwargs) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) @@ -274,12 +287,19 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( - self._data_topics, - self._update_policy_data_callback, + topics=self._data_topics, + callback=self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + on_disconnect=[self.on_disconnect], + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5ae9d93b6 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +29,26 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[ClientAuthenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + self._auth_headers = {} + if self._token != "THIS_IS_A_DEV_SECRET": + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +94,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, @@ -95,7 +112,6 @@ async def _fetch_policy_bundle( self._policy_endpoint_url, headers={ "content-type": "text/plain", - **self._auth_headers, }, params=params, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..d505c52f5 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +44,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[ClientAuthenticator] = None, ): """inits the policy updater. @@ -64,15 +66,21 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data self._server_url = pubsub_url self._token = token - if self._token is None: - self._extra_headers = None - else: - self._extra_headers = [get_authorization_header(self._token)] + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None + self._extra_headers = {} + if self._token is not None: + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] # Pub/Sub topics we subscribe to for policy updates if self._scope_id == "default": self._topics = pubsub_topics_from_directories( @@ -87,7 +95,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +248,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index b27d83d70..97113f109 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,15 +1,15 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..938ac001f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,50 @@ +from abc import abstractmethod +from typing import Optional + +from fastapi import Header +from opal_common.config import opal_common_config +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + +class Authenticator: + @property + def enabled(self): + return self._delegate().enabled + + async def authenticate(self, headers): + if hasattr(self._delegate(), "authenticate") and callable(getattr(self._delegate(), "authenticate")): + await self._delegate().authenticate(headers) + + @abstractmethod + def _delegate(self) -> dict: + pass + +class _ClientAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will authenticate API requests.") + else: + self.__delegate = JWTAuthenticator(self.__verifier()) + + def __verifier(self) -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info("API authentication disabled (public encryption key was not provided)") + + return verifier + + def _delegate(self) -> dict: + return self.__delegate + +class ClientAuthenticator(_ClientAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..9b0ec207f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,46 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + #TODO TODO: maxsize, ttl + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..645e8f9ea --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,157 @@ +import asyncio +import httpx +import jwt +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator: + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return True + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index ab18dd0cb..64c0ea093 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -159,6 +159,28 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str("AUTH_TYPE", None, description="Authentication type.") + OAUTH2_CLIENT_ID = confi.str("OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID.") + OAUTH2_CLIENT_SECRET = confi.str("OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret.") + OAUTH2_TOKEN_URL = confi.str("OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL.") + OAUTH2_INTROSPECT_URL = confi.str("OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL.") + OAUTH2_OPENID_CONFIGURATION_URL = confi.str("OAUTH2_OPENID_CONFIGURATION_URL", None, description="OAuth2 OpenID configuration URL.") + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", 100, description="OAuth2 token validation cache maxsize.") + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_TTL", 5 * 60, description="OAuth2 token validation cache TTL.") + + OAUTH2_EXP_MARGIN = confi.int("OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin.") + OAUTH2_EXACT_MATCH_CLAIMS = confi.str("OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims.") + OAUTH2_REQUIRED_CLAIMS = confi.str("OAUTH2_REQUIRED_CLAIMS", None, description="Comma separated list of required claims.") + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str("OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience") + OAUTH2_JWT_ISSUER = confi.str("OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer") + OAUTH2_JWK_CACHE_MAXSIZE = confi.int("OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize.") + OAUTH2_JWK_CACHE_TTL = confi.int("OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL.") ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fc74223ed..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession @@ -11,6 +11,7 @@ from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") @@ -52,6 +53,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[dict] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -64,6 +67,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = ClientAuthenticator() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -71,7 +77,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..6d8773a1e --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,55 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from opal_server.config import opal_server_config + +class _ServerAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will verify API requests with OAuth2 tokens.") + else: + self.__delegate = JWTAuthenticator(self.__signer()) + + def __signer(self) -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info("OPAL is running in secure mode - will verify API requests with JWT tokens.") + else: + logger.info("OPAL was not provided with JWT encryption keys, cannot verify api requests!") + return signer + + def _delegate(self) -> dict: + return self.__delegate + + def signer(self) -> Optional[JWTSigner]: + if hasattr(self._delegate(), "verifier"): + return self._delegate().verifier + else: + return None + +class ServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) + +class WebsocketServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate()(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..60836994a 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -20,8 +20,9 @@ require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +79,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/security/api.py b/packages/opal-server/opal_server/security/api.py index a17235163..2a562405a 100644 --- a/packages/opal-server/opal_server/security/api.py +++ b/packages/opal-server/opal_server/security/api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status from opal_common.authentication.deps import StaticBearerAuthenticator @@ -7,7 +8,7 @@ from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails -def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): +def init_security_router(signer: Optional[JWTSigner], authenticator: StaticBearerAuthenticator): router = APIRouter() @router.post( @@ -17,7 +18,7 @@ def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthentic dependencies=[Depends(authenticator)], ) async def generate_new_access_token(req: AccessTokenRequest): - if not signer.enabled: + if signer is None or not signer.enabled: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="opal server was not configured with security, cannot generate tokens!", diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 34d9905c3..81a4d5722 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,7 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +21,7 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import ServerAuthenticator, WebsocketServerAuthenticator from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -49,7 +49,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[ServerAuthenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +118,22 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer + if authenticator is not None: + self.authenticator = authenticator else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) - else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticator() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), jwks_url=jwks_url, jwks_static_dir=jwks_static_dir ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticator() + self.pubsub = PubSub(broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +209,17 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) + webhook_router = init_git_webhook_router(self.pubsub.endpoint, self.authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +228,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +237,22 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router(self._scopes, self.authenticator, self.pubsub.endpoint), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..37dd369cb 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -9,3 +9,4 @@ typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 From 2d958c9481c0decc3001a2d815f1010205407dd0 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 47/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..10731adfd 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From db9dae0055dd1f60284dd90573d71ec2e45e79e2 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 11:55:08 +0200 Subject: [PATCH 48/83] Hardcode peer_type = datasource --- .../opal-common/opal_common/authentication/oauth2.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 645e8f9ea..4dd4b9cf0 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -1,8 +1,8 @@ import asyncio -import httpx -import jwt import time +from typing import Optional +import httpx from cachetools import cached, TTLCache from fastapi import Header from httpx import AsyncClient, BasicAuth @@ -10,13 +10,16 @@ from opal_common.authentication.jwk import JWKManager from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.config import opal_common_config -from typing import Optional + class _OAuth2Authenticator: async def authenticate(self, headers): if "Authorization" not in headers: token = await self.token() headers['Authorization'] = f"Bearer {token}" +# logger.info(f".....*****..... Adding headers: {headers}") +# else: +# logger.info(f".....*****..... Authorization header already exists") class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): @@ -72,6 +75,9 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) + #TODO TODO + claims["peer_type"] = "datasource" + return claims def _verify_opaque(self, token: str) -> {}: From 630d28c8cd631ff1dce4d08c39afbf592d0ddf16 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:38:55 +0200 Subject: [PATCH 49/83] Rebase to master --- .../opal-common/opal_common/authentication/jwk.py | 1 - .../opal-common/opal_common/authentication/oauth2.py | 11 ++--------- .../fetcher/providers/http_fetch_provider.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py index 9b0ec207f..182b5cdb9 100644 --- a/packages/opal-common/opal_common/authentication/jwk.py +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -5,7 +5,6 @@ from opal_common.authentication.verifier import Unauthorized class JWKManager: - #TODO TODO: maxsize, ttl def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): self._openid_configuration_url = openid_configuration_url self._jwt_algorithm = jwt_algorithm diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 4dd4b9cf0..aad738b66 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -1,8 +1,7 @@ import asyncio +import httpx import time -from typing import Optional -import httpx from cachetools import cached, TTLCache from fastapi import Header from httpx import AsyncClient, BasicAuth @@ -10,16 +9,13 @@ from opal_common.authentication.jwk import JWKManager from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.config import opal_common_config - +from typing import Optional class _OAuth2Authenticator: async def authenticate(self, headers): if "Authorization" not in headers: token = await self.token() headers['Authorization'] = f"Bearer {token}" -# logger.info(f".....*****..... Adding headers: {headers}") -# else: -# logger.info(f".....*****..... Authorization header already exists") class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): @@ -75,9 +71,6 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) - #TODO TODO - claims["peer_type"] = "datasource" - return claims def _verify_opaque(self, token: str) -> {}: diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 10731adfd..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 707c7661f41734a6b69fd0676dbdecfd2591a0fc Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 50/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..8d2c0edb9 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -14,6 +14,12 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator +from ...http import is_http_error_response +from ...security.sslcontext import get_custom_ssl_context +from ..events import FetcherConfig, FetchEvent +from ..fetch_provider import BaseFetchProvider +from ..logger import get_logger + logger = get_logger("http_fetch_provider") From b5c8451578a6b2c65244b501a35c447d5786e3ff Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Thu, 1 Aug 2024 18:53:53 +0200 Subject: [PATCH 51/83] Add Docker Compose examples with OAuth2 token validations --- .../docker-compose-with-oauth-jwt-token.yml | 93 +++++++++++++++++++ ...docker-compose-with-oauth-opaque-token.yml | 83 +++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 docker/docker-compose-with-oauth-jwt-token.yml create mode 100644 docker/docker-compose-with-oauth-opaque-token.yml diff --git a/docker/docker-compose-with-oauth-jwt-token.yml b/docker/docker-compose-with-oauth-jwt-token.yml new file mode 100644 index 000000000..b62197241 --- /dev/null +++ b/docker/docker-compose-with-oauth-jwt-token.yml @@ -0,0 +1,93 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-oauth-opaque-token.yml b/docker/docker-compose-with-oauth-opaque-token.yml new file mode 100644 index 000000000..7641cd0e8 --- /dev/null +++ b/docker/docker-compose-with-oauth-opaque-token.yml @@ -0,0 +1,83 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" From 28be0973c0782d87f629bffb213cb0ccc28aa205 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:46:26 +0200 Subject: [PATCH 52/83] Remove unused imports --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 8d2c0edb9..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -14,12 +14,6 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger - logger = get_logger("http_fetch_provider") From fb0419ebf5d955b5f0664329805ca2157e182f85 Mon Sep 17 00:00:00 2001 From: eli Date: Tue, 6 Aug 2024 11:38:35 -0700 Subject: [PATCH 53/83] docker-compose install and delete duplicate github token --- .github/workflows/on_release.yml | 6 +++++- .github/workflows/tests.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 2b9461ecc..702a310b9 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -23,7 +23,6 @@ jobs: docker_build_and_publish: runs-on: ubuntu-latest env: - GITHUB_TOKEN: ${{ secrets.TOKEN_GITHUB }} github_token: ${{ secrets.TOKEN_GITHUB }} permissions: id-token: write @@ -47,6 +46,11 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker Compose install + run: | + curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose - name: Echo version tag run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d708b16e0..8638cba36 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,6 +66,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - name: Docker Compose install + run: | + curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + - name: Build client id: build_client uses: docker/build-push-action@v2 From f1a4e84f9a6b294550874ef91882b98e551dfc62 Mon Sep 17 00:00:00 2001 From: Ro'e Katz Date: Wed, 7 Aug 2024 11:12:19 +0300 Subject: [PATCH 54/83] Fix pre-commit trailing whitespace --- .github/workflows/on_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 702a310b9..9cfb2b6ca 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -46,7 +46,7 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - + - name: Docker Compose install run: | curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose From 0d727c35694c0ca96bb339c654311251a65a03c7 Mon Sep 17 00:00:00 2001 From: Oded Date: Tue, 6 Aug 2024 20:28:05 +0300 Subject: [PATCH 55/83] update docs packages to fix snyk alerts --- documentation/package-lock.json | 42 +++++++++++++++++---------------- documentation/package.json | 4 +++- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/documentation/package-lock.json b/documentation/package-lock.json index 62d3cbb7a..262b5fe96 100644 --- a/documentation/package-lock.json +++ b/documentation/package-lock.json @@ -11,8 +11,10 @@ "@docusaurus/core": "3.0.0", "@docusaurus/preset-classic": "3.0.0", "@mdx-js/react": "^3.0.0", + "axios": "^1.6.4", "clsx": "^1.2.1", "docusaurus-plugin-sass": "^0.2.5", + "micromatch": "^4.0.6", "node": "^18.0.0", "prism-react-renderer": "^2.1.0", "react": "^18.0.0", @@ -4126,9 +4128,9 @@ } }, "node_modules/axios": { - "version": "1.6.8", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", - "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.3.tgz", + "integrity": "sha512-Ar7ND9pU99eJ9GpoGQKhKf58GpUOgnzuaB7ueNQ5BMi0p+LZ5oaEnfF999fAArcTIBwXTCHAmGcHOZJaWPq9Nw==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -4335,11 +4337,11 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -6394,9 +6396,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -10380,11 +10382,11 @@ ] }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", + "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -14776,9 +14778,9 @@ } }, "node_modules/webpack-dev-server/node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, @@ -15029,9 +15031,9 @@ } }, "node_modules/ws": { - "version": "7.5.9", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", - "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", "engines": { "node": ">=8.3.0" }, diff --git a/documentation/package.json b/documentation/package.json index 32320080d..89ac97334 100644 --- a/documentation/package.json +++ b/documentation/package.json @@ -23,7 +23,9 @@ "prism-react-renderer": "^2.1.0", "react": "^18.0.0", "react-dom": "^18.0.0", - "sass": "^1.71.1" + "sass": "^1.71.1", + "axios": "^1.6.4", + "micromatch": "^4.0.6" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.0.0" From d7b047ddeda0fb2d42729731bc6b4bc2ba84641d Mon Sep 17 00:00:00 2001 From: venu gopal Date: Wed, 7 Aug 2024 15:48:15 +0530 Subject: [PATCH 56/83] typo fixed in set_url_query_param function docs --- packages/opal-common/opal_common/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/opal-common/opal_common/urls.py b/packages/opal-common/opal_common/urls.py index ba68da20d..404877f7b 100644 --- a/packages/opal-common/opal_common/urls.py +++ b/packages/opal-common/opal_common/urls.py @@ -8,8 +8,8 @@ def set_url_query_param(url: str, param_name: str, param_value: str): >> set_url_query_param('https://api.permit.io/opal/data/config', 'token', 'secret') 'https://api.permit.io/opal/data/config?token=secret' - >> set_url_query_param('https://api.permit.io/opal/data/config&some=var', 'token', 'secret') - 'https://api.permit.io/opal/data/config&some=var?token=secret' + >> set_url_query_param('https://api.permit.io/opal/data/config?some=var', 'token', 'secret') + 'https://api.permit.io/opal/data/config?some=var&token=secret' """ parsed_url: ParseResult = urlparse(url) From 607ed1390846a32fd7adc19c68d1c839ac8e50e3 Mon Sep 17 00:00:00 2001 From: Ro'e Katz Date: Wed, 7 Aug 2024 14:22:46 +0300 Subject: [PATCH 57/83] Dockerfile: Mitigate git vulnerability by disabling symlink support --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 74ac1916b..35f1144a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -143,6 +143,7 @@ USER opal FROM common AS server RUN apt-get update && apt-get install -y openssh-client git && apt-get clean +RUN git config --global core.symlinks false # Mitigate CVE-2024-32002 USER opal From 0d7a39e901c9e08af23fec3be340013f51571c84 Mon Sep 17 00:00:00 2001 From: Eli Moshkovich Date: Wed, 7 Aug 2024 08:49:55 -0700 Subject: [PATCH 58/83] Per-9644 cicd fix pypi url (#639) * 0.7.11 * 0.7.11 wip1 * wip2 * wip3 * version as a placeholder * pre commit fix * wip2 --- .github/workflows/on_release.yml | 18 ++++-------------- packages/__packaging__.py | 4 ++-- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 9cfb2b6ca..eae6d83ac 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -3,7 +3,6 @@ on: release: # job will automatically run after a new "release" is create on github. types: [published] - #Allows you to run this workflow manually from the Actions tab jobs: # this job will build, test and (potentially) push the docker images to docker hub @@ -166,13 +165,10 @@ jobs: - name: Bump version - packaging__.py run: | version_tag=${{ github.event.release.tag_name }} + version_tag=${version_tag#v} # Remove the leading 'v' version_tuple=$(echo $version_tag | sed 's/\./, /g') sed -i "s/VERSION = (.*/VERSION = (${version_tuple})/" packages/__packaging__.py cat packages/__packaging__.py - git config --local user.email "eli@permit.io" - git config --local user.name "elimoshkovich" - git add packages/__packaging__.py - git commit -m "Bump version to ${version_tag}" - name: Cleanup setup.py and Build every sub-packages run: | @@ -208,7 +204,7 @@ jobs: # repository-url: https://test.pypi.org/legacy/ env: name: pypi - url: https://pypi.org/p/opal-common + url: https://pypi.org/p/opal-common/ - name: Publish package distributions to PyPI - Opal-Client uses: pypa/gh-action-pypi-publish@release/v1 @@ -220,7 +216,7 @@ jobs: # repository-url: https://test.pypi.org/legacy/ env: name: pypi - url: https://pypi.org/p/opal-client + url: https://pypi.org/p/opal-client/ - name: Publish package distributions to PyPI - Opal-Server uses: pypa/gh-action-pypi-publish@release/v1 @@ -232,10 +228,4 @@ jobs: # repository-url: https://test.pypi.org/legacy/ env: name: pypi - url: https://pypi.org/p/opal-server - - - name: Push changes of packages/__packaging__.py to GitHub - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.TOKEN_GITHUB }} - branch: master + url: https://pypi.org/p/opal-server/ diff --git a/packages/__packaging__.py b/packages/__packaging__.py index 452772e5a..00901a5ac 100644 --- a/packages/__packaging__.py +++ b/packages/__packaging__.py @@ -8,8 +8,8 @@ """ import os - -VERSION = (0, 7, 10) +# VERSION is a placeholder, the real version is set by the release CI/CD pipeline +VERSION = (0, 0, 0) VERSION_STRING = ".".join(map(str, VERSION)) __version__ = VERSION_STRING From cb7eff8d5e324f0d99f7a30cdc7fe3b1d3a6431f Mon Sep 17 00:00:00 2001 From: roekatz Date: Wed, 7 Aug 2024 19:23:22 +0300 Subject: [PATCH 59/83] Fix __packaging__.py format for pre-commit (#640) --- packages/__packaging__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/__packaging__.py b/packages/__packaging__.py index 00901a5ac..d56cb9982 100644 --- a/packages/__packaging__.py +++ b/packages/__packaging__.py @@ -8,8 +8,8 @@ """ import os -# VERSION is a placeholder, the real version is set by the release CI/CD pipeline -VERSION = (0, 0, 0) + +VERSION = (0, 0, 0) # Placeholder, to be set by CI/CD VERSION_STRING = ".".join(map(str, VERSION)) __version__ = VERSION_STRING From 2223e18a26b61b7ca3524c95774f919a8ce417a3 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 12:02:08 +0200 Subject: [PATCH 60/83] Rebase feature branch onto updated master # Conflicts: # packages/requires.txt --- .../opal-client/opal_client/callbacks/api.py | 4 +- packages/opal-client/opal_client/client.py | 30 ++-- .../opal-client/opal_client/data/updater.py | 38 ++++- .../opal-client/opal_client/policy/fetcher.py | 24 ++- .../opal-client/opal_client/policy/updater.py | 26 ++- .../opal_client/policy_store/api.py | 4 +- .../authentication/authenticator.py | 50 ++++++ .../opal_common/authentication/authz.py | 6 +- .../opal_common/authentication/jwk.py | 46 +++++ .../opal_common/authentication/oauth2.py | 157 ++++++++++++++++++ packages/opal-common/opal_common/config.py | 22 +++ .../fetcher/providers/http_fetch_provider.py | 13 +- .../opal_server/authentication/__init__.py | 0 .../authentication/authenticator.py | 55 ++++++ packages/opal-server/opal_server/data/api.py | 5 +- .../opal_server/policy/webhook/api.py | 4 +- packages/opal-server/opal_server/pubsub.py | 10 +- .../opal-server/opal_server/scopes/api.py | 5 +- .../opal-server/opal_server/security/api.py | 5 +- .../opal-server/opal_server/security/jwks.py | 5 +- packages/opal-server/opal_server/server.py | 52 +++--- packages/requires.txt | 1 + 22 files changed, 468 insertions(+), 94 deletions(-) create mode 100644 packages/opal-common/opal_common/authentication/authenticator.py create mode 100644 packages/opal-common/opal_common/authentication/jwk.py create mode 100644 packages/opal-common/opal_common/authentication/oauth2.py create mode 100644 packages/opal-server/opal_server/authentication/__init__.py create mode 100644 packages/opal-server/opal_server/authentication/authenticator.py diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..9b828cced 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -29,8 +29,7 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +48,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[ClientAuthenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +63,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = ClientAuthenticator() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +122,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -140,6 +144,7 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +167,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +242,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index e288b5963..444219e34 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,6 +24,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig from opal_common.http_utils import is_http_error_response @@ -54,6 +55,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[ClientAuthenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -110,17 +112,18 @@ def __init__( self._callbacks_register, ) self._token = token + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None self._shard_id = shard_id self._server_url = pubsub_url self._data_sources_config_url = data_sources_config_url self._opal_client_id = opal_client_id - self._extra_headers = [] + self._extra_headers = {} if self._token is not None: - self._extra_headers.append(get_authorization_header(self._token)) + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] if self._shard_id is not None: - self._extra_headers.append(("X-Shard-ID", self._shard_id)) - if len(self._extra_headers) == 0: - self._extra_headers = None + self._extra_headers['X-Shard-ID'] = self._shard_id self._stopping = False # custom SSL context (for self-signed certificates) self._custom_ssl_context = get_custom_ssl_context() @@ -132,6 +135,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() async def __aenter__(self): await self.start() @@ -177,8 +184,14 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + try: - async with ClientSession(headers=self._extra_headers) as session: + async with ClientSession(headers=headers) as session: response = await session.get(url, **self._ssl_context_kwargs) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) @@ -274,12 +287,19 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( - self._data_topics, - self._update_policy_data_callback, + topics=self._data_topics, + callback=self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + on_disconnect=[self.on_disconnect], + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5ae9d93b6 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +29,26 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[ClientAuthenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + self._auth_headers = {} + if self._token != "THIS_IS_A_DEV_SECRET": + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +94,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, @@ -95,7 +112,6 @@ async def _fetch_policy_bundle( self._policy_endpoint_url, headers={ "content-type": "text/plain", - **self._auth_headers, }, params=params, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..d505c52f5 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +44,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[ClientAuthenticator] = None, ): """inits the policy updater. @@ -64,15 +66,21 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data self._server_url = pubsub_url self._token = token - if self._token is None: - self._extra_headers = None - else: - self._extra_headers = [get_authorization_header(self._token)] + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None + self._extra_headers = {} + if self._token is not None: + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] # Pub/Sub topics we subscribe to for policy updates if self._scope_id == "default": self._topics = pubsub_topics_from_directories( @@ -87,7 +95,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +248,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index b27d83d70..97113f109 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,15 +1,15 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..938ac001f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,50 @@ +from abc import abstractmethod +from typing import Optional + +from fastapi import Header +from opal_common.config import opal_common_config +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + +class Authenticator: + @property + def enabled(self): + return self._delegate().enabled + + async def authenticate(self, headers): + if hasattr(self._delegate(), "authenticate") and callable(getattr(self._delegate(), "authenticate")): + await self._delegate().authenticate(headers) + + @abstractmethod + def _delegate(self) -> dict: + pass + +class _ClientAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will authenticate API requests.") + else: + self.__delegate = JWTAuthenticator(self.__verifier()) + + def __verifier(self) -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info("API authentication disabled (public encryption key was not provided)") + + return verifier + + def _delegate(self) -> dict: + return self.__delegate + +class ClientAuthenticator(_ClientAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..9b0ec207f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,46 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + #TODO TODO: maxsize, ttl + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..645e8f9ea --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,157 @@ +import asyncio +import httpx +import jwt +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator: + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return True + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index ab18dd0cb..64c0ea093 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -159,6 +159,28 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str("AUTH_TYPE", None, description="Authentication type.") + OAUTH2_CLIENT_ID = confi.str("OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID.") + OAUTH2_CLIENT_SECRET = confi.str("OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret.") + OAUTH2_TOKEN_URL = confi.str("OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL.") + OAUTH2_INTROSPECT_URL = confi.str("OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL.") + OAUTH2_OPENID_CONFIGURATION_URL = confi.str("OAUTH2_OPENID_CONFIGURATION_URL", None, description="OAuth2 OpenID configuration URL.") + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", 100, description="OAuth2 token validation cache maxsize.") + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_TTL", 5 * 60, description="OAuth2 token validation cache TTL.") + + OAUTH2_EXP_MARGIN = confi.int("OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin.") + OAUTH2_EXACT_MATCH_CLAIMS = confi.str("OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims.") + OAUTH2_REQUIRED_CLAIMS = confi.str("OAUTH2_REQUIRED_CLAIMS", None, description="Comma separated list of required claims.") + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str("OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience") + OAUTH2_JWT_ISSUER = confi.str("OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer") + OAUTH2_JWK_CACHE_MAXSIZE = confi.int("OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize.") + OAUTH2_JWK_CACHE_TTL = confi.int("OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL.") ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fc74223ed..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession @@ -11,6 +11,7 @@ from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") @@ -52,6 +53,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[dict] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -64,6 +67,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = ClientAuthenticator() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -71,7 +77,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..6d8773a1e --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,55 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from opal_server.config import opal_server_config + +class _ServerAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will verify API requests with OAuth2 tokens.") + else: + self.__delegate = JWTAuthenticator(self.__signer()) + + def __signer(self) -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info("OPAL is running in secure mode - will verify API requests with JWT tokens.") + else: + logger.info("OPAL was not provided with JWT encryption keys, cannot verify api requests!") + return signer + + def _delegate(self) -> dict: + return self.__delegate + + def signer(self) -> Optional[JWTSigner]: + if hasattr(self._delegate(), "verifier"): + return self._delegate().verifier + else: + return None + +class ServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) + +class WebsocketServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate()(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..60836994a 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -20,8 +20,9 @@ require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +79,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/security/api.py b/packages/opal-server/opal_server/security/api.py index a17235163..2a562405a 100644 --- a/packages/opal-server/opal_server/security/api.py +++ b/packages/opal-server/opal_server/security/api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status from opal_common.authentication.deps import StaticBearerAuthenticator @@ -7,7 +8,7 @@ from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails -def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): +def init_security_router(signer: Optional[JWTSigner], authenticator: StaticBearerAuthenticator): router = APIRouter() @router.post( @@ -17,7 +18,7 @@ def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthentic dependencies=[Depends(authenticator)], ) async def generate_new_access_token(req: AccessTokenRequest): - if not signer.enabled: + if signer is None or not signer.enabled: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="opal server was not configured with security, cannot generate tokens!", diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 34d9905c3..81a4d5722 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,7 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +21,7 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import ServerAuthenticator, WebsocketServerAuthenticator from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -49,7 +49,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[ServerAuthenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +118,22 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer + if authenticator is not None: + self.authenticator = authenticator else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) - else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticator() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), jwks_url=jwks_url, jwks_static_dir=jwks_static_dir ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticator() + self.pubsub = PubSub(broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +209,17 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) + webhook_router = init_git_webhook_router(self.pubsub.endpoint, self.authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +228,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +237,22 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router(self._scopes, self.authenticator, self.pubsub.endpoint), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/requires.txt b/packages/requires.txt index 43f5a740d..c621d6ab0 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -9,3 +9,4 @@ typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 From 65c83029335af7dde77a2375fb65bdd8cff3ff9f Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 61/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..10731adfd 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From f59f7093f3ce4b3b9f09b86954dfa1a508cdcd51 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 11:55:08 +0200 Subject: [PATCH 62/83] Hardcode peer_type = datasource --- .../opal-common/opal_common/authentication/oauth2.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 645e8f9ea..4dd4b9cf0 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -1,8 +1,8 @@ import asyncio -import httpx -import jwt import time +from typing import Optional +import httpx from cachetools import cached, TTLCache from fastapi import Header from httpx import AsyncClient, BasicAuth @@ -10,13 +10,16 @@ from opal_common.authentication.jwk import JWKManager from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.config import opal_common_config -from typing import Optional + class _OAuth2Authenticator: async def authenticate(self, headers): if "Authorization" not in headers: token = await self.token() headers['Authorization'] = f"Bearer {token}" +# logger.info(f".....*****..... Adding headers: {headers}") +# else: +# logger.info(f".....*****..... Authorization header already exists") class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): @@ -72,6 +75,9 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) + #TODO TODO + claims["peer_type"] = "datasource" + return claims def _verify_opaque(self, token: str) -> {}: From 1b46d59e986bcd85374422d9bb3b4cc203a0658d Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:38:55 +0200 Subject: [PATCH 63/83] Rebase to master --- .../opal-common/opal_common/authentication/jwk.py | 1 - .../opal-common/opal_common/authentication/oauth2.py | 11 ++--------- .../fetcher/providers/http_fetch_provider.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py index 9b0ec207f..182b5cdb9 100644 --- a/packages/opal-common/opal_common/authentication/jwk.py +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -5,7 +5,6 @@ from opal_common.authentication.verifier import Unauthorized class JWKManager: - #TODO TODO: maxsize, ttl def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): self._openid_configuration_url = openid_configuration_url self._jwt_algorithm = jwt_algorithm diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py index 4dd4b9cf0..aad738b66 100644 --- a/packages/opal-common/opal_common/authentication/oauth2.py +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -1,8 +1,7 @@ import asyncio +import httpx import time -from typing import Optional -import httpx from cachetools import cached, TTLCache from fastapi import Header from httpx import AsyncClient, BasicAuth @@ -10,16 +9,13 @@ from opal_common.authentication.jwk import JWKManager from opal_common.authentication.verifier import JWTVerifier, Unauthorized from opal_common.config import opal_common_config - +from typing import Optional class _OAuth2Authenticator: async def authenticate(self, headers): if "Authorization" not in headers: token = await self.token() headers['Authorization'] = f"Bearer {token}" -# logger.info(f".....*****..... Adding headers: {headers}") -# else: -# logger.info(f".....*****..... Authorization header already exists") class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): @@ -75,9 +71,6 @@ def verify(self, token: str) -> {}: self._verify_exact_match_claims(claims) self._verify_required_claims(claims) - #TODO TODO - claims["peer_type"] = "datasource" - return claims def _verify_opaque(self, token: str) -> {}: diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 10731adfd..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From e36a67e47ab556cdd1eab5cab0155c1eecb7b1a2 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 64/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..8d2c0edb9 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -14,6 +14,12 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator +from ...http import is_http_error_response +from ...security.sslcontext import get_custom_ssl_context +from ..events import FetcherConfig, FetchEvent +from ..fetch_provider import BaseFetchProvider +from ..logger import get_logger + logger = get_logger("http_fetch_provider") From c003453a272c5dd557f2e78b90e4f006f0943b6d Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Thu, 1 Aug 2024 18:53:53 +0200 Subject: [PATCH 65/83] Add Docker Compose examples with OAuth2 token validations --- .../docker-compose-with-oauth-jwt-token.yml | 93 +++++++++++++++++++ ...docker-compose-with-oauth-opaque-token.yml | 83 +++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 docker/docker-compose-with-oauth-jwt-token.yml create mode 100644 docker/docker-compose-with-oauth-opaque-token.yml diff --git a/docker/docker-compose-with-oauth-jwt-token.yml b/docker/docker-compose-with-oauth-jwt-token.yml new file mode 100644 index 000000000..b62197241 --- /dev/null +++ b/docker/docker-compose-with-oauth-jwt-token.yml @@ -0,0 +1,93 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-oauth-opaque-token.yml b/docker/docker-compose-with-oauth-opaque-token.yml new file mode 100644 index 000000000..7641cd0e8 --- /dev/null +++ b/docker/docker-compose-with-oauth-opaque-token.yml @@ -0,0 +1,83 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" From 10ef61b098c8679071d6d1d53f6dba274102a77e Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:46:26 +0200 Subject: [PATCH 66/83] Remove unused imports --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 8d2c0edb9..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -14,12 +14,6 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger - logger = get_logger("http_fetch_provider") From 0ed5ae29687ffab380dc15c2fb5cc37174910444 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 67/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..10731adfd 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 162bdcb4c30a627a286da8c53367140afd823f0a Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 68/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 10731adfd..7e075b691 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,17 +1,17 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context -from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") From 3553535f2d453562053a7257d14cd4edfe6f2e14 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 12:02:08 +0200 Subject: [PATCH 69/83] Rebase feature branch onto updated master # Conflicts: # packages/opal-common/opal_common/authentication/jwk.py # packages/opal-common/opal_common/authentication/oauth2.py # packages/requires.txt --- .../opal_common/fetcher/providers/http_fetch_provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7e075b691..fbabfdf39 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -12,6 +12,7 @@ from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") From 7b42622152a6c6c3557c8f036956cd7b17a2b7c7 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 70/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fbabfdf39..de89ed66a 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 1d1b4a265b768b860af8faa8ad958b0082aa8095 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:38:55 +0200 Subject: [PATCH 71/83] Rebase to master --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index de89ed66a..fbabfdf39 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 16e5fde9a5c127b1654888efa4b8a311a9933c64 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 72/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fbabfdf39..19ee3324b 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -15,6 +15,12 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator +from ...http import is_http_error_response +from ...security.sslcontext import get_custom_ssl_context +from ..events import FetcherConfig, FetchEvent +from ..fetch_provider import BaseFetchProvider +from ..logger import get_logger + logger = get_logger("http_fetch_provider") From 7100f972de55d0c8dce140b26086d374565f3bc4 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:46:26 +0200 Subject: [PATCH 73/83] Remove unused imports --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 19ee3324b..fbabfdf39 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -15,12 +15,6 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger - logger = get_logger("http_fetch_provider") From 806cd5b930d5ce0642e0fe654a393bfc3db8597e Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 74/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index fbabfdf39..de89ed66a 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 47421cb938098368a8d3563b11ccb839ea45b0cf Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 75/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index de89ed66a..7e075b691 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession @@ -12,7 +12,6 @@ from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context -from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") From 3621a9d2d787c72ff9c9acab8d7a682bb78d2a16 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 76/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7e075b691..b195f9c3f 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -6,7 +6,6 @@ import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config -from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger From 016df4b0dbf01532469292d6d902b98853432fe1 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Fri, 26 Jul 2024 12:02:08 +0200 Subject: [PATCH 77/83] Rebase feature branch onto updated master # Conflicts: # packages/opal-common/opal_common/authentication/jwk.py # packages/opal-common/opal_common/authentication/oauth2.py # packages/requires.txt --- .../opal_common/fetcher/providers/http_fetch_provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index b195f9c3f..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -11,6 +11,7 @@ from opal_common.fetcher.logger import get_logger from opal_common.http_utils import is_http_error_response from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator logger = get_logger("http_fetch_provider") From 091c6170a5772a058c2e9fc1282e094574ae66d6 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 78/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..10731adfd 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 3dc7396b084da6c85bf57a7e3b1159140245c367 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:38:55 +0200 Subject: [PATCH 79/83] Rebase to master --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 10731adfd..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 608582fd40dcb8d0a1170e04646daeff7fe27bd8 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 80/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..8d2c0edb9 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -14,6 +14,12 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator +from ...http import is_http_error_response +from ...security.sslcontext import get_custom_ssl_context +from ..events import FetcherConfig, FetchEvent +from ..fetch_provider import BaseFetchProvider +from ..logger import get_logger + logger = get_logger("http_fetch_provider") From 00d8beedfe2e6f782cd367c645ce21151cfb3da1 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 6 Aug 2024 08:46:26 +0200 Subject: [PATCH 81/83] Remove unused imports --- .../opal_common/fetcher/providers/http_fetch_provider.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 8d2c0edb9..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -14,12 +14,6 @@ from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger - logger = get_logger("http_fetch_provider") From 41427f58bc1267070985f69641cf824a1b76fb57 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 82/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index cb493cf62..10731adfd 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Optional, Union, cast +from typing import Any, Union, cast import httpx from aiohttp import ClientResponse, ClientSession From 1451ff189318e0ff42c6e8271f827d4893b2c015 Mon Sep 17 00:00:00 2001 From: Ondrej Scecina Date: Tue, 25 Jun 2024 13:33:52 +0200 Subject: [PATCH 83/83] Enable OAuth2 authentication. --- .../opal_common/fetcher/providers/http_fetch_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 10731adfd..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,7 +1,7 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession