From 8b691d9d86f7f05973ebf75d1159f6c9819ba904 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:24:28 +0300 Subject: [PATCH 01/10] Changed relative imports to absolute --- packages/opal-client/opal_client/__init__.py | 2 +- packages/opal-client/opal_client/main.py | 2 +- .../policy_store/mock_policy_store_client.py | 2 +- .../opal_common/authentication/casting.py | 1 - packages/opal-common/opal_common/cli/typer_app.py | 2 +- packages/opal-common/opal_common/confi/__init__.py | 2 +- packages/opal-common/opal_common/confi/cli.py | 2 +- packages/opal-common/opal_common/confi/confi.py | 4 ++-- packages/opal-common/opal_common/config.py | 5 ++--- .../opal-common/opal_common/engine/__init__.py | 4 ++-- .../opal-common/opal_common/fetcher/__init__.py | 6 +++--- .../fetcher/engine/base_fetching_engine.py | 6 +++--- .../opal_common/fetcher/engine/core_callbacks.py | 2 +- .../opal_common/fetcher/engine/fetch_worker.py | 8 ++++---- .../opal_common/fetcher/engine/fetching_engine.py | 14 +++++++------- .../opal_common/fetcher/fetch_provider.py | 4 ++-- .../opal_common/fetcher/fetcher_register.py | 10 +++++----- .../opal_common/fetcher/providers/__init__.py | 2 +- .../providers/fastapi_rpc_fetch_provider.py | 6 +++--- .../fetcher/providers/http_fetch_provider.py | 10 +++++----- packages/opal-common/opal_common/logger.py | 12 ++++++------ packages/opal-server/opal_server/main.py | 2 +- scripts/gunicorn_conf.py | 2 -- 23 files changed, 53 insertions(+), 57 deletions(-) diff --git a/packages/opal-client/opal_client/__init__.py b/packages/opal-client/opal_client/__init__.py index c2810c5f7..a1eb3e09d 100644 --- a/packages/opal-client/opal_client/__init__.py +++ b/packages/opal-client/opal_client/__init__.py @@ -1 +1 @@ -from .client import OpalClient +from opal_client.client import OpalClient diff --git a/packages/opal-client/opal_client/main.py b/packages/opal-client/opal_client/main.py index 611cdd741..65f3bb665 100644 --- a/packages/opal-client/opal_client/main.py +++ b/packages/opal-client/opal_client/main.py @@ -1,4 +1,4 @@ -from .client import OpalClient +from opal_client.client import OpalClient client = OpalClient() # expose app for Uvicorn diff --git a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py index 4aa27f0d0..8d6742d4a 100644 --- a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py +++ b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py @@ -8,7 +8,7 @@ from opal_common.schemas.store import JSONPatchAction, StoreTransaction from pydantic import BaseModel -from .base_policy_store_client import BasePolicyStoreClient, JsonableValue +from opal_client.policy_store.base_policy_store_client import BasePolicyStoreClient, JsonableValue class MockPolicyStoreClient(BasePolicyStoreClient): diff --git a/packages/opal-common/opal_common/authentication/casting.py b/packages/opal-common/opal_common/authentication/casting.py index 9713ed2d5..d14a04fc7 100644 --- a/packages/opal-common/opal_common/authentication/casting.py +++ b/packages/opal-common/opal_common/authentication/casting.py @@ -5,7 +5,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception logger = logging.getLogger("opal.authentication") diff --git a/packages/opal-common/opal_common/cli/typer_app.py b/packages/opal-common/opal_common/cli/typer_app.py index 45de0d594..a1d70ff24 100644 --- a/packages/opal-common/opal_common/cli/typer_app.py +++ b/packages/opal-common/opal_common/cli/typer_app.py @@ -1,6 +1,6 @@ import typer -from .commands import all_commands +from opal_common.cli.commands import all_commands def get_typer_app(): diff --git a/packages/opal-common/opal_common/confi/__init__.py b/packages/opal-common/opal_common/confi/__init__.py index ccd3d49ff..114be01c7 100644 --- a/packages/opal-common/opal_common/confi/__init__.py +++ b/packages/opal-common/opal_common/confi/__init__.py @@ -1 +1 @@ -from .confi import * +from opal_common.confi.confi import * diff --git a/packages/opal-common/opal_common/confi/cli.py b/packages/opal-common/opal_common/confi/cli.py index 00e3097ee..cfca25f1c 100644 --- a/packages/opal-common/opal_common/confi/cli.py +++ b/packages/opal-common/opal_common/confi/cli.py @@ -4,7 +4,7 @@ import typer from typer.main import Typer -from .types import ConfiEntry +from opal_common.confi.types import ConfiEntry def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index f391c26a2..8b376be9d 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -19,8 +19,8 @@ from pydantic import BaseModel, ValidationError from typer import Typer -from .cli import get_cli_object_for_config_objects -from .types import ConfiDelay, ConfiEntry, no_cast +from opal_common.confi.cli import get_cli_object_for_config_objects +from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast class Placeholder(object): diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index 7666d47e4..b7d9395b6 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -2,8 +2,7 @@ from sys import prefix from opal_common.authentication.types import EncryptionKeyFormat, JWTAlgorithm - -from .confi import Confi, confi +from opal_common.confi import Confi, confi _LOG_FORMAT_WITHOUT_PID = "{time} | {name: <40}|{level:^6} | {message}\n{exception}" _LOG_FORMAT_WITH_PID = "{time} | {process} | {name: <40}|{level:^6} | {message}\n{exception}" @@ -173,7 +172,7 @@ class OpalCommonConfig(Confi): "HTTP_FETCHER_PROVIDER_CLIENT", "aiohttp", description="The client to use for fetching data, can be either aiohttp or httpx." - "if provided different value, aiohttp will be used.", + "if provided different value, aiohttp will be used.", ) diff --git a/packages/opal-common/opal_common/engine/__init__.py b/packages/opal-common/opal_common/engine/__init__.py index bbc306e62..33d1be247 100644 --- a/packages/opal-common/opal_common/engine/__init__.py +++ b/packages/opal-common/opal_common/engine/__init__.py @@ -1,2 +1,2 @@ -from .parsing import get_rego_package -from .paths import is_data_module, is_policy_module +from opal_common.engine.parsing import get_rego_package +from opal_common.engine.paths import is_data_module, is_policy_module diff --git a/packages/opal-common/opal_common/fetcher/__init__.py b/packages/opal-common/opal_common/fetcher/__init__.py index 84232e236..70a1f643c 100644 --- a/packages/opal-common/opal_common/fetcher/__init__.py +++ b/packages/opal-common/opal_common/fetcher/__init__.py @@ -1,3 +1,3 @@ -from .engine.fetching_engine import FetchingEngine -from .events import FetcherConfig, FetchEvent -from .fetcher_register import FetcherRegister +from opal_common.fetcher.engine.fetching_engine import FetchingEngine +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister diff --git a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py index a30f033d2..22f9325f9 100644 --- a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py @@ -1,8 +1,8 @@ from typing import Coroutine -from ..events import FetcherConfig, FetchEvent -from ..fetcher_register import FetcherRegister -from .core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback class BaseFetchingEngine: diff --git a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py index b083e779e..3da152f14 100644 --- a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py +++ b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py @@ -1,4 +1,4 @@ -from ..events import FetchEvent +from opal_common.fetcher.events import FetchEvent # Callback signatures diff --git a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py index eb816ecf2..460ee1465 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py @@ -1,10 +1,10 @@ import asyncio from typing import Coroutine -from ..events import FetchEvent -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine logger = get_logger("fetch_worker") diff --git a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py index fd50a9f14..cb03693bf 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py @@ -2,13 +2,13 @@ import uuid from typing import Coroutine, Dict, List, Union -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine -from .core_callbacks import OnFetchFailureCallback -from .fetch_worker import fetch_worker +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.fetch_worker import fetch_worker logger = get_logger("engine") diff --git a/packages/opal-common/opal_common/fetcher/fetch_provider.py b/packages/opal-common/opal_common/fetcher/fetch_provider.py index 62ad97532..70b91ea59 100644 --- a/packages/opal-common/opal_common/fetcher/fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/fetch_provider.py @@ -1,7 +1,7 @@ from tenacity import retry, stop, wait -from .events import FetchEvent -from .logger import get_logger +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.logger import get_logger logger = get_logger("opal.providers") diff --git a/packages/opal-common/opal_common/fetcher/fetcher_register.py b/packages/opal-common/opal_common/fetcher/fetcher_register.py index 5bf925160..18ed32f81 100644 --- a/packages/opal-common/opal_common/fetcher/fetcher_register.py +++ b/packages/opal-common/opal_common/fetcher/fetcher_register.py @@ -2,10 +2,10 @@ from opal_common.fetcher.logger import get_logger -from ..config import opal_common_config -from .events import FetchEvent -from .fetch_provider import BaseFetchProvider -from .providers.http_fetch_provider import HttpFetchProvider +from opal_common.config import opal_common_config +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.providers.http_fetch_provider import HttpFetchProvider logger = get_logger("opal.fetcher_register") @@ -30,7 +30,7 @@ def __init__(self, config: Optional[Dict[str, BaseFetchProvider]] = None) -> Non if config is not None: self._config = config else: - from ..emport import emport_objects_by_class + from opal_common.emport import emport_objects_by_class # load fetchers fetchers = [] diff --git a/packages/opal-common/opal_common/fetcher/providers/__init__.py b/packages/opal-common/opal_common/fetcher/providers/__init__.py index 8e1f6bf77..ff1078ced 100644 --- a/packages/opal-common/opal_common/fetcher/providers/__init__.py +++ b/packages/opal-common/opal_common/fetcher/providers/__init__.py @@ -1,3 +1,3 @@ -from ...emport import dynamic_all +from opal_common.emport import dynamic_all __all__ = dynamic_all(__file__) diff --git a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py index 61432b751..94513f9d2 100644 --- a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py @@ -3,9 +3,9 @@ from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger logger = get_logger("rpc_fetch_provider") diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7261b538b..9083f8aa1 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -8,11 +8,11 @@ from opal_common.config import opal_common_config from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger +from opal_common.http_utils import is_http_error_response +from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger logger = get_logger("http_fetch_provider") diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index 5f1229a80..a494491ef 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -3,12 +3,12 @@ from loguru import logger -from .config import opal_common_config -from .logging.filter import ModuleFilter -from .logging.formatter import Formatter -from .logging.intercept import InterceptHandler -from .logging.thirdparty import hijack_uvicorn_logs -from .monitoring.apm import fix_ddtrace_logging +from opal_common.config import opal_common_config +from opal_common.logging.filter import ModuleFilter +from opal_common.logging.formatter import Formatter +from opal_common.logging.intercept import InterceptHandler +from opal_common.logging.thirdparty import hijack_uvicorn_logs +from opal_common.monitoring.apm import fix_ddtrace_logging def configure_logs(): diff --git a/packages/opal-server/opal_server/main.py b/packages/opal-server/opal_server/main.py index 908c4561d..7e61e2a66 100644 --- a/packages/opal-server/opal_server/main.py +++ b/packages/opal-server/opal_server/main.py @@ -1,5 +1,5 @@ def create_app(*args, **kwargs): - from .server import OpalServer + from opal_server.server import OpalServer server = OpalServer(*args, **kwargs) return server.app diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index b40546ae6..0c2c15cb8 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,5 +1,3 @@ -import os - from opal_common.logger import logger From 38446a803ee0cf8ac941ac5444afa34a1c9f911c Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:30:28 +0300 Subject: [PATCH 02/10] Renamed redis_utils module --- packages/opal-server/opal_server/{redis.py => redis_utils.py} | 0 packages/opal-server/opal_server/scopes/scope_repository.py | 2 +- packages/opal-server/opal_server/scopes/task.py | 2 +- packages/opal-server/opal_server/server.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/opal-server/opal_server/{redis.py => redis_utils.py} (100%) diff --git a/packages/opal-server/opal_server/redis.py b/packages/opal-server/opal_server/redis_utils.py similarity index 100% rename from packages/opal-server/opal_server/redis.py rename to packages/opal-server/opal_server/redis_utils.py diff --git a/packages/opal-server/opal_server/scopes/scope_repository.py b/packages/opal-server/opal_server/scopes/scope_repository.py index b3627741c..d9f5d9d20 100644 --- a/packages/opal-server/opal_server/scopes/scope_repository.py +++ b/packages/opal-server/opal_server/scopes/scope_repository.py @@ -1,7 +1,7 @@ from typing import List from opal_common.schemas.scopes import Scope -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB class ScopeNotFoundError(Exception): diff --git a/packages/opal-server/opal_server/scopes/task.py b/packages/opal-server/opal_server/scopes/task.py index b3a577161..83b2b10f0 100644 --- a/packages/opal-server/opal_server/scopes/task.py +++ b/packages/opal-server/opal_server/scopes/task.py @@ -7,7 +7,7 @@ from opal_common.logger import logger from opal_server.config import opal_server_config from opal_server.policy.watcher.task import BasePolicyWatcherTask -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.scope_repository import ScopeRepository from opal_server.scopes.service import ScopesService diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 6a946a8c0..34d9905c3 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -32,7 +32,7 @@ from opal_server.policy.webhook.api import init_git_webhook_router from opal_server.publisher import setup_broadcaster_keepalive_task from opal_server.pubsub import PubSub -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.api import init_scope_router from opal_server.scopes.loader import load_scopes from opal_server.scopes.scope_repository import ScopeRepository From 040162cc11579e51daa8a23be1143b1311378961 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:32:13 +0300 Subject: [PATCH 03/10] Renamed http_utils module --- packages/opal-client/opal_client/callbacks/reporter.py | 2 +- packages/opal-client/opal_client/data/updater.py | 2 +- packages/opal-common/opal_common/{http.py => http_utils.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/opal-common/opal_common/{http.py => http_utils.py} (100%) diff --git a/packages/opal-client/opal_client/callbacks/reporter.py b/packages/opal-client/opal_client/callbacks/reporter.py index 264f45b51..c9f2987b6 100644 --- a/packages/opal-client/opal_client/callbacks/reporter.py +++ b/packages/opal-client/opal_client/callbacks/reporter.py @@ -5,7 +5,7 @@ from opal_client.callbacks.register import CallbackConfig, CallbacksRegister from opal_client.data.fetcher import DataFetcher from opal_common.fetcher.providers.http_fetch_provider import HttpFetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.logger import logger from opal_common.schemas.data import DataUpdateReport diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index d2c81c9ed..e288b5963 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -26,7 +26,7 @@ from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.schemas.data import ( DataEntryReport, DataSourceConfig, diff --git a/packages/opal-common/opal_common/http.py b/packages/opal-common/opal_common/http_utils.py similarity index 100% rename from packages/opal-common/opal_common/http.py rename to packages/opal-common/opal_common/http_utils.py From 8dd6cc1cb9ade1f97e4a3c57e8bcfd956cd122e9 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:32:41 +0300 Subject: [PATCH 04/10] Renamed logging_utils module --- packages/opal-common/opal_common/confi/confi.py | 2 +- packages/opal-common/opal_common/logger.py | 8 ++++---- .../opal_common/{logging => logging_utils}/__init__.py | 0 .../opal_common/{logging => logging_utils}/decorators.py | 0 .../opal_common/{logging => logging_utils}/filter.py | 0 .../opal_common/{logging => logging_utils}/formatter.py | 0 .../opal_common/{logging => logging_utils}/intercept.py | 0 .../opal_common/{logging => logging_utils}/thirdparty.py | 0 8 files changed, 5 insertions(+), 5 deletions(-) rename packages/opal-common/opal_common/{logging => logging_utils}/__init__.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/decorators.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/filter.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/formatter.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/intercept.py (100%) rename packages/opal-common/opal_common/{logging => logging_utils}/thirdparty.py (100%) diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index 8b376be9d..f62576dfa 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -15,7 +15,7 @@ from decouple import Csv, UndefinedValueError, config, text_type, undefined from opal_common.authentication.casting import cast_private_key, cast_public_key from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception +from opal_common.logging_utils.decorators import log_exception from pydantic import BaseModel, ValidationError from typer import Typer diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index a494491ef..2a45a4dc2 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -4,10 +4,10 @@ from loguru import logger from opal_common.config import opal_common_config -from opal_common.logging.filter import ModuleFilter -from opal_common.logging.formatter import Formatter -from opal_common.logging.intercept import InterceptHandler -from opal_common.logging.thirdparty import hijack_uvicorn_logs +from opal_common.logging_utils.filter import ModuleFilter +from opal_common.logging_utils.formatter import Formatter +from opal_common.logging_utils.intercept import InterceptHandler +from opal_common.logging_utils.thirdparty import hijack_uvicorn_logs from opal_common.monitoring.apm import fix_ddtrace_logging diff --git a/packages/opal-common/opal_common/logging/__init__.py b/packages/opal-common/opal_common/logging_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/logging/__init__.py rename to packages/opal-common/opal_common/logging_utils/__init__.py diff --git a/packages/opal-common/opal_common/logging/decorators.py b/packages/opal-common/opal_common/logging_utils/decorators.py similarity index 100% rename from packages/opal-common/opal_common/logging/decorators.py rename to packages/opal-common/opal_common/logging_utils/decorators.py diff --git a/packages/opal-common/opal_common/logging/filter.py b/packages/opal-common/opal_common/logging_utils/filter.py similarity index 100% rename from packages/opal-common/opal_common/logging/filter.py rename to packages/opal-common/opal_common/logging_utils/filter.py diff --git a/packages/opal-common/opal_common/logging/formatter.py b/packages/opal-common/opal_common/logging_utils/formatter.py similarity index 100% rename from packages/opal-common/opal_common/logging/formatter.py rename to packages/opal-common/opal_common/logging_utils/formatter.py diff --git a/packages/opal-common/opal_common/logging/intercept.py b/packages/opal-common/opal_common/logging_utils/intercept.py similarity index 100% rename from packages/opal-common/opal_common/logging/intercept.py rename to packages/opal-common/opal_common/logging_utils/intercept.py diff --git a/packages/opal-common/opal_common/logging/thirdparty.py b/packages/opal-common/opal_common/logging_utils/thirdparty.py similarity index 100% rename from packages/opal-common/opal_common/logging/thirdparty.py rename to packages/opal-common/opal_common/logging_utils/thirdparty.py From c1c6d0feab346fe3a9594b763ab6d05423bea70a Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Sun, 14 Jul 2024 13:32:50 +0300 Subject: [PATCH 05/10] Renamed git_utils module --- .../opal-client/opal_client/policy_store/opa_client.py | 2 +- .../opal_common/{git => git_utils}/__init__.py | 0 .../opal_common/{git => git_utils}/branch_tracker.py | 4 ++-- .../opal_common/{git => git_utils}/bundle_maker.py | 4 ++-- .../opal_common/{git => git_utils}/bundle_utils.py | 0 .../opal_common/{git => git_utils}/commit_viewer.py | 0 .../opal_common/{git => git_utils}/diff_viewer.py | 2 +- .../opal-common/opal_common/{git => git_utils}/env.py | 0 .../opal_common/{git => git_utils}/exceptions.py | 0 .../opal_common/{git => git_utils}/repo_cloner.py | 4 ++-- .../tar_file_to_local_git_extractor.py | 0 .../{git => git_utils}/tests/branch_tracker_test.py | 4 ++-- .../{git => git_utils}/tests/bundle_maker_test.py | 4 ++-- .../{git => git_utils}/tests/commit_viewer_test.py | 2 +- .../opal_common/{git => git_utils}/tests/conftest.py | 0 .../{git => git_utils}/tests/diff_viewer_test.py | 4 ++-- .../{git => git_utils}/tests/repo_cloner_test.py | 4 ++-- .../{git => git_utils}/tests/repo_watcher_test.py | 0 .../opal-common/opal_common/sources/api_policy_source.py | 2 +- .../opal-common/opal_common/sources/git_policy_source.py | 6 +++--- packages/opal-server/opal_server/git_fetcher.py | 2 +- packages/opal-server/opal_server/policy/bundles/api.py | 9 +++++---- .../opal-server/opal_server/policy/watcher/callbacks.py | 4 ++-- .../opal-server/opal_server/policy/watcher/factory.py | 2 +- packages/opal-server/opal_server/scopes/service.py | 2 +- 25 files changed, 31 insertions(+), 30 deletions(-) rename packages/opal-common/opal_common/{git => git_utils}/__init__.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/branch_tracker.py (97%) rename packages/opal-common/opal_common/{git => git_utils}/bundle_maker.py (99%) rename packages/opal-common/opal_common/{git => git_utils}/bundle_utils.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/commit_viewer.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/diff_viewer.py (99%) rename packages/opal-common/opal_common/{git => git_utils}/env.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/exceptions.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/repo_cloner.py (98%) rename packages/opal-common/opal_common/{git => git_utils}/tar_file_to_local_git_extractor.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/tests/branch_tracker_test.py (95%) rename packages/opal-common/opal_common/{git => git_utils}/tests/bundle_maker_test.py (99%) rename packages/opal-common/opal_common/{git => git_utils}/tests/commit_viewer_test.py (98%) rename packages/opal-common/opal_common/{git => git_utils}/tests/conftest.py (100%) rename packages/opal-common/opal_common/{git => git_utils}/tests/diff_viewer_test.py (97%) rename packages/opal-common/opal_common/{git => git_utils}/tests/repo_cloner_test.py (96%) rename packages/opal-common/opal_common/{git => git_utils}/tests/repo_watcher_test.py (100%) diff --git a/packages/opal-client/opal_client/policy_store/opa_client.py b/packages/opal-client/opal_client/policy_store/opa_client.py index 1d8ec0211..54bc94dac 100644 --- a/packages/opal-client/opal_client/policy_store/opa_client.py +++ b/packages/opal-client/opal_client/policy_store/opa_client.py @@ -20,7 +20,7 @@ from opal_client.policy_store.schemas import PolicyStoreAuth from opal_client.utils import exclude_none_fields, proxy_response from opal_common.engine.parsing import get_rego_package -from opal_common.git.bundle_utils import BundleUtils +from opal_common.git_utils.bundle_utils import BundleUtils from opal_common.paths import PathUtils from opal_common.schemas.policy import DataModule, PolicyBundle, RegoModule from opal_common.schemas.store import JSONPatchAction, StoreTransaction, TransactionType diff --git a/packages/opal-common/opal_common/git/__init__.py b/packages/opal-common/opal_common/git_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/git/__init__.py rename to packages/opal-common/opal_common/git_utils/__init__.py diff --git a/packages/opal-common/opal_common/git/branch_tracker.py b/packages/opal-common/opal_common/git_utils/branch_tracker.py similarity index 97% rename from packages/opal-common/opal_common/git/branch_tracker.py rename to packages/opal-common/opal_common/git_utils/branch_tracker.py index 19bba8770..28f692e15 100644 --- a/packages/opal-common/opal_common/git/branch_tracker.py +++ b/packages/opal-common/opal_common/git_utils/branch_tracker.py @@ -3,8 +3,8 @@ from git import GitCommandError, Head, Remote, Repo from git.objects.commit import Commit -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from tenacity import retry, stop_after_attempt, wait_fixed diff --git a/packages/opal-common/opal_common/git/bundle_maker.py b/packages/opal-common/opal_common/git_utils/bundle_maker.py similarity index 99% rename from packages/opal-common/opal_common/git/bundle_maker.py rename to packages/opal-common/opal_common/git_utils/bundle_maker.py index 0b006b6a7..f9d621a89 100644 --- a/packages/opal-common/opal_common/git/bundle_maker.py +++ b/packages/opal-common/opal_common/git_utils/bundle_maker.py @@ -6,7 +6,7 @@ from git import Repo from git.objects import Commit from opal_common.engine import get_rego_package, is_data_module, is_policy_module -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, VersionedDirectory, VersionedFile, @@ -14,7 +14,7 @@ has_extension, is_under_directories, ) -from opal_common.git.diff_viewer import ( +from opal_common.git_utils.diff_viewer import ( DiffViewer, diffed_file_has_extension, diffed_file_is_under_directories, diff --git a/packages/opal-common/opal_common/git/bundle_utils.py b/packages/opal-common/opal_common/git_utils/bundle_utils.py similarity index 100% rename from packages/opal-common/opal_common/git/bundle_utils.py rename to packages/opal-common/opal_common/git_utils/bundle_utils.py diff --git a/packages/opal-common/opal_common/git/commit_viewer.py b/packages/opal-common/opal_common/git_utils/commit_viewer.py similarity index 100% rename from packages/opal-common/opal_common/git/commit_viewer.py rename to packages/opal-common/opal_common/git_utils/commit_viewer.py diff --git a/packages/opal-common/opal_common/git/diff_viewer.py b/packages/opal-common/opal_common/git_utils/diff_viewer.py similarity index 99% rename from packages/opal-common/opal_common/git/diff_viewer.py rename to packages/opal-common/opal_common/git_utils/diff_viewer.py index af5720f5c..ec6dff9d0 100644 --- a/packages/opal-common/opal_common/git/diff_viewer.py +++ b/packages/opal-common/opal_common/git_utils/diff_viewer.py @@ -4,7 +4,7 @@ from git import Repo from git.diff import Diff, DiffIndex from git.objects.commit import Commit -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.paths import PathUtils DiffFilter = Callable[[Diff], bool] diff --git a/packages/opal-common/opal_common/git/env.py b/packages/opal-common/opal_common/git_utils/env.py similarity index 100% rename from packages/opal-common/opal_common/git/env.py rename to packages/opal-common/opal_common/git_utils/env.py diff --git a/packages/opal-common/opal_common/git/exceptions.py b/packages/opal-common/opal_common/git_utils/exceptions.py similarity index 100% rename from packages/opal-common/opal_common/git/exceptions.py rename to packages/opal-common/opal_common/git_utils/exceptions.py diff --git a/packages/opal-common/opal_common/git/repo_cloner.py b/packages/opal-common/opal_common/git_utils/repo_cloner.py similarity index 98% rename from packages/opal-common/opal_common/git/repo_cloner.py rename to packages/opal-common/opal_common/git_utils/repo_cloner.py index 43bda1ba2..76ebb4949 100644 --- a/packages/opal-common/opal_common/git/repo_cloner.py +++ b/packages/opal-common/opal_common/git_utils/repo_cloner.py @@ -8,8 +8,8 @@ from git import GitCommandError, GitError, Repo from opal_common.config import opal_common_config -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from opal_common.utils import get_filepaths_with_glob from tenacity import RetryError, retry, stop, wait diff --git a/packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py b/packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py similarity index 100% rename from packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py rename to packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py diff --git a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py similarity index 95% rename from packages/opal-common/opal_common/git/tests/branch_tracker_test.py rename to packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py index bc17d6b09..751231a0d 100644 --- a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py @@ -18,8 +18,8 @@ from git import Repo from git.objects.commit import Commit -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed def test_pull_with_no_changes(local_repo_clone: Repo): diff --git a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py similarity index 99% rename from packages/opal-common/opal_common/git/tests/bundle_maker_test.py rename to packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py index 63624f6eb..5e77ad0e5 100644 --- a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py @@ -19,8 +19,8 @@ from git import Repo from git.objects import Commit -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer from opal_common.schemas.policy import PolicyBundle, RegoModule OPA_FILE_EXTENSIONS = (".rego", ".json") diff --git a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py similarity index 98% rename from packages/opal-common/opal_common/git/tests/commit_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py index 91d19fcf1..1f9ca522a 100644 --- a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py @@ -19,7 +19,7 @@ from git import Repo from git.objects import Commit -from opal_common.git.commit_viewer import CommitViewer, VersionedNode +from opal_common.git_utils.commit_viewer import CommitViewer, VersionedNode def node_paths(nodes: List[VersionedNode]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/conftest.py b/packages/opal-common/opal_common/git_utils/tests/conftest.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/conftest.py rename to packages/opal-common/opal_common/git_utils/tests/conftest.py diff --git a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py similarity index 97% rename from packages/opal-common/opal_common/git/tests/diff_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py index 6dc77ec4b..974ffaa7e 100644 --- a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py @@ -20,8 +20,8 @@ from git import Diff, Repo from git.objects import Commit -from opal_common.git.commit_viewer import VersionedFile -from opal_common.git.diff_viewer import DiffViewer, diffed_file_is_under_directories +from opal_common.git_utils.commit_viewer import VersionedFile +from opal_common.git_utils.diff_viewer import DiffViewer, diffed_file_is_under_directories def diff_paths(diffs: List[Diff]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py similarity index 96% rename from packages/opal-common/opal_common/git/tests/repo_cloner_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py index ffe6a02fa..567f3707b 100644 --- a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py @@ -18,8 +18,8 @@ from git import Repo from opal_common.confi import Confi -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner VALID_REPO_REMOTE_URL_HTTPS = "https://github.com/permitio/fastapi_websocket_pubsub.git" diff --git a/packages/opal-common/opal_common/git/tests/repo_watcher_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/repo_watcher_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 9b6487907..596e8aba0 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -6,7 +6,7 @@ import aiohttp from fastapi import status from fastapi.exceptions import HTTPException -from opal_common.git.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor +from opal_common.git_utils.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_common.utils import ( diff --git a/packages/opal-common/opal_common/sources/git_policy_source.py b/packages/opal-common/opal_common/sources/git_policy_source.py index bffe8517d..8252cd4ce 100644 --- a/packages/opal-common/opal_common/sources/git_policy_source.py +++ b/packages/opal-common/opal_common/sources/git_policy_source.py @@ -1,9 +1,9 @@ from typing import Optional from git import Repo -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 5ea85c047..36932ee30 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -12,7 +12,7 @@ from ddtrace import tracer from git import Repo from opal_common.async_utils import run_sync -from opal_common.git.bundle_maker import BundleMaker +from opal_common.git_utils.bundle_maker import BundleMaker from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.policy_source import ( diff --git a/packages/opal-server/opal_server/policy/bundles/api.py b/packages/opal-server/opal_server/policy/bundles/api.py index 223e72001..7e25e8ffc 100644 --- a/packages/opal-server/opal_server/policy/bundles/api.py +++ b/packages/opal-server/opal_server/policy/bundles/api.py @@ -4,11 +4,12 @@ import fastapi.responses from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status -from git import Repo +from git.repo import Repo + from opal_common.confi.confi import load_conf_if_none -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_server.config import opal_server_config diff --git a/packages/opal-server/opal_server/policy/watcher/callbacks.py b/packages/opal-server/opal_server/policy/watcher/callbacks.py index 62a30f16b..1b5f65590 100644 --- a/packages/opal-server/opal_server/policy/watcher/callbacks.py +++ b/packages/opal-server/opal_server/policy/watcher/callbacks.py @@ -3,13 +3,13 @@ from typing import List, Optional from git.objects import Commit -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, FileFilter, find_ignore_match, has_extension, ) -from opal_common.git.diff_viewer import DiffViewer +from opal_common.git_utils.diff_viewer import DiffViewer from opal_common.logger import logger from opal_common.paths import PathUtils from opal_common.schemas.policy import ( diff --git a/packages/opal-server/opal_server/policy/watcher/factory.py b/packages/opal-server/opal_server/policy/watcher/factory.py index 10fb1b19c..6d94d6fc4 100644 --- a/packages/opal-server/opal_server/policy/watcher/factory.py +++ b/packages/opal-server/opal_server/policy/watcher/factory.py @@ -3,7 +3,7 @@ from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint from opal_common.confi.confi import load_conf_if_none -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.sources.api_policy_source import ApiPolicySource from opal_common.sources.git_policy_source import GitPolicySource diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 533b397a8..f0104e7bf 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -7,7 +7,7 @@ import git from ddtrace import tracer from fastapi_websocket_pubsub import PubSubEndpoint -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.logger import logger from opal_common.schemas.policy import PolicyUpdateMessageNotification from opal_common.schemas.policy_source import GitPolicyScopeSource From 5f01375cdd4626bd076794e7152328b96d9c9a07 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 19:27:17 +0300 Subject: [PATCH 06/10] Fixed OPAL Cedar Client build CI --- .github/workflows/on_release.yml | 54 +++++++++++++++++++------------- cedar-agent | 2 +- docker/Dockerfile | 8 ++--- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 736a64f0b..e616e11d0 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -109,13 +109,12 @@ jobs: # pushes the *same* docker images that were previously tested as part of e2e sanity test. # each image is pushed with the versioned tag first, if it succeeds the image is pushed with the latest tag as well. - name: Build & Push client - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: client cache-from: type=registry,ref=permitio/opal-client:latest cache-to: type=inline @@ -123,29 +122,13 @@ jobs: permitio/opal-client:latest permitio/opal-client:${{ env.opal_version_tag }} -# - name: Build & Push client cedar -# if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} -# id: build_push_client_cedar -# uses: docker/build-push-action@v4 -# with: -# file: docker/Dockerfile -# platforms: linux/amd64,linux/arm64 -# push: true -# target: client-cedar -# cache-from: type=registry,ref=permitio/opal-client-cedar:latest -# cache-to: type=inline -# tags: | -# permitio/opal-client-cedar:latest -# permitio/opal-client-cedar:${{ env.opal_version_tag }} - - name: Build client-standalone - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client_standalone uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: client-standalone cache-from: type=registry,ref=permitio/opal-client-standalone:latest cache-to: type=inline @@ -154,16 +137,45 @@ jobs: permitio/opal-client-standalone:${{ env.opal_version_tag }} - name: Build server - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_server uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: server cache-from: type=registry,ref=permitio/opal-server:latest cache-to: type=inline tags: | permitio/opal-server:latest permitio/opal-server:${{ env.opal_version_tag }} + + - name: Check if cedar-agent directory exists + id: check_cedar_agent + run: | + if [ -d "cedar-agent" ]; then + echo "exists=true" >> $GITHUB_ENV + else + echo "exists=false" >> $GITHUB_ENV + fi + + - name: Clone cedar-agent repository + if: steps.check_cedar_agent.outputs.exists == 'false' + id: clone_cedar_agent + working-directory: . + run: | + git clone https://github.com/permitio/cedar-agent.git cedar-agent + + - name: Build & Push client cedar + id: build_push_client_cedar + uses: docker/build-push-action@v4 + with: + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + tags: | + permitio/opal-client-cedar:latest + permitio/opal-client-cedar:${{ env.opal_version_tag }} diff --git a/cedar-agent b/cedar-agent index 1838635f1..687efc59e 160000 --- a/cedar-agent +++ b/cedar-agent @@ -1 +1 @@ -Subproject commit 1838635f16ba6db60d16c2ca28cb257e970bdff0 +Subproject commit 687efc59ecc732d1b98fc7789ab803abfc45b94c diff --git a/docker/Dockerfile b/docker/Dockerfile index bccdf3d2c..4723e37a7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,10 +17,10 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # --------------------------------------------------- FROM rust:1.69.0 as cedar-builder COPY cedar-agent /tmp/cedar-agent/ -ARG cargo_flags="-r" -RUN cd /tmp/cedar-agent && \ - cargo build ${cargo_flags} && \ - cp /tmp/cedar-agent/target/*/cedar-agent / +ARG cargo_flags="--release" +RUN cd /tmp/cedar-agent +RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build ${cargo_flags} +RUN cp /tmp/cedar-agent/target/*/cedar-agent / # COMMON IMAGE -------------------------------------- # --------------------------------------------------- From 6ae6b18ccfb63cb632a079883420a5cc6a5c76c3 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 19:45:21 +0300 Subject: [PATCH 07/10] Updated rust version for OPAL Cedar Client docker --- .github/workflows/on_release.yml | 16 ---------------- docker/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index e616e11d0..3733c0fd7 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -150,22 +150,6 @@ jobs: permitio/opal-server:latest permitio/opal-server:${{ env.opal_version_tag }} - - name: Check if cedar-agent directory exists - id: check_cedar_agent - run: | - if [ -d "cedar-agent" ]; then - echo "exists=true" >> $GITHUB_ENV - else - echo "exists=false" >> $GITHUB_ENV - fi - - - name: Clone cedar-agent repository - if: steps.check_cedar_agent.outputs.exists == 'false' - id: clone_cedar_agent - working-directory: . - run: | - git clone https://github.com/permitio/cedar-agent.git cedar-agent - - name: Build & Push client cedar id: build_push_client_cedar uses: docker/build-push-action@v4 diff --git a/docker/Dockerfile b/docker/Dockerfile index 4723e37a7..31ca18012 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # CEDAR AGENT BUILD STAGE --------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM rust:1.69.0 as cedar-builder +FROM rust:1.77-bullseye as cedar-builder COPY cedar-agent /tmp/cedar-agent/ ARG cargo_flags="--release" RUN cd /tmp/cedar-agent From 63c27c8395121af93fa26eaf6f4c2addbcf69adc Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 20:17:05 +0300 Subject: [PATCH 08/10] Fixed docker warnings --- docker/Dockerfile | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 31ca18012..89af2eba0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # BUILD STAGE --------------------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM python:3.10-bookworm as BuildStage +FROM python:3.10-bookworm AS build-stage # from now on, work in the /app directory WORKDIR /app/ # Layer dependency install (for caching) @@ -15,19 +15,18 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # CEDAR AGENT BUILD STAGE --------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM rust:1.77-bullseye as cedar-builder -COPY cedar-agent /tmp/cedar-agent/ -ARG cargo_flags="--release" +FROM rust:1.79 AS cedar-builder +COPY ./cedar-agent /tmp/cedar-agent/ RUN cd /tmp/cedar-agent RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build ${cargo_flags} RUN cp /tmp/cedar-agent/target/*/cedar-agent / # COMMON IMAGE -------------------------------------- # --------------------------------------------------- -FROM python:3.10-slim-bookworm as common +FROM python:3.10-slim-bookworm AS common -# copy libraries from build stage (This won't copy redundant libraries we used in BuildStage) -COPY --from=BuildStage /usr/local /usr/local +# copy libraries from build stage (This won't copy redundant libraries we used in build-stage) +COPY --from=build-stage /usr/local /usr/local # Add non-root user (with home dir at /opal) RUN useradd -m -b / -s /bin/bash opal @@ -61,7 +60,7 @@ CMD ["./start.sh"] # STANDALONE IMAGE ---------------------------------- # --------------------------------------------------- -FROM common as client-standalone +FROM common AS client-standalone # uvicorn config ------------------------------------ # install the opal-client package RUN cd ./packages/opal-client && python setup.py install @@ -88,7 +87,7 @@ VOLUME /opal/backup # IMAGE to extract OPA from official image ---------- # --------------------------------------------------- -FROM alpine:latest as opa-extractor +FROM alpine:latest AS opa-extractor USER root RUN apk update && apk add skopeo tar @@ -106,7 +105,7 @@ RUN skopeo copy "docker://${opa_image}:${opa_tag}" docker-archive:./image.tar && # OPA CLIENT IMAGE ---------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client +FROM client-standalone AS client # Temporarily move back to root for additional setup USER root @@ -123,7 +122,7 @@ USER opal # CEDAR CLIENT IMAGE -------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client-cedar +FROM client-standalone AS client-cedar # Temporarily move back to root for additional setup USER root @@ -142,7 +141,7 @@ USER opal # SERVER IMAGE -------------------------------------- # --------------------------------------------------- -FROM common as server +FROM common AS server RUN apt-get update && apt-get install -y openssh-client git && apt-get clean From ed70c746fe4c2494df14260e1061ed6a49e39144 Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Mon, 22 Jul 2024 20:40:58 +0300 Subject: [PATCH 09/10] Fixed cedar build --- docker/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 89af2eba0..74ac1916b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,10 +16,9 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # split this stage to save time and reduce image size # --------------------------------------------------- FROM rust:1.79 AS cedar-builder -COPY ./cedar-agent /tmp/cedar-agent/ -RUN cd /tmp/cedar-agent -RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build ${cargo_flags} -RUN cp /tmp/cedar-agent/target/*/cedar-agent / +COPY ./cedar-agent /tmp/cedar-agent +WORKDIR /tmp/cedar-agent +RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release # COMMON IMAGE -------------------------------------- # --------------------------------------------------- @@ -128,7 +127,7 @@ FROM client-standalone AS client-cedar USER root # Copy cedar from its build stage -COPY --from=cedar-builder /cedar-agent /bin/cedar-agent +COPY --from=cedar-builder /tmp/cedar-agent/target/*/cedar-agent /bin/cedar-agent # enable inline Cedar agent ENV OPAL_POLICY_STORE_TYPE=CEDAR From 031070144c521af98d19f89e7fe507667199db3a Mon Sep 17 00:00:00 2001 From: Dan Yishai Date: Tue, 23 Jul 2024 14:53:02 +0300 Subject: [PATCH 10/10] Fixed pre-commit --- .../opal_client/policy_store/mock_policy_store_client.py | 6 ++++-- packages/opal-common/opal_common/cli/typer_app.py | 1 - packages/opal-common/opal_common/confi/cli.py | 3 +-- packages/opal-common/opal_common/confi/confi.py | 5 ++--- packages/opal-common/opal_common/config.py | 2 +- .../opal_common/fetcher/engine/base_fetching_engine.py | 2 +- .../opal-common/opal_common/fetcher/engine/fetch_worker.py | 2 +- .../opal_common/fetcher/engine/fetching_engine.py | 6 +++--- packages/opal-common/opal_common/fetcher/fetch_provider.py | 3 +-- .../opal-common/opal_common/fetcher/fetcher_register.py | 3 +-- .../fetcher/providers/fastapi_rpc_fetch_provider.py | 1 - .../opal_common/fetcher/providers/http_fetch_provider.py | 7 +++---- .../opal_common/git_utils/tests/diff_viewer_test.py | 5 ++++- packages/opal-common/opal_common/logger.py | 1 - .../opal-common/opal_common/sources/api_policy_source.py | 4 +++- packages/opal-server/opal_server/policy/bundles/api.py | 1 - 16 files changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py index 8d6742d4a..549dd8435 100644 --- a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py +++ b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py @@ -3,13 +3,15 @@ from typing import Any, Dict, List, Optional import jsonpatch +from opal_client.policy_store.base_policy_store_client import ( + BasePolicyStoreClient, + JsonableValue, +) from opal_client.utils import exclude_none_fields from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.store import JSONPatchAction, StoreTransaction from pydantic import BaseModel -from opal_client.policy_store.base_policy_store_client import BasePolicyStoreClient, JsonableValue - class MockPolicyStoreClient(BasePolicyStoreClient): """A naive mock policy and policy-data store for tests.""" diff --git a/packages/opal-common/opal_common/cli/typer_app.py b/packages/opal-common/opal_common/cli/typer_app.py index a1d70ff24..47d38dd39 100644 --- a/packages/opal-common/opal_common/cli/typer_app.py +++ b/packages/opal-common/opal_common/cli/typer_app.py @@ -1,5 +1,4 @@ import typer - from opal_common.cli.commands import all_commands diff --git a/packages/opal-common/opal_common/confi/cli.py b/packages/opal-common/opal_common/confi/cli.py index cfca25f1c..0ab88e55a 100644 --- a/packages/opal-common/opal_common/confi/cli.py +++ b/packages/opal-common/opal_common/confi/cli.py @@ -2,9 +2,8 @@ import click import typer -from typer.main import Typer - from opal_common.confi.types import ConfiEntry +from typer.main import Typer def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index f62576dfa..cbaa9a587 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -15,13 +15,12 @@ from decouple import Csv, UndefinedValueError, config, text_type, undefined from opal_common.authentication.casting import cast_private_key, cast_public_key from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey +from opal_common.confi.cli import get_cli_object_for_config_objects +from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast from opal_common.logging_utils.decorators import log_exception from pydantic import BaseModel, ValidationError from typer import Typer -from opal_common.confi.cli import get_cli_object_for_config_objects -from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast - class Placeholder(object): """Placeholder instead of default value for decouple.""" diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index b7d9395b6..ab18dd0cb 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -172,7 +172,7 @@ class OpalCommonConfig(Confi): "HTTP_FETCHER_PROVIDER_CLIENT", "aiohttp", description="The client to use for fetching data, can be either aiohttp or httpx." - "if provided different value, aiohttp will be used.", + "if provided different value, aiohttp will be used.", ) diff --git a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py index 22f9325f9..19a636a35 100644 --- a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py @@ -1,8 +1,8 @@ from typing import Coroutine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetcher_register import FetcherRegister -from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback class BaseFetchingEngine: diff --git a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py index 460ee1465..6db97b338 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py @@ -1,10 +1,10 @@ import asyncio from typing import Coroutine +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine from opal_common.fetcher.events import FetchEvent from opal_common.fetcher.fetcher_register import FetcherRegister from opal_common.fetcher.logger import get_logger -from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine logger = get_logger("fetch_worker") diff --git a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py index cb03693bf..b439d4b8d 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py @@ -2,13 +2,13 @@ import uuid from typing import Coroutine, Dict, List, Union +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.fetch_worker import fetch_worker from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.fetcher_register import FetcherRegister from opal_common.fetcher.logger import get_logger -from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine -from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback -from opal_common.fetcher.engine.fetch_worker import fetch_worker logger = get_logger("engine") diff --git a/packages/opal-common/opal_common/fetcher/fetch_provider.py b/packages/opal-common/opal_common/fetcher/fetch_provider.py index 70b91ea59..c05008fcd 100644 --- a/packages/opal-common/opal_common/fetcher/fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/fetch_provider.py @@ -1,7 +1,6 @@ -from tenacity import retry, stop, wait - from opal_common.fetcher.events import FetchEvent from opal_common.fetcher.logger import get_logger +from tenacity import retry, stop, wait logger = get_logger("opal.providers") diff --git a/packages/opal-common/opal_common/fetcher/fetcher_register.py b/packages/opal-common/opal_common/fetcher/fetcher_register.py index 18ed32f81..9abf1322c 100644 --- a/packages/opal-common/opal_common/fetcher/fetcher_register.py +++ b/packages/opal-common/opal_common/fetcher/fetcher_register.py @@ -1,10 +1,9 @@ from typing import Dict, Optional, Type -from opal_common.fetcher.logger import get_logger - from opal_common.config import opal_common_config from opal_common.fetcher.events import FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger from opal_common.fetcher.providers.http_fetch_provider import HttpFetchProvider logger = get_logger("opal.fetcher_register") diff --git a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py index 94513f9d2..4b574a8ea 100644 --- a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py @@ -2,7 +2,6 @@ from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient - from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 9083f8aa1..fc74223ed 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -6,13 +6,12 @@ import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config -from pydantic import validator - -from opal_common.http_utils import is_http_error_response -from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.fetcher.events import FetcherConfig, FetchEvent from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger +from opal_common.http_utils import is_http_error_response +from opal_common.security.sslcontext import get_custom_ssl_context +from pydantic import validator logger = get_logger("http_fetch_provider") diff --git a/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py index 974ffaa7e..bcfbb93be 100644 --- a/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py @@ -21,7 +21,10 @@ from git import Diff, Repo from git.objects import Commit from opal_common.git_utils.commit_viewer import VersionedFile -from opal_common.git_utils.diff_viewer import DiffViewer, diffed_file_is_under_directories +from opal_common.git_utils.diff_viewer import ( + DiffViewer, + diffed_file_is_under_directories, +) def diff_paths(diffs: List[Diff]) -> List[Path]: diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index 2a45a4dc2..8e826abd6 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -2,7 +2,6 @@ import sys from loguru import logger - from opal_common.config import opal_common_config from opal_common.logging_utils.filter import ModuleFilter from opal_common.logging_utils.formatter import Formatter diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 596e8aba0..7adc9ad70 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -6,7 +6,9 @@ import aiohttp from fastapi import status from fastapi.exceptions import HTTPException -from opal_common.git_utils.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor +from opal_common.git_utils.tar_file_to_local_git_extractor import ( + TarFileToLocalGitExtractor, +) from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_common.utils import ( diff --git a/packages/opal-server/opal_server/policy/bundles/api.py b/packages/opal-server/opal_server/policy/bundles/api.py index 7e25e8ffc..ae1da68ef 100644 --- a/packages/opal-server/opal_server/policy/bundles/api.py +++ b/packages/opal-server/opal_server/policy/bundles/api.py @@ -5,7 +5,6 @@ import fastapi.responses from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status from git.repo import Repo - from opal_common.confi.confi import load_conf_if_none from opal_common.git_utils.bundle_maker import BundleMaker from opal_common.git_utils.commit_viewer import CommitViewer