diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 26b1eaaf5..dda827789 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -113,13 +113,12 @@ jobs: # pushes the *same* docker images that were previously tested as part of e2e sanity test. # each image is pushed with the versioned tag first, if it succeeds the image is pushed with the latest tag as well. - name: Build & Push client - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: client cache-from: type=registry,ref=permitio/opal-client:latest cache-to: type=inline @@ -127,29 +126,13 @@ jobs: permitio/opal-client:latest permitio/opal-client:${{ env.opal_version_tag }} -# - name: Build & Push client cedar -# if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} -# id: build_push_client_cedar -# uses: docker/build-push-action@v4 -# with: -# file: docker/Dockerfile -# platforms: linux/amd64,linux/arm64 -# push: true -# target: client-cedar -# cache-from: type=registry,ref=permitio/opal-client-cedar:latest -# cache-to: type=inline -# tags: | -# permitio/opal-client-cedar:latest -# permitio/opal-client-cedar:${{ env.opal_version_tag }} - - name: Build client-standalone - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client_standalone uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: client-standalone cache-from: type=registry,ref=permitio/opal-client-standalone:latest cache-to: type=inline @@ -158,13 +141,12 @@ jobs: permitio/opal-client-standalone:${{ env.opal_version_tag }} - name: Build server - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_server uses: docker/build-push-action@v4 with: file: docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} target: server cache-from: type=registry,ref=permitio/opal-server:latest cache-to: type=inline @@ -172,6 +154,20 @@ jobs: permitio/opal-server:latest permitio/opal-server:${{ env.opal_version_tag }} + - name: Build & Push client cedar + id: build_push_client_cedar + uses: docker/build-push-action@v4 + with: + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + tags: | + permitio/opal-client-cedar:latest + permitio/opal-client-cedar:${{ env.opal_version_tag }} + - name: Python setup if: github.event_name == 'release' && github.event.action == 'created' uses: actions/setup-python@v5 diff --git a/cedar-agent b/cedar-agent index 1838635f1..687efc59e 160000 --- a/cedar-agent +++ b/cedar-agent @@ -1 +1 @@ -Subproject commit 1838635f16ba6db60d16c2ca28cb257e970bdff0 +Subproject commit 687efc59ecc732d1b98fc7789ab803abfc45b94c diff --git a/docker/Dockerfile b/docker/Dockerfile index bccdf3d2c..74ac1916b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # BUILD STAGE --------------------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM python:3.10-bookworm as BuildStage +FROM python:3.10-bookworm AS build-stage # from now on, work in the /app directory WORKDIR /app/ # Layer dependency install (for caching) @@ -15,19 +15,17 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # CEDAR AGENT BUILD STAGE --------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM rust:1.69.0 as cedar-builder -COPY cedar-agent /tmp/cedar-agent/ -ARG cargo_flags="-r" -RUN cd /tmp/cedar-agent && \ - cargo build ${cargo_flags} && \ - cp /tmp/cedar-agent/target/*/cedar-agent / +FROM rust:1.79 AS cedar-builder +COPY ./cedar-agent /tmp/cedar-agent +WORKDIR /tmp/cedar-agent +RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release # COMMON IMAGE -------------------------------------- # --------------------------------------------------- -FROM python:3.10-slim-bookworm as common +FROM python:3.10-slim-bookworm AS common -# copy libraries from build stage (This won't copy redundant libraries we used in BuildStage) -COPY --from=BuildStage /usr/local /usr/local +# copy libraries from build stage (This won't copy redundant libraries we used in build-stage) +COPY --from=build-stage /usr/local /usr/local # Add non-root user (with home dir at /opal) RUN useradd -m -b / -s /bin/bash opal @@ -61,7 +59,7 @@ CMD ["./start.sh"] # STANDALONE IMAGE ---------------------------------- # --------------------------------------------------- -FROM common as client-standalone +FROM common AS client-standalone # uvicorn config ------------------------------------ # install the opal-client package RUN cd ./packages/opal-client && python setup.py install @@ -88,7 +86,7 @@ VOLUME /opal/backup # IMAGE to extract OPA from official image ---------- # --------------------------------------------------- -FROM alpine:latest as opa-extractor +FROM alpine:latest AS opa-extractor USER root RUN apk update && apk add skopeo tar @@ -106,7 +104,7 @@ RUN skopeo copy "docker://${opa_image}:${opa_tag}" docker-archive:./image.tar && # OPA CLIENT IMAGE ---------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client +FROM client-standalone AS client # Temporarily move back to root for additional setup USER root @@ -123,13 +121,13 @@ USER opal # CEDAR CLIENT IMAGE -------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client-cedar +FROM client-standalone AS client-cedar # Temporarily move back to root for additional setup USER root # Copy cedar from its build stage -COPY --from=cedar-builder /cedar-agent /bin/cedar-agent +COPY --from=cedar-builder /tmp/cedar-agent/target/*/cedar-agent /bin/cedar-agent # enable inline Cedar agent ENV OPAL_POLICY_STORE_TYPE=CEDAR @@ -142,7 +140,7 @@ USER opal # SERVER IMAGE -------------------------------------- # --------------------------------------------------- -FROM common as server +FROM common AS server RUN apt-get update && apt-get install -y openssh-client git && apt-get clean diff --git a/packages/opal-client/opal_client/__init__.py b/packages/opal-client/opal_client/__init__.py index c2810c5f7..a1eb3e09d 100644 --- a/packages/opal-client/opal_client/__init__.py +++ b/packages/opal-client/opal_client/__init__.py @@ -1 +1 @@ -from .client import OpalClient +from opal_client.client import OpalClient diff --git a/packages/opal-client/opal_client/callbacks/reporter.py b/packages/opal-client/opal_client/callbacks/reporter.py index 264f45b51..c9f2987b6 100644 --- a/packages/opal-client/opal_client/callbacks/reporter.py +++ b/packages/opal-client/opal_client/callbacks/reporter.py @@ -5,7 +5,7 @@ from opal_client.callbacks.register import CallbackConfig, CallbacksRegister from opal_client.data.fetcher import DataFetcher from opal_common.fetcher.providers.http_fetch_provider import HttpFetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.logger import logger from opal_common.schemas.data import DataUpdateReport diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index d2c81c9ed..e288b5963 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -26,7 +26,7 @@ from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.schemas.data import ( DataEntryReport, DataSourceConfig, diff --git a/packages/opal-client/opal_client/main.py b/packages/opal-client/opal_client/main.py index 611cdd741..65f3bb665 100644 --- a/packages/opal-client/opal_client/main.py +++ b/packages/opal-client/opal_client/main.py @@ -1,4 +1,4 @@ -from .client import OpalClient +from opal_client.client import OpalClient client = OpalClient() # expose app for Uvicorn diff --git a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py index 4aa27f0d0..549dd8435 100644 --- a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py +++ b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py @@ -3,13 +3,15 @@ from typing import Any, Dict, List, Optional import jsonpatch +from opal_client.policy_store.base_policy_store_client import ( + BasePolicyStoreClient, + JsonableValue, +) from opal_client.utils import exclude_none_fields from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.store import JSONPatchAction, StoreTransaction from pydantic import BaseModel -from .base_policy_store_client import BasePolicyStoreClient, JsonableValue - class MockPolicyStoreClient(BasePolicyStoreClient): """A naive mock policy and policy-data store for tests.""" diff --git a/packages/opal-client/opal_client/policy_store/opa_client.py b/packages/opal-client/opal_client/policy_store/opa_client.py index 1d8ec0211..54bc94dac 100644 --- a/packages/opal-client/opal_client/policy_store/opa_client.py +++ b/packages/opal-client/opal_client/policy_store/opa_client.py @@ -20,7 +20,7 @@ from opal_client.policy_store.schemas import PolicyStoreAuth from opal_client.utils import exclude_none_fields, proxy_response from opal_common.engine.parsing import get_rego_package -from opal_common.git.bundle_utils import BundleUtils +from opal_common.git_utils.bundle_utils import BundleUtils from opal_common.paths import PathUtils from opal_common.schemas.policy import DataModule, PolicyBundle, RegoModule from opal_common.schemas.store import JSONPatchAction, StoreTransaction, TransactionType diff --git a/packages/opal-common/opal_common/authentication/casting.py b/packages/opal-common/opal_common/authentication/casting.py index 9713ed2d5..d14a04fc7 100644 --- a/packages/opal-common/opal_common/authentication/casting.py +++ b/packages/opal-common/opal_common/authentication/casting.py @@ -5,7 +5,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception logger = logging.getLogger("opal.authentication") diff --git a/packages/opal-common/opal_common/cli/typer_app.py b/packages/opal-common/opal_common/cli/typer_app.py index 45de0d594..47d38dd39 100644 --- a/packages/opal-common/opal_common/cli/typer_app.py +++ b/packages/opal-common/opal_common/cli/typer_app.py @@ -1,6 +1,5 @@ import typer - -from .commands import all_commands +from opal_common.cli.commands import all_commands def get_typer_app(): diff --git a/packages/opal-common/opal_common/confi/__init__.py b/packages/opal-common/opal_common/confi/__init__.py index ccd3d49ff..114be01c7 100644 --- a/packages/opal-common/opal_common/confi/__init__.py +++ b/packages/opal-common/opal_common/confi/__init__.py @@ -1 +1 @@ -from .confi import * +from opal_common.confi.confi import * diff --git a/packages/opal-common/opal_common/confi/cli.py b/packages/opal-common/opal_common/confi/cli.py index 00e3097ee..0ab88e55a 100644 --- a/packages/opal-common/opal_common/confi/cli.py +++ b/packages/opal-common/opal_common/confi/cli.py @@ -2,10 +2,9 @@ import click import typer +from opal_common.confi.types import ConfiEntry from typer.main import Typer -from .types import ConfiEntry - def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): cli = callback diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index f391c26a2..cbaa9a587 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -15,13 +15,12 @@ from decouple import Csv, UndefinedValueError, config, text_type, undefined from opal_common.authentication.casting import cast_private_key, cast_public_key from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception +from opal_common.confi.cli import get_cli_object_for_config_objects +from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast +from opal_common.logging_utils.decorators import log_exception from pydantic import BaseModel, ValidationError from typer import Typer -from .cli import get_cli_object_for_config_objects -from .types import ConfiDelay, ConfiEntry, no_cast - class Placeholder(object): """Placeholder instead of default value for decouple.""" diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index 7666d47e4..ab18dd0cb 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -2,8 +2,7 @@ from sys import prefix from opal_common.authentication.types import EncryptionKeyFormat, JWTAlgorithm - -from .confi import Confi, confi +from opal_common.confi import Confi, confi _LOG_FORMAT_WITHOUT_PID = "{time} | {name: <40}|{level:^6} | {message}\n{exception}" _LOG_FORMAT_WITH_PID = "{time} | {process} | {name: <40}|{level:^6} | {message}\n{exception}" diff --git a/packages/opal-common/opal_common/engine/__init__.py b/packages/opal-common/opal_common/engine/__init__.py index bbc306e62..33d1be247 100644 --- a/packages/opal-common/opal_common/engine/__init__.py +++ b/packages/opal-common/opal_common/engine/__init__.py @@ -1,2 +1,2 @@ -from .parsing import get_rego_package -from .paths import is_data_module, is_policy_module +from opal_common.engine.parsing import get_rego_package +from opal_common.engine.paths import is_data_module, is_policy_module diff --git a/packages/opal-common/opal_common/fetcher/__init__.py b/packages/opal-common/opal_common/fetcher/__init__.py index 84232e236..70a1f643c 100644 --- a/packages/opal-common/opal_common/fetcher/__init__.py +++ b/packages/opal-common/opal_common/fetcher/__init__.py @@ -1,3 +1,3 @@ -from .engine.fetching_engine import FetchingEngine -from .events import FetcherConfig, FetchEvent -from .fetcher_register import FetcherRegister +from opal_common.fetcher.engine.fetching_engine import FetchingEngine +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister diff --git a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py index a30f033d2..19a636a35 100644 --- a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py @@ -1,8 +1,8 @@ from typing import Coroutine -from ..events import FetcherConfig, FetchEvent -from ..fetcher_register import FetcherRegister -from .core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister class BaseFetchingEngine: diff --git a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py index b083e779e..3da152f14 100644 --- a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py +++ b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py @@ -1,4 +1,4 @@ -from ..events import FetchEvent +from opal_common.fetcher.events import FetchEvent # Callback signatures diff --git a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py index eb816ecf2..6db97b338 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py @@ -1,10 +1,10 @@ import asyncio from typing import Coroutine -from ..events import FetchEvent -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger logger = get_logger("fetch_worker") diff --git a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py index fd50a9f14..b439d4b8d 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py @@ -2,13 +2,13 @@ import uuid from typing import Coroutine, Dict, List, Union -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine -from .core_callbacks import OnFetchFailureCallback -from .fetch_worker import fetch_worker +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.fetch_worker import fetch_worker +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger logger = get_logger("engine") diff --git a/packages/opal-common/opal_common/fetcher/fetch_provider.py b/packages/opal-common/opal_common/fetcher/fetch_provider.py index 62ad97532..c05008fcd 100644 --- a/packages/opal-common/opal_common/fetcher/fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/fetch_provider.py @@ -1,8 +1,7 @@ +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.logger import get_logger from tenacity import retry, stop, wait -from .events import FetchEvent -from .logger import get_logger - logger = get_logger("opal.providers") diff --git a/packages/opal-common/opal_common/fetcher/fetcher_register.py b/packages/opal-common/opal_common/fetcher/fetcher_register.py index 5bf925160..9abf1322c 100644 --- a/packages/opal-common/opal_common/fetcher/fetcher_register.py +++ b/packages/opal-common/opal_common/fetcher/fetcher_register.py @@ -1,11 +1,10 @@ from typing import Dict, Optional, Type +from opal_common.config import opal_common_config +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger - -from ..config import opal_common_config -from .events import FetchEvent -from .fetch_provider import BaseFetchProvider -from .providers.http_fetch_provider import HttpFetchProvider +from opal_common.fetcher.providers.http_fetch_provider import HttpFetchProvider logger = get_logger("opal.fetcher_register") @@ -30,7 +29,7 @@ def __init__(self, config: Optional[Dict[str, BaseFetchProvider]] = None) -> Non if config is not None: self._config = config else: - from ..emport import emport_objects_by_class + from opal_common.emport import emport_objects_by_class # load fetchers fetchers = [] diff --git a/packages/opal-common/opal_common/fetcher/providers/__init__.py b/packages/opal-common/opal_common/fetcher/providers/__init__.py index 8e1f6bf77..ff1078ced 100644 --- a/packages/opal-common/opal_common/fetcher/providers/__init__.py +++ b/packages/opal-common/opal_common/fetcher/providers/__init__.py @@ -1,3 +1,3 @@ -from ...emport import dynamic_all +from opal_common.emport import dynamic_all __all__ = dynamic_all(__file__) diff --git a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py index 61432b751..4b574a8ea 100644 --- a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py @@ -2,10 +2,9 @@ from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient - -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger logger = get_logger("rpc_fetch_provider") diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7261b538b..fc74223ed 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -6,14 +6,13 @@ import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger +from opal_common.http_utils import is_http_error_response +from opal_common.security.sslcontext import get_custom_ssl_context from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger - logger = get_logger("http_fetch_provider") diff --git a/packages/opal-common/opal_common/git/__init__.py b/packages/opal-common/opal_common/git_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/git/__init__.py rename to packages/opal-common/opal_common/git_utils/__init__.py diff --git a/packages/opal-common/opal_common/git/branch_tracker.py b/packages/opal-common/opal_common/git_utils/branch_tracker.py similarity index 97% rename from packages/opal-common/opal_common/git/branch_tracker.py rename to packages/opal-common/opal_common/git_utils/branch_tracker.py index 19bba8770..28f692e15 100644 --- a/packages/opal-common/opal_common/git/branch_tracker.py +++ b/packages/opal-common/opal_common/git_utils/branch_tracker.py @@ -3,8 +3,8 @@ from git import GitCommandError, Head, Remote, Repo from git.objects.commit import Commit -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from tenacity import retry, stop_after_attempt, wait_fixed diff --git a/packages/opal-common/opal_common/git/bundle_maker.py b/packages/opal-common/opal_common/git_utils/bundle_maker.py similarity index 99% rename from packages/opal-common/opal_common/git/bundle_maker.py rename to packages/opal-common/opal_common/git_utils/bundle_maker.py index 0b006b6a7..f9d621a89 100644 --- a/packages/opal-common/opal_common/git/bundle_maker.py +++ b/packages/opal-common/opal_common/git_utils/bundle_maker.py @@ -6,7 +6,7 @@ from git import Repo from git.objects import Commit from opal_common.engine import get_rego_package, is_data_module, is_policy_module -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, VersionedDirectory, VersionedFile, @@ -14,7 +14,7 @@ has_extension, is_under_directories, ) -from opal_common.git.diff_viewer import ( +from opal_common.git_utils.diff_viewer import ( DiffViewer, diffed_file_has_extension, diffed_file_is_under_directories, diff --git a/packages/opal-common/opal_common/git/bundle_utils.py b/packages/opal-common/opal_common/git_utils/bundle_utils.py similarity index 100% rename from packages/opal-common/opal_common/git/bundle_utils.py rename to packages/opal-common/opal_common/git_utils/bundle_utils.py diff --git a/packages/opal-common/opal_common/git/commit_viewer.py b/packages/opal-common/opal_common/git_utils/commit_viewer.py similarity index 100% rename from packages/opal-common/opal_common/git/commit_viewer.py rename to packages/opal-common/opal_common/git_utils/commit_viewer.py diff --git a/packages/opal-common/opal_common/git/diff_viewer.py b/packages/opal-common/opal_common/git_utils/diff_viewer.py similarity index 99% rename from packages/opal-common/opal_common/git/diff_viewer.py rename to packages/opal-common/opal_common/git_utils/diff_viewer.py index af5720f5c..ec6dff9d0 100644 --- a/packages/opal-common/opal_common/git/diff_viewer.py +++ b/packages/opal-common/opal_common/git_utils/diff_viewer.py @@ -4,7 +4,7 @@ from git import Repo from git.diff import Diff, DiffIndex from git.objects.commit import Commit -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.paths import PathUtils DiffFilter = Callable[[Diff], bool] diff --git a/packages/opal-common/opal_common/git/env.py b/packages/opal-common/opal_common/git_utils/env.py similarity index 100% rename from packages/opal-common/opal_common/git/env.py rename to packages/opal-common/opal_common/git_utils/env.py diff --git a/packages/opal-common/opal_common/git/exceptions.py b/packages/opal-common/opal_common/git_utils/exceptions.py similarity index 100% rename from packages/opal-common/opal_common/git/exceptions.py rename to packages/opal-common/opal_common/git_utils/exceptions.py diff --git a/packages/opal-common/opal_common/git/repo_cloner.py b/packages/opal-common/opal_common/git_utils/repo_cloner.py similarity index 98% rename from packages/opal-common/opal_common/git/repo_cloner.py rename to packages/opal-common/opal_common/git_utils/repo_cloner.py index 43bda1ba2..76ebb4949 100644 --- a/packages/opal-common/opal_common/git/repo_cloner.py +++ b/packages/opal-common/opal_common/git_utils/repo_cloner.py @@ -8,8 +8,8 @@ from git import GitCommandError, GitError, Repo from opal_common.config import opal_common_config -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from opal_common.utils import get_filepaths_with_glob from tenacity import RetryError, retry, stop, wait diff --git a/packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py b/packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py similarity index 100% rename from packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py rename to packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py diff --git a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py similarity index 95% rename from packages/opal-common/opal_common/git/tests/branch_tracker_test.py rename to packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py index bc17d6b09..751231a0d 100644 --- a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py @@ -18,8 +18,8 @@ from git import Repo from git.objects.commit import Commit -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed def test_pull_with_no_changes(local_repo_clone: Repo): diff --git a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py similarity index 99% rename from packages/opal-common/opal_common/git/tests/bundle_maker_test.py rename to packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py index 63624f6eb..5e77ad0e5 100644 --- a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py @@ -19,8 +19,8 @@ from git import Repo from git.objects import Commit -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer from opal_common.schemas.policy import PolicyBundle, RegoModule OPA_FILE_EXTENSIONS = (".rego", ".json") diff --git a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py similarity index 98% rename from packages/opal-common/opal_common/git/tests/commit_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py index 91d19fcf1..1f9ca522a 100644 --- a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py @@ -19,7 +19,7 @@ from git import Repo from git.objects import Commit -from opal_common.git.commit_viewer import CommitViewer, VersionedNode +from opal_common.git_utils.commit_viewer import CommitViewer, VersionedNode def node_paths(nodes: List[VersionedNode]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/conftest.py b/packages/opal-common/opal_common/git_utils/tests/conftest.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/conftest.py rename to packages/opal-common/opal_common/git_utils/tests/conftest.py diff --git a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py similarity index 97% rename from packages/opal-common/opal_common/git/tests/diff_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py index 6dc77ec4b..bcfbb93be 100644 --- a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py @@ -20,8 +20,11 @@ from git import Diff, Repo from git.objects import Commit -from opal_common.git.commit_viewer import VersionedFile -from opal_common.git.diff_viewer import DiffViewer, diffed_file_is_under_directories +from opal_common.git_utils.commit_viewer import VersionedFile +from opal_common.git_utils.diff_viewer import ( + DiffViewer, + diffed_file_is_under_directories, +) def diff_paths(diffs: List[Diff]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py similarity index 96% rename from packages/opal-common/opal_common/git/tests/repo_cloner_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py index ffe6a02fa..567f3707b 100644 --- a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py @@ -18,8 +18,8 @@ from git import Repo from opal_common.confi import Confi -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner VALID_REPO_REMOTE_URL_HTTPS = "https://github.com/permitio/fastapi_websocket_pubsub.git" diff --git a/packages/opal-common/opal_common/git/tests/repo_watcher_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/repo_watcher_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py diff --git a/packages/opal-common/opal_common/http.py b/packages/opal-common/opal_common/http_utils.py similarity index 100% rename from packages/opal-common/opal_common/http.py rename to packages/opal-common/opal_common/http_utils.py diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index 5f1229a80..8e826abd6 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -2,13 +2,12 @@ import sys from loguru import logger - -from .config import opal_common_config -from .logging.filter import ModuleFilter -from .logging.formatter import Formatter -from .logging.intercept import InterceptHandler -from .logging.thirdparty import hijack_uvicorn_logs -from .monitoring.apm import fix_ddtrace_logging +from opal_common.config import opal_common_config +from opal_common.logging_utils.filter import ModuleFilter +from opal_common.logging_utils.formatter import Formatter +from opal_common.logging_utils.intercept import InterceptHandler +from opal_common.logging_utils.thirdparty import hijack_uvicorn_logs +from opal_common.monitoring.apm import fix_ddtrace_logging def configure_logs(): diff --git a/packages/opal-common/opal_common/logging/__init__.py b/packages/opal-common/opal_common/logging_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/logging/__init__.py rename to packages/opal-common/opal_common/logging_utils/__init__.py diff --git a/packages/opal-common/opal_common/logging/decorators.py b/packages/opal-common/opal_common/logging_utils/decorators.py similarity index 100% rename from packages/opal-common/opal_common/logging/decorators.py rename to packages/opal-common/opal_common/logging_utils/decorators.py diff --git a/packages/opal-common/opal_common/logging/filter.py b/packages/opal-common/opal_common/logging_utils/filter.py similarity index 100% rename from packages/opal-common/opal_common/logging/filter.py rename to packages/opal-common/opal_common/logging_utils/filter.py diff --git a/packages/opal-common/opal_common/logging/formatter.py b/packages/opal-common/opal_common/logging_utils/formatter.py similarity index 100% rename from packages/opal-common/opal_common/logging/formatter.py rename to packages/opal-common/opal_common/logging_utils/formatter.py diff --git a/packages/opal-common/opal_common/logging/intercept.py b/packages/opal-common/opal_common/logging_utils/intercept.py similarity index 100% rename from packages/opal-common/opal_common/logging/intercept.py rename to packages/opal-common/opal_common/logging_utils/intercept.py diff --git a/packages/opal-common/opal_common/logging/thirdparty.py b/packages/opal-common/opal_common/logging_utils/thirdparty.py similarity index 100% rename from packages/opal-common/opal_common/logging/thirdparty.py rename to packages/opal-common/opal_common/logging_utils/thirdparty.py diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 9b6487907..7adc9ad70 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -6,7 +6,9 @@ import aiohttp from fastapi import status from fastapi.exceptions import HTTPException -from opal_common.git.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor +from opal_common.git_utils.tar_file_to_local_git_extractor import ( + TarFileToLocalGitExtractor, +) from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_common.utils import ( diff --git a/packages/opal-common/opal_common/sources/git_policy_source.py b/packages/opal-common/opal_common/sources/git_policy_source.py index bffe8517d..8252cd4ce 100644 --- a/packages/opal-common/opal_common/sources/git_policy_source.py +++ b/packages/opal-common/opal_common/sources/git_policy_source.py @@ -1,9 +1,9 @@ from typing import Optional from git import Repo -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 5ea85c047..36932ee30 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -12,7 +12,7 @@ from ddtrace import tracer from git import Repo from opal_common.async_utils import run_sync -from opal_common.git.bundle_maker import BundleMaker +from opal_common.git_utils.bundle_maker import BundleMaker from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.policy_source import ( diff --git a/packages/opal-server/opal_server/main.py b/packages/opal-server/opal_server/main.py index 908c4561d..7e61e2a66 100644 --- a/packages/opal-server/opal_server/main.py +++ b/packages/opal-server/opal_server/main.py @@ -1,5 +1,5 @@ def create_app(*args, **kwargs): - from .server import OpalServer + from opal_server.server import OpalServer server = OpalServer(*args, **kwargs) return server.app diff --git a/packages/opal-server/opal_server/policy/bundles/api.py b/packages/opal-server/opal_server/policy/bundles/api.py index 223e72001..ae1da68ef 100644 --- a/packages/opal-server/opal_server/policy/bundles/api.py +++ b/packages/opal-server/opal_server/policy/bundles/api.py @@ -4,11 +4,11 @@ import fastapi.responses from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status -from git import Repo +from git.repo import Repo from opal_common.confi.confi import load_conf_if_none -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_server.config import opal_server_config diff --git a/packages/opal-server/opal_server/policy/watcher/callbacks.py b/packages/opal-server/opal_server/policy/watcher/callbacks.py index 62a30f16b..1b5f65590 100644 --- a/packages/opal-server/opal_server/policy/watcher/callbacks.py +++ b/packages/opal-server/opal_server/policy/watcher/callbacks.py @@ -3,13 +3,13 @@ from typing import List, Optional from git.objects import Commit -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, FileFilter, find_ignore_match, has_extension, ) -from opal_common.git.diff_viewer import DiffViewer +from opal_common.git_utils.diff_viewer import DiffViewer from opal_common.logger import logger from opal_common.paths import PathUtils from opal_common.schemas.policy import ( diff --git a/packages/opal-server/opal_server/policy/watcher/factory.py b/packages/opal-server/opal_server/policy/watcher/factory.py index 10fb1b19c..6d94d6fc4 100644 --- a/packages/opal-server/opal_server/policy/watcher/factory.py +++ b/packages/opal-server/opal_server/policy/watcher/factory.py @@ -3,7 +3,7 @@ from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint from opal_common.confi.confi import load_conf_if_none -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.sources.api_policy_source import ApiPolicySource from opal_common.sources.git_policy_source import GitPolicySource diff --git a/packages/opal-server/opal_server/redis.py b/packages/opal-server/opal_server/redis_utils.py similarity index 100% rename from packages/opal-server/opal_server/redis.py rename to packages/opal-server/opal_server/redis_utils.py diff --git a/packages/opal-server/opal_server/scopes/scope_repository.py b/packages/opal-server/opal_server/scopes/scope_repository.py index b3627741c..d9f5d9d20 100644 --- a/packages/opal-server/opal_server/scopes/scope_repository.py +++ b/packages/opal-server/opal_server/scopes/scope_repository.py @@ -1,7 +1,7 @@ from typing import List from opal_common.schemas.scopes import Scope -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB class ScopeNotFoundError(Exception): diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 533b397a8..f0104e7bf 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -7,7 +7,7 @@ import git from ddtrace import tracer from fastapi_websocket_pubsub import PubSubEndpoint -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.logger import logger from opal_common.schemas.policy import PolicyUpdateMessageNotification from opal_common.schemas.policy_source import GitPolicyScopeSource diff --git a/packages/opal-server/opal_server/scopes/task.py b/packages/opal-server/opal_server/scopes/task.py index b3a577161..83b2b10f0 100644 --- a/packages/opal-server/opal_server/scopes/task.py +++ b/packages/opal-server/opal_server/scopes/task.py @@ -7,7 +7,7 @@ from opal_common.logger import logger from opal_server.config import opal_server_config from opal_server.policy.watcher.task import BasePolicyWatcherTask -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.scope_repository import ScopeRepository from opal_server.scopes.service import ScopesService diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 6a946a8c0..34d9905c3 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -32,7 +32,7 @@ from opal_server.policy.webhook.api import init_git_webhook_router from opal_server.publisher import setup_broadcaster_keepalive_task from opal_server.pubsub import PubSub -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.api import init_scope_router from opal_server.scopes.loader import load_scopes from opal_server.scopes.scope_repository import ScopeRepository diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index b40546ae6..0c2c15cb8 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,5 +1,3 @@ -import os - from opal_common.logger import logger