From f7d59ffeb5f2241a9a0d20926b23221b92b02a38 Mon Sep 17 00:00:00 2001 From: Dan Fuchs Date: Fri, 8 Nov 2024 09:54:01 -0600 Subject: [PATCH] Config from a single YAML file --- src/mobu/config.py | 216 +++++++++++++++---- src/mobu/constants.py | 3 + src/mobu/dependencies/config.py | 64 ++++++ src/mobu/dependencies/github.py | 39 ---- src/mobu/factory.py | 9 +- src/mobu/github_config.py | 119 ---------- src/mobu/handlers/external.py | 8 +- src/mobu/handlers/github_ci_app.py | 20 +- src/mobu/handlers/github_refresh_app.py | 18 +- src/mobu/handlers/internal.py | 11 +- src/mobu/main.py | 98 +++++---- src/mobu/services/business/notebookrunner.py | 5 +- src/mobu/services/business/nublado.py | 3 +- src/mobu/services/business/tap.py | 3 +- src/mobu/services/github_ci/ci_manager.py | 5 +- src/mobu/services/manager.py | 13 +- src/mobu/services/monkey.py | 9 +- src/mobu/status.py | 3 +- src/mobu/storage/gafaelfawr.py | 11 +- tests/autostart_test.py | 42 +--- tests/business/nubladopythonloop_test.py | 4 +- tests/conftest.py | 66 ++---- tests/data/config/autostart.yaml | 34 +++ tests/data/config/base.yaml | 7 + tests/data/config/github_ci_app.yaml | 20 ++ tests/data/config/github_refresh_app.yaml | 12 ++ tests/handlers/index_test.py | 3 +- tests/handlers/internal_test.py | 3 +- tests/monkeyflocker_test.py | 12 +- tests/support/config.py | 27 +++ tests/support/gafaelfawr.py | 3 +- 31 files changed, 501 insertions(+), 389 deletions(-) create mode 100644 src/mobu/dependencies/config.py delete mode 100644 src/mobu/github_config.py create mode 100644 tests/data/config/autostart.yaml create mode 100644 tests/data/config/base.yaml create mode 100644 tests/data/config/github_ci_app.yaml create mode 100644 tests/data/config/github_refresh_app.yaml create mode 100644 tests/support/config.py diff --git a/src/mobu/config.py b/src/mobu/config.py index 3cf09b4a..9352c496 100644 --- a/src/mobu/config.py +++ b/src/mobu/config.py @@ -3,63 +3,159 @@ from __future__ import annotations from pathlib import Path +from textwrap import dedent +from typing import Self -from pydantic import Field, HttpUrl -from pydantic_settings import BaseSettings +import yaml +from pydantic import AliasChoices, Field, HttpUrl +from pydantic.alias_generators import to_camel +from pydantic_settings import BaseSettings, SettingsConfigDict from safir.logging import LogLevel, Profile +from safir.metrics import MetricsConfiguration, metrics_configuration_factory + +from mobu.models.flock import FlockConfig + +from .models.user import User __all__ = [ "Configuration", - "config", + "GitHubCiAppConfig", + "GitHubRefreshAppConfig", ] -class Configuration(BaseSettings): - """Configuration for mobu.""" +class GitHubCiAppConfig(BaseSettings): + """Configuration for GitHub CI app functionality if it is enabled.""" - alert_hook: HttpUrl | None = Field( - None, - title="Slack webhook URL used for sending alerts", + model_config = SettingsConfigDict( + alias_generator=to_camel, extra="forbid", populate_by_name=True + ) + + id: int = Field( + ..., + title="Github CI app id", description=( - "An https URL, which should be considered secret. If not set or" - " set to `None`, this feature will be disabled." + "Found on the GitHub app's settings page (NOT the installation" + " configuration page). For example:" + " https://github.com/organizations/lsst-sqre/settings/apps/mobu-ci-data-dev-lsst-cloud" ), - validation_alias="MOBU_ALERT_HOOK", - examples=["https://slack.example.com/ADFAW1452DAF41/"], + examples=[123456], + validation_alias=AliasChoices("MOBU_GITHUB_CI_APP_ID", "id"), ) - autostart: Path | None = Field( - None, - title="Path to YAML file defining flocks to automatically start", + private_key: str = Field( + ..., + title="Github CI app private key", description=( - "If given, the YAML file must contain a list of flock" - " specifications. All flocks given there will be automatically" - " started when mobu starts." + "Generated when the GitHub app was set up. This should NOT be" + " base64 enocded, and will contain newlines. You can find this" + " in 1Password; check the Phalanx mobu values for more details." + ), + examples=[ + dedent(""" + -----BEGIN RSA PRIVATE KEY----- + abc123MeowMeow456abc123MeowMeow456abc123MeowMeow456abc123MeowMeo + abc123MeowMeow456abc123MeowMeow456abc123MeowMeow456abc123MeowMeo + abc123MeowMeow456abc123MeowMeow456abc123MeowMeow456abc123MeowMeo + etc, etc + -----END RSA PRIVATE KEY----- + """) + ], + validation_alias=AliasChoices( + "MOBU_GITHUB_CI_APP_PRIVATE_KEY", "privateKey" ), - validation_alias="MOBU_AUTOSTART_PATH", - examples=["/etc/mobu/autostart.yaml"], ) - github_ci_app_config_path: Path | None = Field( - None, - title="GitHub CI app config path", + webhook_secret: str = Field( + ..., + title="Github CI app webhook secret", + description=( + "Generated when the GitHub app was set up. You can find this" + " in 1Password; check the Phalanx mobu values for more details." + ), + validation_alias=AliasChoices( + "MOBU_GITHUB_CI_APP_WEBHOOK_SECRET", "webhookSecret" + ), + ) + + users: list[User] = Field( + ..., + title="Environment users for CI jobs to run as.", + description=( + "Must be prefixed with 'bot-', like all mobu users. In " + " environments without Firestore, users have to be provisioned" + " by environment admins, and their usernames, uids, and guids must" + " be specified here. In environments with firestore, only " + " usernames need to be specified, but you still need to explicitly" + " specify as many users as needed to get the amount of concurrency" + " that you want." + ), + ) + + scopes: list[str] = Field( + ..., + title="Gafaelfawr Scopes", + description=( + "A list of Gafaelfawr scopes that will be granted to the" + " user when running notebooks for a GitHub CI app check." + ), + ) + + accepted_github_orgs: list[str] = Field( + [], + title="Allowed GitHub organizations.", + description=( + "Any webhook payload request from a repo in an organization not in" + " this list will get a 403 response." + ), + ) + + +class GitHubRefreshAppConfig(BaseSettings): + """Configuration for GitHub refresh app functionality.""" + + model_config = SettingsConfigDict( + alias_generator=to_camel, extra="forbid", populate_by_name=True + ) + + webhook_secret: str = Field( + ..., + title="Github refresh app webhook secret", description=( - "Path to YAML file defining settings for GitHub CI app" - " integration" + "Generated when the GitHub app was set up. You can find this" + " in 1Password; check the Phalanx mobu values for more details." + ), + validation_alias=AliasChoices( + "MOBU_GITHUB_REFRESH_APP_WEBHOOK_SECRET", "webhookSecret" ), - validation_alias="MOBU_GITHUB_CI_APP_CONFIG_PATH", - examples=["/etc/mobu/github-ci-app.yaml"], ) - github_refresh_app_config_path: Path | None = Field( + accepted_github_orgs: list[str] = Field( + [], + title="Allowed GitHub organizations.", + description=( + "Any webhook payload request from a repo in an organization not in" + " this list will get a 403 response." + ), + ) + + +class Configuration(BaseSettings): + """Configuration for mobu.""" + + model_config = SettingsConfigDict( + alias_generator=to_camel, extra="forbid", populate_by_name=True + ) + + alert_hook: HttpUrl | None = Field( None, - title="GitHub refresh app config path", + title="Slack webhook URL used for sending alerts", description=( - "Path to YAML file defining settings for GitHub refresh app" - " integration" + "An https URL, which should be considered secret. If not set or" + " set to `None`, this feature will be disabled." ), - validation_alias="MOBU_GITHUB_REFRESH_APP_CONFIG_PATH", - examples=["/etc/mobu/github-refresh-app.yaml"], + examples=["https://slack.example.com/ADFAW1452DAF41/"], + validation_alias=AliasChoices("MOBU_ALERT_HOOK", "alertHook"), ) environment_url: HttpUrl | None = Field( @@ -71,7 +167,6 @@ class Configuration(BaseSettings): " suite easier. If it is not set to a valid URL, mobu will abort" " during startup." ), - validation_alias="MOBU_ENVIRONMENT_URL", examples=["https://data.example.org/"], ) @@ -83,8 +178,10 @@ class Configuration(BaseSettings): " get a token for the user. This is only optional to make writing" " tests easier. mobu will abort during startup if it is not set." ), - validation_alias="MOBU_GAFAELFAWR_TOKEN", examples=["gt-vilSCi1ifK_MyuaQgMD2dQ.d6SIJhowv5Hs3GvujOyUig"], + validation_alias=AliasChoices( + "MOBU_GAFAELFAWR_TOKEN", "gafaelfawrToken" + ), ) available_services: set[str] = Field( @@ -96,7 +193,6 @@ class Configuration(BaseSettings): " When we have a service discovery mechanism in place, it should" " be used here." ), - validation_alias="MOBU_AVAILABLE_SERVICES", examples=[{"tap", "ssotap", "butler"}], ) @@ -104,27 +200,63 @@ class Configuration(BaseSettings): "mobu", title="Name of application", description="Doubles as the root HTTP endpoint path.", - validation_alias="MOBU_NAME", + ) + + autostart: list[FlockConfig] = Field( + default=[], + title="Autostart config", + description=( + "Configuration of flocks of monkeys that will run businesses" + " repeatedly as long as Mobu is running." + ), ) path_prefix: str = Field( "/mobu", title="URL prefix for application API", - validation_alias="MOBU_PATH_PREFIX", ) profile: Profile = Field( Profile.development, title="Application logging profile", - validation_alias="MOBU_LOGGING_PROFILE", ) log_level: LogLevel = Field( LogLevel.INFO, title="Log level of the application's logger", - validation_alias="MOBU_LOG_LEVEL", ) + github_ci_app: GitHubCiAppConfig | None = Field( + None, + title="GitHub CI app config", + description=("Configuration for GitHub CI app functionality"), + ) + + github_refresh_app: GitHubRefreshAppConfig | None = Field( + None, + title="GitHub refresh app config", + description=("Configuration for GitHub refresh app functionality"), + ) + + metrics: MetricsConfiguration = Field( + default_factory=metrics_configuration_factory, + title="Metrics configuration", + description="Config for emitting app metrics.", + ) + + @classmethod + def from_file(cls, path: Path) -> Self: + """Construct a Configuration object from a configuration file. + + Parameters + ---------- + path + Path to the configuration file in YAML. -config = Configuration() -"""Configuration for mobu.""" + Returns + ------- + Config + The corresponding `Configuration` object. + """ + with path.open("r") as f: + return cls.model_validate(yaml.safe_load(f)) diff --git a/src/mobu/constants.py b/src/mobu/constants.py index 61e0f712..cfc3c4db 100644 --- a/src/mobu/constants.py +++ b/src/mobu/constants.py @@ -6,6 +6,7 @@ from pathlib import Path __all__ = [ + "CONFIGURATION_PATH", "GITHUB_REPO_CONFIG_PATH", "GITHUB_WEBHOOK_WAIT_SECONDS", "NOTEBOOK_REPO_BRANCH", @@ -15,6 +16,8 @@ "WEBSOCKET_OPEN_TIMEOUT", ] +CONFIGURATION_PATH = Path("/etc/mobu/config.yaml") +"""Default path to configuration.""" GITHUB_REPO_CONFIG_PATH = Path("mobu.yaml") """The path to a config file with repo-specific configuration.""" diff --git a/src/mobu/dependencies/config.py b/src/mobu/dependencies/config.py new file mode 100644 index 00000000..792a896c --- /dev/null +++ b/src/mobu/dependencies/config.py @@ -0,0 +1,64 @@ +"""Config dependency.""" + +import os +from pathlib import Path + +from ..config import Configuration +from ..constants import CONFIGURATION_PATH + +__all__ = [ + "ConfigDependency", + "config_dependency", +] + + +class ConfigDependency: + """Dependency to manage a cached Mobu configuration. + + The controller configuration is read on first request, cached, and + returned to all dependency callers unless `~ConfigDependency.set_path` is + called to change the configuration. + + Parameters + ---------- + path + Path to the Nublado mobu configuration. + """ + + def __init__(self, path: Path = CONFIGURATION_PATH) -> None: + # This is needed for unit tests to specify an alternate config file + # when mobu is started in a separate process from the tests. + if test_path := os.environ.get("MOBU_UNIT_TEST_PATH"): + path = Path(test_path) + self._path = path + self._config: Configuration | None = None + + async def __call__(self) -> Configuration: + return self.config + + @property + def config(self) -> Configuration: + """Load configuration if needed and return it.""" + if self._config is None: + self._config = Configuration.from_file(self._path) + return self._config + + @property + def is_initialized(self) -> bool: + """Whether the configuration has been initialized.""" + return self._config is not None + + def set_path(self, path: Path) -> None: + """Change the configuration path and reload. + + Parameters + ---------- + path + New configuration path. + """ + self._path = path + self._config = Configuration.from_file(path) + + +config_dependency = ConfigDependency() +"""The dependency that will return the global configuration.""" diff --git a/src/mobu/dependencies/github.py b/src/mobu/dependencies/github.py index 0ef70d8d..7aa2e974 100644 --- a/src/mobu/dependencies/github.py +++ b/src/mobu/dependencies/github.py @@ -1,47 +1,10 @@ """Dependencies GitHub CI app functionality.""" -from pathlib import Path - -import yaml - -from ..github_config import GitHubCiAppConfig, GitHubRefreshAppConfig from ..models.user import User from ..services.github_ci.ci_manager import CiManager from .context import ContextDependency -class GitHubCiAppConfigDependency: - """Config for GitHub CI app integration, loaded from a file.""" - - def __init__(self) -> None: - self.config: GitHubCiAppConfig - - def __call__(self) -> GitHubCiAppConfig: - return self.config - - def initialize(self, path: Path) -> None: - self.config = GitHubCiAppConfig.model_validate( - yaml.safe_load(path.read_text()) - ) - - -class GitHubRefreshAppConfigDependency: - """Config for GitHub refresh app integration, loaded from a - file. - """ - - def __init__(self) -> None: - self.config: GitHubRefreshAppConfig - - def __call__(self) -> GitHubRefreshAppConfig: - return self.config - - def initialize(self, path: Path) -> None: - self.config = GitHubRefreshAppConfig.model_validate( - yaml.safe_load(path.read_text()) - ) - - class CiManagerDependency: """A process-global object to manage background CI workers. @@ -103,7 +66,5 @@ def __call__(self) -> CiManager | None: return None -github_refresh_app_config_dependency = GitHubRefreshAppConfigDependency() -github_ci_app_config_dependency = GitHubCiAppConfigDependency() ci_manager_dependency = CiManagerDependency() maybe_ci_manager_dependency = MaybeCiManagerDependency(ci_manager_dependency) diff --git a/src/mobu/factory.py b/src/mobu/factory.py index 58c92e8e..ae7ac06d 100644 --- a/src/mobu/factory.py +++ b/src/mobu/factory.py @@ -7,7 +7,7 @@ from safir.slack.webhook import SlackWebhookClient from structlog.stdlib import BoundLogger -from .config import config +from .dependencies.config import config_dependency from .models.solitary import SolitaryConfig from .services.manager import FlockManager from .services.solitary import Solitary @@ -68,6 +68,7 @@ def __init__( ) -> None: self._context = context self._logger = logger if logger else structlog.get_logger("mobu") + self._config = config_dependency.config def create_slack_webhook_client(self) -> SlackWebhookClient | None: """Create a Slack webhook client if configured for Slack alerting. @@ -78,9 +79,11 @@ def create_slack_webhook_client(self) -> SlackWebhookClient | None: Newly-created Slack client, or `None` if Slack alerting is not configured. """ - if not config.alert_hook: + if not self._config.alert_hook: return None - return SlackWebhookClient(str(config.alert_hook), "Mobu", self._logger) + return SlackWebhookClient( + str(self._config.alert_hook), "Mobu", self._logger + ) def create_solitary(self, solitary_config: SolitaryConfig) -> Solitary: """Create a runner for a solitary monkey. diff --git a/src/mobu/github_config.py b/src/mobu/github_config.py deleted file mode 100644 index 8bf4a074..00000000 --- a/src/mobu/github_config.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Config for GitHub application integrations.""" - -from textwrap import dedent - -from pydantic import Field -from pydantic.alias_generators import to_camel -from pydantic_settings import BaseSettings, SettingsConfigDict - -from .models.user import User - - -class GitHubCiAppConfig(BaseSettings): - """Configuration for GitHub CI app functionality if it is enabled.""" - - model_config = SettingsConfigDict( - alias_generator=to_camel, extra="forbid", populate_by_name=True - ) - - id: int = Field( - ..., - title="Github CI app id", - description=( - "Found on the GitHub app's settings page (NOT the installation" - " configuration page). For example:" - " https://github.com/organizations/lsst-sqre/settings/apps/mobu-ci-data-dev-lsst-cloud" - ), - validation_alias="MOBU_GITHUB_CI_APP_ID", - examples=[123456], - ) - - private_key: str = Field( - ..., - title="Github CI app private key", - description=( - "Generated when the GitHub app was set up. This should NOT be" - " base64 enocded, and will contain newlines. You can find this" - " in 1Password; check the Phalanx mobu values for more details." - ), - validation_alias="MOBU_GITHUB_CI_APP_PRIVATE_KEY", - examples=[ - dedent(""" - -----BEGIN RSA PRIVATE KEY----- - abc123MeowMeow456abc123MeowMeow456abc123MeowMeow456abc123MeowMeo - abc123MeowMeow456abc123MeowMeow456abc123MeowMeow456abc123MeowMeo - abc123MeowMeow456abc123MeowMeow456abc123MeowMeow456abc123MeowMeo - etc, etc - -----END RSA PRIVATE KEY----- - """) - ], - ) - - webhook_secret: str = Field( - ..., - title="Github CI app webhook secret", - description=( - "Generated when the GitHub app was set up. You can find this" - " in 1Password; check the Phalanx mobu values for more details." - ), - validation_alias="MOBU_GITHUB_CI_APP_WEBHOOK_SECRET", - ) - - users: list[User] = Field( - ..., - title="Environment users for CI jobs to run as.", - description=( - "Must be prefixed with 'bot-', like all mobu users. In " - " environments without Firestore, users have to be provisioned" - " by environment admins, and their usernames, uids, and guids must" - " be specified here. In environments with firestore, only " - " usernames need to be specified, but you still need to explicitly" - " specify as many users as needed to get the amount of concurrency" - " that you want." - ), - ) - - scopes: list[str] = Field( - ..., - title="Gafaelfawr Scopes", - description=( - "A list of Gafaelfawr scopes that will be granted to the" - " user when running notebooks for a GitHub CI app check." - ), - ) - - accepted_github_orgs: list[str] = Field( - [], - title="Allowed GitHub organizations.", - description=( - "Any webhook payload request from a repo in an organization not in" - " this list will get a 403 response." - ), - ) - - -class GitHubRefreshAppConfig(BaseSettings): - """Configuration for GitHub refresh app functionality.""" - - model_config = SettingsConfigDict( - alias_generator=to_camel, extra="forbid", populate_by_name=True - ) - - webhook_secret: str = Field( - ..., - title="Github refresh app webhook secret", - description=( - "Generated when the GitHub app was set up. You can find this" - " in 1Password; check the Phalanx mobu values for more details." - ), - validation_alias="MOBU_GITHUB_REFRESH_APP_WEBHOOK_SECRET", - ) - - accepted_github_orgs: list[str] = Field( - [], - title="Allowed GitHub organizations.", - description=( - "Any webhook payload request from a repo in an organization not in" - " this list will get a 403 response." - ), - ) diff --git a/src/mobu/handlers/external.py b/src/mobu/handlers/external.py index c61933bc..e79a25a6 100644 --- a/src/mobu/handlers/external.py +++ b/src/mobu/handlers/external.py @@ -12,7 +12,9 @@ from safir.models import ErrorModel from safir.slack.webhook import SlackRouteErrorHandler -from ..config import config +from mobu.config import Configuration + +from ..dependencies.config import config_dependency from ..dependencies.context import RequestContext, context_dependency from ..dependencies.github import maybe_ci_manager_dependency from ..models.flock import FlockConfig, FlockData, FlockSummary @@ -49,7 +51,9 @@ def render(self, content: Any) -> bytes: response_model_exclude_none=True, summary="Application metadata", ) -async def get_index() -> Index: +async def get_index( + config: Annotated[Configuration, Depends(config_dependency)], +) -> Index: metadata = get_metadata( package_name="mobu", application_name=config.name, diff --git a/src/mobu/handlers/github_ci_app.py b/src/mobu/handlers/github_ci_app.py index b752fdb9..31c96b90 100644 --- a/src/mobu/handlers/github_ci_app.py +++ b/src/mobu/handlers/github_ci_app.py @@ -12,13 +12,11 @@ ) from safir.slack.webhook import SlackRouteErrorHandler +from ..config import Configuration from ..constants import GITHUB_WEBHOOK_WAIT_SECONDS +from ..dependencies.config import config_dependency from ..dependencies.context import RequestContext, anonymous_context_dependency -from ..dependencies.github import ( - ci_manager_dependency, - github_ci_app_config_dependency, -) -from ..github_config import GitHubCiAppConfig +from ..dependencies.github import ci_manager_dependency from ..services.github_ci.ci_manager import CiManager __all__ = ["api_router"] @@ -39,9 +37,7 @@ ) async def post_webhook( context: Annotated[RequestContext, Depends(anonymous_context_dependency)], - ci_app_config: Annotated[ - GitHubCiAppConfig, Depends(github_ci_app_config_dependency) - ], + config: Annotated[Configuration, Depends(config_dependency)], ci_manager: Annotated[CiManager, Depends(ci_manager_dependency)], ) -> None: """Process GitHub webhook events for the mobu CI GitHubApp. @@ -49,18 +45,20 @@ async def post_webhook( Rejects webhooks from organizations that are not explicitly allowed via the mobu config. This should be exposed via a Gafaelfawr anonymous ingress. """ - webhook_secret = ci_app_config.webhook_secret + if config.github_ci_app is None: + raise RuntimeError("GitHub CI app configuration is missing") + webhook_secret = config.github_ci_app.webhook_secret body = await context.request.body() event = Event.from_http( context.request.headers, body, secret=webhook_secret ) owner = event.data.get("organization", {}).get("login") - if owner not in ci_app_config.accepted_github_orgs: + if owner not in config.github_ci_app.accepted_github_orgs: context.logger.debug( "Ignoring GitHub event for unaccepted org", owner=owner, - accepted_orgs=ci_app_config.accepted_github_orgs, + accepted_orgs=config.github_ci_app.accepted_github_orgs, ) raise HTTPException( status_code=403, diff --git a/src/mobu/handlers/github_refresh_app.py b/src/mobu/handlers/github_refresh_app.py index 0aacedbc..ec327776 100644 --- a/src/mobu/handlers/github_refresh_app.py +++ b/src/mobu/handlers/github_refresh_app.py @@ -9,10 +9,10 @@ from safir.github.webhooks import GitHubPushEventModel from safir.slack.webhook import SlackRouteErrorHandler +from ..config import Configuration from ..constants import GITHUB_WEBHOOK_WAIT_SECONDS +from ..dependencies.config import config_dependency from ..dependencies.context import RequestContext, anonymous_context_dependency -from ..dependencies.github import github_refresh_app_config_dependency -from ..github_config import GitHubRefreshAppConfig __all__ = ["api_router"] @@ -32,9 +32,9 @@ ) async def post_webhook( context: Annotated[RequestContext, Depends(anonymous_context_dependency)], - refresh_app_config: Annotated[ - GitHubRefreshAppConfig, - Depends(github_refresh_app_config_dependency), + config: Annotated[ + Configuration, + Depends(config_dependency), ], ) -> None: """Process GitHub webhook events for the mobu refresh GitHub app. @@ -42,18 +42,20 @@ async def post_webhook( Rejects webhooks from organizations that are not explicitly allowed via the mobu config. This should be exposed via a Gafaelfawr anonymous ingress. """ - webhook_secret = refresh_app_config.webhook_secret + if config.github_refresh_app is None: + raise RuntimeError("GitHub refresh app configuration is missing") + webhook_secret = config.github_refresh_app.webhook_secret body = await context.request.body() event = Event.from_http( context.request.headers, body, secret=webhook_secret ) owner = event.data.get("organization", {}).get("login") - if owner not in refresh_app_config.accepted_github_orgs: + if owner not in config.github_refresh_app.accepted_github_orgs: context.logger.debug( "Ignoring GitHub event for unaccepted org", owner=owner, - accepted_orgs=refresh_app_config.accepted_github_orgs, + accepted_orgs=config.github_refresh_app.accepted_github_orgs, ) raise HTTPException( status_code=403, diff --git a/src/mobu/handlers/internal.py b/src/mobu/handlers/internal.py index e9872c4a..425e1399 100644 --- a/src/mobu/handlers/internal.py +++ b/src/mobu/handlers/internal.py @@ -7,11 +7,14 @@ or other information that should not be visible outside the Kubernetes cluster. """ -from fastapi import APIRouter +from typing import Annotated + +from fastapi import APIRouter, Depends from safir.metadata import Metadata, get_metadata from safir.slack.webhook import SlackRouteErrorHandler -from ..config import config +from ..config import Configuration +from ..dependencies.config import config_dependency internal_router = APIRouter(route_class=SlackRouteErrorHandler) """FastAPI router for all internal handlers.""" @@ -30,7 +33,9 @@ response_model_exclude_none=True, summary="Application metadata", ) -async def get_index() -> Metadata: +async def get_index( + config: Annotated[Configuration, Depends(config_dependency)], +) -> Metadata: return get_metadata( package_name="mobu", application_name=config.name, diff --git a/src/mobu/main.py b/src/mobu/main.py index 683ad870..77f4746e 100644 --- a/src/mobu/main.py +++ b/src/mobu/main.py @@ -24,13 +24,9 @@ from safir.slack.webhook import SlackRouteErrorHandler from .asyncio import schedule_periodic -from .config import config +from .dependencies.config import config_dependency from .dependencies.context import context_dependency -from .dependencies.github import ( - ci_manager_dependency, - github_ci_app_config_dependency, - github_refresh_app_config_dependency, -) +from .dependencies.github import ci_manager_dependency from .handlers.external import external_router from .handlers.github_ci_app import api_router as github_ci_app_router from .handlers.github_refresh_app import ( @@ -45,6 +41,7 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Set up and tear down the the base application.""" + config = config_dependency.config if not config.environment_url: raise RuntimeError("MOBU_ENVIRONMENT_URL was not set") if not config.gafaelfawr_token: @@ -56,23 +53,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: status_interval = timedelta(days=1) app.state.periodic_status = schedule_periodic(post_status, status_interval) - if config.github_refresh_app_config_path: - github_refresh_app_config_dependency.initialize( - config.github_refresh_app_config_path - ) - - if config.github_ci_app_config_path: - github_ci_app_config_dependency.initialize( - config.github_ci_app_config_path - ) - ci_app_config = github_ci_app_config_dependency.config - + if config.github_ci_app: ci_manager_dependency.initialize( base_context=context_dependency, - github_app_id=ci_app_config.id, - github_private_key=ci_app_config.private_key, - scopes=ci_app_config.scopes, - users=ci_app_config.users, + github_app_id=config.github_ci_app.id, + github_private_key=config.github_ci_app.private_key, + scopes=config.github_ci_app.scopes, + users=config.github_ci_app.users, ) await ci_manager_dependency.ci_manager.start() @@ -83,35 +70,64 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.state.periodic_status.cancel() -configure_logging( - name="mobu", profile=config.profile, log_level=config.log_level -) -if config.profile == Profile.production: - configure_uvicorn_logging(config.log_level) - +def create_app(*, load_config: bool = True) -> FastAPI: + """Create the FastAPI application. + + This is in a function rather than using a global variable (as is more + typical for FastAPI) because some routing depends on configuration + settings and we therefore want to recreate the application between tests. + + Parameters + ---------- + load_config + If set to `False`, do not try to load the configuration. This is used + primarily for OpenAPI schema generation, where constructing the app is + required but the configuration won't matter. + """ + if load_config: + config = config_dependency.config + path_prefix = config.path_prefix + github_ci_app = config.github_ci_app + github_refresh_app = config.github_refresh_app + + configure_logging( + name="mobu", profile=config.profile, log_level=config.log_level + ) + if config.profile == Profile.production: + configure_uvicorn_logging(config.log_level) + + # Enable Slack alerting for uncaught exceptions. + if config.alert_hook: + logger = structlog.get_logger("mobu") + SlackRouteErrorHandler.initialize( + str(config.alert_hook), "mobu", logger + ) + logger.debug("Initialized Slack webhook") + else: + path_prefix = "/mobu" + github_ci_app = None + github_refresh_app = None -def create_app() -> FastAPI: - """Create the main FastAPI application for mobu.""" app = FastAPI( title="mobu", description=metadata("mobu")["Summary"], version=version("mobu"), - openapi_url=f"{config.path_prefix}/openapi.json", - docs_url=f"{config.path_prefix}/docs", - redoc_url=f"{config.path_prefix}/redoc", + openapi_url=f"{path_prefix}/openapi.json", + docs_url=f"{path_prefix}/docs", + redoc_url=f"{path_prefix}/redoc", lifespan=lifespan, ) # Attach the routers. app.include_router(internal_router) - app.include_router(external_router, prefix=config.path_prefix) + app.include_router(external_router, prefix=path_prefix) - if config.github_ci_app_config_path: + if github_ci_app: app.include_router( - github_ci_app_router, prefix=f"{config.path_prefix}/github/ci" + github_ci_app_router, prefix=f"{path_prefix}/github/ci" ) - if config.github_refresh_app_config_path: + if github_refresh_app: app.include_router( github_refresh_app_router, prefix=f"{config.path_prefix}/github/refresh", @@ -120,14 +136,6 @@ def create_app() -> FastAPI: # Add middleware. app.add_middleware(XForwardedMiddleware) - # Enable Slack alerting for uncaught exceptions. - if config.alert_hook: - logger = structlog.get_logger("mobu") - SlackRouteErrorHandler.initialize( - str(config.alert_hook), "mobu", logger - ) - logger.debug("Initialized Slack webhook") - # Enable the generic exception handler for client errors. app.exception_handler(ClientRequestError)(client_request_error_handler) @@ -136,7 +144,7 @@ def create_app() -> FastAPI: def create_openapi() -> str: """Create the OpenAPI spec for static documentation.""" - app = create_app() + app = create_app(load_config=False) return json.dumps( get_openapi( title=app.title, diff --git a/src/mobu/services/business/notebookrunner.py b/src/mobu/services/business/notebookrunner.py index 593d2b22..be753a8c 100644 --- a/src/mobu/services/business/notebookrunner.py +++ b/src/mobu/services/business/notebookrunner.py @@ -21,8 +21,8 @@ from rubin.nublado.client.models import CodeContext from structlog.stdlib import BoundLogger -from ...config import config from ...constants import GITHUB_REPO_CONFIG_PATH +from ...dependencies.config import config_dependency from ...exceptions import NotebookRepositoryError, RepositoryConfigError from ...models.business.notebookrunner import ( ListNotebookRunnerOptions, @@ -62,6 +62,7 @@ def __init__( logger: BoundLogger, ) -> None: super().__init__(options, user, http_client, logger) + self._config = config_dependency.config self._notebook: Path | None = None self._notebook_paths: list[Path] | None = None self._repo_dir: Path | None = None @@ -155,7 +156,7 @@ def missing_services(self, notebook: Path) -> bool: """ metadata = self.read_notebook_metadata(notebook) missing_services = ( - metadata.required_services - config.available_services + metadata.required_services - self._config.available_services ) if missing_services: msg = "Environment does not provide required services for notebook" diff --git a/src/mobu/services/business/nublado.py b/src/mobu/services/business/nublado.py index bb2980f7..a3c403b2 100644 --- a/src/mobu/services/business/nublado.py +++ b/src/mobu/services/business/nublado.py @@ -17,7 +17,7 @@ from safir.slack.blockkit import SlackException from structlog.stdlib import BoundLogger -from ...config import config +from ...dependencies.config import config_dependency from ...exceptions import ( CodeExecutionError, JupyterProtocolError, @@ -111,6 +111,7 @@ def __init__( ) -> None: super().__init__(options, user, http_client, logger) + config = config_dependency.config if not config.environment_url: raise RuntimeError("environment_url not set") environment_url = str(config.environment_url).rstrip("/") diff --git a/src/mobu/services/business/tap.py b/src/mobu/services/business/tap.py index 5ec01ab7..99f69850 100644 --- a/src/mobu/services/business/tap.py +++ b/src/mobu/services/business/tap.py @@ -12,7 +12,7 @@ from httpx import AsyncClient from structlog.stdlib import BoundLogger -from ...config import config +from ...dependencies.config import config_dependency from ...exceptions import CodeExecutionError, TAPClientError from ...models.business.tap import TAPBusinessData, TAPBusinessOptions from ...models.user import AuthenticatedUser @@ -145,6 +145,7 @@ def _make_client(self, token: str) -> pyvo.dal.TAPService: pyvo.dal.TAPService TAP client object. """ + config = config_dependency.config if not config.environment_url: raise RuntimeError("environment_url not set") tap_url = str(config.environment_url).rstrip("/") + "/api/tap" diff --git a/src/mobu/services/github_ci/ci_manager.py b/src/mobu/services/github_ci/ci_manager.py index e30b7342..9e829310 100644 --- a/src/mobu/services/github_ci/ci_manager.py +++ b/src/mobu/services/github_ci/ci_manager.py @@ -12,7 +12,7 @@ from safir.github import GitHubAppClientFactory from structlog.stdlib import BoundLogger -from ...config import config +from ...dependencies.config import config_dependency from ...models.ci_manager import CiManagerSummary, CiWorkerSummary from ...models.user import User from ...storage.gafaelfawr import GafaelfawrStorage @@ -78,6 +78,7 @@ def __init__( gafaelfawr_storage: GafaelfawrStorage, logger: BoundLogger, ) -> None: + self._config = config_dependency.config self._scopes = scopes self._users = users self._gafaelfawr = gafaelfawr_storage @@ -242,7 +243,7 @@ async def enqueue( ) check_run = await storage.create_check_run( - name=f"Mobu ({config.environment_url})", + name=f"Mobu ({self._config.environment_url})", summary="Waiting for Mobu to run...", ) diff --git a/src/mobu/services/manager.py b/src/mobu/services/manager.py index 9231a02a..3aad00b9 100644 --- a/src/mobu/services/manager.py +++ b/src/mobu/services/manager.py @@ -4,12 +4,11 @@ import asyncio -import yaml from aiojobs import Scheduler from httpx import AsyncClient from structlog.stdlib import BoundLogger -from ..config import config +from ..dependencies.config import config_dependency from ..exceptions import FlockNotFoundError from ..models.flock import FlockConfig, FlockSummary from ..storage.gafaelfawr import GafaelfawrStorage @@ -41,6 +40,7 @@ def __init__( http_client: AsyncClient, logger: BoundLogger, ) -> None: + self._config = config_dependency.config self._gafaelfawr = gafaelfawr_storage self._http_client = http_client self._logger = logger @@ -59,14 +59,7 @@ async def autostart(self) -> None: This function should be called from the startup hook of the FastAPI application. """ - if not config.autostart: - return - with config.autostart.open("r") as f: - autostart = yaml.safe_load(f) - flock_configs = [ - FlockConfig.model_validate(flock) for flock in autostart - ] - for flock_config in flock_configs: + for flock_config in self._config.autostart: await self.start_flock(flock_config) async def start_flock(self, flock_config: FlockConfig) -> Flock: diff --git a/src/mobu/services/monkey.py b/src/mobu/services/monkey.py index 13eaea9f..d6aa0ad1 100644 --- a/src/mobu/services/monkey.py +++ b/src/mobu/services/monkey.py @@ -15,7 +15,7 @@ from safir.slack.webhook import SlackWebhookClient from structlog.stdlib import BoundLogger -from ..config import config +from ..dependencies.config import config_dependency from ..exceptions import MobuMixin from ..models.business.base import BusinessConfig from ..models.business.empty import EmptyLoopConfig @@ -69,6 +69,7 @@ def __init__( http_client: AsyncClient, logger: BoundLogger, ) -> None: + self._config = config_dependency.config self._name = name self._flock = flock self._restart = business_config.restart @@ -117,9 +118,9 @@ def __init__( raise TypeError(msg) self._slack = None - if config.alert_hook: + if self._config.alert_hook: self._slack = SlackWebhookClient( - str(config.alert_hook), "Mobu", self._global_logger + str(self._config.alert_hook), "Mobu", self._global_logger ) async def alert(self, exc: Exception) -> None: @@ -283,7 +284,7 @@ def _build_logger(self, logfile: _TemporaryFileWrapper) -> BoundLogger: logger.setLevel(self._log_level.value) logger.addHandler(file_handler) logger.propagate = False - if config.profile == Profile.development: + if self._config.profile == Profile.development: stream_handler = logging.StreamHandler(stream=sys.stdout) stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) diff --git a/src/mobu/status.py b/src/mobu/status.py index ede95035..67a1fee2 100644 --- a/src/mobu/status.py +++ b/src/mobu/status.py @@ -4,7 +4,7 @@ from safir.slack.blockkit import SlackMessage -from .config import config +from .dependencies.config import config_dependency from .dependencies.context import context_dependency from .factory import Factory @@ -18,6 +18,7 @@ async def post_status() -> None: clear that mobu is alive, but a secondary benefit is to provide some summary statistics. """ + config = config_dependency.config process_context = context_dependency.process_context factory = Factory(process_context) slack = factory.create_slack_webhook_client() diff --git a/src/mobu/storage/gafaelfawr.py b/src/mobu/storage/gafaelfawr.py index 8729ab89..8d46fb27 100644 --- a/src/mobu/storage/gafaelfawr.py +++ b/src/mobu/storage/gafaelfawr.py @@ -11,8 +11,8 @@ from safir.datetime import current_datetime from structlog.stdlib import BoundLogger -from ..config import config from ..constants import TOKEN_LIFETIME, USERNAME_REGEX +from ..dependencies.config import config_dependency from ..exceptions import GafaelfawrParseError, GafaelfawrWebError from ..models.user import AuthenticatedUser, User @@ -82,10 +82,11 @@ class GafaelfawrStorage: def __init__(self, http_client: AsyncClient, logger: BoundLogger) -> None: self._client = http_client self._logger = logger + self._config = config_dependency.config - if not config.environment_url: + if not self._config.environment_url: raise RuntimeError("environment_url not set") - base_url = str(config.environment_url).rstrip("/") + base_url = str(self._config.environment_url).rstrip("/") self._token_url = base_url + "/auth/api/v1/tokens" async def create_service_token( @@ -131,7 +132,9 @@ async def create_service_token( # a better way to do this. r = await self._client.post( self._token_url, - headers={"Authorization": f"Bearer {config.gafaelfawr_token}"}, + headers={ + "Authorization": f"Bearer {self._config.gafaelfawr_token}" + }, json=json.loads(request.model_dump_json(exclude_none=True)), ) r.raise_for_status() diff --git a/tests/autostart_test.py b/tests/autostart_test.py index 8744053c..6bdb8ed0 100644 --- a/tests/autostart_test.py +++ b/tests/autostart_test.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Iterator -from pathlib import Path from unittest.mock import ANY import pytest @@ -11,51 +10,20 @@ from httpx import AsyncClient from rubin.nublado.client.testing import MockJupyter -from mobu.config import config +from mobu.dependencies.config import config_dependency +from .support.config import config_path from .support.gafaelfawr import mock_gafaelfawr from .support.util import wait_for_flock_start -AUTOSTART_CONFIG = """ -- name: basic - count: 10 - user_spec: - username_prefix: bot-mobu-testuser - uid_start: 1000 - gid_start: 2000 - scopes: ["exec:notebook"] - business: - type: EmptyLoop -- name: python - count: 2 - users: - - username: bot-mobu-python - uidnumber: 60000 - - username: bot-mobu-otherpython - uidnumber: 70000 - scopes: ["exec:notebook"] - restart: true - business: - type: NubladoPythonLoop - restart: True - options: - image: - image_class: latest-weekly - size: Large - spawn_settle_time: 0 -""" - @pytest.fixture(autouse=True) -def _configure_autostart( - tmp_path: Path, respx_mock: respx.Router -) -> Iterator[None]: +def _configure_autostart(respx_mock: respx.Router) -> Iterator[None]: """Set up the autostart configuration.""" + config_dependency.set_path(config_path("autostart")) mock_gafaelfawr(respx_mock, any_uid=True) - config.autostart = tmp_path / "autostart.yaml" - config.autostart.write_text(AUTOSTART_CONFIG) yield - config.autostart = None + config_dependency.set_path(config_path("base")) @pytest.mark.asyncio diff --git a/tests/business/nubladopythonloop_test.py b/tests/business/nubladopythonloop_test.py index e6c671f7..df6bd5ad 100644 --- a/tests/business/nubladopythonloop_test.py +++ b/tests/business/nubladopythonloop_test.py @@ -17,7 +17,7 @@ ) from safir.testing.slack import MockSlackWebhook -from mobu.config import config +from mobu.dependencies.config import config_dependency from ..support.gafaelfawr import mock_gafaelfawr from ..support.util import wait_for_business @@ -183,6 +183,7 @@ async def test_hub_failed( slack: MockSlackWebhook, respx_mock: respx.Router, ) -> None: + config = config_dependency.config mock_gafaelfawr(respx_mock) jupyter.fail("bot-mobu-testuser2", JupyterAction.SPAWN) @@ -268,6 +269,7 @@ async def test_redirect_loop( slack: MockSlackWebhook, respx_mock: respx.Router, ) -> None: + config = config_dependency.config mock_gafaelfawr(respx_mock) jupyter.redirect_loop = True diff --git a/tests/conftest.py b/tests/conftest.py index dafe295e..2bca6c1f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,6 @@ from contextlib import asynccontextmanager from pathlib import Path from tempfile import TemporaryDirectory -from textwrap import dedent from unittest.mock import DEFAULT, patch import pytest @@ -17,7 +16,6 @@ from asgi_lifespan import LifespanManager from fastapi import FastAPI from httpx import ASGITransport, AsyncClient -from pydantic import HttpUrl from rubin.nublado.client import NubladoClient from rubin.nublado.client.models import User from rubin.nublado.client.testing import ( @@ -30,10 +28,11 @@ from structlog.stdlib import BoundLogger from mobu import main -from mobu.config import config +from mobu.dependencies.config import config_dependency from mobu.services.business.gitlfs import GitLFSBusiness from mobu.services.business.nublado import _GET_IMAGE, _GET_NODE +from .support.config import config_path from .support.constants import ( TEST_BASE_URL, TEST_GITHUB_CI_APP_PRIVATE_KEY, @@ -71,7 +70,7 @@ def configured_logger() -> BoundLogger: @pytest.fixture(autouse=True) -def _configure(environment_url: str) -> Iterator[None]: +def _configure(environment_url: str, monkeypatch: pytest.MonkeyPatch) -> None: """Set minimal configuration settings. Add an environment URL for testing purposes and create a Gafaelfawr admin @@ -81,13 +80,8 @@ def _configure(environment_url: str) -> Iterator[None]: minimal test configuration and a unique admin token that is replaced after the test runs. """ - config.environment_url = HttpUrl(environment_url) - config.gafaelfawr_token = make_gafaelfawr_token() - config.available_services = {"some_service", "some_other_service"} - yield - config.environment_url = None - config.gafaelfawr_token = None - config.available_services = set() + monkeypatch.setenv("MOBU_GAFAELFAWR_TOKEN", make_gafaelfawr_token()) + config_dependency.set_path(config_path("base")) @pytest.fixture @@ -101,23 +95,6 @@ def _enable_github_ci_app( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> Iterator[None]: """Enable the GitHub CI app functionality.""" - github_config = tmp_path / "github_ci_app_config.yaml" - github_config.write_text( - dedent(""" - users: - - username: bot-mobu-unittest-1 - - username: bot-mobu-unittest-2 - accepted_github_orgs: - - org1 - - org2 - - lsst-sqre - scopes: - - "exec:notebook" - - "exec:portal" - - "read:image" - - "read:tap" - """) - ) monkeypatch.setenv("MOBU_GITHUB_CI_APP_ID", "1") monkeypatch.setenv( "MOBU_GITHUB_CI_APP_WEBHOOK_SECRET", TEST_GITHUB_CI_APP_SECRET @@ -125,7 +102,11 @@ def _enable_github_ci_app( monkeypatch.setenv( "MOBU_GITHUB_CI_APP_PRIVATE_KEY", TEST_GITHUB_CI_APP_PRIVATE_KEY ) - monkeypatch.setattr(config, "github_ci_app_config_path", github_config) + config_dependency.set_path(config_path("github_ci_app")) + + yield + + config_dependency.set_path(config_path("base")) @pytest.fixture @@ -133,23 +114,15 @@ def _enable_github_refresh_app( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> Iterator[None]: """Enable the GitHub refresh app functionality.""" - github_config = tmp_path / "github_ci_app_refresh.yaml" - github_config.write_text( - dedent(""" - accepted_github_orgs: - - org1 - - org2 - - lsst-sqre - """) - ) monkeypatch.setenv("MOBU_GITHUB_REFRESH_APP_ID", "1") monkeypatch.setenv( "MOBU_GITHUB_REFRESH_APP_WEBHOOK_SECRET", TEST_GITHUB_REFRESH_APP_SECRET, ) - monkeypatch.setattr( - config, "github_refresh_app_config_path", github_config - ) + config_dependency.set_path(config_path("github_refresh_app")) + yield + + config_dependency.set_path(config_path("base")) @pytest_asyncio.fixture @@ -259,10 +232,13 @@ async def mock_connect( @pytest.fixture -def slack(respx_mock: respx.Router) -> Iterator[MockSlackWebhook]: - config.alert_hook = HttpUrl("https://slack.example.com/XXXX") - yield mock_slack_webhook(str(config.alert_hook), respx_mock) - config.alert_hook = None +def slack( + respx_mock: respx.Router, monkeypatch: pytest.MonkeyPatch +) -> MockSlackWebhook: + alert_hook = "https://slack.example.com/XXXX" + monkeypatch.setenv("MOBU_ALERT_HOOK", alert_hook) + config_dependency.set_path(config_path("base")) + return mock_slack_webhook(alert_hook, respx_mock) @pytest.fixture diff --git a/tests/data/config/autostart.yaml b/tests/data/config/autostart.yaml new file mode 100644 index 00000000..8dde7a8f --- /dev/null +++ b/tests/data/config/autostart.yaml @@ -0,0 +1,34 @@ +environmentUrl: "https://example.com" +availableServices: + - some_service + - some_other_service +metrics: + appName: mobu + enabled: false +autostart: + - name: basic + count: 10 + user_spec: + username_prefix: bot-mobu-testuser + uid_start: 1000 + gid_start: 2000 + scopes: ["exec:notebook"] + business: + type: EmptyLoop + - name: python + count: 2 + users: + - username: bot-mobu-python + uidnumber: 60000 + - username: bot-mobu-otherpython + uidnumber: 70000 + scopes: ["exec:notebook"] + restart: true + business: + type: NubladoPythonLoop + restart: True + options: + image: + image_class: latest-weekly + size: Large + spawn_settle_time: 0 diff --git a/tests/data/config/base.yaml b/tests/data/config/base.yaml new file mode 100644 index 00000000..0459b5f8 --- /dev/null +++ b/tests/data/config/base.yaml @@ -0,0 +1,7 @@ +environmentUrl: "https://example.com" +availableServices: + - some_service + - some_other_service +metrics: + appName: mobu + enabled: false diff --git a/tests/data/config/github_ci_app.yaml b/tests/data/config/github_ci_app.yaml new file mode 100644 index 00000000..4194d770 --- /dev/null +++ b/tests/data/config/github_ci_app.yaml @@ -0,0 +1,20 @@ +environmentUrl: "https://example.com" +availableServices: + - some_service + - some_other_service +metrics: + appName: mobu + enabled: false +githubCiApp: + users: + - username: bot-mobu-unittest-1 + - username: bot-mobu-unittest-2 + acceptedGithubOrgs: + - org1 + - org2 + - lsst-sqre + scopes: + - "exec:notebook" + - "exec:portal" + - "read:image" + - "read:tap" diff --git a/tests/data/config/github_refresh_app.yaml b/tests/data/config/github_refresh_app.yaml new file mode 100644 index 00000000..e8f5a78a --- /dev/null +++ b/tests/data/config/github_refresh_app.yaml @@ -0,0 +1,12 @@ +environmentUrl: "https://example.com" +availableServices: + - some_service + - some_other_service +metrics: + appName: mobu + enabled: false +githubRefreshApp: + acceptedGithubOrgs: + - org1 + - org2 + - lsst-sqre diff --git a/tests/handlers/index_test.py b/tests/handlers/index_test.py index 99934ffb..57ffbe96 100644 --- a/tests/handlers/index_test.py +++ b/tests/handlers/index_test.py @@ -5,12 +5,13 @@ import pytest from httpx import AsyncClient -from mobu.config import config +from mobu.dependencies.config import config_dependency @pytest.mark.asyncio async def test_get_index(client: AsyncClient) -> None: """Test ``GET /mobu/``.""" + config = config_dependency.config response = await client.get("/mobu/") assert response.status_code == 200 data = response.json() diff --git a/tests/handlers/internal_test.py b/tests/handlers/internal_test.py index a6c19f21..08467414 100644 --- a/tests/handlers/internal_test.py +++ b/tests/handlers/internal_test.py @@ -5,12 +5,13 @@ import pytest from httpx import AsyncClient -from mobu.config import config +from mobu.dependencies.config import config_dependency @pytest.mark.asyncio async def test_get_index(client: AsyncClient) -> None: """Test ``GET /``.""" + config = config_dependency.config response = await client.get("/") assert response.status_code == 200 data = response.json() diff --git a/tests/monkeyflocker_test.py b/tests/monkeyflocker_test.py index 46ddf5b1..9587c24f 100644 --- a/tests/monkeyflocker_test.py +++ b/tests/monkeyflocker_test.py @@ -14,9 +14,10 @@ from click.testing import CliRunner from safir.testing.uvicorn import UvicornProcess, spawn_uvicorn -from mobu.config import config +from mobu.dependencies.config import config_dependency from monkeyflocker.cli import main +from .support.config import config_path from .support.gafaelfawr import make_gafaelfawr_token FLOCK_CONFIG = """ @@ -35,24 +36,23 @@ def monkeyflocker_app( tmp_path: Path, test_filesystem: Path, environment_url: str ) -> Iterator[UvicornProcess]: """Run the application as a separate process for monkeyflocker access.""" + config_dependency.set_path(config_path("base")) + config = config_dependency.config assert config.environment_url config.gafaelfawr_token = make_gafaelfawr_token() uvicorn = spawn_uvicorn( working_directory=tmp_path, factory="tests.support.monkeyflocker:create_app", - env={ - "MOBU_ENVIRONMENT_URL": str(config.environment_url), - "MOBU_GAFAELFAWR_TOKEN": config.gafaelfawr_token, - }, + env={"MOBU_UNIT_TEST_PATH": str(config_path("base"))}, ) yield uvicorn - config.gafaelfawr_token = None uvicorn.process.terminate() def test_start_report_refresh_stop( tmp_path: Path, monkeyflocker_app: UvicornProcess ) -> None: + config = config_dependency.config runner = CliRunner() spec_path = tmp_path / "spec.yaml" spec_path.write_text(FLOCK_CONFIG) diff --git a/tests/support/config.py b/tests/support/config.py new file mode 100644 index 00000000..5f69fc02 --- /dev/null +++ b/tests/support/config.py @@ -0,0 +1,27 @@ +"""Build test configuration for mobu.""" + +from __future__ import annotations + +from pathlib import Path + +__all__ = [ + "config_path", +] + + +def config_path(filename: str) -> Path: + """Return the path to a test configuration file. + + Parameters + ---------- + filename + The base name of a test configuration file or template. + + Returns + ------- + Path + The path to that file. + """ + return ( + Path(__file__).parent.parent / "data" / "config" / (filename + ".yaml") + ) diff --git a/tests/support/gafaelfawr.py b/tests/support/gafaelfawr.py index 05b7a71e..ec441706 100644 --- a/tests/support/gafaelfawr.py +++ b/tests/support/gafaelfawr.py @@ -12,7 +12,7 @@ from httpx import Request, Response from safir.datetime import current_datetime -from mobu.config import config +from mobu.dependencies.config import config_dependency __all__ = ["make_gafaelfawr_token", "mock_gafaelfawr"] @@ -47,6 +47,7 @@ def mock_gafaelfawr( Optionally verifies that the username and UID provided to Gafaelfawr are correct. """ + config = config_dependency.config scopes = scopes or ["exec:notebook"] admin_token = config.gafaelfawr_token assert admin_token