From 416db8cfeeacb5b3381e9a3e5243f931ed94b5cf Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Fri, 1 Nov 2024 10:55:13 +0100 Subject: [PATCH 1/9] Bump conda-store tag to 2024.10.1 --- src/_nebari/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 6e57519fe..013764da6 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -19,7 +19,7 @@ DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE -DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.3.1" +DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.10.1" LATEST_SUPPORTED_PYTHON_VERSION = "3.10" From cf3bfcef6161a3909a66a4ba8329b73cd30b6115 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Thu, 7 Nov 2024 16:41:16 +0100 Subject: [PATCH 2/9] Update imports given conda-store refactor --- .../services/conda-store/config/conda_store_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index ad9b79843..b41f29d25 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -10,7 +10,8 @@ from pathlib import Path import requests -from conda_store_server import api, orm, schema +from conda_store_server import api +from conda_store_server._internal import orm, schema from conda_store_server.server.auth import GenericOAuthAuthentication from conda_store_server.server.dependencies import get_conda_store from conda_store_server.storage import S3Storage From 93d4c11655cb5bb568b8b0990aadd938c7961dbc Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 12 Nov 2024 14:26:33 +0100 Subject: [PATCH 3/9] Vendor conda-store schema --- .../conda-store/config/conda_store_config.py | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index b41f29d25..cc946de5c 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -1,20 +1,44 @@ import dataclasses +import datetime +import functools import json import logging import re import tempfile -import typing import urllib import urllib.parse import urllib.request from pathlib import Path +from typing import Dict, List, Optional, TypeAlias import requests from conda_store_server import api -from conda_store_server._internal import orm, schema from conda_store_server.server.auth import GenericOAuthAuthentication from conda_store_server.server.dependencies import get_conda_store from conda_store_server.storage import S3Storage +from pydantic import BaseModel, Field, constr + +# The following definitions (from line 24 to line 40) have been copied over +# from conda-store source code in order to avoid accessing a "private" schema +# that does not follow a backwards compatibility policy. +ALLOWED_CHARACTERS = "A-Za-z0-9-+_@$&?^~.=" +ARN_ALLOWED = f"^([{ALLOWED_CHARACTERS}*]+)/([{ALLOWED_CHARACTERS}*]+)$" + + +def _datetime_factory(offset: datetime.timedelta): + """utcnow datetime + timezone as string""" + return datetime.datetime.utcnow() + offset + + +RoleBindings: TypeAlias = Dict[constr(regex=ARN_ALLOWED), List[str]] + + +class AuthenticationToken(BaseModel): + exp: datetime.datetime = Field( + default_factory=functools.partial(_datetime_factory, datetime.timedelta(days=1)) + ) + primary_namespace: str = "default" + role_bindings: RoleBindings = {} def conda_store_config(path="/var/lib/conda-store/config.json"): @@ -111,9 +135,7 @@ def _validate_role(self, role): self.log.info(f"role: {role} is {'valid' if valid else 'invalid'}") return valid - def parse_role_and_namespace( - self, text - ) -> typing.Optional[CondaStoreNamespaceRole]: + def parse_role_and_namespace(self, text) -> Optional[CondaStoreNamespaceRole]: # The regex pattern pattern = r"^(\w+)!namespace=([^!]+)$" @@ -128,7 +150,7 @@ def parse_role_and_namespace( else: return None - def parse_scope(self) -> typing.List[CondaStoreNamespaceRole]: + def parse_scope(self) -> List[CondaStoreNamespaceRole]: """Parsed scopes from keycloak role's attribute and returns a list of role/namespace if scopes' syntax is valid otherwise return [] @@ -249,7 +271,7 @@ async def _apply_roles_from_keycloak(self, request, user_data): ) def _filter_duplicate_namespace_roles_with_max_permissions( - self, namespace_roles: typing.List[CondaStoreNamespaceRole] + self, namespace_roles: List[CondaStoreNamespaceRole] ): """Filter duplicate roles in keycloak such that to apply only the one with the highest permissions. @@ -260,7 +282,7 @@ def _filter_duplicate_namespace_roles_with_max_permissions( We need to apply only the role 2 as that one has higher permissions. """ self.log.info("Filtering duplicate roles for same namespace") - namespace_role_mapping: typing.Dict[str:CondaStoreNamespaceRole] = {} + namespace_role_mapping: Dict[str:CondaStoreNamespaceRole] = {} for namespace_role in namespace_roles: namespace = namespace_role.namespace new_role = namespace_role.role @@ -283,7 +305,7 @@ def _filter_duplicate_namespace_roles_with_max_permissions( def _get_permissions_from_keycloak_role( self, keycloak_role - ) -> typing.List[CondaStoreNamespaceRole]: + ) -> List[CondaStoreNamespaceRole]: self.log.info(f"Getting permissions from keycloak role: {keycloak_role}") role_attributes = keycloak_role["attributes"] # scopes returns a list with a value say ["viewer!namespace=pycon,developer!namespace=scipy"] @@ -297,14 +319,14 @@ async def _apply_conda_store_roles_from_keycloak( self.log.info( f"Apply conda store roles from keycloak roles: {conda_store_client_roles}, user: {username}" ) - role_permissions: typing.List[CondaStoreNamespaceRole] = [] + role_permissions: List[CondaStoreNamespaceRole] = [] for conda_store_client_role in conda_store_client_roles: role_permissions += self._get_permissions_from_keycloak_role( conda_store_client_role ) self.log.info("Filtering duplicate namespace role for max permissions") - filtered_namespace_role: typing.List[CondaStoreNamespaceRole] = ( + filtered_namespace_role: List[CondaStoreNamespaceRole] = ( self._filter_duplicate_namespace_roles_with_max_permissions( role_permissions ) @@ -357,9 +379,7 @@ def _get_conda_store_client_roles_for_user( return client_roles_rich def _get_current_entity_bindings(self, username): - entity = schema.AuthenticationToken( - primary_namespace=username, role_bindings={} - ) + entity = AuthenticationToken(primary_namespace=username, role_bindings={}) self.log.info(f"entity: {entity}") entity_bindings = self.authorization.get_entity_bindings(entity) self.log.info(f"current entity_bindings: {entity_bindings}") @@ -387,7 +407,7 @@ async def authenticate(self, request): # superadmin gets access to everything if "conda_store_superadmin" in user_data.get("roles", []): - return schema.AuthenticationToken( + return AuthenticationToken( primary_namespace=username, role_bindings={"*/*": {"admin"}}, ) @@ -423,10 +443,9 @@ async def authenticate(self, request): for namespace in namespaces: _namespace = api.get_namespace(db, name=namespace) if _namespace is None: - db.add(orm.Namespace(name=namespace)) - db.commit() + api.create_namespace(db, name=namespace) - return schema.AuthenticationToken( + return AuthenticationToken( primary_namespace=username, role_bindings=role_bindings, ) From 05d80a27a0a2f60b4c4de16dff21c9ab8ef1fa23 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 12 Nov 2024 18:31:26 +0100 Subject: [PATCH 4/9] Revert vendoring conda-store schema --- .../conda-store/config/conda_store_config.py | 55 ++++++------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index cc946de5c..b41f29d25 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -1,44 +1,20 @@ import dataclasses -import datetime -import functools import json import logging import re import tempfile +import typing import urllib import urllib.parse import urllib.request from pathlib import Path -from typing import Dict, List, Optional, TypeAlias import requests from conda_store_server import api +from conda_store_server._internal import orm, schema from conda_store_server.server.auth import GenericOAuthAuthentication from conda_store_server.server.dependencies import get_conda_store from conda_store_server.storage import S3Storage -from pydantic import BaseModel, Field, constr - -# The following definitions (from line 24 to line 40) have been copied over -# from conda-store source code in order to avoid accessing a "private" schema -# that does not follow a backwards compatibility policy. -ALLOWED_CHARACTERS = "A-Za-z0-9-+_@$&?^~.=" -ARN_ALLOWED = f"^([{ALLOWED_CHARACTERS}*]+)/([{ALLOWED_CHARACTERS}*]+)$" - - -def _datetime_factory(offset: datetime.timedelta): - """utcnow datetime + timezone as string""" - return datetime.datetime.utcnow() + offset - - -RoleBindings: TypeAlias = Dict[constr(regex=ARN_ALLOWED), List[str]] - - -class AuthenticationToken(BaseModel): - exp: datetime.datetime = Field( - default_factory=functools.partial(_datetime_factory, datetime.timedelta(days=1)) - ) - primary_namespace: str = "default" - role_bindings: RoleBindings = {} def conda_store_config(path="/var/lib/conda-store/config.json"): @@ -135,7 +111,9 @@ def _validate_role(self, role): self.log.info(f"role: {role} is {'valid' if valid else 'invalid'}") return valid - def parse_role_and_namespace(self, text) -> Optional[CondaStoreNamespaceRole]: + def parse_role_and_namespace( + self, text + ) -> typing.Optional[CondaStoreNamespaceRole]: # The regex pattern pattern = r"^(\w+)!namespace=([^!]+)$" @@ -150,7 +128,7 @@ def parse_role_and_namespace(self, text) -> Optional[CondaStoreNamespaceRole]: else: return None - def parse_scope(self) -> List[CondaStoreNamespaceRole]: + def parse_scope(self) -> typing.List[CondaStoreNamespaceRole]: """Parsed scopes from keycloak role's attribute and returns a list of role/namespace if scopes' syntax is valid otherwise return [] @@ -271,7 +249,7 @@ async def _apply_roles_from_keycloak(self, request, user_data): ) def _filter_duplicate_namespace_roles_with_max_permissions( - self, namespace_roles: List[CondaStoreNamespaceRole] + self, namespace_roles: typing.List[CondaStoreNamespaceRole] ): """Filter duplicate roles in keycloak such that to apply only the one with the highest permissions. @@ -282,7 +260,7 @@ def _filter_duplicate_namespace_roles_with_max_permissions( We need to apply only the role 2 as that one has higher permissions. """ self.log.info("Filtering duplicate roles for same namespace") - namespace_role_mapping: Dict[str:CondaStoreNamespaceRole] = {} + namespace_role_mapping: typing.Dict[str:CondaStoreNamespaceRole] = {} for namespace_role in namespace_roles: namespace = namespace_role.namespace new_role = namespace_role.role @@ -305,7 +283,7 @@ def _filter_duplicate_namespace_roles_with_max_permissions( def _get_permissions_from_keycloak_role( self, keycloak_role - ) -> List[CondaStoreNamespaceRole]: + ) -> typing.List[CondaStoreNamespaceRole]: self.log.info(f"Getting permissions from keycloak role: {keycloak_role}") role_attributes = keycloak_role["attributes"] # scopes returns a list with a value say ["viewer!namespace=pycon,developer!namespace=scipy"] @@ -319,14 +297,14 @@ async def _apply_conda_store_roles_from_keycloak( self.log.info( f"Apply conda store roles from keycloak roles: {conda_store_client_roles}, user: {username}" ) - role_permissions: List[CondaStoreNamespaceRole] = [] + role_permissions: typing.List[CondaStoreNamespaceRole] = [] for conda_store_client_role in conda_store_client_roles: role_permissions += self._get_permissions_from_keycloak_role( conda_store_client_role ) self.log.info("Filtering duplicate namespace role for max permissions") - filtered_namespace_role: List[CondaStoreNamespaceRole] = ( + filtered_namespace_role: typing.List[CondaStoreNamespaceRole] = ( self._filter_duplicate_namespace_roles_with_max_permissions( role_permissions ) @@ -379,7 +357,9 @@ def _get_conda_store_client_roles_for_user( return client_roles_rich def _get_current_entity_bindings(self, username): - entity = AuthenticationToken(primary_namespace=username, role_bindings={}) + entity = schema.AuthenticationToken( + primary_namespace=username, role_bindings={} + ) self.log.info(f"entity: {entity}") entity_bindings = self.authorization.get_entity_bindings(entity) self.log.info(f"current entity_bindings: {entity_bindings}") @@ -407,7 +387,7 @@ async def authenticate(self, request): # superadmin gets access to everything if "conda_store_superadmin" in user_data.get("roles", []): - return AuthenticationToken( + return schema.AuthenticationToken( primary_namespace=username, role_bindings={"*/*": {"admin"}}, ) @@ -443,9 +423,10 @@ async def authenticate(self, request): for namespace in namespaces: _namespace = api.get_namespace(db, name=namespace) if _namespace is None: - api.create_namespace(db, name=namespace) + db.add(orm.Namespace(name=namespace)) + db.commit() - return AuthenticationToken( + return schema.AuthenticationToken( primary_namespace=username, role_bindings=role_bindings, ) From a4e331ad724959238c95782d79a12086dde5e908 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 12 Nov 2024 18:51:03 +0100 Subject: [PATCH 5/9] Replace orm call with api call --- .../services/conda-store/config/conda_store_config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index b41f29d25..fe9751437 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -11,7 +11,7 @@ import requests from conda_store_server import api -from conda_store_server._internal import orm, schema +from conda_store_server._internal import schema from conda_store_server.server.auth import GenericOAuthAuthentication from conda_store_server.server.dependencies import get_conda_store from conda_store_server.storage import S3Storage @@ -423,8 +423,7 @@ async def authenticate(self, request): for namespace in namespaces: _namespace = api.get_namespace(db, name=namespace) if _namespace is None: - db.add(orm.Namespace(name=namespace)) - db.commit() + api.create_namespace(db, name=namespace) return schema.AuthenticationToken( primary_namespace=username, From 887a3f3e3bfdf5c544a023168c6da12bb1ffc89b Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 17 Dec 2024 11:18:23 -0500 Subject: [PATCH 6/9] Upgrade conda-store to 2024.11.2 --- src/_nebari/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index b5a17c878..f30da0f87 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -16,7 +16,7 @@ DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE -DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.10.1" +DEFAULT_CONDA_STORE_IMAGE_TAG = "2024.11.2" LATEST_SUPPORTED_PYTHON_VERSION = "3.10" From 195789d4673dd09dd90f7b26ce203ebde66f3660 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 17 Dec 2024 11:44:04 -0500 Subject: [PATCH 7/9] Update get_conda_store import --- .../services/conda-store/config/conda_store_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index fe9751437..0c1ed4500 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -12,8 +12,8 @@ import requests from conda_store_server import api from conda_store_server._internal import schema +from conda_store_server._internal.server.dependencies import get_conda_store from conda_store_server.server.auth import GenericOAuthAuthentication -from conda_store_server.server.dependencies import get_conda_store from conda_store_server.storage import S3Storage From 903808885fe1e2d0f176074ab103025a48de1189 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 17 Dec 2024 17:08:51 -0500 Subject: [PATCH 8/9] Add explicit db.commit --- .../kubernetes/services/conda-store/config/conda_store_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index 0c1ed4500..3e33fe40e 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -424,6 +424,7 @@ async def authenticate(self, request): _namespace = api.get_namespace(db, name=namespace) if _namespace is None: api.create_namespace(db, name=namespace) + db.commit() return schema.AuthenticationToken( primary_namespace=username, From 67e739aa0356494490fffbfd78c85a504f7ca7f8 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 7 Jan 2025 18:53:52 +0100 Subject: [PATCH 9/9] Use ensure_namespace function to create new namespace --- .../services/conda-store/config/conda_store_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py index 3e33fe40e..f14c35297 100644 --- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py +++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/conda-store/config/conda_store_config.py @@ -423,8 +423,7 @@ async def authenticate(self, request): for namespace in namespaces: _namespace = api.get_namespace(db, name=namespace) if _namespace is None: - api.create_namespace(db, name=namespace) - db.commit() + api.ensure_namespace(db, name=namespace) return schema.AuthenticationToken( primary_namespace=username,