Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adapt to new version of the zero to jupyterhub chart (3.1.0) #186

Merged
merged 16 commits into from
Oct 24, 2023
Merged

Adapt to new version of the zero to jupyterhub chart (3.1.0) #186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
3088a15
Update to z2jh chart v2.0.0
etejedor Jul 31, 2023
6dc4c50
Adapt spawner-related hooks to async KubeSpawner
etejedor Jul 31, 2023
e43f7c1
Update to z2jh chart v3.0.1
etejedor Aug 16, 2023
a1eca44
Remove CERN-specific configuration for auth from swan chart
etejedor Aug 16, 2023
3013430
Update renamed configuration parameter
etejedor Aug 16, 2023
fb86780
Set read:metrics scope for prometheus-service-monitor service
etejedor Aug 30, 2023
d33cb7d
Prevent redirection from /metrics to /hub/metrics
etejedor Aug 30, 2023
c6cd3fe
Allow all authenticated users to log in
etejedor Aug 31, 2023
f4f5997
Remove swan_user label
diocas Sep 1, 2023
ee62864
Cleanup
diocas Sep 1, 2023
15a11d0
Safe label username
diocas Aug 31, 2023
b5df246
Update to z2jh chart v3.1.0
etejedor Oct 17, 2023
99d5748
Disable debug messages from the user's Jupyter server
etejedor Oct 17, 2023
9755e9e
Add a role with specific permissions for the swan culler service
etejedor Oct 17, 2023
26eac6d
Do not attempt to delete non-existing pvc
etejedor Oct 19, 2023
366b588
Don't refresh the token on spawn
diocas Oct 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 66 additions & 64 deletions swan-cern/files/swan_config_cern.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,22 @@
import os, subprocess

from kubernetes import client
from kubernetes.client.rest import ApiException
import escapism

from kubernetes_asyncio.client.models import (
V1EmptyDirVolumeSource,
V1EnvVar,
V1EnvVarSource,
V1ConfigMapVolumeSource,
V1Container,
V1KeyToPath,
V1ObjectFieldSelector,
V1ObjectMeta,
V1Secret,
V1SecretVolumeSource,
V1Volume,
V1VolumeMount,
)

from kubernetes_asyncio.client.rest import ApiException

"""
Class handling KubeSpawner.modify_pod_hook(spawner,pod) call
Expand All @@ -10,7 +25,7 @@

class SwanPodHookHandlerProd(SwanPodHookHandler):

def get_swan_user_pod(self):
async def get_swan_user_pod(self):
super().get_swan_user_pod()

# ATTENTION Spark requires this side container, so we need to create it!!
Expand All @@ -19,17 +34,19 @@ def get_swan_user_pod(self):
# not self.spawner.local_home:

# get eos token
eos_secret_name = self._init_eos_secret()
eos_secret_name = await self._init_eos_secret()

# init user containers (notebook and side-container)
self._init_eos_containers(eos_secret_name)

return self.pod

def _init_eos_secret(self):
username = self.spawner.user.name
user_uid = self.spawner.user_uid
eos_secret_name ='eos-tokens-%s' % username
async def _init_eos_secret(self):

username = escapism.escape(
self.spawner.user.name, safe = self.spawner.safe_chars, escape_char = '-'
).lower()
eos_secret_name = 'eos-tokens-%s' % username

try:
# Retrieve eos token for user
Expand All @@ -39,34 +56,29 @@ def _init_eos_secret(self):
except Exception as e:
raise ValueError("Could not create required user credential")


# ITHADOOP-819 - Ports need to be opened using service creation, and later assigning allocated service nodeport to a pod
# Create V1Secret with eos token
try:
secret_data = client.V1Secret()

secret_meta = client.V1ObjectMeta()
secret_meta.name = eos_secret_name
secret_meta.namespace = swan_container_namespace
secret_meta.labels = {
"swan_user": username
}
secret_data.metadata = secret_meta
secret_data.data = {}
secret_data.data['krb5cc'] = eos_token_base64
secret_data = V1Secret()

secret_meta = V1ObjectMeta()
secret_meta.name = eos_secret_name
secret_meta.namespace = swan_container_namespace
secret_meta.labels = {
"swan_user": username
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we align the label name to the same as the container?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What name exactly?

}
secret_data.metadata = secret_meta
secret_data.data = {}
secret_data.data['krb5cc'] = eos_token_base64

try:
# eos-tokens secret is cleaned when user session ends, so try creating it
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Btw, this reminds me that we split the code... This means that we have similar logic in 2 different places (we assume here that it will be created, but it's done somewhere else... And in that place we assumed it was created here...). I'm ok with leaving it here for now, but it's not the ideal solution. Let's open a PR upstream to fix the async call and then we put them back together (unless you think they should all live in the spawner).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, creation and deletion are in different places now. I'd prefer they are all in the spawner, but that does not mean they can't be hooks. Regarding the contribution upstream, I added a comment to this task https://its.cern.ch/jira/browse/SWAN-85

await self.spawner.api.create_namespaced_secret(swan_container_namespace, secret_data)
except ApiException:
# A secret with the same name exists, probably a remnant of a wrongly-terminated session, then replace it
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we have a log object? This should be logged as a warning (since even wrongly terminated sessions should still remove it, no?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wrongly terminated sessions should remove it, yes, since the removal code that is part of stop is in a finally block. I guess it could happen that the hub crashes and the stop method is never called for the user session. I can add the warning anyway, though, we should have access to the logger of the spawner from here.

try:
self.spawner.api.read_namespaced_secret(eos_secret_name, swan_container_namespace)
exists = True
except ApiException:
exists = False

if exists:
self.spawner.api.replace_namespaced_secret(eos_secret_name, swan_container_namespace, secret_data)
else:
self.spawner.api.create_namespaced_secret(swan_container_namespace, secret_data)
except ApiException as e:
raise Exception("Could not create required eos secret: %s\n" % e)
await self.spawner.api.replace_namespaced_secret(eos_secret_name, swan_container_namespace, secret_data)
except ApiException as e:
raise Exception("Could not create required eos secret: %s\n" % e)

return eos_secret_name

Expand All @@ -82,23 +94,23 @@ def _init_eos_containers(self, eos_secret_name):

# Shared directory between notebook and side-container for tokens with correct privileges
self.pod.spec.volumes.append(
client.V1Volume(
V1Volume(
name='shared-pod-volume',
empty_dir=client.V1EmptyDirVolumeSource(
empty_dir=V1EmptyDirVolumeSource(
medium='Memory'
)
)
)
side_container_volume_mounts.append(
client.V1VolumeMount(
V1VolumeMount(
name='shared-pod-volume',
mount_path='/srv/notebook'
)
)

# Mount shared tokens volume that contains tokens with correct permissions
notebook_container.volume_mounts.append(
client.V1VolumeMount(
V1VolumeMount(
name='shared-pod-volume',
mount_path='/srv/notebook'
)
Expand All @@ -107,15 +119,15 @@ def _init_eos_containers(self, eos_secret_name):
# pod volume to mount generated eos tokens and
# side-container volume mount with generated tokens
self.pod.spec.volumes.append(
client.V1Volume(
V1Volume(
name=eos_secret_name,
secret=client.V1SecretVolumeSource(
secret=V1SecretVolumeSource(
secret_name='eos-tokens-%s' % username,
)
)
)
side_container_volume_mounts.append(
client.V1VolumeMount(
V1VolumeMount(
name=eos_secret_name,
mount_path='/srv/side-container/eos'
)
Expand All @@ -124,7 +136,7 @@ def _init_eos_containers(self, eos_secret_name):
# define eos kerberos credentials path for Jupyter server in notebook container
notebook_container.env = self._add_or_replace_by_name(
notebook_container.env,
client.V1EnvVar(
V1EnvVar(
name='KRB5CCNAME',
value='/srv/notebook/tokens/krb5cc'
),
Expand All @@ -133,7 +145,7 @@ def _init_eos_containers(self, eos_secret_name):
# define eos kerberos credentials path for notebook and terminal processes in notebook container
notebook_container.env = self._add_or_replace_by_name(
notebook_container.env,
client.V1EnvVar(
V1EnvVar(
name='KRB5CCNAME_NB_TERM',
value='/srv/notebook/tokens/writable/krb5cc_nb_term'
),
Expand All @@ -142,10 +154,10 @@ def _init_eos_containers(self, eos_secret_name):
# Set server hostname of the pod running jupyterhub
notebook_container.env = self._add_or_replace_by_name(
notebook_container.env,
client.V1EnvVar(
V1EnvVar(
name='SERVER_HOSTNAME',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
value_from=V1EnvVarSource(
field_ref=V1ObjectFieldSelector(
field_path='spec.nodeName'
)
)
Expand All @@ -155,12 +167,12 @@ def _init_eos_containers(self, eos_secret_name):
# append as first (it will be first to spawn) side container which currently:
# - refreshes the kerberos token and adjust permissions for the user
self.pod.spec.volumes.append(
client.V1Volume(
V1Volume(
name='side-container-scripts',
config_map=client.V1ConfigMapVolumeSource(
config_map=V1ConfigMapVolumeSource(
name='swan-scripts-cern',
items=[
client.V1KeyToPath(
V1KeyToPath(
key='side_container_tokens_perm.sh',
path='side_container_tokens_perm.sh',
)
Expand All @@ -170,7 +182,7 @@ def _init_eos_containers(self, eos_secret_name):
)
)
side_container_volume_mounts.append(
client.V1VolumeMount(
V1VolumeMount(
name='side-container-scripts',
mount_path='/srv/side-container/side_container_tokens_perm.sh',
sub_path='side_container_tokens_perm.sh',
Expand All @@ -179,7 +191,7 @@ def _init_eos_containers(self, eos_secret_name):

env = self.spawner.get_env()
pod_spec_containers.append(
client.V1Container(
V1Container(
name='side-container',
image='cern/cc7-base:20181210',
command=['/srv/side-container/side_container_tokens_perm.sh'],
Expand All @@ -203,18 +215,18 @@ def _init_eos_containers(self, eos_secret_name):
# This is defined in the configuration to allow overring iindependently
# of which config file is loaded first
# c.SwanKubeSpawner.modify_pod_hook = swan_pod_hook
def swan_pod_hook_prod(spawner, pod):
async def swan_pod_hook_prod(spawner, pod):
"""
:param spawner: Swan Kubernetes Spawner
:type spawner: swanspawner.SwanKubeSpawner
:param pod: default pod definition set by jupyterhub
:type pod: client.V1Pod
:type pod: V1Pod

:returns: dynamically customized pod specification for user session
:rtype: client.V1Pod
:rtype: V1Pod
"""
pod_hook_handler = SwanPodHookHandlerProd(spawner, pod)
return pod_hook_handler.get_swan_user_pod()
return await pod_hook_handler.get_swan_user_pod()


swan_cull_period = get_config('custom.cull.every', 600)
Expand All @@ -223,15 +235,5 @@ def swan_pod_hook_prod(spawner, pod):

c.SwanKubeSpawner.modify_pod_hook = swan_pod_hook_prod

def swan_cern_post_stop_hook(spawner):
# Delete Kubernetes Secret storing eos kerberos ticket of the user
username = spawner.user.name
eos_secret_name = f"eos-tokens-{username}"
swan_container_namespace = os.environ.get('POD_NAMESPACE', 'default')
spawner.log.info('Deleting secret %s', eos_secret_name)
spawner.api.delete_namespaced_secret(eos_secret_name, swan_container_namespace)

c.SwanKubeSpawner.post_stop_hook = swan_cern_post_stop_hook

# Required for swan systemuser.sh
c.SwanKubeSpawner.cmd = None
Loading