Skip to content

Commit

Permalink
Merge pull request #4971 from sgibson91/cilogon-cleanup
Browse files Browse the repository at this point in the history 
Cilogon cleanup
  • Loading branch information
@sgibson91
sgibson91 authored Oct 17, 2024
2 parents 942feef + ef38052 commit 71036c4
Show file tree
Hide file tree
Showing 3 changed files with 206 additions and 58 deletions.
4 changes: 2 additions & 2 deletions deployer/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,9 +369,9 @@ This allows us to validate that all required values are present and have the cor
### The `cilogon-client` sub-command for CILogon OAuth client management
Deployer sub-command for managing CILogon clients for 2i2c hubs.

#### `cilogon-client create/delete/get/get-all/update`
#### `cilogon-client create/delete/get/get-all/update/cleanup`

create/delete/get/get-all/update/ CILogon clients using the 2i2c administrative client provided by CILogon.
create/delete/get/get-all/update/cleanup CILogon clients using the 2i2c administrative client provided by CILogon.

### The `exec` sub-command for executing shells and debugging commands

Expand Down
251 changes: 195 additions & 56 deletions deployer/commands/cilogon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@
- `delete` a CILogon client application when a hub is removed or changes auth methods
- `get` details about an existing hub CILogon client
- `get-all` existing 2i2c CILogon client applications
- `cleanup` duplicated CILogon applications
"""

import base64
import json
from collections import Counter
from pathlib import Path

import requests
Expand All @@ -27,6 +29,8 @@
from deployer.cli_app import cilogon_client_app
from deployer.utils.file_acquisition import (
build_absolute_path_to_hub_encrypted_config_file,
find_absolute_path_to_cluster_file,
get_cluster_names_list,
get_decrypted_file,
persist_config_in_encrypted_file,
remove_jupyterhub_hub_config_key_from_encrypted_file,
Expand Down Expand Up @@ -264,52 +268,19 @@ def get_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None):
return client_details


def delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None):
"""Deletes the client associated with the id.
Args:
id (str): Id of the client to delete
def delete_client(admin_id, admin_secret, client_id=None):
"""Deletes a CILogon client.
Returns status code if response.ok
or None if the `delete` request returned a status code not in the range 200-299.
See: https://github.com/ncsa/OA4MP/blob/HEAD/oa4mp-server-admin-oauth2/src/main/scripts/oidc-cm-scripts/cm-delete.sh
"""
config_filename = build_absolute_path_to_hub_encrypted_config_file(
cluster_name, hub_name
)

if not client_id:
if Path(config_filename).is_file():
client_id = load_client_id_from_file(config_filename)
# Nothing to do if no client has been found
if not client_id:
print_colour(
"No `client_id` to delete was provided and couldn't find any in `config_filename`",
"red",
)
return
else:
print_colour(
f"No `client_id` to delete was provided and couldn't find any {config_filename} file",
"red",
)
return
if client_id is None:
print("Deleting a CILogon client for unknown ID")
else:
if not stored_client_id_same_with_cilogon_records(
admin_id,
admin_secret,
cluster_name,
hub_name,
client_id,
):
print_colour(
"CILogon records are different than the client app stored in the configuration file. Consider updating the file.",
"red",
)
return
print(f"Deleting the CILogon client details for {client_id}...")

print(f"Deleting the CILogon client details for {client_id}...")
headers = build_request_headers(admin_id, admin_secret)
response = requests.delete(build_request_url(client_id), headers=headers, timeout=5)
if not response.ok:
Expand All @@ -318,19 +289,6 @@ def delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id=None

print_colour("Done!")

# Delete client credentials from config file also if file exists
if Path(config_filename).is_file():
print(f"Deleting the CILogon client details from the {config_filename} also...")
key = "CILogonOAuthenticator"
try:
remove_jupyterhub_hub_config_key_from_encrypted_file(config_filename, key)
except KeyError:
print_colour(f"No {key} found to delete from {config_filename}", "yellow")
return
print_colour(f"CILogonAuthenticator config removed from {config_filename}")
if not Path(config_filename).is_file():
print_colour(f"Empty {config_filename} file also deleted.", "yellow")


def get_all_clients(admin_id, admin_secret):
print("Getting all existing CILogon client applications...")
Expand All @@ -344,8 +302,67 @@ def get_all_clients(admin_id, admin_secret):
return

clients = response.json()
for c in clients["clients"]:
print(c)
return [c for c in clients["clients"]]


def find_duplicated_clients(clients):
"""Determine duplicated CILogon clients by comparing client names
Args:
clients (list[dict]): A list of dictionaries containing information about
the existing CILogon clients. Generated by get_all_clients function.
Returns:
list: A list of duplicated client names
"""
client_names = [c["name"] for c in clients]
client_names_count = Counter(client_names)
return [k for k, v in client_names_count.items() if v > 1]


def find_orphaned_clients(clients):
"""Find CILogon clients for which an associated cluster or hub no longer
exists and can safely be deleted.
Args:
clients (list[dict]): A list of existing CILogon client info
Returns:
list[dict]: A list of 'orphaned' CILogon clients which don't have an
associated cluster or hub, which can be deleted
"""
clients_to_be_deleted = []
clusters = get_cluster_names_list()

for client in clients:
cluster = next((cl for cl in clusters if cl in client["name"]), "")

if cluster:
cluster_config_file = find_absolute_path_to_cluster_file(cluster)
with open(cluster_config_file) as f:
cluster_config = yaml.load(f)

hub = next(
(
hub["name"]
for hub in cluster_config["hubs"]
if hub["name"] in client["name"]
),
"",
)

if not hub:
print(
f"A hub pertaining to client {client['name']} does NOT exist. Marking client for deletion."
)
clients_to_be_deleted.append(client)
else:
print(
f"A cluster pertaining to client {client['name']} does NOT exist. Marking client for deletion."
)
clients_to_be_deleted.append(client)

return clients_to_be_deleted


def get_2i2c_cilogon_admin_credentials():
Expand Down Expand Up @@ -415,7 +432,13 @@ def get(
def get_all():
"""Retrieve details about all existing 2i2c CILogon clients."""
admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()
get_all_clients(admin_id, admin_secret)
clients = get_all_clients(admin_id, admin_secret)
for c in clients:
print(c)

# Our plan with CILogon only permits 100 clients, so provide feedback on that
# number here. Change this if our plan updates.
print_colour(f"{len(clients)} / 100 clients used", "yellow")


@cilogon_client_app.command()
Expand All @@ -432,7 +455,123 @@ def delete(
""",
),
):
"""Delete an existing CILogon client. This deletes both the CILogon client application,
and the client credentials from the configuration file."""
"""
Delete an existing CILogon client. This deletes both the CILogon client application,
and the client credentials from the configuration file.
"""
config_filename = build_absolute_path_to_hub_encrypted_config_file(
cluster_name, hub_name
)
admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()

if not client_id:
if Path(config_filename).is_file():
client_id = load_client_id_from_file(config_filename)
# Nothing to do if no client has been found
if not client_id:
print_colour(
"No `client_id` to delete was provided and couldn't find any in `config_filename`",
"red",
)
return
else:
print_colour(
f"No `client_id` to delete was provided and couldn't find any {config_filename} file",
"red",
)
return
else:
if not stored_client_id_same_with_cilogon_records(
admin_id,
admin_secret,
cluster_name,
hub_name,
client_id,
):
print_colour(
"CILogon records are different than the client app stored in the configuration file. Consider updating the file.",
"red",
)
return

delete_client(admin_id, admin_secret, cluster_name, hub_name, client_id)

# Delete client credentials from config file also if file exists
if Path(config_filename).is_file():
print(f"Deleting the CILogon client details from the {config_filename} also...")
key = "CILogonOAuthenticator"
try:
remove_jupyterhub_hub_config_key_from_encrypted_file(config_filename, key)
except KeyError:
print_colour(f"No {key} found to delete from {config_filename}", "yellow")
return
print_colour(f"CILogonAuthenticator config removed from {config_filename}")
if not Path(config_filename).is_file():
print_colour(f"Empty {config_filename} file also deleted.", "yellow")


@cilogon_client_app.command()
def cleanup(
delete: bool = typer.Option(
False, help="Proceed with deleting duplicated CILogon apps"
)
):
"""Identify duplicated CILogon clients and which ID is being actively used in config,
and optionally delete unused duplicates.
Args:
delete (bool, optional): Delete unused duplicate CILogon apps. Defaults to False.
"""
clients_to_be_deleted = []

admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()
clients = get_all_clients(admin_id, admin_secret)
duplicated_clients = find_duplicated_clients(clients)

# Cycle over each duplicated client name
for duped_client in duplicated_clients:
# Finds all the client IDs associated with a duplicated name
ids = [c["client_id"] for c in clients if c["name"] == duped_client]

# Establish the cluster and hub name from the client name and build the
# absolute path to the encrypted hub values file
cluster_name, hub_name = duped_client.split("-")
config_filename = build_absolute_path_to_hub_encrypted_config_file(
cluster_name, hub_name
)

with get_decrypted_file(config_filename) as decrypted_path:
with open(decrypted_path) as f:
secret_config = yaml.load(f)

if (
"CILogonOAuthenticator"
not in secret_config["jupyterhub"]["hub"]["config"].keys()
):
print(
f"Hub {hub_name} on cluster {cluster_name} doesn't use CILogonOAuthenticator."
)
else:
# Extract the client ID *currently in use* from the encrypted config and remove it from the list of IDs
config_client_id = secret_config["jupyterhub"]["hub"]["config"][
"CILogonOAuthenticator"
]["client_id"]
ids.remove(config_client_id)

clients_to_be_deleted.extend(
[{"client_name": duped_client, "client_id": id} for id in ids]
)

# Remove the duplicated clients from the client list
clients = [c for c in clients if c["name"] != duped_client]

orphaned_clients = find_orphaned_clients(clients)
clients_to_be_deleted.extend(orphaned_clients)

print_colour("CILogon clients to be deleted...")
for c in clients_to_be_deleted:
print(c)

if delete:
for c in clients_to_be_deleted:
delete_client(admin_id, admin_secret, client_id=c["client_id"])
9 changes: 9 additions & 0 deletions deployer/utils/file_acquisition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,3 +248,12 @@ def get_all_cluster_yaml_files():
for path in CONFIG_CLUSTERS_PATH.glob("**/cluster.yaml")
if "templates" not in path.as_posix()
}


def get_cluster_names_list():
"""
Returns a list of all the clusters currently listed under config/clusters
"""
return [
d.name for d, _, _ in CONFIG_CLUSTERS_PATH.walk() if "templates" not in str(d)
]

0 comments on commit 71036c4

Please sign in to comment.