From 19689600816cd66753796aa3ec6aa4ff23a3dfc4 Mon Sep 17 00:00:00 2001 From: hugo Date: Fri, 30 Sep 2022 14:08:51 -0700 Subject: [PATCH 1/8] wip --- examples/registering-users/README.md | 5 ++ examples/registering-users/register.py | 81 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 examples/registering-users/README.md create mode 100644 examples/registering-users/register.py diff --git a/examples/registering-users/README.md b/examples/registering-users/README.md new file mode 100644 index 00000000..f961eed6 --- /dev/null +++ b/examples/registering-users/README.md @@ -0,0 +1,5 @@ +# Registering Users + +This example illustrates how to write a script to register users in Saturn Cloud. + +This example takes a list of email addresses (passed in via an `EMAILS_FOR_ACCOUNTS` environment variable). It will check if a user account exists or not, and if not, create an account for the user. The reason we are passing this in via an ENV var is so that we can pass this via the Saturn secrets manager. diff --git a/examples/registering-users/register.py b/examples/registering-users/register.py new file mode 100644 index 00000000..6eec90fa --- /dev/null +++ b/examples/registering-users/register.py @@ -0,0 +1,81 @@ + +from urllib.parse import urlencode + +import requests + + +# this should be populated by the secrets manager +EMAILS_FOR_ACCOUNTS = os.getenv("EMAILS_FOR_ACCOUNTS") + +# this should be populated by Saturn. This script will only work if +# run from an account that has admin access +BASE_URL = os.getenv("BASE_URL") +SATURN_TOKEN = os.getenv("SATURN_TOKEN") +saturn_headers = {"Authorization": f"token {SATURN_TOKEN}"} + +def check_for_account_by_email(email: str) -> bool: + url = f"{BASE_URL}/api/users" + query_string = urlencode(dict(q=f"email:{email}", page=1, size=1)) + url = url + "?" + query_string + response = requests.get(url, headers=saturn_headers) + results = response.json()['users'] + if results: + return True + return False + + +def check_for_account_by_username(username: str) -> bool: + url = f"{BASE_URL}/api/users" + query_string = urlencode(dict(q=f"username:{username}", page=1, size=1)) + url = url + "?" + query_string + response = requests.get(url, headers=saturn_headers) + results = response.json()['users'] + + if results: + return True + return False + + +def make_unique_username(email: str) -> str: + candidate_username = email.split('@')[0] + candidate_username = "".join(c for c in candidate_username if c.isalnum()) + + # we'll try 100 integers until we get a unique name + for c in range(100): + to_try = candidate_username + if c: + to_try = candidate_username + str(c) + if not check_for_account_by_username(to_try): + return to_try + raise ValueError(f'unable to find username for {candidate_username}') + + +def make_account(username: str, email: str): + url = f"{BASE_URL}/api/users" + body = dict( + username=username, + email=email, + admin=False, + locked=False, + send_reset_email=False, + prevent_duplicate_emails=True, + ) + response = requests.post(url, json=body, headers=saturn_headers) + print(response.json()) + + +def ensure_account_exists(email: str) -> None: + if check_for_account_by_email(email): + return + username = make_unique_username(email) + make_account(username, email) + + +def run(): + for email in EMAILS_FOR_ACCOUNTS.split('\n'): + if email: + ensure_account_exists(email) + + +if __name__ == "__main__": + run() From 9a062045ad68313c1882748ff3e2712fb67716d2 Mon Sep 17 00:00:00 2001 From: hugo Date: Mon, 17 Oct 2022 09:53:11 -0700 Subject: [PATCH 2/8] wip --- examples/registering-images/README.md | 12 ++++++ examples/registering-images/register.py | 57 +++++++++++++++++++++++++ examples/registering-users/register.py | 1 + 3 files changed, 70 insertions(+) create mode 100644 examples/registering-images/README.md create mode 100644 examples/registering-images/register.py diff --git a/examples/registering-images/README.md b/examples/registering-images/README.md new file mode 100644 index 00000000..6fc5f6a4 --- /dev/null +++ b/examples/registering-images/README.md @@ -0,0 +1,12 @@ +# Registering Images + +This example illustrates how to write a script to register images in Saturn Cloud. + +This example takes a CSV of images (passed in via an `IMAGES` environment variable). + +The first column is the name of the image in ECR. The second is the name of the image in Saturn Cloud. We assume we want +to register images under the Saturn Cloud user that is running this script. + +This script will query ECR for image tags for the image. For each tag, it will check if the image version exists in Saturn Cloud. If it does not - it will create the image version. + +The script will NOT create images - those must already exist. diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py new file mode 100644 index 00000000..9bf06b88 --- /dev/null +++ b/examples/registering-images/register.py @@ -0,0 +1,57 @@ +import os +from urllib.parse import urlencode + +import requests +import boto3 + + +# this should be populated by the secrets manager +IMAGES = os.getenv("IMAGES") + + +# this should be populated by Saturn. +BASE_URL = os.getenv("BASE_URL") +SATURN_TOKEN = os.getenv("SATURN_TOKEN") +saturn_headers = {"Authorization": f"token {SATURN_TOKEN}"} + +def list_images(ecr_image_name: str): + ecr = boto3.client('ecr') + + repository = ecr.describe_repositories(repositoryNames=[ecr_image_name])[ + 'repositories' + ][0] + repository_uri = repository['repositoryUri'] + + list_images = ecr.get_paginator("list_images") + for page in list_images.paginate(RepositoryName=ecr_image_name): + for image_id in page['imageIds']: + tag = image_id.get('imageTag', None) + if tag: + yield dict(image_uri=f"{repository_uri}:{tag}", image_tag=tag) + + +def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): + is_gpu = is_gpu.lower() == 'true' + + ecr_images = ecr.list_images(ecr_image_name) + for image in ecr_images: + image_uri = image['image_uri'] + image_tag = image['image_tag'] + url = f"{BASE_URL}/api/images" + requests.post(url, json={ + image_uri: image_uri, + version: image_tag, + is_new_version: True, + image: { + is_external: True, + is_gpu: is_gpu, + visibility: 'account', + image: saturn_image_name + } + }) + + +def run(): + for line in IMAGES.split('\n'): + ecr_image_name, saturn_image_name, is_gpu = line.spit(',') + register(ecr_image_name, saturn_image_name, is_gpu) diff --git a/examples/registering-users/register.py b/examples/registering-users/register.py index 6eec90fa..253108d4 100644 --- a/examples/registering-users/register.py +++ b/examples/registering-users/register.py @@ -1,3 +1,4 @@ +import os from urllib.parse import urlencode From eafabf7a260a2bb0f32ab6aced4514cc65a26cf4 Mon Sep 17 00:00:00 2001 From: hugo Date: Tue, 18 Oct 2022 08:43:40 -0700 Subject: [PATCH 3/8] wip --- examples/registering-images/register.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py index 9bf06b88..a3401053 100644 --- a/examples/registering-images/register.py +++ b/examples/registering-images/register.py @@ -1,12 +1,14 @@ import os from urllib.parse import urlencode +import json import requests import boto3 +# Currently only works against the 2022.05.01 API. # this should be populated by the secrets manager -IMAGES = os.getenv("IMAGES") +IMAGE_SPEC = json.loads(os.getenv("IMAGE_SPEC")) # this should be populated by Saturn. @@ -31,13 +33,17 @@ def list_images(ecr_image_name: str): def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): + url = f"{BASE_URL}/api/images?page_size=-1" + existing_images = requests.get(url).json()['images'] + existing_image_uris = set([x['image_uri'] for x in existing_images]) is_gpu = is_gpu.lower() == 'true' - + url = f"{BASE_URL}/api/images" ecr_images = ecr.list_images(ecr_image_name) for image in ecr_images: image_uri = image['image_uri'] + if image_uri in existing_image_uris: + continue image_tag = image['image_tag'] - url = f"{BASE_URL}/api/images" requests.post(url, json={ image_uri: image_uri, version: image_tag, @@ -52,6 +58,8 @@ def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): def run(): - for line in IMAGES.split('\n'): - ecr_image_name, saturn_image_name, is_gpu = line.spit(',') + for image_spec in IMAGE_SPEC: + ecr_image_name = image_spec['ecr_image_name'] + saturn_image_name = image_spec['saturn_image_name'] + is_gpu = image_spec['is_gpu'] register(ecr_image_name, saturn_image_name, is_gpu) From caed217f7c26ae2d63aa49a61e87156ec8b8d941 Mon Sep 17 00:00:00 2001 From: hugo Date: Tue, 18 Oct 2022 08:44:59 -0700 Subject: [PATCH 4/8] dry run --- examples/registering-images/register.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py index a3401053..c357334a 100644 --- a/examples/registering-images/register.py +++ b/examples/registering-images/register.py @@ -9,6 +9,7 @@ # this should be populated by the secrets manager IMAGE_SPEC = json.loads(os.getenv("IMAGE_SPEC")) +DRY_RUN = True # this should be populated by Saturn. @@ -44,7 +45,7 @@ def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): if image_uri in existing_image_uris: continue image_tag = image['image_tag'] - requests.post(url, json={ + payload = { image_uri: image_uri, version: image_tag, is_new_version: True, @@ -54,7 +55,11 @@ def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): visibility: 'account', image: saturn_image_name } - }) + } + if DRY_RUN: + print(f'REGISTER image_uri {payload}') + else: + requests.post(url, json=payload) def run(): From 59bfcbabee9ee67cec8fa098d742f9d8646d51a2 Mon Sep 17 00:00:00 2001 From: hugo Date: Tue, 18 Oct 2022 09:24:00 -0700 Subject: [PATCH 5/8] update --- examples/registering-images/register.py | 63 +++++++++++++++++-------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py index c357334a..f1ddeea4 100644 --- a/examples/registering-images/register.py +++ b/examples/registering-images/register.py @@ -1,6 +1,9 @@ import os from urllib.parse import urlencode import json +import click +import time +from datetime import timedelta import requests import boto3 @@ -8,15 +11,16 @@ # Currently only works against the 2022.05.01 API. # this should be populated by the secrets manager -IMAGE_SPEC = json.loads(os.getenv("IMAGE_SPEC")) -DRY_RUN = True - +with open("/home/jovyan/image_spec.json") as f: + IMAGE_SPEC = json.load(f) +DRY_RUN = os.getenv('DRY_RUN', 'TRUE').lower() == 'true' # this should be populated by Saturn. BASE_URL = os.getenv("BASE_URL") SATURN_TOKEN = os.getenv("SATURN_TOKEN") saturn_headers = {"Authorization": f"token {SATURN_TOKEN}"} + def list_images(ecr_image_name: str): ecr = boto3.client('ecr') @@ -26,7 +30,7 @@ def list_images(ecr_image_name: str): repository_uri = repository['repositoryUri'] list_images = ecr.get_paginator("list_images") - for page in list_images.paginate(RepositoryName=ecr_image_name): + for page in list_images.paginate(repositoryName=ecr_image_name): for image_id in page['imageIds']: tag = image_id.get('imageTag', None) if tag: @@ -35,36 +39,55 @@ def list_images(ecr_image_name: str): def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): url = f"{BASE_URL}/api/images?page_size=-1" - existing_images = requests.get(url).json()['images'] + existing_images = requests.get(url, headers=saturn_headers).json()['images'] existing_image_uris = set([x['image_uri'] for x in existing_images]) - is_gpu = is_gpu.lower() == 'true' url = f"{BASE_URL}/api/images" - ecr_images = ecr.list_images(ecr_image_name) + ecr_images = list_images(ecr_image_name) for image in ecr_images: image_uri = image['image_uri'] if image_uri in existing_image_uris: continue image_tag = image['image_tag'] payload = { - image_uri: image_uri, - version: image_tag, - is_new_version: True, - image: { - is_external: True, - is_gpu: is_gpu, - visibility: 'account', - image: saturn_image_name + "image_uri": image_uri, + "version": image_tag, + "is_new_version": True, + "image": { + "is_external": True, + "is_gpu": is_gpu, + "visibility": 'org', + "name": saturn_image_name } } - if DRY_RUN: - print(f'REGISTER image_uri {payload}') - else: - requests.post(url, json=payload) + print(f"REGISTER image_uri {payload['image_uri']}, {payload['image']['name']}") + if not DRY_RUN: + print(requests.post(url, headers=saturn_headers, json=payload).json()) -def run(): +def sync(): for image_spec in IMAGE_SPEC: ecr_image_name = image_spec['ecr_image_name'] saturn_image_name = image_spec['saturn_image_name'] is_gpu = image_spec['is_gpu'] register(ecr_image_name, saturn_image_name, is_gpu) + + +@click.group() +def cli(): + pass + + +@cli.command() +def run_once(): + sync() + + +@cli.command() +def run(): + while True: + sync() + time.sleep(30) + + +if __name__ == "__main__": + cli() \ No newline at end of file From 580d2dfa0c7f23b870814816cfec3e7be650fdf8 Mon Sep 17 00:00:00 2001 From: hugo Date: Tue, 15 Nov 2022 15:25:06 -0800 Subject: [PATCH 6/8] updating for 10.01 api --- examples/registering-images/register.py | 67 ++++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py index f1ddeea4..671244a2 100644 --- a/examples/registering-images/register.py +++ b/examples/registering-images/register.py @@ -1,4 +1,5 @@ import os +from typing import Dict, Optional from urllib.parse import urlencode import json import click @@ -8,17 +9,20 @@ import requests import boto3 -# Currently only works against the 2022.05.01 API. # this should be populated by the secrets manager with open("/home/jovyan/image_spec.json") as f: IMAGE_SPEC = json.load(f) + + DRY_RUN = os.getenv('DRY_RUN', 'TRUE').lower() == 'true' + # this should be populated by Saturn. BASE_URL = os.getenv("BASE_URL") SATURN_TOKEN = os.getenv("SATURN_TOKEN") saturn_headers = {"Authorization": f"token {SATURN_TOKEN}"} +SATURN_USERNAME = os.getenv("SATURN_USERNAME") def list_images(ecr_image_name: str): @@ -37,39 +41,52 @@ def list_images(ecr_image_name: str): yield dict(image_uri=f"{repository_uri}:{tag}", image_tag=tag) -def register(ecr_image_name: str, saturn_image_name: str, is_gpu: str): - url = f"{BASE_URL}/api/images?page_size=-1" - existing_images = requests.get(url, headers=saturn_headers).json()['images'] - existing_image_uris = set([x['image_uri'] for x in existing_images]) - url = f"{BASE_URL}/api/images" +def make_url(path: str, queries: Optional[Dict[str, str]] = None) -> str: + if queries: + return f"{BASE_URL}/{path}?" + urlencode(queries) + else: + return f"{BASE_URL}/{path}" + + + +def register(image_uri: str, version: str, saturn_image_name: str, dry_run: bool = False): + q = f"owner:{SATURN_USERNAME} name:{saturn_image_name}" + url = make_url("api/images", dict(q=q, page_size="-1")) + images = requests.get(url, headers=saturn_headers).json() + images = [x for x in images['images'] if x['name'] == saturn_image_name] + if not images: + raise ValueError(f'no image found for {q}') + elif len(images) > 1: + raise ValueError(f'multiple images found for {q}') + image = images[0] + image_id = image['id'] + + q = f"version:{version}" + url = make_url(f"api/images/{image_id}/tags", dict(q=q, page_size="-1")) + + tags = requests.get(url, headers=saturn_headers).json()['image_tags'] + if image_uri in [x['image_uri'] for x in tags]: + print(f'found {image_uri}') + return + + print(f"REGISTER {image_uri} {image}") + if not dry_run: + requests.post(url, json={'image_uri': image_uri}, headers=saturn_headers) + + +def register_all(ecr_image_name: str, saturn_image_name: str): ecr_images = list_images(ecr_image_name) for image in ecr_images: image_uri = image['image_uri'] - if image_uri in existing_image_uris: - continue image_tag = image['image_tag'] - payload = { - "image_uri": image_uri, - "version": image_tag, - "is_new_version": True, - "image": { - "is_external": True, - "is_gpu": is_gpu, - "visibility": 'org', - "name": saturn_image_name - } - } - print(f"REGISTER image_uri {payload['image_uri']}, {payload['image']['name']}") - if not DRY_RUN: - print(requests.post(url, headers=saturn_headers, json=payload).json()) + register(image_uri, image_tag, saturn_image_name, dry_run=DRY_RUN) def sync(): for image_spec in IMAGE_SPEC: ecr_image_name = image_spec['ecr_image_name'] saturn_image_name = image_spec['saturn_image_name'] - is_gpu = image_spec['is_gpu'] - register(ecr_image_name, saturn_image_name, is_gpu) + register_all(ecr_image_name, saturn_image_name) @click.group() @@ -90,4 +107,4 @@ def run(): if __name__ == "__main__": - cli() \ No newline at end of file + cli() From 99146b7dbe1b097e2b9d4984fdf84509dcf7a950 Mon Sep 17 00:00:00 2001 From: saturn Date: Tue, 6 Feb 2024 15:15:27 +0000 Subject: [PATCH 7/8] wip --- examples/registering-images/register.py | 76 ++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py index 671244a2..0c37b5a4 100644 --- a/examples/registering-images/register.py +++ b/examples/registering-images/register.py @@ -1,5 +1,5 @@ import os -from typing import Dict, Optional +from typing import Dict, Optional, List from urllib.parse import urlencode import json import click @@ -14,6 +14,10 @@ with open("/home/jovyan/image_spec.json") as f: IMAGE_SPEC = json.load(f) + +with open("/home/jovyan/base_image_spec.json") as f: + BASE_IMAGE_SPEC = json.load(f) + DRY_RUN = os.getenv('DRY_RUN', 'TRUE').lower() == 'true' @@ -25,7 +29,12 @@ SATURN_USERNAME = os.getenv("SATURN_USERNAME") -def list_images(ecr_image_name: str): +def list_images(ecr_image_name: str) -> List[Dict[str, str]]: + """ + for a name in ECR, yield a list of dicts, with + - image_uri + - image_tag + """ ecr = boto3.client('ecr') repository = ecr.describe_repositories(repositoryNames=[ecr_image_name])[ @@ -50,6 +59,12 @@ def make_url(path: str, queries: Optional[Dict[str, str]] = None) -> str: def register(image_uri: str, version: str, saturn_image_name: str, dry_run: bool = False): + """ + looks up Saturn image_id from saturn_image_name. + looks up available image tags from saturn matching the saturn_image_name + if image_uri has not yet been registered, then create a new + ImageTag object with image_uri and version under saturn_image_name + """ q = f"owner:{SATURN_USERNAME} name:{saturn_image_name}" url = make_url("api/images", dict(q=q, page_size="-1")) images = requests.get(url, headers=saturn_headers).json() @@ -71,17 +86,74 @@ def register(image_uri: str, version: str, saturn_image_name: str, dry_run: bool print(f"REGISTER {image_uri} {image}") if not dry_run: + url = make_url(f"api/images/{image_id}/tags") requests.post(url, json={'image_uri': image_uri}, headers=saturn_headers) + +def get_all_tags(saturn_image_id: str) -> List[Dict]: + url = make_url(f"api/images/{saturn_image_id}/tags", dict(page_size="-1")) + tags = requests.get(url, headers=saturn_headers).json()['image_tags'] + return tags + + +def delete_all_tags(saturn_image_id: str, tags: List[Dict], dry_run: bool=False): + tags = get_all_tags(saturn_image_id) + for t in tags: + url = make_url(f"api/images/{saturn_image_id}/tags/{t['id']}") + print('delete', url) + if not dry_run: + requests.delete(url, headers=saturn_headers).json() + + +def register_by_id(image_uri: str, version: str, saturn_image_id: str, dry_run: bool = False): + """ + Create a new ImageTag object with image_uri and version under saturn_image_name + """ + if not dry_run: + url = make_url(f"api/images/{saturn_image_id}/tags") + requests.post(url, json={'image_uri': image_uri}, headers=saturn_headers) + +def register_base_image(ecr_image_name: str, saturn_image_id: str): + """ + For a given ecr image name, and a saturn_image_id which corresponds to a base image. + 1. find the latest tag + 2. delete all image_tags associated with saturn_image_id + we do this because the saturn image UI is bad at handling a large number of images tags + so we just keep the most recent one + 3. Register a new image tag under saturn_image_id + """ + ecr_images = list(list_images(ecr_image_name)) + ecr_image = sorted(ecr_images, key=lambda x: x['image_tag'])[-1] + image_uri = ecr_image['image_uri'] + image_tag = ecr_image['image_tag'] + tags = get_all_tags(saturn_image_id) + if image_uri in [x['image_uri'] for x in tags]: + print(f'found {image_uri}') + return + delete_all_tags(saturn_image_id, tags, dry_run=DRY_RUN) + register_by_id(image_uri, image_tag, saturn_image_id, dry_run=DRY_RUN) + + def register_all(ecr_image_name: str, saturn_image_name: str): + """ + for a given ecr image name, retrieve all image_uris/tags from ECR. + attempt to register all in Saturn + """ ecr_images = list_images(ecr_image_name) for image in ecr_images: image_uri = image['image_uri'] image_tag = image['image_tag'] register(image_uri, image_tag, saturn_image_name, dry_run=DRY_RUN) + +def sync_base(): + for image_spec in BASE_IMAGE_SPEC: + ecr_image_name = image_spec['ecr_image_name'] + saturn_image_id = image_spec['saturn_image_id'] + register_base_image(ecr_image_name, saturn_image_id) + def sync(): for image_spec in IMAGE_SPEC: ecr_image_name = image_spec['ecr_image_name'] From 8fcec14a420dd3d3547b6866add13bf7f923a233 Mon Sep 17 00:00:00 2001 From: saturn Date: Tue, 6 Feb 2024 15:57:36 +0000 Subject: [PATCH 8/8] wip --- examples/registering-images/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/registering-images/register.py b/examples/registering-images/register.py index 0c37b5a4..cab018c8 100644 --- a/examples/registering-images/register.py +++ b/examples/registering-images/register.py @@ -102,7 +102,7 @@ def delete_all_tags(saturn_image_id: str, tags: List[Dict], dry_run: bool=False) url = make_url(f"api/images/{saturn_image_id}/tags/{t['id']}") print('delete', url) if not dry_run: - requests.delete(url, headers=saturn_headers).json() + resp = requests.delete(url, headers=saturn_headers) def register_by_id(image_uri: str, version: str, saturn_image_id: str, dry_run: bool = False): @@ -169,12 +169,14 @@ def cli(): @cli.command() def run_once(): sync() + sync_base() @cli.command() def run(): while True: sync() + sync_base() time.sleep(30)