From d2924578034c4c8a20b013dfba3b25f0966e3c13 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 21:51:23 -0500 Subject: [PATCH 01/27] Initial config setup --- tljh/config-schema.json | 1 + tljh/config.py | 8 ++++++++ tljh/requirements-hub-env.txt | 2 ++ 3 files changed, 11 insertions(+) create mode 100644 tljh/config-schema.json diff --git a/tljh/config-schema.json b/tljh/config-schema.json new file mode 100644 index 00000000..15591da4 --- /dev/null +++ b/tljh/config-schema.json @@ -0,0 +1 @@ +file:///home/bradfojb/Desktop/Personal-Repos/the-littlest-jupyterhub/schema.json \ No newline at end of file diff --git a/tljh/config.py b/tljh/config.py index d308e9e8..09ecef06 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -154,6 +154,13 @@ def remove_item_from_config(config, property_path, value): return config_copy +def validate_config(config): + import json + import jsonschema + config_schema = json.load("config-schema.json") + jsonschema.validate(instance=config, schema=config_schema) + + def show_config(config_path): """ Pretty print config from given config_path @@ -180,6 +187,7 @@ def set_config_value(config_path, key_path, value): config = {} config = set_item_in_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index 62f39f4b..cde66e4d 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -17,6 +17,8 @@ jupyterhub-tmpauthenticator>=1.0.0,<2 oauthenticator[azuread]>=16.0.4,<17 jupyterhub-idle-culler>=1.2.1,<2 +jsonschema + # pycurl is installed to improve reliability and performance for when JupyterHub # makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when # making requests over tornado's SimpleHTTPClient automatically if pycurl is From 78d4b7fbc4a4fe170a6cd2275865c88b02b0da02 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 22:52:16 -0500 Subject: [PATCH 02/27] Test config setup --- tljh/config-schema.json | 143 +++++++++++++++++++++++++++++++++- tljh/config.py | 25 +++--- tljh/requirements-hub-env.txt | 2 - 3 files changed, 158 insertions(+), 12 deletions(-) diff --git a/tljh/config-schema.json b/tljh/config-schema.json index 15591da4..4563d192 100644 --- a/tljh/config-schema.json +++ b/tljh/config-schema.json @@ -1 +1,142 @@ -file:///home/bradfojb/Desktop/Personal-Repos/the-littlest-jupyterhub/schema.json \ No newline at end of file +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Littlest JupyterHub YAML config file", + "definitions": { + "Users": { + "type": "object", + "properties": { + "extra_user_groups": { + "type": "object", + "items": { + "type": "string" + } + }, + "allowed": { + "type": "array", + "items": { + "type": "string" + } + }, + "banned": { + "type": "array", + "items": { + "type": "string" + } + }, + "admin": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "HTTP": { + "type": "object", + "additionalProperties": false, + "properties": { + "address": { + "type": "string", + "format": "ipv4" + }, + "port": { + "type": "integer" + } + } + }, + "HTTPS": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "address": { + "type": "string", + "format": "ipv4" + }, + "port": { + "type": "integer" + }, + "tls": { + "$ref": "#/definitions/TLS" + }, + "letsencrypt": { + "$ref": "#/definitions/LetsEncrypt" + } + } + }, + "LetsEncrypt": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "type": "string", + "format": "email" + }, + "domains": { + "type": "array", + "items": { + "type": "string", + "format": "hostname" + } + } + } + }, + "TLS": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "type": "string" + }, + "cert": { + "type": "string" + } + }, + "required": ["key", "cert"] + }, + "Limits": { + "description": "User CPU and memory limits.", + "type": "object", + "additionalProperties": false, + "properties": { + "memory": { + "type": "string" + }, + "cpu": { + "type": "integer" + } + } + }, + "UserEnvironment": { + "type": "object", + "additionalProperties": false, + "properties": { + "default_app": { + "type": "string", + "enum": ["jupyterlab", "classic"], + "default": "jupyterlab" + } + } + } + }, + "properties": { + "additionalProperties": false, + "user_environment": { + "$ref": "#/definitions/UserEnvironment" + }, + "users": { + "$ref": "#/definitions/Users" + }, + "limits": { + "$ref": "#/definitions/Limits" + }, + "https": { + "$ref": "#/definitions/HTTPS" + }, + "http": { + "$ref": "#/definitions/HTTP" + } + } +} diff --git a/tljh/config.py b/tljh/config.py index 09ecef06..8f0767a1 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,10 +155,18 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): - import json - import jsonschema - config_schema = json.load("config-schema.json") - jsonschema.validate(instance=config, schema=config_schema) + import json, jsonschema + + cwd = os.getcwd() + config_schema_file = os.path.join(cwd, "config-schema.json") + with open(config_schema_file) as f: + config_schema = json.load(f) + + try: + jsonschema.validate(instance=config, schema=config_schema) + except jsonschema.exceptions.ValidationError as e: + print(e) + exit() def show_config(config_path): @@ -179,14 +187,13 @@ def set_config_value(config_path, key_path, value): Set key at key_path in config_path to value """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) except FileNotFoundError: config = {} - config = set_item_in_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: @@ -198,7 +205,6 @@ def unset_config_value(config_path, key_path): Unset key at key_path in config_path """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) @@ -206,6 +212,7 @@ def unset_config_value(config_path, key_path): config = {} config = unset_item_from_config(config, key_path) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) @@ -216,7 +223,6 @@ def add_config_value(config_path, key_path, value): Add value to list at key_path """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) @@ -224,6 +230,7 @@ def add_config_value(config_path, key_path, value): config = {} config = add_item_to_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) @@ -234,7 +241,6 @@ def remove_config_value(config_path, key_path, value): Remove value from list at key_path """ # FIXME: Have a file lock here - # FIXME: Validate schema here try: with open(config_path) as f: config = yaml.load(f) @@ -242,6 +248,7 @@ def remove_config_value(config_path, key_path, value): config = {} config = remove_item_from_config(config, key_path, value) + validate_config(config) with open(config_path, "w") as f: yaml.dump(config, f) diff --git a/tljh/requirements-hub-env.txt b/tljh/requirements-hub-env.txt index cde66e4d..62f39f4b 100644 --- a/tljh/requirements-hub-env.txt +++ b/tljh/requirements-hub-env.txt @@ -17,8 +17,6 @@ jupyterhub-tmpauthenticator>=1.0.0,<2 oauthenticator[azuread]>=16.0.4,<17 jupyterhub-idle-culler>=1.2.1,<2 -jsonschema - # pycurl is installed to improve reliability and performance for when JupyterHub # makes web requests. JupyterHub will use tornado's CurlAsyncHTTPClient when # making requests over tornado's SimpleHTTPClient automatically if pycurl is From fb01dea5e4feab9c122c014873d0473bf6083e7c Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:01:36 -0500 Subject: [PATCH 03/27] Working schema path --- tljh/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 8f0767a1..9bcd200c 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -157,8 +157,8 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): import json, jsonschema - cwd = os.getcwd() - config_schema_file = os.path.join(cwd, "config-schema.json") + pd = os.path.abspath(os.path.join(__file__, os.pardir)) + config_schema_file = os.path.join(pd, "config-schema.json") with open(config_schema_file) as f: config_schema = json.load(f) From ef5c6c56b79bde9bd5652dd49e4d708fdea77ba7 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:06:31 -0500 Subject: [PATCH 04/27] `print` human readable error message --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 9bcd200c..dce1bae8 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -165,7 +165,7 @@ def validate_config(config): try: jsonschema.validate(instance=config, schema=config_schema) except jsonschema.exceptions.ValidationError as e: - print(e) + print(e.message) exit() From 929536de7bdaf40125e4e035c28165ff5ff4c7a4 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:07:38 -0500 Subject: [PATCH 05/27] Run `pre-commit` hooks --- tljh/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index dce1bae8..3dbbc878 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,7 +155,9 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): - import json, jsonschema + import json + + import jsonschema pd = os.path.abspath(os.path.join(__file__, os.pardir)) config_schema_file = os.path.join(pd, "config-schema.json") From 5a0de137d2b8a63dbcc9b6c26b7770f7985dfe48 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:22:57 -0500 Subject: [PATCH 06/27] Do not allow `additionalProperties` for `Users` --- tljh/config-schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tljh/config-schema.json b/tljh/config-schema.json index 4563d192..d983dc33 100644 --- a/tljh/config-schema.json +++ b/tljh/config-schema.json @@ -4,6 +4,7 @@ "definitions": { "Users": { "type": "object", + "additionalProperties": false, "properties": { "extra_user_groups": { "type": "object", From 166eba67351bcfe1a6bc34b7c9a657ab644b6250 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 12:48:31 -0500 Subject: [PATCH 07/27] Switch from JSON to py --- tljh/config-schema.json | 143 ---------------------------------------- tljh/config.py | 10 +-- tljh/config_schema.py | 77 ++++++++++++++++++++++ 3 files changed, 79 insertions(+), 151 deletions(-) delete mode 100644 tljh/config-schema.json create mode 100644 tljh/config_schema.py diff --git a/tljh/config-schema.json b/tljh/config-schema.json deleted file mode 100644 index d983dc33..00000000 --- a/tljh/config-schema.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Littlest JupyterHub YAML config file", - "definitions": { - "Users": { - "type": "object", - "additionalProperties": false, - "properties": { - "extra_user_groups": { - "type": "object", - "items": { - "type": "string" - } - }, - "allowed": { - "type": "array", - "items": { - "type": "string" - } - }, - "banned": { - "type": "array", - "items": { - "type": "string" - } - }, - "admin": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "HTTP": { - "type": "object", - "additionalProperties": false, - "properties": { - "address": { - "type": "string", - "format": "ipv4" - }, - "port": { - "type": "integer" - } - } - }, - "HTTPS": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "address": { - "type": "string", - "format": "ipv4" - }, - "port": { - "type": "integer" - }, - "tls": { - "$ref": "#/definitions/TLS" - }, - "letsencrypt": { - "$ref": "#/definitions/LetsEncrypt" - } - } - }, - "LetsEncrypt": { - "type": "object", - "additionalProperties": false, - "properties": { - "email": { - "type": "string", - "format": "email" - }, - "domains": { - "type": "array", - "items": { - "type": "string", - "format": "hostname" - } - } - } - }, - "TLS": { - "type": "object", - "additionalProperties": false, - "properties": { - "key": { - "type": "string" - }, - "cert": { - "type": "string" - } - }, - "required": ["key", "cert"] - }, - "Limits": { - "description": "User CPU and memory limits.", - "type": "object", - "additionalProperties": false, - "properties": { - "memory": { - "type": "string" - }, - "cpu": { - "type": "integer" - } - } - }, - "UserEnvironment": { - "type": "object", - "additionalProperties": false, - "properties": { - "default_app": { - "type": "string", - "enum": ["jupyterlab", "classic"], - "default": "jupyterlab" - } - } - } - }, - "properties": { - "additionalProperties": false, - "user_environment": { - "$ref": "#/definitions/UserEnvironment" - }, - "users": { - "$ref": "#/definitions/Users" - }, - "limits": { - "$ref": "#/definitions/Limits" - }, - "https": { - "$ref": "#/definitions/HTTPS" - }, - "http": { - "$ref": "#/definitions/HTTP" - } - } -} diff --git a/tljh/config.py b/tljh/config.py index 3dbbc878..09ee92e0 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,14 +155,8 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): - import json - import jsonschema - - pd = os.path.abspath(os.path.join(__file__, os.pardir)) - config_schema_file = os.path.join(pd, "config-schema.json") - with open(config_schema_file) as f: - config_schema = json.load(f) + from config_schema import config_schema try: jsonschema.validate(instance=config, schema=config_schema) @@ -314,7 +308,7 @@ def parse_value(value_str): return float(value_str) elif value_str.lower() == "true": return True - elif value_str.lower() == "false": + elif value_str.lower() == "False": return False else: # it's a string diff --git a/tljh/config_schema.py b/tljh/config_schema.py new file mode 100644 index 00000000..b0595b37 --- /dev/null +++ b/tljh/config_schema.py @@ -0,0 +1,77 @@ +config_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Littlest JupyterHub YAML config file", + "definitions": { + "Users": { + "type": "object", + "additionalProperties": False, + "properties": { + "extra_user_groups": {"type": "object", "items": {"type": "string"}}, + "allowed": {"type": "array", "items": {"type": "string"}}, + "banned": {"type": "array", "items": {"type": "string"}}, + "admin": {"type": "array", "items": {"type": "string"}}, + }, + }, + "HTTP": { + "type": "object", + "additionalProperties": False, + "properties": { + "address": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + }, + }, + "HTTPS": { + "type": "object", + "additionalProperties": False, + "properties": { + "enabled": {"type": "boolean"}, + "address": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + "tls": {"$ref": "#/definitions/TLS"}, + "letsencrypt": {"$ref": "#/definitions/LetsEncrypt"}, + }, + }, + "LetsEncrypt": { + "type": "object", + "additionalProperties": False, + "properties": { + "email": {"type": "string", "format": "email"}, + "domains": { + "type": "array", + "items": {"type": "string", "format": "hostname"}, + }, + }, + }, + "TLS": { + "type": "object", + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "cert": {"type": "string"}}, + "required": ["key", "cert"], + }, + "Limits": { + "description": "User CPU and memory limits.", + "type": "object", + "additionalProperties": False, + "properties": {"memory": {"type": "string"}, "cpu": {"type": "integer"}}, + }, + "UserEnvironment": { + "type": "object", + "additionalProperties": False, + "properties": { + "default_app": { + "type": "string", + "enum": ["jupyterlab", "classic"], + "default": "jupyterlab", + } + }, + }, + }, + "properties": { + "additionalProperties": False, + "user_environment": {"$ref": "#/definitions/UserEnvironment"}, + "users": {"$ref": "#/definitions/Users"}, + "limits": {"$ref": "#/definitions/Limits"}, + "https": {"$ref": "#/definitions/HTTPS"}, + "http": {"$ref": "#/definitions/HTTP"}, + }, +} From fa363658dfef926f2b94af0be52787821812899d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:08:27 -0500 Subject: [PATCH 08/27] Update schema based on values in `configurer.py` --- tljh/config.py | 3 ++- tljh/config_schema.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 09ee92e0..1c4967bc 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -156,7 +156,8 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): import jsonschema - from config_schema import config_schema + + from .config_schema import config_schema try: jsonschema.validate(instance=config, schema=config_schema) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index b0595b37..2a0c5f0e 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Littlest JupyterHub YAML config file", "definitions": { + "BaseURL": { + "type": "string", + }, "Users": { "type": "object", "additionalProperties": False, @@ -12,6 +15,26 @@ "admin": {"type": "array", "items": {"type": "string"}}, }, }, + "Services": { + "type": "object", + "properties": { + "cull": { + "type": "object", + "additionalProperties": False, + "properties": { + "enabled": {"type": "boolean"}, + "timeout": {"type": "integer"}, + "every": {"type": "integer"}, + "concurrency": {"type": "integer"}, + "users": { + "type": "boolean", + }, + "max_age": {"type": "integer"}, + "remove_named_servers": {"type": "boolean"}, + }, + } + }, + }, "HTTP": { "type": "object", "additionalProperties": False, @@ -65,13 +88,30 @@ } }, }, + "TraefikAPI": { + "type": "object", + "additionalProperties": False, + "properties": { + "ip": {"type": "string", "format": "ipv4"}, + "port": {"type": "integer"}, + "username": { + "type": "string", + }, + "password": { + "type": "string", + }, + }, + }, }, "properties": { "additionalProperties": False, + "base_url": {"$ref": "#/definitions/BaseURL"}, "user_environment": {"$ref": "#/definitions/UserEnvironment"}, "users": {"$ref": "#/definitions/Users"}, "limits": {"$ref": "#/definitions/Limits"}, "https": {"$ref": "#/definitions/HTTPS"}, "http": {"$ref": "#/definitions/HTTP"}, + "traefik_api": {"$ref": "#/definitions/TraefikAPI"}, + "services": {"$ref": "#/definitions/Services"}, }, } From d0c9aa263aea28ee9cf5e24fb9b0255c01df8ac1 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:18:39 -0500 Subject: [PATCH 09/27] Remove `TLS` required properties --- tljh/config_schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index 2a0c5f0e..6480c7f5 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -69,7 +69,6 @@ "type": "object", "additionalProperties": False, "properties": {"key": {"type": "string"}, "cert": {"type": "string"}}, - "required": ["key", "cert"], }, "Limits": { "description": "User CPU and memory limits.", From 4912cffe65c0150c1fc91de0ad83c7006e0e880d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:27:58 -0500 Subject: [PATCH 10/27] Fix `parse_value` --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 1c4967bc..7083dccb 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -309,7 +309,7 @@ def parse_value(value_str): return float(value_str) elif value_str.lower() == "true": return True - elif value_str.lower() == "False": + elif value_str.lower() == "false": return False else: # it's a string From 90602674582419c6ba1cae46e2836390222cf076 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Sat, 3 Feb 2024 13:39:17 -0500 Subject: [PATCH 11/27] Add `LetsEncrypt.staging`; run `pre-commit` --- tljh/config_schema.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index 6480c7f5..e7ae2ba8 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -26,9 +26,7 @@ "timeout": {"type": "integer"}, "every": {"type": "integer"}, "concurrency": {"type": "integer"}, - "users": { - "type": "boolean", - }, + "users": {"type": "boolean"}, "max_age": {"type": "integer"}, "remove_named_servers": {"type": "boolean"}, }, @@ -63,6 +61,7 @@ "type": "array", "items": {"type": "string", "format": "hostname"}, }, + "staging": {"type": "boolean"}, }, }, "TLS": { @@ -93,12 +92,8 @@ "properties": { "ip": {"type": "string", "format": "ipv4"}, "port": {"type": "integer"}, - "username": { - "type": "string", - }, - "password": { - "type": "string", - }, + "username": {"type": "string"}, + "password": {"type": "string"}, }, }, }, From 1f7d6d1c55e9bc9cc7defc66249523d9541e8a6a Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:52:31 -0500 Subject: [PATCH 12/27] Add doc string for `validate_config` --- tljh/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tljh/config.py b/tljh/config.py index 7083dccb..8537cb32 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -155,6 +155,9 @@ def remove_item_from_config(config, property_path, value): def validate_config(config): + """ + Validate changes to the config with tljh-config against the schema + """ import jsonschema from .config_schema import config_schema From 48fe4403726417209fac2f862fc6d5048ead0dfe Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:36:49 -0400 Subject: [PATCH 13/27] Add `--[no-]validation` flag for `tljh-config` --- tljh/config.py | 50 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 8537cb32..849fdb5c 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -154,7 +154,7 @@ def remove_item_from_config(config, property_path, value): return config_copy -def validate_config(config): +def validate_config(config, validate): """ Validate changes to the config with tljh-config against the schema """ @@ -165,8 +165,13 @@ def validate_config(config): try: jsonschema.validate(instance=config, schema=config_schema) except jsonschema.exceptions.ValidationError as e: - print(e.message) - exit() + if validate: + print( + f"Config validation error: {e.message}.\n" + "You can still apply this change without validation by re-running your command with the --no-validate flag.\n" + "If you think this validation error is incorrect, please report it to https://github.com/jupyterhub/the-littlest-jupyterhub/issues." + ) + exit() def show_config(config_path): @@ -182,7 +187,7 @@ def show_config(config_path): yaml.dump(config, sys.stdout) -def set_config_value(config_path, key_path, value): +def set_config_value(config_path, key_path, value, validate): """ Set key at key_path in config_path to value """ @@ -194,13 +199,13 @@ def set_config_value(config_path, key_path, value): config = {} config = set_item_in_config(config, key_path, value) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) -def unset_config_value(config_path, key_path): +def unset_config_value(config_path, key_path, validate): """ Unset key at key_path in config_path """ @@ -212,13 +217,13 @@ def unset_config_value(config_path, key_path): config = {} config = unset_item_from_config(config, key_path) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) -def add_config_value(config_path, key_path, value): +def add_config_value(config_path, key_path, value, validate): """ Add value to list at key_path """ @@ -230,13 +235,13 @@ def add_config_value(config_path, key_path, value): config = {} config = add_item_to_config(config, key_path, value) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) -def remove_config_value(config_path, key_path, value): +def remove_config_value(config_path, key_path, value, validate): """ Remove value from list at key_path """ @@ -248,7 +253,7 @@ def remove_config_value(config_path, key_path, value): config = {} config = remove_item_from_config(config, key_path, value) - validate_config(config) + validate_config(config, validate) with open(config_path, "w") as f: yaml.dump(config, f) @@ -351,6 +356,12 @@ def main(argv=None): argparser.add_argument( "--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file" ) + argparser.add_argument( + "--validate", + action=argparse.BooleanOptionalAction, + help="Validate the TLJH config", + ) + subparsers = argparser.add_subparsers(dest="action") show_parser = subparsers.add_parser("show", help="Show current configuration") @@ -395,16 +406,25 @@ def main(argv=None): args = argparser.parse_args(argv) + if args.validate == None: + args.validate = True + if args.action == "show": show_config(args.config_path) elif args.action == "set": - set_config_value(args.config_path, args.key_path, parse_value(args.value)) + set_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "unset": - unset_config_value(args.config_path, args.key_path) + unset_config_value(args.config_path, args.key_path, args.validate) elif args.action == "add-item": - add_config_value(args.config_path, args.key_path, parse_value(args.value)) + add_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "remove-item": - remove_config_value(args.config_path, args.key_path, parse_value(args.value)) + remove_config_value( + args.config_path, args.key_path, parse_value(args.value), args.validate + ) elif args.action == "reload": reload_component(args.component) else: From 743f729902ec14d22f81420f58f9772077982d36 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:45:36 -0400 Subject: [PATCH 14/27] Update `set_config_value` tests --- tests/test_traefik.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/test_traefik.py b/tests/test_traefik.py index f9502662..380ac4c2 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -64,12 +64,12 @@ def test_default_config(tmpdir, tljh_dir): def test_letsencrypt_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True) + config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org" + config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org", True ) config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"] + config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"], True ) traefik.ensure_traefik_config(str(state_dir)) @@ -138,9 +138,13 @@ def test_letsencrypt_config(tljh_dir): def test_manual_ssl_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True) - config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") - config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") + config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) + config.set_config_value( + config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True + ) + config.set_config_value( + config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True + ) traefik.ensure_traefik_config(str(state_dir)) cfg = _read_static_config(state_dir) @@ -244,12 +248,16 @@ def test_extra_config(tmpdir, tljh_dir): def test_listen_address(tmpdir, tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True) - config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") - config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") + config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) + config.set_config_value( + config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True + ) + config.set_config_value( + config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True + ) - config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1") - config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1") + config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1", True) + config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1", True) traefik.ensure_traefik_config(str(state_dir)) From 46e404556801c978a8218957f28cacbf8f796818 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:08:21 -0400 Subject: [PATCH 15/27] Update `test_proxy` tests --- integration-tests/test_proxy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index cc9d7662..f758655d 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -63,9 +63,9 @@ def test_manual_https(preserve_config): "/CN=tljh.jupyer.org", ] ) - set_config_value(CONFIG_FILE, "https.enabled", True) - set_config_value(CONFIG_FILE, "https.tls.key", key) - set_config_value(CONFIG_FILE, "https.tls.cert", cert) + set_config_value(CONFIG_FILE, "https.enabled", True, True) + set_config_value(CONFIG_FILE, "https.tls.key", key, True) + set_config_value(CONFIG_FILE, "https.tls.cert", cert, True) reload_component("proxy") for i in range(10): time.sleep(i) @@ -89,7 +89,7 @@ def test_manual_https(preserve_config): # cleanup shutil.rmtree(ssl_dir) - set_config_value(CONFIG_FILE, "https.enabled", False) + set_config_value(CONFIG_FILE, "https.enabled", False, True) reload_component("proxy") From f921acc1833ccca2064663b1369538c446feecf8 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:09:07 -0400 Subject: [PATCH 16/27] Bump `ubuntu` image to `22.04` --- integration-tests/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/Dockerfile b/integration-tests/Dockerfile index c1c73d8c..eb116eb6 100644 --- a/integration-tests/Dockerfile +++ b/integration-tests/Dockerfile @@ -1,5 +1,5 @@ # Systemd inside a Docker container, for CI only -ARG BASE_IMAGE=ubuntu:20.04 +ARG BASE_IMAGE=ubuntu:22.04 FROM $BASE_IMAGE # DEBIAN_FRONTEND is set to avoid being asked for input and hang during build: @@ -29,8 +29,8 @@ RUN systemctl set-default multi-user.target STOPSIGNAL SIGRTMIN+3 # Uncomment these lines for a development install -#ENV TLJH_BOOTSTRAP_DEV=yes -#ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src -#ENV PATH=/opt/tljh/hub/bin:${PATH} +# ENV TLJH_BOOTSTRAP_DEV=yes +# ENV TLJH_BOOTSTRAP_PIP_SPEC=/srv/src +# ENV PATH=/opt/tljh/hub/bin:${PATH} CMD ["/bin/bash", "-c", "exec /lib/systemd/systemd --log-target=journal 3>&1"] From 4ddd798928a25d3fc5d8faad441eebfb2d45b1f2 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:08:28 -0400 Subject: [PATCH 17/27] Add docstring from `config_schema.py` --- tljh/config_schema.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tljh/config_schema.py b/tljh/config_schema.py index e7ae2ba8..0b12c8f9 100644 --- a/tljh/config_schema.py +++ b/tljh/config_schema.py @@ -1,3 +1,9 @@ +""" +The schema against which the TLJH config file can be validated. + +Validation occurs when changing values with tljh-config. +""" + config_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Littlest JupyterHub YAML config file", From 196208ae58bf83b4c400d549023d8f936eece5b1 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 13:30:32 -0400 Subject: [PATCH 18/27] Remove `argparse.BooleanOptionalAction` for Python 3.8 compatibility --- tljh/config.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 849fdb5c..3f942c84 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -356,11 +356,17 @@ def main(argv=None): argparser.add_argument( "--config-path", default=CONFIG_FILE, help="Path to TLJH config.yaml file" ) + + argparser.add_argument( + "--validate", action="store_true", help="Validate the TLJH config" + ) argparser.add_argument( - "--validate", - action=argparse.BooleanOptionalAction, - help="Validate the TLJH config", + "--no-validate", + dest="validate", + action="store_false", + help="Do not validate the TLJH config", ) + argparser.set_defaults(validate=True) subparsers = argparser.add_subparsers(dest="action") From 51f847053510e239c92b3d24793dfe38a3dab321 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:05:32 -0400 Subject: [PATCH 19/27] Fix unit tests with `pip` --- .github/workflows/unit-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 3b5f8573..ae77c290 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -59,6 +59,7 @@ jobs: apt-get update apt-get install --yes \ python3-venv \ + python3-pip \ bzip2 \ git From c578a7bec09612eb010e18866ea4631387e6dcee Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:10:16 -0400 Subject: [PATCH 20/27] Update `step` `name` --- .github/workflows/unit-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index ae77c290..154bd1c7 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -53,7 +53,7 @@ jobs: with: python-version: "${{ matrix.python_version }}" - - name: Install venv, git and setup venv + - name: Install venv, git, pip and setup venv run: | export DEBIAN_FRONTEND=noninteractive apt-get update From 67dd3c8abe985253b7da402ff195ad4429232c54 Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:02:55 -0400 Subject: [PATCH 21/27] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 3f942c84..6dc2ed2d 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -223,7 +223,7 @@ def unset_config_value(config_path, key_path, validate): yaml.dump(config, f) -def add_config_value(config_path, key_path, value, validate): +def add_config_value(config_path, key_path, value, validate=True): """ Add value to list at key_path """ From b94a281ff87d751c69fdd2aada5a6f8e65ccb67a Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:03:05 -0400 Subject: [PATCH 22/27] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 6dc2ed2d..59117035 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -187,7 +187,7 @@ def show_config(config_path): yaml.dump(config, sys.stdout) -def set_config_value(config_path, key_path, value, validate): +def set_config_value(config_path, key_path, value, validate=True): """ Set key at key_path in config_path to value """ From 5ae31ce169327cf9713394f93651b700b3e4abe4 Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:03:14 -0400 Subject: [PATCH 23/27] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index 59117035..1f68c83d 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -205,7 +205,7 @@ def set_config_value(config_path, key_path, value, validate=True): yaml.dump(config, f) -def unset_config_value(config_path, key_path, validate): +def unset_config_value(config_path, key_path, validate=True): """ Unset key at key_path in config_path """ From 7474b876f1f8647b68559d71e94ba6d99e27ba1a Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:04:29 -0400 Subject: [PATCH 24/27] Remove unneeded `validate` code --- tljh/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tljh/config.py b/tljh/config.py index 1f68c83d..b4964008 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -412,9 +412,6 @@ def main(argv=None): args = argparser.parse_args(argv) - if args.validate == None: - args.validate = True - if args.action == "show": show_config(args.config_path) elif args.action == "set": From 9bcfa7032697a74befebda43e444efde1a3fb087 Mon Sep 17 00:00:00 2001 From: Jordan <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:05:16 -0400 Subject: [PATCH 25/27] Update tljh/config.py Co-authored-by: Erik Sundell --- tljh/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tljh/config.py b/tljh/config.py index b4964008..633cc361 100644 --- a/tljh/config.py +++ b/tljh/config.py @@ -241,7 +241,7 @@ def add_config_value(config_path, key_path, value, validate=True): yaml.dump(config, f) -def remove_config_value(config_path, key_path, value, validate): +def remove_config_value(config_path, key_path, value, validate=True): """ Remove value from list at key_path """ From 38a01e840689e1ee526259f2b32ba2cb123ae690 Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:09:18 -0400 Subject: [PATCH 26/27] Use default `true` for `validate` --- integration-tests/test_proxy.py | 8 ++++---- tests/test_traefik.py | 30 +++++++++++------------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/integration-tests/test_proxy.py b/integration-tests/test_proxy.py index f758655d..cc9d7662 100644 --- a/integration-tests/test_proxy.py +++ b/integration-tests/test_proxy.py @@ -63,9 +63,9 @@ def test_manual_https(preserve_config): "/CN=tljh.jupyer.org", ] ) - set_config_value(CONFIG_FILE, "https.enabled", True, True) - set_config_value(CONFIG_FILE, "https.tls.key", key, True) - set_config_value(CONFIG_FILE, "https.tls.cert", cert, True) + set_config_value(CONFIG_FILE, "https.enabled", True) + set_config_value(CONFIG_FILE, "https.tls.key", key) + set_config_value(CONFIG_FILE, "https.tls.cert", cert) reload_component("proxy") for i in range(10): time.sleep(i) @@ -89,7 +89,7 @@ def test_manual_https(preserve_config): # cleanup shutil.rmtree(ssl_dir) - set_config_value(CONFIG_FILE, "https.enabled", False, True) + set_config_value(CONFIG_FILE, "https.enabled", False) reload_component("proxy") diff --git a/tests/test_traefik.py b/tests/test_traefik.py index 380ac4c2..b060be4c 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -64,12 +64,12 @@ def test_default_config(tmpdir, tljh_dir): def test_letsencrypt_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) + config.set_config_value(config.CONFIG_FILE, "https.enabled") config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org", True + config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org" ) config.set_config_value( - config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"], True + config.CONFIG_FILE, "https.letsencrypt.domains", ["testing.jovyan.org"] ) traefik.ensure_traefik_config(str(state_dir)) @@ -138,13 +138,9 @@ def test_letsencrypt_config(tljh_dir): def test_manual_ssl_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) - config.set_config_value( - config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True - ) - config.set_config_value( - config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True - ) + config.set_config_value(config.CONFIG_FILE, "https.enabled", True) + config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") + config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") traefik.ensure_traefik_config(str(state_dir)) cfg = _read_static_config(state_dir) @@ -248,16 +244,12 @@ def test_extra_config(tmpdir, tljh_dir): def test_listen_address(tmpdir, tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled", True, True) - config.set_config_value( - config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key", True - ) - config.set_config_value( - config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert", True - ) + config.set_config_value(config.CONFIG_FILE, "https.enabled", True) + config.set_config_value(config.CONFIG_FILE, "https.tls.key", "/path/to/ssl.key") + config.set_config_value(config.CONFIG_FILE, "https.tls.cert", "/path/to/ssl.cert") - config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1", True) - config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1", True) + config.set_config_value(config.CONFIG_FILE, "http.address", "127.0.0.1") + config.set_config_value(config.CONFIG_FILE, "https.address", "127.0.0.1") traefik.ensure_traefik_config(str(state_dir)) From 5469e21e74bcddd8731bac947ef7331a030ede5d Mon Sep 17 00:00:00 2001 From: Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:19:45 -0400 Subject: [PATCH 27/27] Fix removal of `https.enabled` --- tests/test_traefik.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_traefik.py b/tests/test_traefik.py index b060be4c..f9502662 100644 --- a/tests/test_traefik.py +++ b/tests/test_traefik.py @@ -64,7 +64,7 @@ def test_default_config(tmpdir, tljh_dir): def test_letsencrypt_config(tljh_dir): state_dir = config.STATE_DIR - config.set_config_value(config.CONFIG_FILE, "https.enabled") + config.set_config_value(config.CONFIG_FILE, "https.enabled", True) config.set_config_value( config.CONFIG_FILE, "https.letsencrypt.email", "fake@jupyter.org" )