From c26057d65ea0fd552c46c54f53b02b17d6525bae Mon Sep 17 00:00:00 2001 From: Maxime Jublou Date: Thu, 19 Oct 2023 11:14:06 +0200 Subject: [PATCH 1/2] feat: Automate ABI installation --- custom/jupyter_server_config.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/custom/jupyter_server_config.py b/custom/jupyter_server_config.py index bbcefcb5..7ab1c6c4 100644 --- a/custom/jupyter_server_config.py +++ b/custom/jupyter_server_config.py @@ -75,6 +75,7 @@ def naasStarter(): time.sleep(ONE_HOUR) + def naasTemplates(): while True: logging.info("Refreshing templates") @@ -93,6 +94,43 @@ def naasTemplates(): time.sleep(ONE_HOUR) +def naasABIInstaller(): + while True: + try: + logging.info("Refreshing ABI setup") + + config = [ + { + 'repository': 'https://github.com/jupyter-naas/abi.git', + 'target': '/home/ftp/__abi__', + 'version': 'main', + 'entrypoints': ['models/__chat_plugin__.ipynb'] + } + ] + + for c in config: + os.system( + f"git clone {c['repository']} {c['target']}|| (cd /home/ftp/.naas/awesome-notebooks && git reset --hard && git pull)" + ) + + os.system( + f"cd {c['target']} && git reset --hard && git checkout {c['version']} && git pull" + ) + for entrypoint in c['entrypoints']: + working_directory = os.path.join(c['target'], '/'.join(entrypoint.split('/')[:1])) + + execute_cmd = f"cd {working_directory} && jupyter nbconvert --execute {entrypoint.split('/')[-1]} --to notebook --output {entrypoint.split('/')[-1]}.setup-execution.ipynb" + + os.system( + execute_cmd + ) + except Exception as e: + logging.error(f'Exception while installing ABI', e) + FIVE_MINUTES = 300 + time.sleep(FIVE_MINUTES) + + + runner = threading.Thread(target=naasRunner, args=(naas_port,)) runner.start() @@ -101,3 +139,6 @@ def naasTemplates(): templates = threading.Thread(target=naasTemplates, args=()) templates.start() + +abiInstaller = threading.Thread(target=naasABIInstaller, args=()) +abiInstaller.start() \ No newline at end of file From 2a812ba8b753ba8826269cfd425b298eed4238ab Mon Sep 17 00:00:00 2001 From: Maxime Jublou Date: Tue, 24 Oct 2023 11:27:28 +0200 Subject: [PATCH 2/2] feat: Auto install ABI on the machine and keep it up to date --- custom/jupyter_server_config.py | 60 ++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/custom/jupyter_server_config.py b/custom/jupyter_server_config.py index 7ab1c6c4..e87b240c 100644 --- a/custom/jupyter_server_config.py +++ b/custom/jupyter_server_config.py @@ -6,6 +6,7 @@ import logging import threading import time +from glob import glob c = get_config() c.ServerApp.ip = "0.0.0.0" @@ -93,6 +94,29 @@ def naasTemplates(): ) time.sleep(ONE_HOUR) +def get_current_commit(dir_path): + commit_hash = None + try: + # Run the Git command to get the current commit hash + commit_hash = subprocess.check_output(['git', '-C', dir_path, 'rev-parse', 'HEAD']).decode().strip() + except subprocess.CalledProcessError as e: + print(f"Failed to get current commit: {e}") + + return commit_hash + +def store_last_installed_version(dir_path, version, install_log_file='.last_installed_version'): + installed_version_file = os.path.join(dir_path, install_log_file) + with open(installed_version_file, 'w') as f: + f.write(version) + return installed_version_file + +def get_last_installed_version(dir_path, install_log_file='.last_installed_version'): + last_installed_version = None + installed_version_file = os.path.join(dir_path, install_log_file) + if os.path.exists(installed_version_file): + with open(installed_version_file, 'r') as f: + last_installed_version = f.read() + return last_installed_version def naasABIInstaller(): while True: @@ -103,27 +127,47 @@ def naasABIInstaller(): { 'repository': 'https://github.com/jupyter-naas/abi.git', 'target': '/home/ftp/__abi__', - 'version': 'main', - 'entrypoints': ['models/__chat_plugin__.ipynb'] + 'version': 'main' } ] for c in config: + # Clone repository os.system( - f"git clone {c['repository']} {c['target']}|| (cd /home/ftp/.naas/awesome-notebooks && git reset --hard && git pull)" + f"git clone {c['repository']} {c['target']}" ) + # Get last installed version + last_installed_version = get_last_installed_version(c['target']) + print(last_installed_version) + + # Reset hard and pull latest version. os.system( f"cd {c['target']} && git reset --hard && git checkout {c['version']} && git pull" ) - for entrypoint in c['entrypoints']: - working_directory = os.path.join(c['target'], '/'.join(entrypoint.split('/')[:1])) + + # Grab current commit version + current_commmit = get_current_commit(c['target']) + print(current_commmit) + + # Check if current commit is different than last installed version. + # If yes then we execute all targeted notebooks. + if current_commmit != last_installed_version: - execute_cmd = f"cd {working_directory} && jupyter nbconvert --execute {entrypoint.split('/')[-1]} --to notebook --output {entrypoint.split('/')[-1]}.setup-execution.ipynb" - os.system( - execute_cmd + f"cd {c['target']} && pip install --user -r requirements.txt" ) + + entrypoints = glob(os.path.join(c['target'], '**', '__plugin__.ipynb'), recursive=True) + for entrypoint in entrypoints: + working_directory = '/'.join(entrypoint.split('/')[:-1]) + + execute_cmd = f"cd {working_directory} && papermill {entrypoint.split('/')[-1]} {entrypoint.split('/')[-1]}.setup-execution.ipynb" + + os.system( + execute_cmd + ) + store_last_installed_version(c['target'], current_commmit) except Exception as e: logging.error(f'Exception while installing ABI', e) FIVE_MINUTES = 300