From db964140a80ed3a38241c196eb40d6f220d86b6d Mon Sep 17 00:00:00 2001 From: Shivani Date: Fri, 5 Apr 2024 18:01:49 -0400 Subject: [PATCH 1/9] Add clear.py to cupid-clear branch --- cupid/clear.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 cupid/clear.py diff --git a/cupid/clear.py b/cupid/clear.py new file mode 100755 index 0000000..4899b1c --- /dev/null +++ b/cupid/clear.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +import os +import yaml + +def clearFolder(folderPath): + #Clears all contents in the specified folder at folderPath (i.e. computed_notebooks) + try: + # Iterate over all items in the folder + for item in os.listdir(folderPath): + itemPath = os.path.join(folderPath, item) + # If item is a file, delete it + if os.path.isfile(itemPath): + os.remove(itemPath) + # If item is a directory, recursively clear it + elif os.path.isdir(itemPath): + clearFolder(itemPath) + # After deleting all items, remove the folder itself + os.rmdir(folderPath) + print(f"All contents in {folderPath} have been cleared.") + except Exception as e: + print(f"Error occurred while clearing contents of the file at path {folderPath}: {e}") + +def readConfigFile(configFilePath): + #Given the file path to config.yml, this function reads the config file content and + #returns the val of the run_dir string with '/computed_notebooks' appended to it + try: + with open(configFilePath, 'r') as file: + configContent = yaml.safe_load(file) + run_dir = configContent.get('data_sources', {}).get('run_dir') + if run_dir: + #Append '/computed_notebooks' to the run_dir value if it is not empty + fullPath = os.path.join(run_dir, 'computed_notebooks') + return fullPath + else: #run_dir is empty/wasn't found in config file so return error + raise ValueError("'run_dir' was not found in the config file.") + except FileNotFoundError: + print(f"config.yml at path'{configFilePath}' not found.") + except Exception as e: + print(f"Error occurred while reading config file at path '{configFilePath}': {e}") + return None + +#Entry point to this script +def clear(): + #Get the current working directory and add 'config.yml' to the path to + #obtain the path to the config.yml file in the current working directory + currWorkingDir = os.getcwd() + configFilePath = os.path.join(currWorkingDir, 'config.yml') + + run_dir = readConfigFile(configFilePath) + + if run_dir: + clearFolder(run_dir) \ No newline at end of file From 7b2c7c4e31b1b2ff36d80e0510b739c60d17b1cb Mon Sep 17 00:00:00 2001 From: Shivani Date: Fri, 5 Apr 2024 18:11:52 -0400 Subject: [PATCH 2/9] Add pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8120bdb..3e2db81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,4 @@ documentation = "https://nbscuid.readthedocs.io" [project.scripts] cupid-run = "cupid.run:run" cupid-build = "cupid.build:build" +cupid-clear = "cupid.clear:clear" From d8e2ce144dc821102823e8aa27389c243522cbc3 Mon Sep 17 00:00:00 2001 From: Shivani Date: Wed, 10 Apr 2024 13:11:54 -0400 Subject: [PATCH 3/9] Modified clear.py and README.md --- README.md | 9 +++++++++ cupid/clear.py | 27 ++++++++++++++------------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e6b04a6..9359d6d 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,14 @@ $ cupid-build config.yml # Will build HTML from Jupyter Book After the last step is finished, you can use Jupyter to view generated notebooks in `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run` or you can view `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run/_build/html/index.html` in a web browser. +Furthermore, to clear the generated notebooks folder which was generated by the cupid-run and cupid-build commands, you can run the following command: + +``` bash +$ cupid-clear config.yml +``` + +This will clear the computed notebooks folder which is at the location pointed to by the run_dir variable in the config.yml file. + ### CUPiD Options Most of CUPiD's configuration is done via the `config.yml` file, but there are a few command line options as well: @@ -92,3 +100,4 @@ client ### Timeseries File Generation CUPiD also has the capability to generate single variable timeseries files from history files for all components. To run timeseries, edit the `config.yml` file's timeseries section to fit your preferences, and then run `cupid-run config.yml -ts`. + diff --git a/cupid/clear.py b/cupid/clear.py index 4899b1c..9de477c 100755 --- a/cupid/clear.py +++ b/cupid/clear.py @@ -1,6 +1,6 @@ #!/usr/bin/env python import os -import yaml +import cupid.util def clearFolder(folderPath): #Clears all contents in the specified folder at folderPath (i.e. computed_notebooks) @@ -19,20 +19,21 @@ def clearFolder(folderPath): print(f"All contents in {folderPath} have been cleared.") except Exception as e: print(f"Error occurred while clearing contents of the file at path {folderPath}: {e}") - + def readConfigFile(configFilePath): #Given the file path to config.yml, this function reads the config file content and #returns the val of the run_dir string with '/computed_notebooks' appended to it try: - with open(configFilePath, 'r') as file: - configContent = yaml.safe_load(file) - run_dir = configContent.get('data_sources', {}).get('run_dir') - if run_dir: - #Append '/computed_notebooks' to the run_dir value if it is not empty - fullPath = os.path.join(run_dir, 'computed_notebooks') - return fullPath - else: #run_dir is empty/wasn't found in config file so return error - raise ValueError("'run_dir' was not found in the config file.") + #Obtain the contents of the config.yml file and extract the run_dir variable + control = cupid.util.get_control_dict(configFilePath) + run_dir = control['data_sources']['run_dir'] + + if run_dir: + #Append '/computed_notebooks' to the run_dir value if it is not empty + fullPath = os.path.join(run_dir, 'computed_notebooks') + return fullPath + else: #run_dir is empty/wasn't found in config file so return error + raise ValueError("'run_dir' was not found in the config file.") except FileNotFoundError: print(f"config.yml at path'{configFilePath}' not found.") except Exception as e: @@ -45,8 +46,8 @@ def clear(): #obtain the path to the config.yml file in the current working directory currWorkingDir = os.getcwd() configFilePath = os.path.join(currWorkingDir, 'config.yml') - + run_dir = readConfigFile(configFilePath) - + if run_dir: clearFolder(run_dir) \ No newline at end of file From d7b2f6a6f97b10a4b76fc75ec2c111760a2f4539 Mon Sep 17 00:00:00 2001 From: Shivani Date: Thu, 11 Apr 2024 12:19:21 -0400 Subject: [PATCH 4/9] Updated clear.py and README.md --- README.md | 4 ++-- cupid/clear.py | 48 ++++++++++++++++++++---------------------------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9359d6d..28896da 100644 --- a/README.md +++ b/README.md @@ -61,13 +61,13 @@ $ cupid-build config.yml # Will build HTML from Jupyter Book After the last step is finished, you can use Jupyter to view generated notebooks in `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run` or you can view `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run/_build/html/index.html` in a web browser. -Furthermore, to clear the generated notebooks folder which was generated by the cupid-run and cupid-build commands, you can run the following command: +Furthermore, to clear the generated notebooks folder which was generated by the 'cupid-run' and 'cupid-build' commands, you can run the following command: ``` bash $ cupid-clear config.yml ``` -This will clear the computed notebooks folder which is at the location pointed to by the run_dir variable in the config.yml file. +This will clear the 'computed_notebooks' folder which is at the location pointed to by the 'run_dir' variable in the 'config.yml' file. ### CUPiD Options diff --git a/cupid/clear.py b/cupid/clear.py index 9de477c..cee47a3 100755 --- a/cupid/clear.py +++ b/cupid/clear.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import os +import click import cupid.util def clearFolder(folderPath): @@ -18,36 +19,27 @@ def clearFolder(folderPath): os.rmdir(folderPath) print(f"All contents in {folderPath} have been cleared.") except Exception as e: - print(f"Error occurred while clearing contents of the file at path {folderPath}: {e}") + print(f"Error occurred while clearing contents in {folderPath}: {e}") -def readConfigFile(configFilePath): +def readConfigFile(config_path): #Given the file path to config.yml, this function reads the config file content and #returns the val of the run_dir string with '/computed_notebooks' appended to it - try: - #Obtain the contents of the config.yml file and extract the run_dir variable - control = cupid.util.get_control_dict(configFilePath) - run_dir = control['data_sources']['run_dir'] - - if run_dir: - #Append '/computed_notebooks' to the run_dir value if it is not empty - fullPath = os.path.join(run_dir, 'computed_notebooks') - return fullPath - else: #run_dir is empty/wasn't found in config file so return error - raise ValueError("'run_dir' was not found in the config file.") - except FileNotFoundError: - print(f"config.yml at path'{configFilePath}' not found.") - except Exception as e: - print(f"Error occurred while reading config file at path '{configFilePath}': {e}") - return None + + #Obtain the contents of the config.yml file and extract the run_dir variable + control = cupid.util.get_control_dict(config_path) + run_dir = control['data_sources'].get('run_dir', None) + + if run_dir: + #Append '/computed_notebooks' to the run_dir value if it is not empty + fullPath = os.path.join(run_dir, 'computed_notebooks') + return fullPath + + else: #run_dir is empty/wasn't found in config file so return error + raise ValueError("'run_dir' was empty/not found in the config file.") +@click.command() +@click.argument('config_path') #Entry point to this script -def clear(): - #Get the current working directory and add 'config.yml' to the path to - #obtain the path to the config.yml file in the current working directory - currWorkingDir = os.getcwd() - configFilePath = os.path.join(currWorkingDir, 'config.yml') - - run_dir = readConfigFile(configFilePath) - - if run_dir: - clearFolder(run_dir) \ No newline at end of file +def clear(config_path): + run_dir = readConfigFile(config_path) + clearFolder(run_dir) \ No newline at end of file From a9c025a8ce17aeceed545bfa8ff641142b414519 Mon Sep 17 00:00:00 2001 From: Shivani Date: Thu, 11 Apr 2024 12:28:40 -0400 Subject: [PATCH 5/9] Updated README.md again --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 28896da..7ba5e77 100644 --- a/README.md +++ b/README.md @@ -61,13 +61,13 @@ $ cupid-build config.yml # Will build HTML from Jupyter Book After the last step is finished, you can use Jupyter to view generated notebooks in `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run` or you can view `${CUPID_ROOT}/examples/coupled-model/computed_notebooks/quick-run/_build/html/index.html` in a web browser. -Furthermore, to clear the generated notebooks folder which was generated by the 'cupid-run' and 'cupid-build' commands, you can run the following command: +Furthermore, to clear the `computed_notebooks` folder which was generated by the `cupid-run` and `cupid-build` commands, you can run the following command: ``` bash $ cupid-clear config.yml ``` -This will clear the 'computed_notebooks' folder which is at the location pointed to by the 'run_dir' variable in the 'config.yml' file. +This will clear the `computed_notebooks` folder which is at the location pointed to by the `run_dir` variable in the `config.yml` file. ### CUPiD Options From 0cb7d6d6454be0367a36255fecd73c7ea6831bdd Mon Sep 17 00:00:00 2001 From: Shivani Date: Thu, 11 Apr 2024 12:41:09 -0400 Subject: [PATCH 6/9] Added try/except blocks again to clear.py --- cupid/clear.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/cupid/clear.py b/cupid/clear.py index cee47a3..83e2761 100755 --- a/cupid/clear.py +++ b/cupid/clear.py @@ -24,18 +24,20 @@ def clearFolder(folderPath): def readConfigFile(config_path): #Given the file path to config.yml, this function reads the config file content and #returns the val of the run_dir string with '/computed_notebooks' appended to it - - #Obtain the contents of the config.yml file and extract the run_dir variable - control = cupid.util.get_control_dict(config_path) - run_dir = control['data_sources'].get('run_dir', None) - - if run_dir: - #Append '/computed_notebooks' to the run_dir value if it is not empty - fullPath = os.path.join(run_dir, 'computed_notebooks') - return fullPath - - else: #run_dir is empty/wasn't found in config file so return error - raise ValueError("'run_dir' was empty/not found in the config file.") + try: + #Obtain the contents of the config.yml file and extract the run_dir variable + control = cupid.util.get_control_dict(config_path) + run_dir = control['data_sources'].get('run_dir', None) + + if run_dir: + #Append '/computed_notebooks' to the run_dir value if it is not empty + fullPath = os.path.join(run_dir, 'computed_notebooks') + return fullPath + + else: #run_dir is empty/wasn't found in config file so return error + raise ValueError("'run_dir' was empty/not found in the config file.") + except: + raise FileNotFoundError(f"{config_path} not found") @click.command() @click.argument('config_path') From b6297e5e509cd960f655f4ac53abf7e5852fdca6 Mon Sep 17 00:00:00 2001 From: Shivani Date: Thu, 11 Apr 2024 13:20:59 -0400 Subject: [PATCH 7/9] Added try/except blocks to util.py and modified clear.py --- cupid/clear.py | 28 ++++++++++++++-------------- cupid/util.py | 11 ++++++++--- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/cupid/clear.py b/cupid/clear.py index 83e2761..49a6239 100755 --- a/cupid/clear.py +++ b/cupid/clear.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import os +import sys import click import cupid.util @@ -24,20 +25,19 @@ def clearFolder(folderPath): def readConfigFile(config_path): #Given the file path to config.yml, this function reads the config file content and #returns the val of the run_dir string with '/computed_notebooks' appended to it - try: - #Obtain the contents of the config.yml file and extract the run_dir variable - control = cupid.util.get_control_dict(config_path) - run_dir = control['data_sources'].get('run_dir', None) - - if run_dir: - #Append '/computed_notebooks' to the run_dir value if it is not empty - fullPath = os.path.join(run_dir, 'computed_notebooks') - return fullPath - - else: #run_dir is empty/wasn't found in config file so return error - raise ValueError("'run_dir' was empty/not found in the config file.") - except: - raise FileNotFoundError(f"{config_path} not found") + + #Obtain the contents of the config.yml file and extract the run_dir variable + control = cupid.util.get_control_dict(config_path) + run_dir = control['data_sources'].get('run_dir', None) + + if run_dir: + #Append '/computed_notebooks' to the run_dir value if it is not empty + fullPath = os.path.join(run_dir, 'computed_notebooks') + return fullPath + + else: #run_dir is empty/wasn't found in config file so return error + print("ERROR: 'run_dir' was empty/not found in the config file.") + sys.exit(1) @click.command() @click.argument('config_path') diff --git a/cupid/util.py b/cupid/util.py index 35fd448..340f2b8 100644 --- a/cupid/util.py +++ b/cupid/util.py @@ -4,6 +4,7 @@ import pathlib import subprocess import json +import sys import yaml import jupyter_client import papermill as pm @@ -74,9 +75,13 @@ def execute_managed_notebook(cls, nb_man, kernel_name, **kwargs): def get_control_dict(config_path): - with open(config_path, "r") as fid: - control = yaml.safe_load(fid) - + try: + with open(config_path, "r") as fid: + control = yaml.safe_load(fid) + except FileNotFoundError: + print(f"ERROR: {config_path} not found") + sys.exit(1) + # theoretically ploomber should manage this kernel checking by itself, but this seems to add # the default kernel to info where necessary. currently a bit messy with copy pasting in # script stuff. From 092c1ccc34e4088ee64f1276ee410f196e4ace5a Mon Sep 17 00:00:00 2001 From: Shivani Date: Thu, 11 Apr 2024 16:13:42 -0400 Subject: [PATCH 8/9] Removed clearFolder and added rmtree in clear.py --- cupid/clear.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/cupid/clear.py b/cupid/clear.py index 49a6239..7e06627 100755 --- a/cupid/clear.py +++ b/cupid/clear.py @@ -3,24 +3,7 @@ import sys import click import cupid.util - -def clearFolder(folderPath): - #Clears all contents in the specified folder at folderPath (i.e. computed_notebooks) - try: - # Iterate over all items in the folder - for item in os.listdir(folderPath): - itemPath = os.path.join(folderPath, item) - # If item is a file, delete it - if os.path.isfile(itemPath): - os.remove(itemPath) - # If item is a directory, recursively clear it - elif os.path.isdir(itemPath): - clearFolder(itemPath) - # After deleting all items, remove the folder itself - os.rmdir(folderPath) - print(f"All contents in {folderPath} have been cleared.") - except Exception as e: - print(f"Error occurred while clearing contents in {folderPath}: {e}") +import shutil def readConfigFile(config_path): #Given the file path to config.yml, this function reads the config file content and @@ -44,4 +27,6 @@ def readConfigFile(config_path): #Entry point to this script def clear(config_path): run_dir = readConfigFile(config_path) - clearFolder(run_dir) \ No newline at end of file + #Delete the 'computed_notebooks' folder and all the contents inside of it + shutil.rmtree(run_dir) + print(f"All contents in {run_dir} have been cleared.") \ No newline at end of file From 1648661ee6f07ee6973cb8c18d5d132c3f3edc21 Mon Sep 17 00:00:00 2001 From: Shivani Date: Thu, 11 Apr 2024 16:52:16 -0400 Subject: [PATCH 9/9] Added docstring to clear.py --- cupid/clear.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cupid/clear.py b/cupid/clear.py index 7e06627..869980c 100755 --- a/cupid/clear.py +++ b/cupid/clear.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import os -import sys import click import cupid.util import shutil @@ -19,13 +18,18 @@ def readConfigFile(config_path): return fullPath else: #run_dir is empty/wasn't found in config file so return error - print("ERROR: 'run_dir' was empty/not found in the config file.") - sys.exit(1) + raise ValueError("'run_dir' was empty/not found in the config file.") @click.command() @click.argument('config_path') #Entry point to this script def clear(config_path): + """Clears the contents of the 'computed_notebooks' folder at the location specified by the 'run_dir' variable in the 'config.yml' file. + + Args: config_path - The path to the config.yml file. + + """ + run_dir = readConfigFile(config_path) #Delete the 'computed_notebooks' folder and all the contents inside of it shutil.rmtree(run_dir)