From fe6007fd7f209bc1677cca6ab7581e5f4f3b935e Mon Sep 17 00:00:00 2001 From: Kushashwa Ravi Shrimali Date: Mon, 13 Jun 2022 19:53:07 +0530 Subject: [PATCH 1/3] Renaming --- README.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9357bd6..56e916a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@
-A lightning component to run a Hyper-Parameter engine on a given config for RandomSearch and GridSearch strategies. +A lightning component to generate Hyper Parameter Space on a given config for Random Search and Grid Search strategies. ______________________________________________________________________ @@ -12,8 +12,8 @@ ______________________________________________________________________ Use these instructions to install: ```bash -git clone https://github.com/PyTorchLightning/lightning-hp-engine.git -cd lightning-hp-engine +git clone https://github.com/PyTorchLightning/LAI-Hyper-Parameter-Space-Generator.git +cd LAI-Hyper-Parameter-Space-Generator pip install -r requirements.txt pip install -e . ``` @@ -40,8 +40,8 @@ from lightning.frontend import StreamlitFrontend from lightning.utilities.state import AppState -from lightning_hp_engine import LightningHPEngine -from lightning_hp_engine import RandomSearchStrategy, GridSearchStrategy +from hp_space_generator import HPSpaceGenerator +from hp_space_generator import RandomSearchStrategy, GridSearchStrategy class DoSomethingExtra(LightningWork): @@ -80,10 +80,10 @@ class Visualizer(LightningFlow): return StreamlitFrontend(render_fn=_render_fn) -class HPEComponent(LightningFlow): +class HPComponent(LightningFlow): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.hpe = LightningHPEngine() + self.space_generator = HPSpaceGenerator() self.work_random_search = DoSomethingExtra() self.work_grid_search = DoSomethingExtra() self.visualize = Visualizer() @@ -93,27 +93,26 @@ class HPEComponent(LightningFlow): # If you want, you can write your own preprocess functions, and pass should_preprocess=False # in the class instantiation. - hpe_config = { + hp_config = { "backbone": ["prajjwal1/bert-tiny", "pra1/bert-medium"], "learning_rate": [0.000001, 0.1], } # work_cls allows you to use the hyper-paramaters from the given num_runs - # basically .run() method is called with the hpes from the HP Engine component after this - self.hpe.run(hpe_dict=hpe_config, num_runs=5, + self.space_generator.run(hp_dict=hp_config, num_runs=5, work=self.work_random_search, strategy=RandomSearchStrategy(should_preprocess=True)) - # self.hpe.run(hpe_dict=hpe_config, num_runs=5, + # self.space_generator.run(hp_dict=hp_config, num_runs=5, # strategy=GridSearchStrategy(should_preprocess=False), work=self.work_grid_search) if self.work_random_search.has_succeeded: - self.visualize.run(self.hpe.results) + self.visualize.run(self.space_generator.results) def configure_layout(self): - return {"name": "hpe Output", "content": self.visualize} + return {"name": "generated Hyper Parameter Space", "content": self.visualize} # To launch the hpe Component -app = LightningApp(HPEComponent(), debug=True) +app = LightningApp(HPComponent(), debug=True) ``` From d8611f65971c3a1e2785a47e90aa31034c885eae Mon Sep 17 00:00:00 2001 From: Kushashwa Ravi Shrimali Date: Mon, 13 Jun 2022 20:06:31 +0530 Subject: [PATCH 2/3] Full rename --- hp_space_generator/__init__.py | 6 ++++ .../component.py | 10 +++--- .../grid_search_strategy.py | 6 ++-- .../random_search_strategy.py | 30 ++++++++-------- .../search_strategy.py | 10 +++--- .../setup_tools.py | 0 lightning_hp_engine/__init__.py | 6 ---- setup.py | 8 ++--- tests/test_hp_space_generator.py | 36 +++++++++++++++++++ tests/test_hpe.py | 36 ------------------- 10 files changed, 74 insertions(+), 74 deletions(-) create mode 100644 hp_space_generator/__init__.py rename {lightning_hp_engine => hp_space_generator}/component.py (80%) rename {lightning_hp_engine => hp_space_generator}/grid_search_strategy.py (83%) rename {lightning_hp_engine => hp_space_generator}/random_search_strategy.py (60%) rename {lightning_hp_engine => hp_space_generator}/search_strategy.py (68%) rename {lightning_hp_engine => hp_space_generator}/setup_tools.py (100%) delete mode 100644 lightning_hp_engine/__init__.py create mode 100644 tests/test_hp_space_generator.py delete mode 100644 tests/test_hpe.py diff --git a/hp_space_generator/__init__.py b/hp_space_generator/__init__.py new file mode 100644 index 0000000..55e603d --- /dev/null +++ b/hp_space_generator/__init__.py @@ -0,0 +1,6 @@ +from hp_space_generator.component import HPSpaceGenerator +from hp_space_generator.search_strategy import SearchStrategy +from hp_space_generator.random_search_strategy import RandomSearchStrategy +from hp_space_generator.grid_search_strategy import GridSearchStrategy + +__all__ = ["HPSpaceGenerator", "SearchStrategy", "RandomSearchStrategy", "GridSearchStrategy"] diff --git a/lightning_hp_engine/component.py b/hp_space_generator/component.py similarity index 80% rename from lightning_hp_engine/component.py rename to hp_space_generator/component.py index 6e2d484..10c4466 100644 --- a/lightning_hp_engine/component.py +++ b/hp_space_generator/component.py @@ -1,7 +1,7 @@ from lightning import LightningFlow -class LightningHPEngine(LightningFlow): +class HPSpaceGenerator(LightningFlow): """ The HP Engine Component is used to suggest a list of configurations (hyper-parameters) to run with some config from the user for any task. @@ -12,22 +12,22 @@ class LightningHPEngine(LightningFlow): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.hpe_dict = {} + self.hp_dict = {} self.num_runs = 0 self.results = None # Having strategy as a class allows the users to define their own strategy class - def run(self, hpe_dict: dict, num_runs: int=1, strategy=None, work=None, work_kwargs={}, + def run(self, hp_dict: dict, num_runs: int=1, strategy=None, work=None, work_kwargs={}, *args, **kwargs): if self.results is None: - self.hpe_dict = hpe_dict + self.hp_dict = hp_dict self.num_runs = num_runs # Thought: maybe we should receive an object, and not the class...? # Let them instantiate it with the args they want to store? # Yep^^ done! :) for run_id in range(num_runs): - strategy.run(run_id=run_id, hpe_config_dict=hpe_dict, *args, **kwargs) + strategy.run(run_id=run_id, hp_config_dict=hp_dict, *args, **kwargs) # Now pass the runs to the given work_cls assert len(strategy.runs) > 0, "The strategy class did not generate any runs! Probably something went wrong..." diff --git a/lightning_hp_engine/grid_search_strategy.py b/hp_space_generator/grid_search_strategy.py similarity index 83% rename from lightning_hp_engine/grid_search_strategy.py rename to hp_space_generator/grid_search_strategy.py index fb08b0a..d1c827a 100644 --- a/lightning_hp_engine/grid_search_strategy.py +++ b/hp_space_generator/grid_search_strategy.py @@ -9,11 +9,11 @@ class GridSearchStrategy(SearchStrategy): def __init__(self, should_preprocess=False, *args, **kwargs): super().__init__(should_preprocess=should_preprocess, *args, **kwargs) - def run(self, hpe_config_dict: Dict[str, Any], run_id=-1): + def run(self, hp_config_dict: Dict[str, Any], run_id=-1): if self.should_preprocess: - hpe_config_dict = self.preprocess(hpe_config_dict) + hp_config_dict = self.preprocess(hp_config_dict) - self.runs.extend(self.generate_runs(run_id, hpe_config_dict)) + self.runs.extend(self.generate_runs(run_id, hp_config_dict)) def preprocess(self, hpe_dict): """ diff --git a/lightning_hp_engine/random_search_strategy.py b/hp_space_generator/random_search_strategy.py similarity index 60% rename from lightning_hp_engine/random_search_strategy.py rename to hp_space_generator/random_search_strategy.py index a5ab1ae..e6d090a 100644 --- a/lightning_hp_engine/random_search_strategy.py +++ b/hp_space_generator/random_search_strategy.py @@ -3,43 +3,43 @@ from ray import tune -from lightning_hp_engine import SearchStrategy +from hp_space_generator import SearchStrategy class RandomSearchStrategy(SearchStrategy): - def run(self, run_id: int, hpe_config_dict: Dict[str, Any], *args, **kwargs): + def run(self, run_id: int, hp_config_dict: Dict[str, Any], *args, **kwargs): # To ensure that we don't preprocessing again, we need to pass the pre-processed dict in the run method maybe? # Currently for each run in the given num_runs, pre-processing will happen each time! # Maybe do this in __init__()? if self.should_preprocess: - hpe_config_dict = self.preprocess(hpe_config_dict) - self.runs.extend(self.generate_runs(run_id, hpe_config_dict)) + hp_config_dict = self.preprocess(hp_config_dict) + self.runs.extend(self.generate_runs(run_id, hp_config_dict)) - def preprocess(self, hpe_dict): - # Pre-process incoming hpe_dict into a dict with values which can be supported by _generate_runs + def preprocess(self, hp_dict): + # Pre-process incoming hp_dict into a dict with values which can be supported by _generate_runs # This currently assumes you are using a Random Strategy, feel free to override it with something else - preprocessed_hpe_dict = {} - for key, val in hpe_dict.items(): + preprocessed_hp_dict = {} + for key, val in hp_dict.items(): if key == "backbone": if isinstance(val, str): - preprocessed_hpe_dict[key] = val + preprocessed_hp_dict[key] = val elif isinstance(val, list): - preprocessed_hpe_dict[key] = tune.choice(val) + preprocessed_hp_dict[key] = tune.choice(val) else: raise TypeError(f"Expected either List or a string for backbone but got {type(val)}") elif key == "learning_rate": if not isinstance(val, list) or len(val) != 2: raise ValueError(f"Expected a list of two numbers (float/int) but got {val}") - preprocessed_hpe_dict[key] = tune.uniform(val[0], val[1]) + preprocessed_hp_dict[key] = tune.uniform(val[0], val[1]) else: - preprocessed_hpe_dict[key] = val - return preprocessed_hpe_dict + preprocessed_hp_dict[key] = val + return preprocessed_hp_dict - def generate_runs(self, run_id: int, hpe_dict: Dict[str, Any]): + def generate_runs(self, run_id: int, hp_dict: Dict[str, Any]): runs = [] model_config = {} - for key, domain in hpe_dict.items(): + for key, domain in hp_dict.items(): if hasattr(domain, "sample"): model_config[key] = domain.sample() else: diff --git a/lightning_hp_engine/search_strategy.py b/hp_space_generator/search_strategy.py similarity index 68% rename from lightning_hp_engine/search_strategy.py rename to hp_space_generator/search_strategy.py index 934349b..ccc97ce 100644 --- a/lightning_hp_engine/search_strategy.py +++ b/hp_space_generator/search_strategy.py @@ -10,11 +10,11 @@ def __init__(self, should_preprocess=True, *args, **kwargs): self.should_preprocess = should_preprocess self.runs = [] - def run(self, hpe_config_dict: Dict[str, Any], num_runs=1, *args, **kwargs): + def run(self, hp_config_dict: Dict[str, Any], num_runs=1, *args, **kwargs): raise NotImplementedError("You need to implement the run method") - def preprocess(hpe_config_dict, *args, **kwargs): - raise NotImplementedError("You need to implement preprocess(hpe_config_dict) method") + def preprocess(hp_config_dict, *args, **kwargs): + raise NotImplementedError("You need to implement preprocess(hp_config_dict) method") - def generate_runs(num_runs: int, hpe_config_dict: Dict[str, Any], *args, **kwargs): - raise NotImplementedError("You need to implement generate_runs(num_runs, hpe_config_dict) method") + def generate_runs(num_runs: int, hp_config_dict: Dict[str, Any], *args, **kwargs): + raise NotImplementedError("You need to implement generate_runs(num_runs, hp_config_dict) method") diff --git a/lightning_hp_engine/setup_tools.py b/hp_space_generator/setup_tools.py similarity index 100% rename from lightning_hp_engine/setup_tools.py rename to hp_space_generator/setup_tools.py diff --git a/lightning_hp_engine/__init__.py b/lightning_hp_engine/__init__.py deleted file mode 100644 index 28acb85..0000000 --- a/lightning_hp_engine/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from lightning_hp_engine.component import LightningHPEngine -from lightning_hp_engine.search_strategy import SearchStrategy -from lightning_hp_engine.random_search_strategy import RandomSearchStrategy -from lightning_hp_engine.grid_search_strategy import GridSearchStrategy - -__all__ = ["LightningHPEngine", "SearchStrategy", "RandomSearchStrategy", "GridSearchStrategy"] diff --git a/setup.py b/setup.py index 6f2594d..51aab3b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ _PATH_ROOT = os.path.dirname(__file__) -def _load_py_module(fname, pkg="lightning_hp_engine"): +def _load_py_module(fname, pkg="hp_space_generator"): spec = spec_from_file_location( os.path.join(pkg, fname), os.path.join(_PATH_ROOT, pkg, fname) ) @@ -22,12 +22,12 @@ def _load_py_module(fname, pkg="lightning_hp_engine"): REQUIREMENTS = [req.strip() for req in open("requirements.txt").readlines()] setup( - name="lightning_hp_engine", + name="hp_space_generator", version="0.0.1", - description="Run Hyper Parameter Engine around the given choices of hyper-parameters with Grid Search and Random Search strategies", + description="Generator Hyper Parameter space around the given choices of hyper-parameters with Grid Search and Random Search strategies", author="Kushashwa Ravi Shrimali, Ethan Harris", author_email="kush@grid.ai", - url="https://github.com/PyTorchLightning/LAI-hyper-parameter-engine", + url="https://github.com/PyTorchLightning/LAI-Hyper-Parameter-Space-Generator", packages=find_packages(exclude=["tests", "tests.*"]), install_requires=setup_tools._load_requirements(_PATH_ROOT), ) diff --git a/tests/test_hp_space_generator.py b/tests/test_hp_space_generator.py new file mode 100644 index 0000000..b544184 --- /dev/null +++ b/tests/test_hp_space_generator.py @@ -0,0 +1,36 @@ +from hp_space_generator import HPSpaceGenerator, RandomSearchStrategy, GridSearchStrategy + +from lightning import LightningWork + +class DoSomethingExtra(LightningWork): + def __init__(self): + super().__init__(run_once=True) + self.hp_list = [] + + def run(self, hp_list): + self.hp_list.extend(hp_list) + + +def test_hp_space_generator_random(): + # Test the HP Component for text classification + hp_config = { + "backbone": ["prajjwal1/bert-tiny", "pra1/bert-medium"], + "learning_rate": [0.00001, 0.01], + } + + space_generator = HPSpaceGenerator() + work_obj = DoSomethingExtra() + space_generator.run(hp_config, num_runs=2, work=work_obj, strategy=RandomSearchStrategy()) + assert len(work_obj.hp_list) != 0, "Didn't generate results..." + +def test_hp_space_generator_grid(): + # Test the HP Space Generator Component for text classification + hp_config = { + "backbone": ["prajjwal1/bert-tiny", "pra1/bert-medium"], + "learning_rate": [0.00001, 0.01], + } + + space_generator = HPSpaceGenerator() + work_obj = DoSomethingExtra() + space_generator.run(hp_config, num_runs=2, work=work_obj, strategy=GridSearchStrategy(should_preprocess=False)) + assert len(work_obj.hp_list) != 0, "Didn't generate results..." diff --git a/tests/test_hpe.py b/tests/test_hpe.py deleted file mode 100644 index fcc0476..0000000 --- a/tests/test_hpe.py +++ /dev/null @@ -1,36 +0,0 @@ -from lightning_hp_engine import LightningHPEngine, RandomSearchStrategy, GridSearchStrategy - -from lightning import LightningWork - -class DoSomethingExtra(LightningWork): - def __init__(self): - super().__init__(run_once=True) - self.hpe_list = [] - - def run(self, hpe_list): - self.hpe_list.extend(hpe_list) - - -def test_lightning_hp_engine_random(): - # Test the HP Engine Component for text classification - hpe_config = { - "backbone": ["prajjwal1/bert-tiny", "pra1/bert-medium"], - "learning_rate": [0.00001, 0.01], - } - - hpe = LightningHPEngine() - work_obj = DoSomethingExtra() - hpe.run(hpe_config, num_runs=2, work=work_obj, strategy=RandomSearchStrategy()) - assert len(work_obj.hpe_list) != 0, "Didn't generate results..." - -def test_lightning_hp_engine_grid(): - # Test the HP Engine Component for text classification - hpe_config = { - "backbone": ["prajjwal1/bert-tiny", "pra1/bert-medium"], - "learning_rate": [0.00001, 0.01], - } - - hpe = LightningHPEngine() - work_obj = DoSomethingExtra() - hpe.run(hpe_config, num_runs=2, work=work_obj, strategy=GridSearchStrategy(should_preprocess=False)) - assert len(work_obj.hpe_list) != 0, "Didn't generate results..." From 3190c338c0fae4187122f3e57465d020a3a89195 Mon Sep 17 00:00:00 2001 From: Kushashwa Ravi Shrimali Date: Mon, 13 Jun 2022 20:08:06 +0530 Subject: [PATCH 3/3] Rename in the CI --- .github/workflows/ci-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index fddab83..734d8e3 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -72,7 +72,7 @@ jobs: - name: Tests run: | - coverage run --source lightning_hp_engine -m py.test lightning_hp_engine tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml + coverage run --source hp_space_generator -m py.test hp_space_generator tests -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml - name: Statistics if: success()