psuhign updates

smartsim/_core/generation/generator.py

@@ -28,6 +28,7 @@
 import os
 import pathlib
 import pickle
+import time
 import subprocess
 import sys
 import typing as t
@@ -52,37 +53,37 @@ class _GenerableProtocol(t.Protocol):
 
 
 Job_Path = namedtuple("Job_Path", ["run_path", "out_path", "err_path"])
-"""Paths related to the Jobs execution."""
+"""Paths related to the Job's execution."""
 
 
 class Generator:
     """The primary responsibility of the Generator class is to create the directory structure
     for a SmartSim Job and to build and execute file operation commands."""
 
-    run = "run"
+    run_directory = "run"
     """The name of the directory where run-related files are stored."""
-    log = "log"
+    log_directory = "log"
     """The name of the directory where log files are stored."""
 
     def __init__(self, root: pathlib.Path) -> None:
         """Initialize a Generator object
 
-        The Generator class constructs a Jobs directory structure, including:
+        The Generator class constructs a Job's directory structure, including:
 
         - The run and log directories
         - Output and error files
         - The "smartsim_params.txt" settings file
 
         Additionally, it manages symlinking, copying, and configuring files associated
-        with a Jobs entity.
+        with a Job's entity.
 
-        :param root: The Jobs base path
+        :param root: The Job's base path
         """
         self.root = root
         """The root path under which to generate files"""
 
     def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path:
-        """Build and return a Jobs base directory. The path is created by combining the
+        """Build and return a Job's base directory. The path is created by combining the
         root directory with the Job type (derived from the class name),
         the name attribute of the Job, and an index to differentiate between multiple
         Job runs.
@@ -96,29 +97,27 @@ def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path:
         return pathlib.Path(job_path)
 
     def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path:
-        """Build and return a Jobs run directory. The path is formed by combining
+        """Build and return a Job's run directory. The path is formed by combining
         the base directory with the `run` class-level variable, where run specifies
         the name of the job's run folder.
 
         :param job: The job object
         :param job_index: The index of the Job
         :returns: The built file path for the Job run folder
         """
-        path = self._build_job_base_path(job, job_index) / self.run
-        path.mkdir(exist_ok=False, parents=True)
+        path = self._build_job_base_path(job, job_index) / self.run_directory
         return pathlib.Path(path)
 
     def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path:
-        """Build and return a Jobs log directory. The path is formed by combining
+        """Build and return a Job's log directory. The path is formed by combining
         the base directory with the `log` class-level variable, where log specifies
         the name of the job's log folder.
 
         :param job: The job object
         :param job_index: The index of the Job
         :returns: The built file path for the Job run folder
         """
-        path = self._build_job_base_path(job, job_index) / self.log
-        path.mkdir(exist_ok=False, parents=True)
+        path = self._build_job_base_path(job, job_index) / self.log_directory
         return pathlib.Path(path)
 
     @staticmethod
@@ -134,7 +133,7 @@ def _build_log_file_path(log_path: pathlib.Path) -> pathlib.Path:
     @staticmethod
     def _build_out_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path:
         """Build and return the path to the output file. The path is created by combining
-        the Jobs log directory with the job name and appending the `.out` extension.
+        the Job's log directory with the job name and appending the `.out` extension.
 
         :param log_path: Path to log directory
         :param job_name: Name of the Job
@@ -146,7 +145,7 @@ def _build_out_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path:
     @staticmethod
     def _build_err_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path:
         """Build and return the path to the error file. The path is created by combining
-        the Jobs log directory with the job name and appending the `.err` extension.
+        the Job's log directory with the job name and appending the `.err` extension.
 
         :param log_path: Path to log directory
         :param job_name: Name of the Job
@@ -172,39 +171,41 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path:
         job_path = self._build_job_run_path(job, job_index)
         log_path = self._build_job_log_path(job, job_index)
 
+        out_file = self._build_out_file_path(log_path, job.entity.name)
+        err_file = self._build_err_file_path(log_path, job.entity.name)
+
+        cmd_list = self._build_commands(job, job_path, log_path)
+
+        self._execute_commands(cmd_list)
+
         with open(
             self._build_log_file_path(log_path), mode="w", encoding="utf-8"
         ) as log_file:
             dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
             log_file.write(f"Generation start date and time: {dt_string}\n")
 
-        out_file = self._build_out_file_path(log_path, job.entity.name)
-        err_file = self._build_err_file_path(log_path, job.entity.name)
-
-        cmd_list = self._build_commands(job, job_path)
-        if cmd_list:
-            self._execute_commands(cmd_list)
-
         return Job_Path(job_path, out_file, err_file)
 
     @classmethod
     def _build_commands(
-        cls, job: Job, job_path: pathlib.Path
-    ) -> t.Optional[CommandList]:
-        """Build file operation commands for all files attached to a Job's entity.
+        cls, job: Job, job_path: pathlib.Path, log_path: pathlib.Path
+    ) -> CommandList:
+        """Build file operation commands for a Job's entity.
 
         This method constructs commands for copying, symlinking, and writing tagged files
-        associated with the Job's entity. It aggregates these commands into a CommandList
+        associated with the Job's entity. This method builds the constructs the commands to
+        generate the Job's run and log directory. It aggregates these commands into a CommandList
         to return.
 
         :param job: The job object
         :param job_path: The file path for the Job run folder
-        :return: A CommandList containing the file operation commands, or None if the entity
-            does not support file operations.
+        :return: A CommandList containing the file operation commands
         """
+        cmd_list = CommandList()
+        cmd_list.commands.append(cls._mkdir_file(job_path))
+        cmd_list.commands.append(cls._mkdir_file(log_path))
         entity = job.entity
         if isinstance(entity, _GenerableProtocol):
-            cmd_list = CommandList()
             helpers: t.List[
                 t.Callable[[EntityFiles | None, pathlib.Path], CommandList | None]
             ] = [
@@ -220,8 +221,7 @@ def _build_commands(
                 if return_cmd_list:
                     cmd_list.commands.extend(return_cmd_list.commands)
 
-            return cmd_list
-        return None
+        return cmd_list
 
     @classmethod
     def _execute_commands(cls, cmd_list: CommandList) -> None:
@@ -234,6 +234,11 @@ def _execute_commands(cls, cmd_list: CommandList) -> None:
         """
         for cmd in cmd_list:
             subprocess.run(cmd.command)
+
+    @staticmethod
+    def _mkdir_file(file_path: pathlib.Path) -> Command:
+        cmd = Command(["mkdir", "-p", str(file_path)])
+        return cmd
 
     @staticmethod
     def _copy_files(

tests/test_generator.py

@@ -205,7 +205,7 @@ def test_generate_job(
     """Test Generator.generate_job returns correct paths"""
     mock_index = 1
     job_paths = generator_instance.generate_job(mock_job, mock_index)
-    assert job_paths.run_path.name == Generator.run
+    assert job_paths.run_path.name == Generator.run_directory
     assert job_paths.out_path.name == f"{mock_job.entity.name}.out"
     assert job_paths.err_path.name == f"{mock_job.entity.name}.err"
 
@@ -225,7 +225,7 @@ def test_build_commands(
             "smartsim._core.generation.Generator._write_tagged_files"
         ) as mock_write_tagged_files,
     ):
-        generator_instance._build_commands(mock_job, pathlib.Path(test_dir))
+        generator_instance._build_commands(mock_job, pathlib.Path(test_dir) / generator_instance.run_directory, pathlib.Path(test_dir) / generator_instance.log_directory)
         mock_copy_files.assert_called_once()
         mock_symlink_files.assert_called_once()
         mock_write_tagged_files.assert_called_once()
@@ -242,6 +242,12 @@ def test_execute_commands(generator_instance: Generator):
         generator_instance._execute_commands(cmd_list)
         run_process.assert_called_once()
 
+def test_mkdir_file(generator_instance: Generator, test_dir: str):
+    """Test Generator._mkdir_file returns correct type and value"""
+    cmd = generator_instance._mkdir_file(pathlib.Path(test_dir))
+    assert isinstance(cmd, Command)
+    assert cmd.command == ["mkdir", "-p", test_dir]
+
 
 def test_copy_file(generator_instance: Generator, fileutils):
     """Test Generator._copy_files helper function with file"""
@@ -353,9 +359,9 @@ def test_generate_ensemble_directory_start(
     jobs_dir_path = pathlib.Path(test_dir) / run_dir[0] / "jobs"
     list_of_job_dirs = jobs_dir_path.iterdir()
     for job in list_of_job_dirs:
-        run_path = jobs_dir_path / job / Generator.run
+        run_path = jobs_dir_path / job / Generator.run_directory
         assert run_path.is_dir()
-        log_path = jobs_dir_path / job / Generator.log
+        log_path = jobs_dir_path / job / Generator.log_directory
         assert log_path.is_dir()
     ids.clear()
 
@@ -378,7 +384,7 @@ def test_generate_ensemble_copy(
     jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs"
     job_dir = jobs_dir.iterdir()
     for ensemble_dir in job_dir:
-        copy_folder_path = jobs_dir / ensemble_dir / Generator.run / "to_copy_dir"
+        copy_folder_path = jobs_dir / ensemble_dir / Generator.run_directory / "to_copy_dir"
         assert copy_folder_path.is_dir()
     ids.clear()
 
@@ -449,8 +455,8 @@ def _check_generated(param_0, param_1, dir):
             line = f.readline()
             assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"'
 
-    _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run)
-    _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run)
-    _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run)
-    _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run)
+    _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run_directory)
+    _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run_directory)
+    _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory)
+    _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory)
     ids.clear()