From c67dc3b76c7dd9981c8ae8de8439bb771ae30062 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Mon, 6 May 2024 22:22:12 -0700 Subject: [PATCH] create test dirs by copying test files from static dir into temp dirs instead of uploading test dirs for job files manager tests --- conftest.py | 7 ++ .../job-1 => static}/helloworld-1.html | 0 .../job-1 => static}/helloworld-1.ipynb | 0 .../job-1 => static}/helloworld.ipynb | 0 .../job-2 => static}/helloworld.tar.gz | Bin .../tests/test_job_files_manager.py | 107 +++++++++++------- 6 files changed, 72 insertions(+), 42 deletions(-) rename jupyter_scheduler/tests/{test_staging_dir/job-1 => static}/helloworld-1.html (100%) rename jupyter_scheduler/tests/{test_staging_dir/job-1 => static}/helloworld-1.ipynb (100%) rename jupyter_scheduler/tests/{test_staging_dir/job-1 => static}/helloworld.ipynb (100%) rename jupyter_scheduler/tests/{test_staging_dir/job-2 => static}/helloworld.tar.gz (100%) diff --git a/conftest.py b/conftest.py index f39a54c74..253b796fd 100644 --- a/conftest.py +++ b/conftest.py @@ -23,6 +23,13 @@ def jp_scheduler_root_dir(tmp_path): return root_dir +@pytest.fixture +def jp_scheduler_output_dir(jp_scheduler_root_dir): + output_dir = jp_scheduler_root_dir / "jobs" + output_dir.mkdir() + return output_dir + + @pytest.fixture def jp_scheduler_staging_dir(jp_data_dir): staging_area = jp_data_dir / "scheduler_staging_area" diff --git a/jupyter_scheduler/tests/test_staging_dir/job-1/helloworld-1.html b/jupyter_scheduler/tests/static/helloworld-1.html similarity index 100% rename from jupyter_scheduler/tests/test_staging_dir/job-1/helloworld-1.html rename to jupyter_scheduler/tests/static/helloworld-1.html diff --git a/jupyter_scheduler/tests/test_staging_dir/job-1/helloworld-1.ipynb b/jupyter_scheduler/tests/static/helloworld-1.ipynb similarity index 100% rename from jupyter_scheduler/tests/test_staging_dir/job-1/helloworld-1.ipynb rename to jupyter_scheduler/tests/static/helloworld-1.ipynb diff --git a/jupyter_scheduler/tests/test_staging_dir/job-1/helloworld.ipynb b/jupyter_scheduler/tests/static/helloworld.ipynb similarity index 100% rename from jupyter_scheduler/tests/test_staging_dir/job-1/helloworld.ipynb rename to jupyter_scheduler/tests/static/helloworld.ipynb diff --git a/jupyter_scheduler/tests/test_staging_dir/job-2/helloworld.tar.gz b/jupyter_scheduler/tests/static/helloworld.tar.gz similarity index 100% rename from jupyter_scheduler/tests/test_staging_dir/job-2/helloworld.tar.gz rename to jupyter_scheduler/tests/static/helloworld.tar.gz diff --git a/jupyter_scheduler/tests/test_job_files_manager.py b/jupyter_scheduler/tests/test_job_files_manager.py index e6fcb5d6d..66a9727b7 100644 --- a/jupyter_scheduler/tests/test_job_files_manager.py +++ b/jupyter_scheduler/tests/test_job_files_manager.py @@ -60,65 +60,88 @@ async def test_copy_from_staging(): ) -HERE = Path(__file__).parent.resolve() -OUTPUTS_DIR = os.path.join(HERE, "test_files_output") +@pytest.fixture +def staging_dir_with_notebook_job(static_test_files_dir, jp_scheduler_staging_dir): + staging_dir = jp_scheduler_staging_dir / "job-1" + job_filenames = ["helloworld-1.ipynb", "helloworld-1.html", "helloworld.ipynb"] + + staged_job_files = [] + staging_dir.mkdir() + for job_filename in job_filenames: + staged_job_file = shutil.copy2(static_test_files_dir / job_filename, staging_dir) + staged_job_files.append(staged_job_file) + + return staged_job_files @pytest.fixture -def clear_outputs_dir(): - yield - shutil.rmtree(OUTPUTS_DIR) - # rmtree() is not synchronous; wait until it has finished running - while os.path.isdir(OUTPUTS_DIR): - time.sleep(0.01) - - -@pytest.mark.parametrize( - "output_formats, output_filenames, staging_paths, output_dir, redownload", - [ - ( - ["ipynb", "html"], - { +def staging_dir_with_tar_job(static_test_files_dir, jp_scheduler_staging_dir): + staging_dir = jp_scheduler_staging_dir / "job-2" + job_tar_file = static_test_files_dir / "helloworld.tar.gz" + + staging_dir.mkdir() + staged_tar_file = shutil.copy2(job_tar_file, staging_dir) + + return staged_tar_file + + +@pytest.fixture +def downloader_parameters( + staging_dir_with_notebook_job, + staging_dir_with_tar_job, + request, + jp_scheduler_output_dir, +): + job_1_ipynb_file_path, job_1_html_file_path, job_1_input_file_path = ( + staging_dir_with_notebook_job + ) + job_2_tar_file_path = staging_dir_with_tar_job + index = request.param + parameters = [ + { + "output_formats": ["ipynb", "html"], + "output_filenames": { "ipynb": "job-1/helloworld-out.ipynb", "html": "job-1/helloworld-out.html", "input": "job-1/helloworld-input.ipynb", }, - { - "ipynb": os.path.join(HERE, "test_staging_dir", "job-1", "helloworld-1.ipynb"), - "html": os.path.join(HERE, "test_staging_dir", "job-1", "helloworld-1.html"), - "input": os.path.join(HERE, "test_staging_dir", "job-1", "helloworld.ipynb"), + "staging_paths": { + "ipynb": job_1_ipynb_file_path, + "html": job_1_html_file_path, + "input": job_1_input_file_path, }, - OUTPUTS_DIR, - False, - ), - ( - ["ipynb", "html"], - { + "output_dir": jp_scheduler_output_dir, + "redownload": False, + }, + { + "output_formats": ["ipynb", "html"], + "output_filenames": { "ipynb": "job-2/helloworld-1.ipynb", "html": "job-2/helloworld-1.html", "input": "job-2/helloworld.ipynb", }, - { - "tar.gz": os.path.join(HERE, "test_staging_dir", "job-2", "helloworld.tar.gz"), + "staging_paths": { + "tar.gz": job_2_tar_file_path, "ipynb": "job-2/helloworld-1.ipynb", "html": "job-2/helloworld-1.html", "input": "job-2/helloworld.ipynb", }, - OUTPUTS_DIR, - False, - ), - ], -) -def test_downloader_download( - clear_outputs_dir, output_formats, output_filenames, staging_paths, output_dir, redownload -): - downloader = Downloader( - output_formats=output_formats, - output_filenames=output_filenames, - staging_paths=staging_paths, - output_dir=output_dir, - redownload=redownload, + "output_dir": jp_scheduler_output_dir, + "redownload": False, + }, + ] + return parameters[index] + + +@pytest.mark.parametrize("downloader_parameters", [0, 1], indirect=True) +def test_downloader_download(downloader_parameters): + output_formats, output_filenames, staging_paths, output_dir = ( + downloader_parameters["output_formats"], + downloader_parameters["output_filenames"], + downloader_parameters["staging_paths"], + downloader_parameters["output_dir"], ) + downloader = Downloader(**downloader_parameters) downloader.download() assert os.path.exists(output_dir)