Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TaskVineExecutor: write function data to temp directory #3592

Merged
merged 7 commits into from
Sep 6, 2024
Merged

TaskVineExecutor: write function data to temp directory #3592

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b75a3ae
function data in temp
colinthomas-z80 Aug 19, 2024
2c18d6c
use getpass for username
colinthomas-z80 Aug 19, 2024
c201ec1
use tempfile module
colinthomas-z80 Aug 20, 2024
9f6b037
flake etc
colinthomas-z80 Aug 20, 2024
5ec7cdb
Merge branch 'master' into tmp_function_data
benclifford Aug 21, 2024
cd7229f
Merge branch 'master' into tmp_function_data
benclifford Sep 4, 2024
bfa9bd9
Merge branch 'master' into tmp_function_data
benclifford Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions parsl/executors/taskvine/executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
high-throughput system for delegating Parsl tasks to thousands of remote machines
"""

import getpass
import hashlib
import inspect
import itertools
Expand All @@ -18,6 +19,7 @@
import threading
import uuid
from concurrent.futures import Future
from datetime import datetime
from typing import List, Literal, Optional, Union

# Import other libraries
Expand Down Expand Up @@ -215,9 +217,9 @@ def __create_data_and_logging_dirs(self):

# Create directories for data and results
log_dir = os.path.join(run_dir, self.label)
self._function_data_dir = os.path.join(run_dir, self.label, "function_data")
os.makedirs(log_dir)
os.makedirs(self._function_data_dir)
tmp_prefix = f'{self.label}-{getpass.getuser()}-{datetime.now().strftime("%Y%m%d%H%M%S%f")}-'
self._function_data_dir = tempfile.TemporaryDirectory(prefix=tmp_prefix)

# put TaskVine logs outside of a Parsl run as TaskVine caches between runs while
# Parsl does not.
Expand All @@ -227,7 +229,7 @@ def __create_data_and_logging_dirs(self):

# factory logs go with manager logs regardless
self.factory_config.scratch_dir = self.manager_config.vine_log_dir
logger.debug(f"Function data directory: {self._function_data_dir}, log directory: {log_dir}")
logger.debug(f"Function data directory: {self._function_data_dir.name}, log directory: {log_dir}")
logger.debug(
f"TaskVine manager log directory: {self.manager_config.vine_log_dir}, "
f"factory log directory: {self.factory_config.scratch_dir}")
Expand Down Expand Up @@ -293,7 +295,7 @@ def _path_in_task(self, executor_task_id, *path_components):
'map': Pickled file with a dict between local parsl names, and remote taskvine names.
"""
task_dir = "{:04d}".format(executor_task_id)
return os.path.join(self._function_data_dir, task_dir, *path_components)
return os.path.join(self._function_data_dir.name, task_dir, *path_components)

def submit(self, func, resource_specification, *args, **kwargs):
"""Processes the Parsl app by its arguments and submits the function
Expand Down
Loading