Skip to content

Commit

Permalink
[HWORKS-559] Specify directory to download logs (#183)
Browse files Browse the repository at this point in the history
  • Loading branch information
gibchikafa authored Nov 13, 2023
1 parent 1d3dd86 commit aae195d
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 5 deletions.
4 changes: 4 additions & 0 deletions python/hopsworks/client/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ class OpenSearchException(Exception):
"""Generic opensearch exception"""


class JobExecutionException(Exception):
"""Generic job executions exception"""


class ExternalClientError(TypeError):
"""Raised when external client cannot be initialized due to missing arguments."""

Expand Down
13 changes: 11 additions & 2 deletions python/hopsworks/engine/execution_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,34 @@
import time
import uuid

from hopsworks.client.exceptions import JobExecutionException


class ExecutionEngine:
def __init__(self, project_id=None):
self._dataset_api = dataset_api.DatasetApi(project_id)
self._execution_api = execution_api.ExecutionsApi(project_id)
self._log = logging.getLogger(__name__)

def download_logs(self, execution):
def download_logs(self, execution, path=None):
"""Download execution logs to current directory
:param execution: execution to download logs for
:type execution: Execution
:param path: path to download the logs
:type path: str
:return: downloaded stdout and stderr log path
:rtype: str, str
:raises: JobExecutionException if path is provided but does not exist
"""
if path is not None and not os.path.exists(path):
raise JobExecutionException("Path {} does not exist".format(path))
elif path is None:
path = os.getcwd()

job_logs_dir = "logs-job-{}-exec-{}_{}".format(
execution.job_name, str(execution.id), str(uuid.uuid4())[:16]
)
download_log_dir = os.path.join(os.getcwd(), job_logs_dir)
download_log_dir = os.path.join(path, job_logs_dir)

if not os.path.exists(download_log_dir):
os.mkdir(download_log_dir)
Expand Down
8 changes: 5 additions & 3 deletions python/hopsworks/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,8 @@ def success(self):
return True
return None

def download_logs(self):
def download_logs(self, path=None):
"""Download stdout and stderr logs for the execution
Example for downloading and printing the logs
```python
Expand All @@ -195,11 +194,14 @@ def download_logs(self):
print(err_fd.read())
```
# Arguments
path: path to download the logs. must be `str`
# Returns
`str`. Path to downloaded log for stdout.
`str`. Path to downloaded log for stderr.
"""
return self._execution_engine.download_logs(self)
return self._execution_engine.download_logs(self, path)

def delete(self):
"""Delete the execution
Expand Down

0 comments on commit aae195d

Please sign in to comment.